@loro-dev/flock 3.1.0 → 4.1.0

package/src/index.ts

@@ -1,6 +1,7 @@
 import {
   newFlock,
   get_ffi,
+  get_entry_ffi,
   put_json_ffi,
   put_with_meta_ffi,
   delete_ffi,
@@ -8,7 +9,9 @@ import {
   set_peer_id,
   export_json_ffi,
   import_json_ffi,
+  import_json_str_ffi,
   version_ffi,
+  inclusiveVersion_ffi,
   get_max_physical_time_ffi,
   peer_id_ffi,
   kv_to_json_ffi,
@@ -32,6 +35,16 @@ type RawEventEntry = {
   payload?: RawEventPayload;
 };
 type RawEventBatch = { source?: string; events?: RawEventEntry[] };
+type RawEntryClock = {
+  physicalTime?: number;
+  logicalCounter?: number;
+  peerId?: string;
+};
+type RawEntryInfo = {
+  data?: Value;
+  metadata?: MetadataMap;
+  clock?: RawEntryClock;
+};
 
 type MaybePromise<T> = T | Promise<T>;
 
@@ -57,7 +70,17 @@ export type VersionVectorEntry = {
   logicalCounter: number;
 };
 
-export type VersionVector = Record<string, VersionVectorEntry>;
+export interface VersionVector {
+  [peer: string]: VersionVectorEntry | undefined;
+}
+
+export function encodeVersionVector(vector: VersionVector): Uint8Array {
+  return encodeVersionVectorBinary(vector);
+}
+
+export function decodeVersionVector(bytes: Uint8Array): VersionVector {
+  return decodeVersionVectorBinary(bytes);
+}
 
 export type Value =
   | string
@@ -77,7 +100,10 @@ export type ExportRecord = {
   m?: MetadataMap;
 };
 
-export type ExportBundle = { version: number, entries: Record<string, ExportRecord> };
+export type ExportBundle = {
+  version: number;
+  entries: Record<string, ExportRecord>;
+};
 
 export type EntryClock = {
   physicalTime: number;
@@ -85,6 +111,12 @@ export type EntryClock = {
   peerId: string;
 };
 
+export type EntryInfo = {
+  data?: Value;
+  metadata: MetadataMap;
+  clock: EntryClock;
+};
+
 export type ExportPayload = {
   data?: Value;
   metadata?: MetadataMap;
@@ -182,6 +214,7 @@ export type EventBatch = {
 };
 
 const textEncoder = new TextEncoder();
+const textDecoder = new TextDecoder();
 
 function utf8ByteLength(value: string): number {
   return textEncoder.encode(value).length;
@@ -193,7 +226,10 @@ function isValidPeerId(peerId: unknown): peerId is string {
 
 function createRandomPeerId(): string {
   const id = new Uint8Array(32);
-  if (typeof crypto !== "undefined" && typeof crypto.getRandomValues === "function") {
+  if (
+    typeof crypto !== "undefined" &&
+    typeof crypto.getRandomValues === "function"
+  ) {
     crypto.getRandomValues(id);
   } else {
     for (let i = 0; i < 32; i += 1) {
@@ -208,46 +244,277 @@ function normalizePeerId(peerId?: string): string {
     return createRandomPeerId();
   }
   if (!isValidPeerId(peerId)) {
-    throw new TypeError("peerId must be a UTF-8 string under 128 bytes");
+    throw new TypeError("peerId must be a UTF-8 string under 128 bytes.");
   }
   return peerId;
 }
 
-function encodeVersionVector(vv?: VersionVector): RawVersionVector | undefined {
-  if (!vv) {
-    return undefined;
+type EncodableVersionVectorEntry = {
+  peer: string;
+  peerBytes: Uint8Array;
+  timestamp: number;
+  counter: number;
+};
+
+function comparePeerBytes(a: Uint8Array, b: Uint8Array): number {
+  if (a === b) {
+    return 0;
   }
-  const raw: RawVersionVector = {};
-  for (const [peer, entry] of Object.entries(vv)) {
-    if (!entry) {
-      continue;
+  const limit = Math.min(a.length, b.length);
+  for (let i = 0; i < limit; i += 1) {
+    const diff = a[i] - b[i];
+    if (diff !== 0) {
+      return diff;
     }
-    if (!isValidPeerId(peer)) {
+  }
+  return a.length - b.length;
+}
+
+function collectEncodableVersionVectorEntries(
+  vv?: VersionVector,
+): EncodableVersionVectorEntry[] {
+  if (!vv || typeof vv !== "object") {
+    return [];
+  }
+  const entries: EncodableVersionVectorEntry[] = [];
+  for (const [peer, entry] of Object.entries(vv)) {
+    if (!entry || !isValidPeerId(peer)) {
       continue;
     }
     const { physicalTime, logicalCounter } = entry;
-    if (typeof physicalTime !== "number" || Number.isNaN(physicalTime)) {
+    if (
+      typeof physicalTime !== "number" ||
+      !Number.isFinite(physicalTime) ||
+      typeof logicalCounter !== "number" ||
+      !Number.isFinite(logicalCounter)
+    ) {
       continue;
     }
-    if (!Number.isFinite(logicalCounter)) {
-      continue;
+    const peerBytes = textEncoder.encode(peer);
+    entries.push({
+      peer,
+      peerBytes,
+      timestamp: Math.trunc(physicalTime),
+      counter: Math.max(0, Math.trunc(logicalCounter)),
+    });
+  }
+  entries.sort((a, b) => {
+    if (a.timestamp !== b.timestamp) {
+      return a.timestamp - b.timestamp;
+    }
+    const peerCmp = comparePeerBytes(a.peerBytes, b.peerBytes);
+    if (peerCmp !== 0) {
+      return peerCmp;
     }
-    raw[peer] = [physicalTime, Math.trunc(logicalCounter)];
+    return a.counter - b.counter;
+  });
+  return entries;
+}
+
+function writeUnsignedLeb128(value: number, out: number[]): void {
+  if (!Number.isFinite(value) || value < 0) {
+    throw new TypeError("leb128 values must be finite and non-negative");
+  }
+  let remaining = Math.trunc(value);
+  if (remaining === 0) {
+    out.push(0);
+    return;
+  }
+  while (remaining > 0) {
+    const byte = remaining % 0x80;
+    remaining = Math.floor(remaining / 0x80);
+    out.push(remaining > 0 ? byte | 0x80 : byte);
+  }
+}
+
+function writeVarStringBytes(bytes: Uint8Array, out: number[]): void {
+  writeUnsignedLeb128(bytes.length, out);
+  for (let i = 0; i < bytes.length; i += 1) {
+    out.push(bytes[i]);
+  }
+}
+
+const VERSION_VECTOR_MAGIC = new Uint8Array([86, 69, 86, 69]); // "VEVE"
+
+function encodeVersionVectorBinary(vv?: VersionVector): Uint8Array {
+  const entries = collectEncodableVersionVectorEntries(vv);
+  const buffer: number[] = Array.from(VERSION_VECTOR_MAGIC);
+  if (entries.length === 0) {
+    return Uint8Array.from(buffer);
+  }
+
+  let lastTimestamp = 0;
+  for (let i = 0; i < entries.length; i += 1) {
+    const entry = entries[i];
+    if (entry.timestamp < 0) {
+      throw new TypeError("timestamp must be non-negative");
+    }
+    if (i === 0) {
+      writeUnsignedLeb128(entry.timestamp, buffer);
+      lastTimestamp = entry.timestamp;
+    } else {
+      const delta = entry.timestamp - lastTimestamp;
+      if (delta < 0) {
+        throw new TypeError("version vector timestamps must be non-decreasing");
+      }
+      writeUnsignedLeb128(delta, buffer);
+      lastTimestamp = entry.timestamp;
+    }
+
+    writeUnsignedLeb128(entry.counter, buffer);
+    writeVarStringBytes(entry.peerBytes, buffer);
+  }
+
+  return Uint8Array.from(buffer);
+}
+
+function decodeUnsignedLeb128(
+  bytes: Uint8Array,
+  offset: number,
+): [number, number] {
+  let result = 0;
+  let multiplier = 1;
+  let consumed = 0;
+  while (offset + consumed < bytes.length) {
+    const byte = bytes[offset + consumed];
+    consumed += 1;
+    // Use arithmetic instead of bitwise operations to avoid 32-bit overflow.
+    // JavaScript bitwise operators convert to 32-bit signed integers,
+    // which breaks for values >= 2^31.
+    result += (byte & 0x7f) * multiplier;
+    if ((byte & 0x80) === 0) {
+      break;
+    }
+    multiplier *= 128;
+  }
+  return [result, consumed];
+}
+
+function decodeVarString(bytes: Uint8Array, offset: number): [string, number] {
+  const [length, used] = decodeUnsignedLeb128(bytes, offset);
+  const start = offset + used;
+  const end = start + length;
+  if (end > bytes.length) {
+    throw new TypeError("varString length exceeds buffer");
+  }
+  const slice = bytes.subarray(start, end);
+  return [textDecoder.decode(slice), used + length];
+}
+
+function hasMagic(bytes: Uint8Array): boolean {
+  return (
+    bytes.length >= 4 &&
+    bytes[0] === VERSION_VECTOR_MAGIC[0] &&
+    bytes[1] === VERSION_VECTOR_MAGIC[1] &&
+    bytes[2] === VERSION_VECTOR_MAGIC[2] &&
+    bytes[3] === VERSION_VECTOR_MAGIC[3]
+  );
+}
+
+function decodeLegacyVersionVector(bytes: Uint8Array): VersionVector {
+  let offset = 0;
+  const [count, usedCount] = decodeUnsignedLeb128(bytes, offset);
+  offset += usedCount;
+  const [baseTimestamp, usedBase] = decodeUnsignedLeb128(bytes, offset);
+  offset += usedBase;
+  const vv: VersionVector = {};
+  for (let i = 0; i < count; i += 1) {
+    const [peer, usedPeer] = decodeVarString(bytes, offset);
+    offset += usedPeer;
+    if (!isValidPeerId(peer)) {
+      throw new TypeError("invalid peer id in encoded version vector");
+    }
+    const [delta, usedDelta] = decodeUnsignedLeb128(bytes, offset);
+    offset += usedDelta;
+    const [counter, usedCounter] = decodeUnsignedLeb128(bytes, offset);
+    offset += usedCounter;
+    vv[peer] = {
+      physicalTime: baseTimestamp + delta,
+      logicalCounter: counter,
+    };
+  }
+  return vv;
+}
+
+function decodeNewVersionVector(bytes: Uint8Array): VersionVector {
+  let offset = 4;
+  const vv: VersionVector = {};
+  if (offset === bytes.length) {
+    return vv;
+  }
+
+  const [firstTimestamp, usedTs] = decodeUnsignedLeb128(bytes, offset);
+  offset += usedTs;
+  const [firstCounter, usedCounter] = decodeUnsignedLeb128(bytes, offset);
+  offset += usedCounter;
+  const [firstPeer, usedPeer] = decodeVarString(bytes, offset);
+  offset += usedPeer;
+  if (!isValidPeerId(firstPeer)) {
+    throw new TypeError("invalid peer id in encoded version vector");
+  }
+  vv[firstPeer] = {
+    physicalTime: firstTimestamp,
+    logicalCounter: firstCounter,
+  };
+
+  let lastTimestamp = firstTimestamp;
+  while (offset < bytes.length) {
+    const [delta, usedDelta] = decodeUnsignedLeb128(bytes, offset);
+    offset += usedDelta;
+    const [counter, usedCtr] = decodeUnsignedLeb128(bytes, offset);
+    offset += usedCtr;
+    const [peer, usedPeerLen] = decodeVarString(bytes, offset);
+    offset += usedPeerLen;
+    if (!isValidPeerId(peer)) {
+      throw new TypeError("invalid peer id in encoded version vector");
+    }
+    const timestamp = lastTimestamp + delta;
+    if (timestamp < lastTimestamp) {
+      throw new TypeError("version vector timestamps must be non-decreasing");
+    }
+    vv[peer] = { physicalTime: timestamp, logicalCounter: counter };
+    lastTimestamp = timestamp;
+  }
+
+  return vv;
+}
+
+function decodeVersionVectorBinary(bytes: Uint8Array): VersionVector {
+  if (hasMagic(bytes)) {
+    return decodeNewVersionVector(bytes);
+  }
+  return decodeLegacyVersionVector(bytes);
+}
+
+function encodeVersionVectorForFfi(
+  vv?: VersionVector,
+): RawVersionVector | undefined {
+  if (!vv) {
+    return undefined;
+  }
+  const raw: RawVersionVector = {};
+  for (const entry of collectEncodableVersionVectorEntries(vv)) {
+    raw[entry.peer] = [entry.timestamp, entry.counter];
   }
   return raw;
 }
 
-function normalizePruneBefore(pruneTombstonesBefore?: number): number | undefined {
+function normalizePruneBefore(
+  pruneTombstonesBefore?: number,
+): number | undefined {
   if (pruneTombstonesBefore === undefined) {
     return undefined;
   }
-  if (typeof pruneTombstonesBefore !== "number" || !Number.isFinite(pruneTombstonesBefore)) {
+  if (
+    typeof pruneTombstonesBefore !== "number" ||
+    !Number.isFinite(pruneTombstonesBefore)
+  ) {
     return undefined;
   }
   return pruneTombstonesBefore;
 }
 
-function decodeVersionVector(raw: unknown): VersionVector {
+function decodeVersionVectorFromRaw(raw: unknown): VersionVector {
   if (raw === null || typeof raw !== "object") {
     return {};
   }
@@ -263,7 +530,10 @@ function decodeVersionVector(raw: unknown): VersionVector {
     if (typeof physicalTime !== "number" || !Number.isFinite(physicalTime)) {
       continue;
     }
-    if (typeof logicalCounter !== "number" || !Number.isFinite(logicalCounter)) {
+    if (
+      typeof logicalCounter !== "number" ||
+      !Number.isFinite(logicalCounter)
+    ) {
       continue;
     }
     result[peer] = {
@@ -284,6 +554,23 @@ function encodeBound(bound?: ScanBound): Record<string, unknown> | undefined {
   return { kind: bound.kind, key: bound.key.slice() };
 }
 
+function decodeEntryInfo(raw: unknown): EntryInfo | undefined {
+  if (!raw || typeof raw !== "object") {
+    return undefined;
+  }
+  const info = raw as RawEntryInfo;
+  const clock = normalizeEntryClock(info.clock);
+  if (!clock) {
+    return undefined;
+  }
+  const metadata = normalizeMetadataMap(info.metadata);
+  const result: EntryInfo = { metadata, clock };
+  if ("data" in info) {
+    result.data = cloneJson(info.data as Value);
+  }
+  return result;
+}
+
 function decodeEventBatch(raw: unknown): EventBatch {
   if (!raw || typeof raw !== "object") {
     return { source: "local", events: [] };
@@ -338,8 +625,9 @@ function normalizeRawEventPayload(
   return result;
 }
 
-const structuredCloneFn: (<T>(value: T) => T) | undefined =
-  (globalThis as typeof globalThis & { structuredClone?: <T>(value: T) => T }).structuredClone;
+const structuredCloneFn: (<T>(value: T) => T) | undefined = (
+  globalThis as typeof globalThis & { structuredClone?: <T>(value: T) => T }
+).structuredClone;
 
 function cloneJson<T>(value: T): T {
   if (value === undefined) {
@@ -367,10 +655,16 @@ function cloneMetadata(metadata: unknown): MetadataMap | undefined {
   return cloneJson(metadata as MetadataMap);
 }
 
+function normalizeMetadataMap(metadata: unknown): MetadataMap {
+  const cloned = cloneMetadata(metadata);
+  return cloned ?? {};
+}
+
 function decodeClock(record: ExportRecord): EntryClock {
   const rawClock = typeof record.c === "string" ? record.c : "";
   const firstComma = rawClock.indexOf(",");
-  const secondComma = firstComma === -1 ? -1 : rawClock.indexOf(",", firstComma + 1);
+  const secondComma =
+    firstComma === -1 ? -1 : rawClock.indexOf(",", firstComma + 1);
   if (firstComma === -1 || secondComma === -1) {
     return { physicalTime: 0, logicalCounter: 0, peerId: "" };
   }
@@ -387,6 +681,29 @@ function decodeClock(record: ExportRecord): EntryClock {
   };
 }
 
+function normalizeEntryClock(
+  clock: RawEntryClock | undefined,
+): EntryClock | undefined {
+  if (!clock || typeof clock !== "object") {
+    return undefined;
+  }
+  const { physicalTime, logicalCounter, peerId } = clock;
+  if (typeof physicalTime !== "number" || !Number.isFinite(physicalTime)) {
+    return undefined;
+  }
+  if (typeof logicalCounter !== "number" || !Number.isFinite(logicalCounter)) {
+    return undefined;
+  }
+  if (!isValidPeerId(peerId)) {
+    return undefined;
+  }
+  return {
+    physicalTime,
+    logicalCounter: Math.trunc(logicalCounter),
+    peerId,
+  };
+}
+
 function createExportPayload(record: ExportRecord): ExportPayload {
   const payload: ExportPayload = {};
   if (record.d !== undefined) {
@@ -399,10 +716,7 @@ function createExportPayload(record: ExportRecord): ExportPayload {
   return payload;
 }
 
-function createPutPayload(
-  value: Value,
-  metadata?: MetadataMap,
-): ExportPayload {
+function createPutPayload(value: Value, metadata?: MetadataMap): ExportPayload {
   const payload: ExportPayload = { data: cloneJson(value) };
   const cleanMetadata = cloneMetadata(metadata);
   if (cleanMetadata !== undefined) {
@@ -511,10 +825,7 @@ function isExportOptions(value: unknown): value is ExportOptions {
     value !== null &&
     (Object.prototype.hasOwnProperty.call(value, "hooks") ||
       Object.prototype.hasOwnProperty.call(value, "from") ||
-      Object.prototype.hasOwnProperty.call(
-        value,
-        "pruneTombstonesBefore",
-      ) ||
+      Object.prototype.hasOwnProperty.call(value, "pruneTombstonesBefore") ||
       Object.prototype.hasOwnProperty.call(value, "peerId"))
   );
 }
@@ -564,7 +875,13 @@ export class Flock {
     now?: number,
   ): void {
     const metadataClone = cloneMetadata(metadata);
-    put_with_meta_ffi(this.inner, key, JSON.stringify(value), metadataClone, now);
+    put_with_meta_ffi(
+      this.inner,
+      key,
+      JSON.stringify(value),
+      metadataClone,
+      now,
+    );
   }
 
   private async putWithMetaWithHooks(
@@ -601,15 +918,19 @@ export class Flock {
 
   /**
    * Put a value into the flock. If the given entry already exists, this insert will be skipped.
-   * @param key 
-   * @param value 
-   * @param now 
+   * @param key
+   * @param value
+   * @param now
    */
   put(key: KeyPart[], value: Value, now?: number): void {
     put_json_ffi(this.inner, key, JSON.stringify(value), now);
   }
 
-  putWithMeta(key: KeyPart[], value: Value, options?: PutWithMetaOptions): void | Promise<void> {
+  putWithMeta(
+    key: KeyPart[],
+    value: Value,
+    options?: PutWithMetaOptions,
+  ): void | Promise<void> {
     const opts = options ?? {};
     if (opts.hooks?.transform) {
       return this.putWithMetaWithHooks(key, value, opts);
@@ -623,8 +944,8 @@ export class Flock {
 
   /**
    * Delete a value from the flock. If the given entry does not exist, this delete will be skipped.
-   * @param key 
-   * @param now 
+   * @param key
+   * @param now
    */
   delete(key: KeyPart[], now?: number): void {
     delete_ffi(this.inner, key, now);
@@ -634,12 +955,41 @@ export class Flock {
     return get_ffi(this.inner, key) as Value | undefined;
   }
 
+  /**
+   * Returns the full entry payload (data, metadata, and clock) for a key.
+   *
+   * Unlike `get`, this distinguishes between a missing key (`undefined`) and a
+   * tombstone (returns the clock and metadata with `data` omitted). Metadata is
+   * cloned and defaults to `{}` when absent.
+   */
+  getEntry(key: KeyPart[]): EntryInfo | undefined {
+    const raw = get_entry_ffi(this.inner, key) as RawEntryInfo | undefined;
+    return decodeEntryInfo(raw);
+  }
+
   merge(other: Flock): void {
     merge(this.inner, other.inner);
   }
 
+  /**
+   * Returns the exclusive version vector, which only includes peers that have
+   * at least one entry in the current state. This is consistent with the state
+   * after export and re-import.
+   *
+   * Use this version when sending to other peers for incremental sync.
+   */
   version(): VersionVector {
-    return decodeVersionVector(version_ffi(this.inner));
+    return decodeVersionVectorFromRaw(version_ffi(this.inner));
+  }
+
+  /**
+   * Returns the inclusive version vector, which includes all peers ever seen,
+   * even if their entries have been overridden by other peers.
+   *
+   * Use this version when checking if you have received all data from another peer.
+   */
+  inclusiveVersion(): VersionVector {
+    return decodeVersionVectorFromRaw(inclusiveVersion_ffi(this.inner));
   }
 
   private exportJsonInternal(
@@ -648,16 +998,19 @@ export class Flock {
     peerId?: string,
   ): ExportBundle {
     const pruneBefore = normalizePruneBefore(pruneTombstonesBefore);
-    const normalizedPeerId = peerId !== undefined && isValidPeerId(peerId) ? peerId : undefined;
+    const normalizedPeerId =
+      peerId !== undefined && isValidPeerId(peerId) ? peerId : undefined;
     return export_json_ffi(
       this.inner,
-      encodeVersionVector(from),
+      encodeVersionVectorForFfi(from),
       pruneBefore,
       normalizedPeerId,
     ) as ExportBundle;
   }
 
-  private async exportJsonWithHooks(options: ExportOptions): Promise<ExportBundle> {
+  private async exportJsonWithHooks(
+    options: ExportOptions,
+  ): Promise<ExportBundle> {
     const base = this.exportJsonInternal(
       options.from,
       options.pruneTombstonesBefore,
@@ -684,10 +1037,7 @@ export class Flock {
 
   exportJson(): ExportBundle;
   exportJson(from: VersionVector): ExportBundle;
-  exportJson(
-    from: VersionVector,
-    pruneTombstonesBefore: number,
-  ): ExportBundle;
+  exportJson(from: VersionVector, pruneTombstonesBefore: number): ExportBundle;
   exportJson(options: ExportOptions): Promise<ExportBundle>;
   exportJson(
     arg?: VersionVector | ExportOptions,
@@ -703,11 +1053,15 @@ export class Flock {
   }
 
   private importJsonInternal(bundle: ExportBundle): ImportReport {
-    const report = import_json_ffi(this.inner, bundle) as RawImportReport | undefined;
+    const report = import_json_ffi(this.inner, bundle) as
+      | RawImportReport
+      | undefined;
     return decodeImportReport(report);
   }
 
-  private async importJsonWithHooks(options: ImportOptions): Promise<ImportReport> {
+  private async importJsonWithHooks(
+    options: ImportOptions,
+  ): Promise<ImportReport> {
     const preprocess = options.hooks?.preprocess;
     const working = preprocess ? cloneBundle(options.bundle) : options.bundle;
     const skippedByHooks: Array<{ key: KeyPart[]; reason: string }> = [];
@@ -747,6 +1101,13 @@ export class Flock {
     return this.importJsonInternal(arg);
   }
 
+  importJsonStr(bundle: string): ImportReport {
+    const report = import_json_str_ffi(this.inner, bundle) as
+      | RawImportReport
+      | undefined;
+    return decodeImportReport(report);
+  }
+
   getMaxPhysicalTime(): number {
     return Number(get_max_physical_time_ffi(this.inner));
   }
@@ -787,7 +1148,9 @@ export class Flock {
     const start = encodeBound(options.start);
     const end = encodeBound(options.end);
     const prefix = options.prefix ? options.prefix.slice() : undefined;
-    const rows = scan_ffi(this.inner, start, end, prefix) as RawScanRow[] | undefined;
+    const rows = scan_ffi(this.inner, start, end, prefix) as
+      | RawScanRow[]
+      | undefined;
     if (!Array.isArray(rows)) {
       return [];
     }