@vrtmrz/livesync-commonlib 0.1.0-rc.0

package/dist/replication/journal/JournalSyncCore.js

@@ -0,0 +1,737 @@
+import {
+  LOG_LEVEL_INFO,
+  LOG_LEVEL_NOTICE,
+  LOG_LEVEL_VERBOSE,
+  LOG_LEVEL_DEBUG,
+  DEFAULT_SYNC_PARAMETERS,
+  ProtocolVersions,
+  DOCID_JOURNAL_SYNC_PARAMETERS,
+  E2EEAlgorithms
+} from "../../common/types.js";
+import { Logger } from "../../common/logger.js";
+import {
+  concatUInt8Array,
+  escapeNewLineFromString,
+  setAllItems,
+  unescapeNewLineFromString
+} from "../../common/utils.js";
+import { shareRunningResult } from "octagonal-wheels/concurrency/lock";
+import { wrappedDeflate, wrappedInflate } from "../../pouchdb/compress.js";
+import { CheckPointInfoDefault } from "./JournalSyncTypes.js";
+import {
+  clearHandlers,
+  createSyncParamsHanderForServer,
+  SyncParamsFetchError,
+  SyncParamsNotFoundError,
+  SyncParamsUpdateError
+} from "../SyncParamsHandler.js";
+import { REMOTE_CHUNK_FETCHED } from "../../pouchdb/LiveSyncLocalDB.js";
+import { decryptBinary, encryptBinary } from "octagonal-wheels/encryption/encryption";
+import {
+  encryptBinary as encryptBinaryHKDF,
+  decryptBinary as decryptBinaryHKDF
+} from "octagonal-wheels/encryption/hkdf";
+const RECORD_SPLIT = `
+`;
+const UNIT_SPLIT = ``;
+const te = new TextEncoder();
+function serializeDoc(doc) {
+  if (doc._id.startsWith("h:")) {
+    const data = doc.data;
+    const writeData = escapeNewLineFromString(data);
+    return te.encode(`~${doc._id}${UNIT_SPLIT}${writeData}${RECORD_SPLIT}`);
+  }
+  return te.encode(JSON.stringify(doc) + RECORD_SPLIT);
+}
+class JournalSyncCore {
+  _settings;
+  storage;
+  get db() {
+    return this.env.services.database.localDatabase.localDatabase;
+  }
+  get currentSettings() {
+    return this.env.services.setting.currentSettings();
+  }
+  hash = "";
+  processReplication;
+  batchSize = 100;
+  env;
+  store;
+  requestedStop = false;
+  getInitialSyncParameters() {
+    return Promise.resolve({
+      ...DEFAULT_SYNC_PARAMETERS,
+      protocolVersion: ProtocolVersions.ADVANCED_E2EE,
+      pbkdf2salt: ""
+    });
+  }
+  async getSyncParameters() {
+    try {
+      const downloadedData = await this.storage.download(DOCID_JOURNAL_SYNC_PARAMETERS, true);
+      if (!downloadedData) {
+        throw new SyncParamsNotFoundError(`Missing sync parameters`);
+      }
+      const downloadedSyncParams = JSON.parse(new TextDecoder().decode(downloadedData));
+      return downloadedSyncParams;
+    } catch (ex) {
+      Logger(`Could not retrieve remote sync parameters`, LOG_LEVEL_INFO);
+      throw SyncParamsFetchError.fromError(ex);
+    }
+  }
+  async putSyncParameters(params) {
+    try {
+      const data = new TextEncoder().encode(JSON.stringify(params));
+      if (await this.storage.upload(DOCID_JOURNAL_SYNC_PARAMETERS, data, "application/json")) {
+        return true;
+      }
+      throw new SyncParamsUpdateError(`Could not store remote sync parameters`);
+    } catch (ex) {
+      Logger(`Could not upload sync parameters`, LOG_LEVEL_INFO);
+      Logger(ex, LOG_LEVEL_VERBOSE);
+      throw SyncParamsUpdateError.fromError(ex);
+    }
+  }
+  getHash(settings) {
+    return btoa(
+      encodeURI([settings.endpoint, `${settings.bucket}${settings.bucketPrefix}`, settings.region].join())
+    );
+  }
+  constructor(settings, store, env, storage) {
+    this._settings = settings;
+    this.env = env;
+    this.processReplication = async (docs) => await env.services.replication.parseSynchroniseResult(docs);
+    this.store = store;
+    this.hash = this.getHash(settings);
+    this.storage = storage;
+    clearHandlers();
+  }
+  async downloadJson(key) {
+    try {
+      const data = await this.storage.download(key, true);
+      if (!data) return false;
+      return JSON.parse(new TextDecoder().decode(data));
+    } catch (ex) {
+      Logger(`Could not download json ${key}`);
+      Logger(ex, LOG_LEVEL_VERBOSE);
+      return false;
+    }
+  }
+  async uploadJson(key, body) {
+    try {
+      const data = new TextEncoder().encode(JSON.stringify(body));
+      return await this.storage.upload(key, data, "application/json");
+    } catch (ex) {
+      Logger(`Could not upload json ${key}`);
+      Logger(ex, LOG_LEVEL_VERBOSE);
+      return false;
+    }
+  }
+  applyNewConfig(settings, store, env) {
+    this._settings = settings;
+    this.env = env;
+    this.processReplication = async (docs) => await env.services.replication.parseSynchroniseResult(docs);
+    this.store = store;
+    this.hash = this.getHash(settings);
+    this.storage.applyNewConfig(settings);
+    clearHandlers();
+  }
+  updateInfo(info) {
+    const old = this.env.services.replicator.replicationStatics.value;
+    this.env.services.replicator.replicationStatics.value = {
+      sent: info.sent ?? old.sent,
+      arrived: info.arrived ?? old.arrived,
+      maxPullSeq: info.maxPullSeq ?? old.maxPullSeq,
+      maxPushSeq: info.maxPushSeq ?? old.maxPushSeq,
+      lastSyncPullSeq: info.lastSyncPullSeq ?? old.lastSyncPullSeq,
+      lastSyncPushSeq: info.lastSyncPushSeq ?? old.lastSyncPushSeq,
+      syncStatus: info.syncStatus ?? old.syncStatus
+    };
+  }
+  async updateCheckPointInfo(func) {
+    const checkPointKey = `bucketsync-checkpoint-${this.hash}`;
+    const old = await this.getCheckpointInfo();
+    const newInfo = func(old);
+    this._currentCheckPointInfo = newInfo;
+    await this.store.set(checkPointKey, newInfo);
+    return newInfo;
+  }
+  _currentCheckPointInfo = { ...CheckPointInfoDefault };
+  async getCheckpointInfo() {
+    const checkPointKey = `bucketsync-checkpoint-${this.hash}`;
+    const old = await this.store.get(checkPointKey) || {};
+    const items = ["knownIDs", "sentIDs", "receivedFiles", "sentFiles"];
+    for (const key of items) {
+      if (!(key in old)) {
+        continue;
+      }
+      const value = old[key];
+      if (value instanceof Set) {
+        continue;
+      }
+      if (Array.isArray(value)) {
+        old[key] = new Set(value);
+        continue;
+      }
+      if (value && typeof value === "object") {
+        old[key] = new Set(Object.keys(value));
+        continue;
+      }
+      old[key] = /* @__PURE__ */ new Set();
+    }
+    this._currentCheckPointInfo = { ...CheckPointInfoDefault, ...old };
+    return this._currentCheckPointInfo;
+  }
+  resetAllCaches() {
+    clearHandlers();
+  }
+  async resetCheckpointInfo() {
+    await this.updateCheckPointInfo((info) => ({ ...CheckPointInfoDefault }));
+    clearHandlers();
+  }
+  getJournalEpochFromSyncParams(params) {
+    return `${params.protocolVersion}:${params.pbkdf2salt}`;
+  }
+  async ensureCheckpointCachesAreFresh() {
+    let journalEpoch = "";
+    try {
+      const params = await this.getSyncParameters();
+      journalEpoch = this.getJournalEpochFromSyncParams(params);
+    } catch {
+      return;
+    }
+    const current = await this.getCheckpointInfo();
+    if (current.journalEpoch === journalEpoch) {
+      return;
+    }
+    const lastSentFile = [...current.sentFiles].sort().pop();
+    if (!lastSentFile) {
+      await this.updateCheckPointInfo((info) => ({ ...info, journalEpoch }));
+      return;
+    }
+    let remoteWipeConfirmed;
+    try {
+      const probe = await this.storage.listFiles(lastSentFile.slice(0, -1), 1);
+      remoteWipeConfirmed = probe[0] !== lastSentFile;
+    } catch {
+      remoteWipeConfirmed = true;
+    }
+    if (!remoteWipeConfirmed) {
+      await this.updateCheckPointInfo((info) => ({ ...info, journalEpoch }));
+      Logger(`Journal epoch changed (remote files still present). Epoch updated; caches kept.`, LOG_LEVEL_NOTICE);
+      return;
+    }
+    Logger(`Journal epoch changed and remote wipe confirmed. Clearing dedupe caches.`, LOG_LEVEL_NOTICE);
+    await this.updateCheckPointInfo((info) => ({
+      ...info,
+      journalEpoch,
+      knownIDs: /* @__PURE__ */ new Set(),
+      sentIDs: /* @__PURE__ */ new Set(),
+      receivedFiles: /* @__PURE__ */ new Set(),
+      sentFiles: /* @__PURE__ */ new Set()
+    }));
+    clearHandlers();
+  }
+  async isAvailable() {
+    return await this.storage.isAvailable();
+  }
+  async resetBucket() {
+    let files = [];
+    try {
+      do {
+        files = await this.storage.listFiles("", 100);
+        if (files.length == 0) {
+          break;
+        }
+        await this.storage.deleteFiles(files);
+      } while (files.length != 0);
+      clearHandlers();
+    } catch (ex) {
+      Logger(`WARNING! Could not delete files.`, LOG_LEVEL_NOTICE, "reset-bucket");
+      Logger(ex, LOG_LEVEL_VERBOSE);
+    }
+    const journals = await this._getRemoteJournals();
+    if (journals.length == 0) {
+      Logger("Nothing to delete!", LOG_LEVEL_NOTICE);
+      return true;
+    }
+    await this.storage.deleteFiles(journals);
+    Logger(`${journals.length} items has been deleted!`, LOG_LEVEL_NOTICE);
+    await this.resetCheckpointInfo();
+    return true;
+  }
+  getRemoteKey() {
+    return this.getHash(this._settings);
+  }
+  async getReplicationPBKDF2Salt(refresh) {
+    const server = this.getRemoteKey();
+    const manager = createSyncParamsHanderForServer(server, {
+      put: (params) => this.putSyncParameters(params),
+      get: () => this.getSyncParameters(),
+      create: () => this.getInitialSyncParameters()
+    });
+    return await manager.getPBKDF2Salt(refresh);
+  }
+  isEncryptionPrevented(fileName) {
+    if (fileName.endsWith(DOCID_JOURNAL_SYNC_PARAMETERS)) return true;
+    return false;
+  }
+  async decryptDataV2(encrypted, set) {
+    const salt = await this.getReplicationPBKDF2Salt();
+    return await decryptBinaryHKDF(encrypted, set.passphrase, salt);
+  }
+  async decryptDataV1(encrypted, set) {
+    return await decryptBinary(
+      encrypted,
+      set.passphrase,
+      set.useDynamicIterationCount
+    );
+  }
+  async decryptDownloaded(key, encrypted, set) {
+    const u = new Uint8Array(encrypted);
+    try {
+      if (!set.encrypt || set.passphrase == "" || this.isEncryptionPrevented(key)) {
+        return u;
+      }
+      if (set.E2EEAlgorithm === E2EEAlgorithms.ForceV1) {
+        return await this.decryptDataV1(u, set);
+      }
+      const decrypted = await this.decryptDataV2(u, set);
+      return decrypted;
+    } catch (ex) {
+      Logger(`Failed to decrypt in v2. Falling back to v1: ${key}`, LOG_LEVEL_INFO);
+      try {
+        const r = await this.decryptDataV1(u, set);
+        Logger(`Decrypted in v1: ${key}`, LOG_LEVEL_VERBOSE);
+        return r;
+      } catch (ex2) {
+        Logger(`Could not decrypt in v1: ${key}`, LOG_LEVEL_VERBOSE);
+        Logger(ex, LOG_LEVEL_VERBOSE);
+        Logger(ex2, LOG_LEVEL_VERBOSE);
+        throw ex2;
+      }
+    }
+  }
+  async encryptForUpload(key, data, set) {
+    if (!set.encrypt || set.passphrase == "" || this.isEncryptionPrevented(key)) {
+      return data;
+    }
+    if (set.E2EEAlgorithm === E2EEAlgorithms.V2) {
+      const salt = await this.getReplicationPBKDF2Salt();
+      return await encryptBinaryHKDF(data, set.passphrase, salt);
+    } else {
+      return await encryptBinary(data, set.passphrase, set.useDynamicIterationCount);
+    }
+  }
+  getDocKey(doc) {
+    if (doc && doc._id.startsWith("h:")) {
+      return doc._id;
+    }
+    return doc._id + "-" + doc._rev;
+  }
+  async _createJournalPack(override) {
+    const checkPointInfo = await this.getCheckpointInfo();
+    const from = override || checkPointInfo.lastLocalSeq;
+    Logger(`Journal reading from seq:${from}`, LOG_LEVEL_VERBOSE);
+    let knownKeyCount = 0;
+    const allChangesTask = this.db.changes({
+      live: false,
+      since: override || from,
+      conflicts: true,
+      limit: this.batchSize,
+      return_docs: true,
+      attachments: false,
+      style: "all_docs"
+      // NOTE: Do NOT add a filter function here that tests the winning-revision doc.
+      // With style:"all_docs", each change entry can carry multiple leaf revisions
+      // (e.g. the winner plus a newly-created tombstone for a resolved conflict).
+      // A filter based on the winner's key would incorrectly suppress the entire entry
+      // even when one of the other leaf revisions (e.g. the tombstone) has never been
+      // sent.  Per-revision deduplication is handled correctly by the second filter
+      // applied after bulkGet below.
+    });
+    const allChanges = await allChangesTask;
+    if (allChanges.results.length == 0) {
+      return { changes: [], hasNext: false, packLastSeq: allChanges.last_seq };
+    }
+    const bd = await this.db.bulkGet({
+      docs: allChanges.results.map((e) => e.changes.map((change) => ({ id: e.id, rev: change.rev }))).flat(),
+      revs: true
+    });
+    const packLastSeq = allChanges.last_seq;
+    const dbInfo = await this.db.info();
+    const hasNext = packLastSeq < dbInfo.update_seq;
+    const docs = bd.results.map((e) => e.docs).flat();
+    const docChanges = docs.filter((e) => "ok" in e).map((e) => e.ok).filter((doc) => {
+      const key = this.getDocKey(doc);
+      if (this._currentCheckPointInfo.knownIDs.has(key)) {
+        knownKeyCount++;
+        return false;
+      }
+      if (this._currentCheckPointInfo.sentIDs.has(key)) {
+        knownKeyCount++;
+        return false;
+      }
+      return true;
+    });
+    Logger(
+      `Checked ${allChanges.results.length} changed entries, selected ${docChanges.length} docs (${knownKeyCount} keys already known)`,
+      LOG_LEVEL_DEBUG
+    );
+    return { changes: docChanges, hasNext, packLastSeq };
+  }
+  _createSendReadableStream(startSeq, logLevel, MSG_KEY) {
+    let currentLastSeq = startSeq;
+    return new ReadableStream({
+      pull: async (controller) => {
+        while (true) {
+          if (this.requestedStop) {
+            Logger("Packing Journal : Stop requested", logLevel, MSG_KEY);
+            controller.close();
+            return;
+          }
+          const { changes, hasNext, packLastSeq } = await this._createJournalPack(currentLastSeq);
+          currentLastSeq = packLastSeq;
+          if (changes.length > 0) {
+            controller.enqueue({ changes, packLastSeq });
+            return;
+          }
+          if (!hasNext) {
+            controller.close();
+            return;
+          }
+        }
+      }
+    });
+  }
+  _createSendCompressTransformStream(startSeq, seqToProcess, logLevel, MSG_KEY, stats) {
+    const maxOutBufLength = 250;
+    const maxBinarySize = 1024 * 1024 * 10;
+    let outBuf = [];
+    let binarySize = 0;
+    let batchSentIDs = [];
+    let lastProcessedSeq = startSeq;
+    return new TransformStream({
+      transform: async (chunk, controller) => {
+        lastProcessedSeq = chunk.packLastSeq;
+        const currentSeq = lastProcessedSeq - startSeq;
+        Logger(`Packing Journal: ${currentSeq} / ${seqToProcess}`, logLevel, MSG_KEY);
+        for (const row of chunk.changes) {
+          const serialized = serializeDoc(row);
+          batchSentIDs.push(this.getDocKey(row));
+          binarySize += serialized.length;
+          outBuf.push(serialized);
+          stats.packedDocs++;
+          if (outBuf.length > maxOutBufLength || binarySize > maxBinarySize) {
+            const sendBuf = concatUInt8Array(outBuf);
+            const bin = await wrappedDeflate(sendBuf, { consume: true, level: 8 });
+            controller.enqueue({ bin, packLastSeq: chunk.packLastSeq, sentIDs: [...batchSentIDs] });
+            outBuf = [];
+            binarySize = 0;
+            batchSentIDs = [];
+          }
+        }
+      },
+      flush: async (controller) => {
+        if (outBuf.length > 0) {
+          const sendBuf = concatUInt8Array(outBuf);
+          const bin = await wrappedDeflate(sendBuf, { consume: true, level: 8 });
+          controller.enqueue({ bin, packLastSeq: lastProcessedSeq, sentIDs: [...batchSentIDs] });
+        }
+      }
+    });
+  }
+  _createSendUploadWritableStream(max, logLevel, MSG_KEY, stats) {
+    let sentFilesCount = 0;
+    return new WritableStream({
+      write: async (chunk) => {
+        const sendTimeStamp = Date.now();
+        const filename = `${sendTimeStamp}-docs.jsonl.gz`;
+        const mime = "application/octet-stream";
+        const encryptedBin = await this.encryptForUpload(filename, chunk.bin, this.currentSettings);
+        const ret = await this.storage.upload(filename, encryptedBin, mime);
+        if (!ret) {
+          throw new Error(`Could not send journalPack to the bucket (${filename})`);
+        }
+        sentFilesCount++;
+        stats.uploadedFiles++;
+        this.updateInfo({
+          sent: sentFilesCount,
+          maxPushSeq: max,
+          lastSyncPushSeq: chunk.packLastSeq
+        });
+        await this.updateCheckPointInfo((info) => ({
+          ...info,
+          lastLocalSeq: chunk.packLastSeq,
+          sentIDs: setAllItems(info.sentIDs, chunk.sentIDs),
+          sentFiles: info.sentFiles.add(filename)
+        }));
+        Logger(`Uploading journal: ${sentFilesCount} / ...`, logLevel, MSG_KEY);
+      }
+    });
+  }
+  async sendLocalJournal(showMessage = false) {
+    this.updateInfo({ syncStatus: "JOURNAL_SEND" });
+    return await shareRunningResult("send_journal_stream", async () => {
+      this.requestedStop = false;
+      const logLevel = showMessage ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO;
+      const MSG_KEY = "pack_journal";
+      const max = (await this.db.info()).update_seq;
+      const checkPointInfo = await this.getCheckpointInfo();
+      const startSeq = checkPointInfo.lastLocalSeq;
+      const seqToProcess = max - startSeq;
+      Logger(`Packing Journal: Start sending`, logLevel, MSG_KEY);
+      const stats = { packedDocs: 0, uploadedFiles: 0 };
+      const readable = this._createSendReadableStream(startSeq, logLevel, MSG_KEY);
+      const transform = this._createSendCompressTransformStream(startSeq, seqToProcess, logLevel, MSG_KEY, stats);
+      const writable = this._createSendUploadWritableStream(max, logLevel, `${MSG_KEY}_upload`, stats);
+      try {
+        await readable.pipeThrough(transform).pipeTo(writable);
+        if (seqToProcess != 0) {
+          Logger(
+            `Packing Journal: Finished. Processed ${stats.packedDocs} doc(s) into ${stats.uploadedFiles} chunk(s)`,
+            logLevel,
+            MSG_KEY
+          );
+          if (stats.uploadedFiles > 0) {
+            Logger(
+              `Uploading journal: All ${stats.uploadedFiles} chunk(s) uploaded`,
+              logLevel,
+              `${MSG_KEY}_upload`
+            );
+          }
+        } else {
+          Logger(`Packing Journal: No journals to be packed!`, logLevel, MSG_KEY);
+        }
+        this.updateInfo({ syncStatus: "COMPLETED" });
+        return true;
+      } catch (ex) {
+        Logger(`Packing Journal Error`, logLevel);
+        Logger(ex, LOG_LEVEL_VERBOSE);
+        this.updateInfo({ syncStatus: "ERRORED" });
+        return false;
+      }
+    });
+  }
+  async _getRemoteJournals() {
+    const checkPointInfo = await this.getCheckpointInfo();
+    const StartAfter = [...checkPointInfo.receivedFiles.keys()].sort(
+      (a, b) => b.localeCompare(a, void 0, { numeric: true })
+    )[0];
+    const files = (await this.storage.listFiles(StartAfter)).filter((e) => !e.startsWith("_"));
+    if (!files) return [];
+    return files.sort((a, b) => a.localeCompare(b, void 0, { numeric: true }));
+  }
+  async processDocuments(allDocs) {
+    let applyTotal = 0;
+    let wholeItems = 0;
+    try {
+      const chunks = [];
+      const docs = [];
+      allDocs.forEach((e) => {
+        if (e._id.startsWith("h:")) {
+          chunks.push(e);
+        } else {
+          docs.push(e);
+        }
+      });
+      try {
+        const e1 = (await this.db.allDocs({ include_docs: true, keys: [...chunks.map((e) => e._id)] })).rows;
+        const e2 = e1.map((e) => e.id ?? void 0);
+        const existChunks = new Set(e2.filter((e) => e !== void 0));
+        const saveChunks = chunks.filter((e) => !existChunks.has(e._id)).map((e) => ({ ...e, _rev: void 0 }));
+        const ret = await this.db.bulkDocs(saveChunks, { new_edits: true });
+        const saveError = ret.filter((e) => "error" in e).map((e) => e.id);
+        saveChunks.filter((e) => saveError.indexOf(e._id) === -1).forEach(
+          (doc) => this.env.services.context.events.emitEvent(REMOTE_CHUNK_FETCHED, doc)
+        );
+        await this.updateCheckPointInfo((info) => ({
+          ...info,
+          knownIDs: setAllItems(
+            info.knownIDs,
+            chunks.map((e) => this.getDocKey(e))
+          )
+        }));
+      } catch (ex) {
+        Logger(`Applying chunks failed`, LOG_LEVEL_INFO);
+        Logger(ex, LOG_LEVEL_VERBOSE);
+      }
+      const params = docs.map((e) => [e._id, [e._rev]]);
+      const docsRevs = params.reduce(
+        (acc, [id, revs]) => {
+          return { ...acc, [id]: [...acc[id] ?? [], ...revs] };
+        },
+        {}
+      );
+      const diffRevs = await this.db.revsDiff(docsRevs);
+      const saveDocs = docs.filter(
+        (e) => e._id in diffRevs && "missing" in diffRevs[e._id] && (diffRevs[e._id].missing?.indexOf(e._rev) ?? 0) !== -1
+      );
+      await this.db.bulkDocs(saveDocs, { new_edits: false });
+      const writeDoc = !this.env.services.setting.currentSettings().suspendParseReplicationResult;
+      if (writeDoc) {
+        await this.processReplication(saveDocs);
+      }
+      await this.updateCheckPointInfo((info) => ({
+        ...info,
+        knownIDs: setAllItems(
+          info.knownIDs,
+          docs.map((e) => this.getDocKey(e))
+        )
+      }));
+      applyTotal += saveDocs.length;
+      wholeItems += docs.length;
+      Logger(
+        `Applied ${applyTotal} of ${wholeItems} docs (${wholeItems - applyTotal} skipped)`,
+        LOG_LEVEL_VERBOSE
+      );
+      return true;
+    } catch (ex) {
+      Logger(`Applying journal failed`, LOG_LEVEL_INFO);
+      Logger(ex, LOG_LEVEL_VERBOSE);
+      return false;
+    }
+  }
+  _createReceiveReadableStream(files) {
+    return new ReadableStream({
+      pull: (controller) => {
+        if (this.requestedStop || files.length === 0) {
+          controller.close();
+          return;
+        }
+        const file = files.shift();
+        if (file) {
+          controller.enqueue(file);
+        }
+      }
+    });
+  }
+  _createReceiveTransformStream(logLevel) {
+    let count = 0;
+    return new TransformStream({
+      transform: async (key, controller) => {
+        count++;
+        Logger(`Receiving Journal: ${count}`, logLevel, "receivejournal");
+        const checkPointInfo = await this.getCheckpointInfo();
+        if (checkPointInfo.sentFiles.has(key) || checkPointInfo.receivedFiles.has(key)) {
+          Logger(`Receiving Journal: ${key} is already processed`, LOG_LEVEL_VERBOSE);
+          await this.updateCheckPointInfo((info) => ({
+            ...info,
+            receivedFiles: info.receivedFiles.add(key)
+          }));
+          return;
+        }
+        try {
+          const encryptedData = await this.storage.download(key, true);
+          if (encryptedData === false) {
+            throw new Error("Download Error");
+          }
+          const data = await this.decryptDownloaded(
+            key,
+            encryptedData,
+            this.currentSettings
+          );
+          const decompressed = await wrappedInflate(new Uint8Array(data), { consume: true });
+          if (decompressed.length == 0) {
+            controller.enqueue({ key, docs: [] });
+            return;
+          }
+          let idxFrom = 0;
+          let idxTo = 0;
+          const d = new TextDecoder();
+          const result = [];
+          do {
+            idxTo = decompressed.indexOf(10, idxFrom);
+            if (idxTo == -1) break;
+            const piece = decompressed.slice(idxFrom, idxTo);
+            const strPiece = d.decode(piece);
+            if (strPiece.startsWith("~")) {
+              const [idPart, dataPart] = strPiece.substring(1).split(UNIT_SPLIT);
+              result.push({
+                _id: idPart,
+                data: unescapeNewLineFromString(dataPart),
+                type: "leaf",
+                _rev: ""
+              });
+            } else {
+              result.push(JSON.parse(strPiece));
+            }
+            idxFrom = idxTo + 1;
+          } while (idxTo > 0);
+          controller.enqueue({ key, docs: result });
+        } catch (ex) {
+          controller.error(ex);
+        }
+      }
+    });
+  }
+  _createReceiveWritableStream() {
+    let downloaded = 0;
+    return new WritableStream({
+      write: async (chunk) => {
+        const { key, docs } = chunk;
+        if (docs.length > 0) {
+          const success = await this.processDocuments(docs);
+          if (!success) {
+            throw new Error(`Could not process downloaded journals for ${key}`);
+          }
+        }
+        await this.updateCheckPointInfo((info) => ({
+          ...info,
+          receivedFiles: info.receivedFiles.add(key)
+        }));
+        downloaded++;
+        this.updateInfo({ arrived: downloaded, maxPullSeq: downloaded, lastSyncPullSeq: downloaded });
+        Logger(`Processing journal: ${key} has been processed`, LOG_LEVEL_INFO);
+      }
+    });
+  }
+  async receiveRemoteJournal(showMessage = false) {
+    this.updateInfo({ syncStatus: "JOURNAL_RECEIVE" });
+    return await shareRunningResult("receive_journal_stream", async () => {
+      this.requestedStop = false;
+      const logLevel = showMessage ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO;
+      Logger("Receiving Journal: Getting list of remote journal", logLevel, "receivejournal");
+      const files = await this._getRemoteJournals();
+      if (files.length == 0) {
+        Logger(`Receiving Journal: No journals needs to be downloaded`, logLevel, "receivejournal");
+        this.updateInfo({ syncStatus: "COMPLETED" });
+        return true;
+      }
+      const readable = this._createReceiveReadableStream(files);
+      const transform = this._createReceiveTransformStream(logLevel);
+      const writable = this._createReceiveWritableStream();
+      try {
+        await readable.pipeThrough(transform).pipeTo(writable);
+        this.updateInfo({ syncStatus: "COMPLETED" });
+        return true;
+      } catch (ex) {
+        Logger(`Receive Journal Error`, logLevel);
+        Logger(ex, LOG_LEVEL_VERBOSE);
+        this.updateInfo({ syncStatus: "ERRORED" });
+        return false;
+      }
+    });
+  }
+  async sync(showResult = false) {
+    return await shareRunningResult("replicate", async () => {
+      this.requestedStop = false;
+      const receiveResult = await this.receiveRemoteJournal(showResult);
+      if (this.requestedStop) return;
+      if (!receiveResult) {
+        const logLevel = showResult ? LOG_LEVEL_NOTICE : LOG_LEVEL_INFO;
+        Logger(
+          `Could not receive remote journal, so we prevent sending local journals to prevent unwanted mass transfers`,
+          logLevel
+        );
+        return;
+      }
+      return await this.sendLocalJournal(showResult);
+    }) ?? false;
+  }
+  requestStop() {
+    this.requestedStop = true;
+  }
+}
+export {
+  JournalSyncCore
+};