@comapeo/core 2.0.1 → 2.2.0

package/src/sync/sync-api.js

@@ -1,4 +1,5 @@
 import { TypedEmitter } from 'tiny-typed-emitter'
+import WebSocket from 'ws'
 import { SyncState } from './sync-state.js'
 import { PeerSyncController } from './peer-sync-controller.js'
 import { Logger } from '../logger.js'
@@ -8,15 +9,25 @@ import {
   PRESYNC_NAMESPACES,
 } from '../constants.js'
 import { ExhaustivenessError, assert, keyToId, noop } from '../utils.js'
+import { getOwn } from '../lib/get-own.js'
+import { wsCoreReplicator } from '../lib/ws-core-replicator.js'
 import { NO_ROLE_ID } from '../roles.js'
 /** @import { CoreOwnership as CoreOwnershipDoc } from '@comapeo/schema' */
+/** @import * as http from 'node:http' */
 /** @import { CoreOwnership } from '../core-ownership.js' */
 /** @import { OpenedNoiseStream } from '../lib/noise-secret-stream-helpers.js' */
+/** @import { BlobFilter, ReplicationStream } from '../types.js' */
 
 export const kHandleDiscoveryKey = Symbol('handle discovery key')
 export const kSyncState = Symbol('sync state')
 export const kRequestFullStop = Symbol('background')
 export const kRescindFullStopRequest = Symbol('foreground')
+export const kWaitForInitialSyncWithPeer = Symbol(
+  'wait for initial sync with peer'
+)
+export const kSetBlobDownloadFilter = Symbol('set isArchiveDevice')
+export const kAddBlobWantRange = Symbol('add blob want range')
+export const kClearBlobWantRanges = Symbol('clear blob want ranges')
 
 /**
  * @typedef {'initial' | 'full'} SyncType
@@ -65,6 +76,7 @@ export class SyncApi extends TypedEmitter {
   /** @type {Map<string, PeerSyncController>} */
   #pscByPeerId = new Map()
   #wantsToSyncData = false
+  #wantsToConnectToServers = false
   #hasRequestedFullStop = false
   /** @type {SyncEnabledState} */
   #previousSyncEnabledState = 'none'
@@ -77,22 +89,41 @@ export class SyncApi extends TypedEmitter {
   /** @type {Map<import('protomux'), Set<Buffer>>} */
   #pendingDiscoveryKeys = new Map()
   #l
+  #getServerWebsocketUrls
+  #getReplicationStream
+  /** @type {Map<string, WebSocket>} */
+  #serverWebsockets = new Map()
+  /** @type {null | BlobFilter} */
+  #blobDownloadFilter = null
 
   /**
-   *
    * @param {object} opts
    * @param {import('../core-manager/index.js').CoreManager} opts.coreManager
    * @param {CoreOwnership} opts.coreOwnership
    * @param {import('../roles.js').Roles} opts.roles
+   * @param {() => Promise<Iterable<string>>} opts.getServerWebsocketUrls
+   * @param {() => ReplicationStream} opts.getReplicationStream
+   * @param {null | BlobFilter} opts.blobDownloadFilter
    * @param {number} [opts.throttleMs]
    * @param {Logger} [opts.logger]
    */
-  constructor({ coreManager, throttleMs = 200, roles, logger, coreOwnership }) {
+  constructor({
+    coreManager,
+    throttleMs = 200,
+    roles,
+    getServerWebsocketUrls,
+    getReplicationStream,
+    logger,
+    coreOwnership,
+    blobDownloadFilter,
+  }) {
     super()
     this.#l = Logger.create('syncApi', logger)
     this.#coreManager = coreManager
     this.#coreOwnership = coreOwnership
     this.#roles = roles
+    this.#getServerWebsocketUrls = getServerWebsocketUrls
+    this.#getReplicationStream = getReplicationStream
     this[kSyncState] = new SyncState({
       coreManager,
       throttleMs,
@@ -104,6 +135,8 @@ export class SyncApi extends TypedEmitter {
       this.#updateState(namespaceSyncState)
     })
 
+    this[kSetBlobDownloadFilter](blobDownloadFilter)
+
     this.#coreManager.creatorCore.on('peer-add', this.#handlePeerAdd)
     this.#coreManager.creatorCore.on('peer-remove', this.#handlePeerDisconnect)
 
@@ -123,6 +156,37 @@ export class SyncApi extends TypedEmitter {
       .catch(noop)
   }
 
+  /** @param {import('../types.js').BlobFilter | null} blobDownloadFilter */
+  [kSetBlobDownloadFilter](blobDownloadFilter) {
+    this.#blobDownloadFilter = blobDownloadFilter
+    if (!blobDownloadFilter) return // No download intents = intend to download everything
+    for (const peer of this.#coreManager.creatorCore.peers) {
+      this.#coreManager.sendDownloadIntents(blobDownloadFilter, peer)
+    }
+  }
+
+  /**
+   * Add some blob blocks this peer wants.
+   *
+   * @param {string} peerId
+   * @param {number} start
+   * @param {number} length
+   * @returns {void}
+   */
+  [kAddBlobWantRange](peerId, start, length) {
+    this[kSyncState].addBlobWantRange(peerId, start, length)
+  }
+
+  /**
+   * Clear the blob blocks this peer wants.
+   *
+   * @param {string} peerId
+   * @returns {void}
+   */
+  [kClearBlobWantRanges](peerId) {
+    this[kSyncState].clearBlobWantRanges(peerId)
+  }
+
   /** @type {import('../local-peers.js').LocalPeersEvents['discovery-key']} */
   [kHandleDiscoveryKey](discoveryKey, protomux) {
     const peerSyncController = this.#peerSyncControllers.get(protomux)
@@ -274,6 +338,74 @@ export class SyncApi extends TypedEmitter {
     this.emit('sync-state', this.#getState(namespaceSyncState))
   }
 
+  /**
+   * @returns {void}
+   */
+  connectServers() {
+    this.#wantsToConnectToServers = true
+
+    this.#getServerWebsocketUrls()
+      .then((urls) => {
+        const hasDisconnectedSinceWebsocketUrlsRequestFinished =
+          !this.#wantsToConnectToServers
+        if (hasDisconnectedSinceWebsocketUrlsRequestFinished) return
+
+        for (const url of urls) {
+          const existingWebsocket = this.#serverWebsockets.get(url)
+          if (
+            existingWebsocket &&
+            (existingWebsocket.readyState === WebSocket.OPEN ||
+              existingWebsocket.readyState === WebSocket.CONNECTING)
+          ) {
+            continue
+          }
+
+          const websocket = new WebSocket(url)
+
+          /** @param {Error} err */
+          const onWebsocketError = (err) => {
+            this.#l.log('Ignoring WebSocket error to %s: %o', url, err)
+          }
+          websocket.on('error', onWebsocketError)
+
+          /**
+           * @param {unknown} _req
+           * @param {http.IncomingMessage} res
+           */
+          const onWebsocketUnexpectedResponse = (_req, res) => {
+            this.#l.log(
+              'Ignoring unexpected %d WebSocket response to %s',
+              res.statusCode,
+              url
+            )
+          }
+          websocket.on('unexpected-response', onWebsocketUnexpectedResponse)
+
+          const replicationStream = this.#getReplicationStream()
+          wsCoreReplicator(websocket, replicationStream)
+
+          this.#serverWebsockets.set(url, websocket)
+          websocket.once('close', () => {
+            websocket.off('error', onWebsocketError)
+            websocket.off('unexpected-response', onWebsocketUnexpectedResponse)
+            this.#serverWebsockets.delete(url)
+          })
+        }
+      })
+      .catch(noop)
+  }
+
+  /**
+   * @returns {void}
+   */
+  disconnectServers() {
+    for (const websocket of this.#serverWebsockets.values()) {
+      websocket.close()
+    }
+    this.#serverWebsockets.clear()
+    this.#wantsToConnectToServers = false
+  }
+
   /**
    * Start syncing data cores.
    *
@@ -348,6 +480,40 @@ export class SyncApi extends TypedEmitter {
     })
   }
 
+  /**
+   * @param {string} deviceId
+   * @param {AbortSignal} abortSignal
+   * @returns {Promise<void>}
+   */
+  async [kWaitForInitialSyncWithPeer](deviceId, abortSignal) {
+    abortSignal.throwIfAborted()
+
+    const state = this[kSyncState].getState()
+    if (isInitiallySyncedWithPeer(state, deviceId)) return
+
+    return new Promise((resolve, reject) => {
+      /** @param {import('./sync-state.js').State} state */
+      const onState = (state) => {
+        if (isInitiallySyncedWithPeer(state, deviceId)) {
+          cleanup()
+          resolve()
+        }
+      }
+      const onAbort = () => {
+        cleanup()
+        reject(abortSignal.reason)
+      }
+
+      const cleanup = () => {
+        this[kSyncState].off('state', onState)
+        abortSignal.removeEventListener('abort', onAbort)
+      }
+
+      this[kSyncState].on('state', onState)
+      abortSignal.addEventListener('abort', onAbort)
+    })
+  }
+
   #clearAutostopDataSyncTimeoutIfExists() {
     if (this.#autostopDataSyncTimeout) {
       clearTimeout(this.#autostopDataSyncTimeout)
@@ -363,7 +529,7 @@ export class SyncApi extends TypedEmitter {
    * will then handle validation of role records to ensure that the peer is
    * actually still part of the project.
    *
-   * @param {{ protomux: import('protomux')<OpenedNoiseStream> }} peer
+   * @param {import('../types.js').HypercorePeer & { protomux: import('protomux')<OpenedNoiseStream> }} peer
    */
   #handlePeerAdd = (peer) => {
     const { protomux } = peer
@@ -374,6 +540,9 @@ export class SyncApi extends TypedEmitter {
       )
       return
     }
+    if (this.#blobDownloadFilter) {
+      this.#coreManager.sendDownloadIntents(this.#blobDownloadFilter, peer)
+    }
     const peerSyncController = new PeerSyncController({
       protomux,
       coreManager: this.#coreManager,
@@ -524,6 +693,31 @@ function isSynced(state, type, peerSyncControllers) {
   return true
 }
 
+/**
+ * @param {import('./sync-state.js').State} state
+ * @param {string} peerId
+ */
+function isInitiallySyncedWithPeer(state, peerId) {
+  for (const ns of PRESYNC_NAMESPACES) {
+    const remoteDeviceSyncState = getOwn(state[ns].remoteStates, peerId)
+    if (!remoteDeviceSyncState) return false
+
+    switch (remoteDeviceSyncState.status) {
+      case 'starting':
+        return false
+      case 'started':
+      case 'stopped': {
+        const { want, wanted } = remoteDeviceSyncState
+        if (want || wanted) return false
+        break
+      }
+      default:
+        throw new ExhaustivenessError(remoteDeviceSyncState.status)
+    }
+  }
+  return true
+}
+
 /**
  * @param {import('./sync-state.js').State} namespaceSyncState
  * @param {Iterable<PeerSyncController>} peerSyncControllers

package/src/sync/sync-state.js

@@ -68,6 +68,24 @@ export class SyncState extends TypedEmitter {
     ])
   }
 
+  /**
+   * @param {string} peerId
+   * @param {number} start
+   * @param {number} length
+   * @returns {void}
+   */
+  addBlobWantRange(peerId, start, length) {
+    this.#syncStates.blob.addWantRange(peerId, start, length)
+  }
+
+  /**
+   * @param {string} peerId
+   * @returns {void}
+   */
+  clearBlobWantRanges(peerId) {
+    this.#syncStates.blob.clearWantRanges(peerId)
+  }
+
   #handleUpdate = () => {
     this.emit('state', this.getState())
   }

package/src/translation-api.js

@@ -1,7 +1,7 @@
 import { and, sql } from 'drizzle-orm'
 import { kCreateWithDocId, kSelect } from './datatype/index.js'
 import { hashObject } from './utils.js'
-import { NotFoundError } from './errors.js'
+import { nullIfNotFound } from './errors.js'
 import { omit } from './lib/omit.js'
 /** @import { Translation, TranslationValue } from '@comapeo/schema' */
 /** @import { SetOptional } from 'type-fest' */
@@ -50,15 +50,11 @@ export default class TranslationApi {
   async put(value) {
     const identifiers = omit(value, ['message'])
     const docId = hashObject(identifiers)
-    try {
-      const doc = await this.#dataType.getByDocId(docId)
+    const doc = await this.#dataType.getByDocId(docId).catch(nullIfNotFound)
+    if (doc) {
       return await this.#dataType.update(doc.versionId, value)
-    } catch (e) {
-      if (e instanceof NotFoundError) {
-        return await this.#dataType[kCreateWithDocId](docId, value)
-      } else {
-        throw new Error(`Error on translation ${e}`)
-      }
+    } else {
+      return await this.#dataType[kCreateWithDocId](docId, value)
     }
   }
 

package/src/types.ts

@@ -14,6 +14,8 @@ import { Duplex } from 'streamx'
 import RandomAccessStorage from 'random-access-storage'
 import { DefaultListener, ListenerSignature } from 'tiny-typed-emitter'
 import type { NAMESPACES } from './constants.js'
+import type { Readable } from 'stream'
+import type { HyperdriveEntry } from 'hyperdrive'
 
 export type Namespace = (typeof NAMESPACES)[number]
 
@@ -41,12 +43,13 @@ export type BlobId = Simplify<
   }>
 >
 
-type ArrayAtLeastOne<T> = [T, ...T[]]
-
 export type BlobFilter = RequireAtLeastOne<{
-  [KeyType in BlobType]: ArrayAtLeastOne<BlobVariant<KeyType>>
+  [KeyType in BlobType]: Array<BlobVariant<KeyType>>
 }>
 
+/** Map of blob types to array of blob variants */
+export type GenericBlobFilter = Record<string, string[]>
+
 export type MapeoDocMap = {
   [K in MapeoDoc['schemaName']]: Extract<MapeoDoc, { schemaName: K }>
 }
@@ -146,3 +149,9 @@ export type DefaultEmitterEvents<
   newListener: (event: keyof L, listener: L[keyof L]) => void
   removeListener: (event: keyof L, listener: L[keyof L]) => void
 }
+
+export type BlobStoreEntriesStream = Readable & {
+  [Symbol.asyncIterator](): AsyncIterableIterator<
+    HyperdriveEntry & { driveId: string }
+  >
+}

package/dist/blob-store/live-download.d.ts

@@ -1,107 +0,0 @@
-/**
- * Reduce multiple states into one. Factored out for unit testing because I
- * don't trust my coding. Probably a smarter way to do this, but this works.
- *
- * @param {Iterable<{ state: BlobDownloadState | BlobDownloadStateError }>} liveDownloads
- * @param {{ signal?: AbortSignal }} options
- * @returns {BlobDownloadState | BlobDownloadStateError}
- */
-export function combineStates(liveDownloads: Iterable<{
-    state: BlobDownloadState | BlobDownloadStateError;
-}>, { signal }?: {
-    signal?: AbortSignal;
-}): BlobDownloadState | BlobDownloadStateError;
-/**
- * @typedef {object} BlobDownloadState
- * @property {number} haveCount The number of files already downloaded
- * @property {number} haveBytes The bytes already downloaded
- * @property {number} wantCount The number of files pending download
- * @property {number} wantBytes The bytes pending download
- * @property {null} error If status = 'error' then this will be an Error object
- * @property {'checking' | 'downloading' | 'downloaded' | 'aborted'} status
- */
-/** @typedef {Omit<BlobDownloadState, 'error' | 'status'> & { status: 'error', error: Error }} BlobDownloadStateError */
-/**
- * @typedef {object} BlobDownloadEvents
- * @property {(state: BlobDownloadState | BlobDownloadStateError ) => void} state Emitted with the current download state whenever it changes (not emitted during initial 'checking' status)
- */
-/**
- * LiveDownload class
- * @extends {TypedEmitter<BlobDownloadEvents>}
- */
-export class LiveDownload extends TypedEmitter<BlobDownloadEvents> {
-    /**
-     * Like drive.download() but 'live', and for multiple drives
-     * @param {Iterable<import('hyperdrive')>} drives
-     * @param {import('./index.js').InternalDriveEmitter} emitter
-     * @param {object} options
-     * @param {import('../types.js').BlobFilter} [options.filter] Filter blobs of specific types and/or sizes to download
-     * @param {AbortSignal} [options.signal]
-     */
-    constructor(drives: Iterable<import("hyperdrive")>, emitter: import("./index.js").InternalDriveEmitter, { filter, signal }: {
-        filter?: import("../types.js").BlobFilter | undefined;
-        signal?: AbortSignal | undefined;
-    });
-    /**
-     * @returns {BlobDownloadState | BlobDownloadStateError}
-     */
-    get state(): BlobDownloadState | BlobDownloadStateError;
-    #private;
-}
-/**
- * LiveDownload class
- * @extends {TypedEmitter<BlobDownloadEvents>}
- */
-export class DriveLiveDownload extends TypedEmitter<BlobDownloadEvents> {
-    /**
-     * Like drive.download() but 'live',
-     * @param {import('hyperdrive')} drive
-     * @param {object} options
-     * @param {import('../types.js').BlobFilter} [options.filter] Filter blobs of specific types and/or sizes to download
-     * @param {AbortSignal} [options.signal]
-     */
-    constructor(drive: import("hyperdrive"), { filter, signal }?: {
-        filter?: import("../types.js").BlobFilter | undefined;
-        signal?: AbortSignal | undefined;
-    });
-    /**
-     * @returns {BlobDownloadState | BlobDownloadStateError}
-     */
-    get state(): BlobDownloadState | BlobDownloadStateError;
-    #private;
-}
-export type BlobDownloadState = {
-    /**
-     * The number of files already downloaded
-     */
-    haveCount: number;
-    /**
-     * The bytes already downloaded
-     */
-    haveBytes: number;
-    /**
-     * The number of files pending download
-     */
-    wantCount: number;
-    /**
-     * The bytes pending download
-     */
-    wantBytes: number;
-    /**
-     * If status = 'error' then this will be an Error object
-     */
-    error: null;
-    status: "checking" | "downloading" | "downloaded" | "aborted";
-};
-export type BlobDownloadStateError = Omit<BlobDownloadState, "error" | "status"> & {
-    status: "error";
-    error: Error;
-};
-export type BlobDownloadEvents = {
-    /**
-     * Emitted with the current download state whenever it changes (not emitted during initial 'checking' status)
-     */
-    state: (state: BlobDownloadState | BlobDownloadStateError) => void;
-};
-import { TypedEmitter } from 'tiny-typed-emitter';
-//# sourceMappingURL=live-download.d.ts.map

package/dist/blob-store/live-download.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"live-download.d.ts","sourceRoot":"","sources":["../../src/blob-store/live-download.js"],"names":[],"mappings":"AA+RA;;;;;;;GAOG;AACH,6CAJW,QAAQ,CAAC;IAAE,KAAK,EAAE,iBAAiB,GAAG,sBAAsB,CAAA;CAAE,CAAC,eAC/D;IAAE,MAAM,CAAC,EAAE,WAAW,CAAA;CAAE,GACtB,iBAAiB,GAAG,sBAAsB,CAqCtD;AApUD;;;;;;;;GAQG;AAEH,wHAAwH;AAExH;;;GAGG;AAEH;;;GAGG;AACH;IAKE;;;;;;;OAOG;IACH,oBANW,QAAQ,CAAC,OAAO,YAAY,CAAC,CAAC,WAC9B,OAAO,YAAY,EAAE,oBAAoB,sBAEjD;QAAmD,MAAM;QAC3B,MAAM;KAAC,EAiCvC;IAED;;OAEG;IACH,wDAEC;;CACF;AAED;;;GAGG;AACH;IAaE;;;;;;OAMG;IACH,mBALW,OAAO,YAAY,CAAC,uBAE5B;QAAmD,MAAM;QAC3B,MAAM;KAAC,EAmBvC;IAED;;OAEG;IACH,wDAyBC;;CAqIF;;;;;eArRa,MAAM;;;;eACN,MAAM;;;;eACN,MAAM;;;;eACN,MAAM;;;;WACN,IAAI;YACJ,UAAU,GAAG,aAAa,GAAG,YAAY,GAAG,SAAS;;qCAGrD,IAAI,CAAC,iBAAiB,EAAE,OAAO,GAAG,QAAQ,CAAC,GAAG;IAAE,MAAM,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,KAAK,CAAA;CAAE;;;;;WAI/E,CAAC,KAAK,EAAE,iBAAiB,GAAG,sBAAsB,KAAM,IAAI;;6BApB7C,oBAAoB"}

package/dist/lib/timing-safe-equal.d.ts

@@ -1,15 +0,0 @@
-/**
- * Compare two values in constant time.
- *
- * Useful when you want to avoid leaking data.
- *
- * Like `crypto.timingSafeEqual`, but works with strings and doesn't throw if
- * lengths differ.
- *
- * @template {string | NodeJS.ArrayBufferView} T
- * @param {T} a
- * @param {T} b
- * @returns {boolean}
- */
-export default function timingSafeEqual<T extends string | NodeJS.ArrayBufferView>(a: T, b: T): boolean;
-//# sourceMappingURL=timing-safe-equal.d.ts.map

package/dist/lib/timing-safe-equal.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"timing-safe-equal.d.ts","sourceRoot":"","sources":["../../src/lib/timing-safe-equal.js"],"names":[],"mappings":"AAaA;;;;;;;;;;;;GAYG;AACH,wCAL+C,CAAC,SAAlC,MAAM,GAAG,MAAM,CAAC,eAAgB,KACnC,CAAC,KACD,CAAC,GACC,OAAO,CASnB"}