@comapeo/core 2.0.1 → 2.1.0

package/src/generated/extensions.ts

@@ -66,6 +66,20 @@ export function haveExtension_NamespaceToNumber(object: HaveExtension_Namespace)
   }
 }
 
+/** A map of blob types and variants that a peer intends to download */
+export interface DownloadIntentExtension {
+  downloadIntents: { [key: string]: DownloadIntentExtension_DownloadIntent };
+}
+
+export interface DownloadIntentExtension_DownloadIntent {
+  variants: string[];
+}
+
+export interface DownloadIntentExtension_DownloadIntentsEntry {
+  key: string;
+  value: DownloadIntentExtension_DownloadIntent | undefined;
+}
+
 function createBaseProjectExtension(): ProjectExtension {
   return { authCoreKeys: [] };
 }
@@ -194,6 +208,173 @@ export const HaveExtension = {
   },
 };
 
+function createBaseDownloadIntentExtension(): DownloadIntentExtension {
+  return { downloadIntents: {} };
+}
+
+export const DownloadIntentExtension = {
+  encode(message: DownloadIntentExtension, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer {
+    Object.entries(message.downloadIntents).forEach(([key, value]) => {
+      DownloadIntentExtension_DownloadIntentsEntry.encode({ key: key as any, value }, writer.uint32(10).fork())
+        .ldelim();
+    });
+    return writer;
+  },
+
+  decode(input: _m0.Reader | Uint8Array, length?: number): DownloadIntentExtension {
+    const reader = input instanceof _m0.Reader ? input : _m0.Reader.create(input);
+    let end = length === undefined ? reader.len : reader.pos + length;
+    const message = createBaseDownloadIntentExtension();
+    while (reader.pos < end) {
+      const tag = reader.uint32();
+      switch (tag >>> 3) {
+        case 1:
+          if (tag !== 10) {
+            break;
+          }
+
+          const entry1 = DownloadIntentExtension_DownloadIntentsEntry.decode(reader, reader.uint32());
+          if (entry1.value !== undefined) {
+            message.downloadIntents[entry1.key] = entry1.value;
+          }
+          continue;
+      }
+      if ((tag & 7) === 4 || tag === 0) {
+        break;
+      }
+      reader.skipType(tag & 7);
+    }
+    return message;
+  },
+
+  create<I extends Exact<DeepPartial<DownloadIntentExtension>, I>>(base?: I): DownloadIntentExtension {
+    return DownloadIntentExtension.fromPartial(base ?? ({} as any));
+  },
+  fromPartial<I extends Exact<DeepPartial<DownloadIntentExtension>, I>>(object: I): DownloadIntentExtension {
+    const message = createBaseDownloadIntentExtension();
+    message.downloadIntents = Object.entries(object.downloadIntents ?? {}).reduce<
+      { [key: string]: DownloadIntentExtension_DownloadIntent }
+    >((acc, [key, value]) => {
+      if (value !== undefined) {
+        acc[key] = DownloadIntentExtension_DownloadIntent.fromPartial(value);
+      }
+      return acc;
+    }, {});
+    return message;
+  },
+};
+
+function createBaseDownloadIntentExtension_DownloadIntent(): DownloadIntentExtension_DownloadIntent {
+  return { variants: [] };
+}
+
+export const DownloadIntentExtension_DownloadIntent = {
+  encode(message: DownloadIntentExtension_DownloadIntent, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer {
+    for (const v of message.variants) {
+      writer.uint32(10).string(v!);
+    }
+    return writer;
+  },
+
+  decode(input: _m0.Reader | Uint8Array, length?: number): DownloadIntentExtension_DownloadIntent {
+    const reader = input instanceof _m0.Reader ? input : _m0.Reader.create(input);
+    let end = length === undefined ? reader.len : reader.pos + length;
+    const message = createBaseDownloadIntentExtension_DownloadIntent();
+    while (reader.pos < end) {
+      const tag = reader.uint32();
+      switch (tag >>> 3) {
+        case 1:
+          if (tag !== 10) {
+            break;
+          }
+
+          message.variants.push(reader.string());
+          continue;
+      }
+      if ((tag & 7) === 4 || tag === 0) {
+        break;
+      }
+      reader.skipType(tag & 7);
+    }
+    return message;
+  },
+
+  create<I extends Exact<DeepPartial<DownloadIntentExtension_DownloadIntent>, I>>(
+    base?: I,
+  ): DownloadIntentExtension_DownloadIntent {
+    return DownloadIntentExtension_DownloadIntent.fromPartial(base ?? ({} as any));
+  },
+  fromPartial<I extends Exact<DeepPartial<DownloadIntentExtension_DownloadIntent>, I>>(
+    object: I,
+  ): DownloadIntentExtension_DownloadIntent {
+    const message = createBaseDownloadIntentExtension_DownloadIntent();
+    message.variants = object.variants?.map((e) => e) || [];
+    return message;
+  },
+};
+
+function createBaseDownloadIntentExtension_DownloadIntentsEntry(): DownloadIntentExtension_DownloadIntentsEntry {
+  return { key: "", value: undefined };
+}
+
+export const DownloadIntentExtension_DownloadIntentsEntry = {
+  encode(message: DownloadIntentExtension_DownloadIntentsEntry, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer {
+    if (message.key !== "") {
+      writer.uint32(10).string(message.key);
+    }
+    if (message.value !== undefined) {
+      DownloadIntentExtension_DownloadIntent.encode(message.value, writer.uint32(18).fork()).ldelim();
+    }
+    return writer;
+  },
+
+  decode(input: _m0.Reader | Uint8Array, length?: number): DownloadIntentExtension_DownloadIntentsEntry {
+    const reader = input instanceof _m0.Reader ? input : _m0.Reader.create(input);
+    let end = length === undefined ? reader.len : reader.pos + length;
+    const message = createBaseDownloadIntentExtension_DownloadIntentsEntry();
+    while (reader.pos < end) {
+      const tag = reader.uint32();
+      switch (tag >>> 3) {
+        case 1:
+          if (tag !== 10) {
+            break;
+          }
+
+          message.key = reader.string();
+          continue;
+        case 2:
+          if (tag !== 18) {
+            break;
+          }
+
+          message.value = DownloadIntentExtension_DownloadIntent.decode(reader, reader.uint32());
+          continue;
+      }
+      if ((tag & 7) === 4 || tag === 0) {
+        break;
+      }
+      reader.skipType(tag & 7);
+    }
+    return message;
+  },
+
+  create<I extends Exact<DeepPartial<DownloadIntentExtension_DownloadIntentsEntry>, I>>(
+    base?: I,
+  ): DownloadIntentExtension_DownloadIntentsEntry {
+    return DownloadIntentExtension_DownloadIntentsEntry.fromPartial(base ?? ({} as any));
+  },
+  fromPartial<I extends Exact<DeepPartial<DownloadIntentExtension_DownloadIntentsEntry>, I>>(
+    object: I,
+  ): DownloadIntentExtension_DownloadIntentsEntry {
+    const message = createBaseDownloadIntentExtension_DownloadIntentsEntry();
+    message.key = object.key ?? "";
+    message.value = (object.value !== undefined && object.value !== null)
+      ? DownloadIntentExtension_DownloadIntent.fromPartial(object.value)
+      : undefined;
+    return message;
+  },
+};
+
 declare const self: any | undefined;
 declare const window: any | undefined;
 declare const global: any | undefined;

package/src/index.js

@@ -3,9 +3,19 @@ import {
   COORDINATOR_ROLE_ID,
   MEMBER_ROLE_ID,
 } from './roles.js'
+import { kProjectReplicate } from './mapeo-project.js'
 export { plugin as CoMapeoMapsFastifyPlugin } from './fastify-plugins/maps.js'
 export { FastifyController } from './fastify-controller.js'
 export { MapeoManager } from './mapeo-manager.js'
+/** @import { MapeoProject } from './mapeo-project.js' */
+
+/**
+ * @param {MapeoProject} project
+ * @param {Parameters<MapeoProject.prototype[kProjectReplicate]>} args
+ * @returns {ReturnType<MapeoProject.prototype[kProjectReplicate]>}
+ */
+export const replicateProject = (project, ...args) =>
+  project[kProjectReplicate](...args)
 
 export const roles = /** @type {const} */ ({
   CREATOR_ROLE_ID,

package/src/invite-api.js

@@ -3,7 +3,7 @@ import { pEvent } from 'p-event'
 import { InviteResponse_Decision } from './generated/rpc.js'
 import { assert, keyToId, noop } from './utils.js'
 import HashMap from './lib/hashmap.js'
-import timingSafeEqual from './lib/timing-safe-equal.js'
+import timingSafeEqual from 'string-timing-safe-equal'
 import { Logger } from './logger.js'
 /** @import { MapBuffers } from './types.js' */
 /**

package/src/lib/drizzle-helpers.js

@@ -0,0 +1,79 @@
+import { sql } from 'drizzle-orm'
+import { assert } from '../utils.js'
+import { migrate as drizzleMigrate } from 'drizzle-orm/better-sqlite3/migrator'
+import { DRIZZLE_MIGRATIONS_TABLE } from '../constants.js'
+/** @import { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' */
+
+/**
+ * @param {unknown} queryResult
+ * @returns {number}
+ */
+const getNumberResult = (queryResult) => {
+  assert(
+    queryResult &&
+      typeof queryResult === 'object' &&
+      'result' in queryResult &&
+      typeof queryResult.result === 'number',
+    'expected query to return proper result'
+  )
+  return queryResult.result
+}
+
+/**
+ * Get the number of rows in a table using `SELECT COUNT(*)`.
+ * Returns 0 if the table doesn't exist.
+ *
+ * @param {BetterSQLite3Database} db
+ * @param {string} tableName
+ * @returns {number}
+ */
+const safeCountTableRows = (db, tableName) =>
+  db.transaction((tx) => {
+    const existsQuery = sql`
+      SELECT EXISTS (
+        SELECT 1
+        FROM sqlite_schema
+        WHERE type IS 'table'
+        AND name IS ${tableName}
+      ) AS result
+    `
+    const existsResult = tx.get(existsQuery)
+    const exists = getNumberResult(existsResult)
+    if (!exists) return 0
+
+    const countQuery = sql`
+      SELECT COUNT(*) AS result
+      FROM ${sql.identifier(tableName)}
+    `
+    const countResult = tx.get(countQuery)
+    return getNumberResult(countResult)
+  })
+
+/**
+ * @internal
+ * @typedef {'initialized database' | 'migrated' | 'no migration'} MigrationResult
+ */
+
+/**
+ * Wrapper around Drizzle's migration function. Returns what happened during
+ * migration; did a migration occur?
+ *
+ * @param {BetterSQLite3Database} db
+ * @param {object} options
+ * @param {string} options.migrationsFolder
+ * @returns {MigrationResult}
+ */
+export const migrate = (db, { migrationsFolder }) => {
+  const migrationsBefore = safeCountTableRows(db, DRIZZLE_MIGRATIONS_TABLE)
+  drizzleMigrate(db, {
+    migrationsFolder,
+    migrationsTable: DRIZZLE_MIGRATIONS_TABLE,
+  })
+  const migrationsAfter = safeCountTableRows(db, DRIZZLE_MIGRATIONS_TABLE)
+
+  if (migrationsAfter === migrationsBefore) return 'no migration'
+
+  if (migrationsBefore === 0) return 'initialized database'
+
+  return 'migrated'
+}

package/src/lib/error.js

@@ -0,0 +1,47 @@
+/**
+ * Create an `Error` with a `code` property.
+ *
+ * @example
+ * const err = new ErrorWithCode('INVALID_DATA', 'data was invalid')
+ * err.message
+ * // => 'data was invalid'
+ * err.code
+ * // => 'INVALID_DATA'
+ */
+export class ErrorWithCode extends Error {
+  /**
+   * @param {string} code
+   * @param {string} message
+   * @param {object} [options]
+   * @param {unknown} [options.cause]
+   */
+  constructor(code, message, options) {
+    super(message, options)
+    /** @readonly */ this.code = code
+  }
+}
+
+/**
+ * Get the error message from an object if possible.
+ * Otherwise, stringify the argument.
+ *
+ * @param {unknown} maybeError
+ * @returns {string}
+ * @example
+ * try {
+ *   // do something
+ * } catch (err) {
+ *   console.error(getErrorMessage(err))
+ * }
+ */
+export function getErrorMessage(maybeError) {
+  if (maybeError && typeof maybeError === 'object' && 'message' in maybeError) {
+    try {
+      const { message } = maybeError
+      if (typeof message === 'string') return message
+    } catch (_err) {
+      // Ignored
+    }
+  }
+  return 'unknown error'
+}

package/src/lib/get-own.js

@@ -0,0 +1,10 @@
+/**
+ * @template {object} T
+ * @template {keyof T} K
+ * @param {T} obj
+ * @param {K} key
+ * @returns {undefined | T[K]}
+ */
+export function getOwn(obj, key) {
+  return Object.hasOwn(obj, key) ? obj[key] : undefined
+}

package/src/lib/is-hostname-ip-address.js

@@ -0,0 +1,26 @@
+import { isIPv4, isIPv6 } from 'node:net'
+
+/**
+ * Is this hostname an IP address?
+ *
+ * @param {string} hostname
+ * @returns {boolean}
+ * @example
+ * isHostnameIpAddress('100.64.0.42')
+ * // => false
+ *
+ * isHostnameIpAddress('[2001:0db8:85a3:0000:0000:8a2e:0370:7334]')
+ * // => true
+ *
+ * isHostnameIpAddress('example.com')
+ * // => false
+ */
+export function isHostnameIpAddress(hostname) {
+  if (isIPv4(hostname)) return true
+
+  if (hostname.startsWith('[') && hostname.endsWith(']')) {
+    return isIPv6(hostname.slice(1, -1))
+  }
+
+  return false
+}

package/src/lib/ws-core-replicator.js

@@ -0,0 +1,47 @@
+import { pipeline } from 'node:stream/promises'
+import { Transform } from 'node:stream'
+import { createWebSocketStream } from 'ws'
+/** @import { WebSocket } from 'ws' */
+/** @import { ReplicationStream } from '../types.js' */
+
+/**
+ * @param {WebSocket} ws
+ * @param {ReplicationStream} replicationStream
+ * @returns {Promise<void>}
+ */
+export function wsCoreReplicator(ws, replicationStream) {
+  // This is purely to satisfy typescript at its worst. `pipeline` expects a
+  // NodeJS ReadWriteStream, but our replicationStream is a streamx Duplex
+  // stream. The difference is that streamx does not implement the
+  // `setEncoding`, `unpipe`, `wrap` or `isPaused` methods. The `pipeline`
+  // function does not depend on any of these methods (I have read through the
+  // NodeJS source code at cebf21d (v22.9.0) to confirm this), so we can safely
+  // cast the stream to a NodeJS ReadWriteStream.
+  const _replicationStream = /** @type {NodeJS.ReadWriteStream} */ (
+    /** @type {unknown} */ (replicationStream)
+  )
+  return pipeline(
+    _replicationStream,
+    wsSafetyTransform(ws),
+    createWebSocketStream(ws),
+    _replicationStream
+  )
+}
+
+/**
+ * Avoid writing data to a closing or closed websocket, which would result in an
+ * error. Instead we drop the data and wait for the stream close/end events to
+ * propagate and close the streams cleanly.
+ *
+ * @param {WebSocket} ws
+ */
+function wsSafetyTransform(ws) {
+  return new Transform({
+    transform(chunk, encoding, callback) {
+      if (ws.readyState === ws.CLOSED || ws.readyState === ws.CLOSING) {
+        return callback()
+      }
+      callback(null, chunk)
+    },
+  })
+}

package/src/mapeo-manager.js

@@ -16,10 +16,11 @@ import {
   kBlobStore,
   kClearDataIfLeft,
   kProjectLeave,
+  kSetIsArchiveDevice,
   kSetOwnDeviceInfo,
 } from './mapeo-project.js'
 import {
-  localDeviceInfoTable,
+  deviceSettingsTable,
   projectKeysTable,
   projectSettingsTable,
 } from './schema/client.js'
@@ -44,7 +45,6 @@ import { LocalPeers } from './local-peers.js'
 import { InviteApi } from './invite-api.js'
 import { LocalDiscovery } from './discovery/local-discovery.js'
 import { Roles } from './roles.js'
-import NoiseSecretStream from '@hyperswarm/secret-stream'
 import { Logger } from './logger.js'
 import {
   kSyncState,
@@ -52,6 +52,7 @@ import {
   kRescindFullStopRequest,
 } from './sync/sync-api.js'
 /** @import { ProjectSettingsValue as ProjectValue } from '@comapeo/schema' */
+/** @import NoiseSecretStream from '@hyperswarm/secret-stream' */
 /** @import { SetNonNullable } from 'type-fest' */
 /** @import { CoreStorage, Namespace } from './types.js' */
 /** @import { DeviceInfoParam } from './schema/client.js' */
@@ -79,9 +80,6 @@ export const DEFAULT_FALLBACK_MAP_FILE_PATH = require.resolve(
 export const DEFAULT_ONLINE_STYLE_URL =
   'https://demotiles.maplibre.org/style.json'
 
-export const kRPC = Symbol('rpc')
-export const kManagerReplicate = Symbol('replicate manager')
-
 /**
  * @typedef {Omit<import('./local-peers.js').PeerInfo, 'protomux'>} PublicPeerInfo
  */
@@ -221,33 +219,10 @@ export class MapeoManager extends TypedEmitter {
     this.#localDiscovery.on('connection', this.#replicate.bind(this))
   }
 
-  /**
-   * MapeoRPC instance, used for tests
-   */
-  get [kRPC]() {
-    return this.#localPeers
-  }
-
   get deviceId() {
     return this.#deviceId
   }
 
-  /**
-   * Create a Mapeo replication stream. This replication connects the Mapeo RPC
-   * channel and allows invites. All active projects will sync automatically to
-   * this replication stream. Only use for local (trusted) connections, because
-   * the RPC channel key is public. To sync a specific project without
-   * connecting RPC, use project[kProjectReplication].
-   *
-   * @param {boolean} isInitiator
-   */
-  [kManagerReplicate](isInitiator) {
-    const noiseStream = new NoiseSecretStream(isInitiator, undefined, {
-      keyPair: this.#keyManager.getIdentityKeypair(),
-    })
-    return this.#replicate(noiseStream)
-  }
-
   /**
    * @param {'blobs' | 'icons' | 'maps'} mediaType
    * @returns {Promise<string>}
@@ -507,6 +482,7 @@ export class MapeoManager extends TypedEmitter {
   async #createProjectInstance(projectKeys) {
     validateProjectKeys(projectKeys)
     const projectId = keyToId(projectKeys.projectKey)
+    const isArchiveDevice = this.getIsArchiveDevice()
     const project = new MapeoProject({
       ...this.#projectStorage(projectId),
       ...projectKeys,
@@ -517,6 +493,7 @@ export class MapeoManager extends TypedEmitter {
       localPeers: this.#localPeers,
       logger: this.#loggerBase,
       getMediaBaseUrl: this.#getMediaBaseUrl.bind(this),
+      isArchiveDevice,
     })
     await project[kClearDataIfLeft]()
     return project
@@ -579,7 +556,7 @@ export class MapeoManager extends TypedEmitter {
    * downloaded their proof of project membership and the project config.
    *
    * @param {Pick<import('./generated/rpc.js').ProjectJoinDetails, 'projectKey' | 'encryptionKeys'> & { projectName: string }} projectJoinDetails
-   * @param {{ waitForSync?: boolean }} [opts] For internal use in tests, set opts.waitForSync = false to not wait for sync during addProject()
+   * @param {{ waitForSync?: boolean }} [opts] Set opts.waitForSync = false to not wait for sync during addProject()
    * @returns {Promise<string>}
    */
   addProject = async (
@@ -733,9 +710,7 @@ export class MapeoManager extends TypedEmitter {
   }
 
   /**
-   * @typedef {Exclude<
-   * import('./schema/client.js').DeviceInfoParam['deviceType'],
-   * 'selfHostedServer'>} RPCDeviceType
+   * @typedef {import('./schema/client.js').DeviceInfoParam['deviceType']} RPCDeviceType
    */
 
   /**
@@ -746,10 +721,10 @@ export class MapeoManager extends TypedEmitter {
   async setDeviceInfo(deviceInfo) {
     const values = { deviceId: this.#deviceId, deviceInfo }
     this.#db
-      .insert(localDeviceInfoTable)
+      .insert(deviceSettingsTable)
       .values(values)
       .onConflictDoUpdate({
-        target: localDeviceInfoTable.deviceId,
+        target: deviceSettingsTable.deviceId,
         set: values,
       })
       .run()
@@ -762,13 +737,22 @@ export class MapeoManager extends TypedEmitter {
       })
     )
 
-    await Promise.all(
-      this.#localPeers.peers
-        .filter(({ status }) => status === 'connected')
-        .map((peer) =>
-          this.#localPeers.sendDeviceInfo(peer.deviceId, deviceInfo)
-        )
-    )
+    if (deviceInfo.deviceType !== 'selfHostedServer') {
+      // We have to make a copy of this because TypeScript can't guarantee that
+      // `deviceInfo` won't be mutated by the time it gets to the
+      // `sendDeviceInfo` call below.
+      const deviceInfoToSend = {
+        ...deviceInfo,
+        deviceType: deviceInfo.deviceType,
+      }
+      await Promise.all(
+        this.#localPeers.peers
+          .filter(({ status }) => status === 'connected')
+          .map((peer) =>
+            this.#localPeers.sendDeviceInfo(peer.deviceId, deviceInfoToSend)
+          )
+      )
+    }
 
     this.#l.log('set device info %o', deviceInfo)
   }
@@ -784,8 +768,8 @@ export class MapeoManager extends TypedEmitter {
   getDeviceInfo() {
     const row = this.#db
       .select()
-      .from(localDeviceInfoTable)
-      .where(eq(localDeviceInfoTable.deviceId, this.#deviceId))
+      .from(deviceSettingsTable)
+      .where(eq(deviceSettingsTable.deviceId, this.#deviceId))
       .get()
     return {
       deviceId: this.#deviceId,
@@ -794,6 +778,50 @@ export class MapeoManager extends TypedEmitter {
     }
   }
 
+  /**
+   * Set whether this device is an archive device. Archive devices will download
+   * all media during sync, where-as non-archive devices will not download media
+   * original variants, and only download preview and thumbnail variants.
+   * @param {boolean} isArchiveDevice
+   * @returns {void}
+   */
+  setIsArchiveDevice(isArchiveDevice) {
+    const values = { deviceId: this.#deviceId, isArchiveDevice }
+    const result = this.#db
+      .insert(deviceSettingsTable)
+      .values(values)
+      .onConflictDoUpdate({
+        target: deviceSettingsTable.deviceId,
+        set: values,
+      })
+      .run()
+    if (!result || result.changes === 0) {
+      throw new Error('Failed to set isArchiveDevice')
+    }
+    for (const project of this.#activeProjects.values()) {
+      project[kSetIsArchiveDevice](isArchiveDevice)
+    }
+  }
+
+  /**
+   * Get whether this device is an archive device. Archive devices will download
+   * all media during sync, where-as non-archive devices will not download media
+   * original variants, and only download preview and thumbnail variants.
+   * @returns {boolean} isArchiveDevice
+   */
+  getIsArchiveDevice() {
+    const row = this.#db
+      .select()
+      .from(deviceSettingsTable)
+      .where(eq(deviceSettingsTable.deviceId, this.#deviceId))
+      .get()
+    if (typeof row?.isArchiveDevice === 'boolean') {
+      return row.isArchiveDevice
+    } else {
+      return true
+    }
+  }
+
   /**
    * @returns {InviteApi}
    */