@comapeo/core 1.0.0

package/drizzle/project/meta/_journal.json

@@ -0,0 +1,13 @@
+{
+  "version": "5",
+  "dialect": "sqlite",
+  "entries": [
+    {
+      "idx": 0,
+      "version": "5",
+      "when": 1726514275142,
+      "tag": "0000_spooky_lady_ursula",
+      "breakpoints": true
+    }
+  ]
+}

package/package.json

@@ -0,0 +1,202 @@
+{
+  "name": "@comapeo/core",
+  "version": "1.0.0",
+  "description": "Offline p2p mapping library",
+  "main": "src/index.js",
+  "types": "dist/index.d.ts",
+  "type": "module",
+  "scripts": {
+    "lint": "eslint --cache .",
+    "format": "prettier . --write",
+    "test": "npm-run-all lint test:prettier build:types type test:buildConfigs test:unit test:e2e test:types",
+    "test:prettier": "prettier --check .",
+    "test:buildConfigs": "node scripts/build-config-fixtures.js",
+    "test:unit": "node --test tests/*.js tests/**/*.js",
+    "test:e2e": "node --test test-e2e/*.js test-e2e/**/*.js",
+    "test:types": "tsc -p test-types/tsconfig.json",
+    "build:types": "tsc -p tsconfig.npm.json && cpy 'src/**/*.d.ts' dist",
+    "bench": "nanobench benchmarks/*.js",
+    "type": "tsc",
+    "doc": "rimraf docs/api/md && typedoc --plugin typedoc-plugin-markdown --plugin typedoc-plugin-missing-exports --out docs/api/md",
+    "doc:publish": "rimraf docs/api/html && typedoc --plugin typedoc-plugin-missing-exports --out docs/api/html",
+    "protobuf": "node ./scripts/build-messages.js",
+    "db:generate:project": "drizzle-kit generate:sqlite --schema src/schema/project.js --out drizzle/project",
+    "db:generate:client": "drizzle-kit generate:sqlite --schema src/schema/client.js --out drizzle/client",
+    "prepack": "npm run build:types",
+    "prepare": "husky install"
+  },
+  "files": [
+    "src",
+    "dist",
+    "drizzle"
+  ],
+  "prettier": {
+    "semi": false,
+    "singleQuote": true
+  },
+  "lint-staged": {
+    "*.js": [
+      "eslint --cache"
+    ],
+    "*": [
+      "prettier --check"
+    ]
+  },
+  "eslintConfig": {
+    "env": {
+      "commonjs": true,
+      "es2022": true,
+      "node": true
+    },
+    "extends": "eslint:recommended",
+    "parserOptions": {
+      "ecmaVersion": 13,
+      "sourceType": "module"
+    },
+    "rules": {
+      "curly": [
+        "error",
+        "multi-line"
+      ],
+      "eqeqeq": "error",
+      "default-case": "error",
+      "default-case-last": "error",
+      "prefer-const": "error",
+      "no-unused-vars": [
+        "error",
+        {
+          "varsIgnorePattern": "^_",
+          "argsIgnorePattern": "^_"
+        }
+      ],
+      "no-restricted-imports": [
+        "error",
+        {
+          "paths": [
+            {
+              "name": "assert",
+              "message": "Prefer importing node:assert/strict."
+            },
+            {
+              "name": "node:assert",
+              "message": "Prefer importing node:assert/strict."
+            }
+          ]
+        }
+      ],
+      "no-var": "error"
+    },
+    "ignorePatterns": [
+      "docs/*"
+    ]
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/digidem/comapeo-core.git"
+  },
+  "authors": [
+    "Andrew Chou <achou@awana.digital>",
+    "Evan Hahn <ehahn@awana.digital>",
+    "Gregor MacLennan <gmaclennan@awana.digital>",
+    "Seth Vincent",
+    "Tomás Ciccola <tciccola@awana.digital>"
+  ],
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/digidem/comapeo-core/issues"
+  },
+  "homepage": "https://github.com/digidem/comapeo-core#readme",
+  "devDependencies": {
+    "@bufbuild/buf": "^1.26.1",
+    "@mapeo/default-config": "5.0.0",
+    "@mapeo/mock-data": "2.0.0",
+    "@sinonjs/fake-timers": "^10.0.2",
+    "@types/b4a": "^1.6.0",
+    "@types/bogon": "^1.0.2",
+    "@types/compact-encoding": "^2.15.0",
+    "@types/debug": "^4.1.8",
+    "@types/json-schema": "^7.0.11",
+    "@types/json-stable-stringify": "^1.0.36",
+    "@types/nanobench": "^3.0.0",
+    "@types/node": "^18.19.33",
+    "@types/sinonjs__fake-timers": "^8.1.2",
+    "@types/streamx": "^2.9.5",
+    "@types/sub-encoder": "^2.1.0",
+    "@types/throttle-debounce": "^5.0.0",
+    "@types/varint": "^6.0.1",
+    "@types/yauzl-promise": "^4.0.0",
+    "@types/yazl": "^2.4.5",
+    "bitfield": "^4.1.0",
+    "cpy": "^10.1.0",
+    "cpy-cli": "^5.0.0",
+    "drizzle-kit": "^0.20.14",
+    "eslint": "^8.57.0",
+    "husky": "^8.0.0",
+    "iterpal": "^0.4.0",
+    "lint-staged": "^14.0.1",
+    "mapeo-offline-map": "^2.0.0",
+    "math-random-seed": "^2.0.0",
+    "nanobench": "^3.0.0",
+    "npm-run-all": "^4.1.5",
+    "prettier": "^2.8.8",
+    "random-access-file": "^4.0.7",
+    "random-access-memory": "^6.2.1",
+    "rimraf": "^5.0.5",
+    "tempy": "^3.1.0",
+    "ts-proto": "^1.156.7",
+    "typedoc": "^0.26.6",
+    "typedoc-plugin-markdown": "^4.2.5",
+    "typedoc-plugin-missing-exports": "^3.0.0",
+    "typescript": "^5.5.4",
+    "yazl": "^2.5.1"
+  },
+  "dependencies": {
+    "@digidem/types": "^2.3.0",
+    "@electron/asar": "^3.2.8",
+    "@fastify/error": "^3.4.1",
+    "@fastify/static": "^7.0.3",
+    "@fastify/type-provider-typebox": "^4.0.0",
+    "@hyperswarm/secret-stream": "^6.1.2",
+    "@mapeo/crypto": "1.0.0-alpha.10",
+    "@comapeo/schema": "1.0.0",
+    "@mapeo/sqlite-indexer": "1.0.0-alpha.9",
+    "@sinclair/typebox": "^0.29.6",
+    "b4a": "^1.6.3",
+    "bcp-47": "^2.1.0",
+    "better-sqlite3": "^8.7.0",
+    "big-sparse-array": "^1.0.3",
+    "bogon": "^1.1.0",
+    "compact-encoding": "^2.12.0",
+    "corestore": "^6.8.4",
+    "debug": "^4.3.4",
+    "dot-prop": "^9.0.0",
+    "drizzle-orm": "^0.30.8",
+    "fastify": ">= 4",
+    "fastify-plugin": "^4.5.1",
+    "hyperblobs": "2.3.0",
+    "hypercore": "10.17.0",
+    "hypercore-crypto": "3.4.2",
+    "hyperdrive": "11.5.3",
+    "json-stable-stringify": "^1.1.1",
+    "magic-bytes.js": "^1.10.0",
+    "map-obj": "^5.0.2",
+    "mime": "^4.0.3",
+    "multi-core-indexer": "^1.0.0-alpha.10",
+    "p-defer": "^4.0.0",
+    "p-event": "^6.0.1",
+    "p-timeout": "^6.1.2",
+    "protobufjs": "^7.2.3",
+    "protomux": "^3.4.1",
+    "quickbit-universal": "^2.2.0",
+    "sodium-universal": "^4.0.0",
+    "start-stop-state-machine": "^1.2.0",
+    "streamx": "^2.19.0",
+    "sub-encoder": "^2.1.1",
+    "throttle-debounce": "^5.0.0",
+    "tiny-typed-emitter": "^2.1.0",
+    "type-fest": "^4.5.0",
+    "undici": "^6.13.0",
+    "varint": "^6.0.0",
+    "yauzl-promise": "^4.0.0"
+  }
+}

package/src/blob-api.js

@@ -0,0 +1,139 @@
+import fs from 'node:fs'
+// @ts-expect-error - pipelinePromise missing from streamx types
+import { Transform, pipelinePromise as pipeline } from 'streamx'
+import { createHash, randomBytes } from 'node:crypto'
+/** @import { BlobId, BlobType } from './types.js' */
+
+/**
+ * Location coordinate data. Based on [Expo's `LocationObjectCoords`][0].
+ * [0]: https://docs.expo.dev/versions/latest/sdk/location/#locationobjectcoords
+ *
+ * @typedef {object} LocationObjectCoords
+ * @prop {number | null} accuracy
+ * @prop {number | null} altitude
+ * @prop {number | null} altitudeAccuracy
+ * @prop {number | null} heading
+ * @prop {number} latitude
+ * @prop {number} longitude
+ * @prop {number | null} speed
+ */
+
+/**
+ * Location metadata for a blob. Based on [Expo's `LocationObject`][0].
+ * [0]: https://docs.expo.dev/versions/latest/sdk/location/#locationobject
+ *
+ * @typedef {object} LocationObject
+ * @prop {LocationObjectCoords} coords
+ * @prop {boolean} [mocked]
+ * @prop {number} timestamp
+ */
+
+/**
+ * @typedef {object} Metadata
+ * @prop {string} mimeType
+ * @prop {number} timestamp
+ * @prop {LocationObject} [location]
+ */
+
+export class BlobApi {
+  #blobStore
+  #getMediaBaseUrl
+
+  /**
+   * @param {object} options
+   * @param {import('./blob-store/index.js').BlobStore} options.blobStore
+   * @param {() => Promise<string>} options.getMediaBaseUrl
+   */
+  constructor({ blobStore, getMediaBaseUrl }) {
+    this.#blobStore = blobStore
+    this.#getMediaBaseUrl = getMediaBaseUrl
+  }
+
+  /**
+   * Get a url for a blob based on its BlobId
+   * @param {BlobId} blobId
+   * @returns {Promise<string>}
+   */
+  async getUrl(blobId) {
+    const { driveId, type, variant, name } = blobId
+
+    let base = await this.#getMediaBaseUrl()
+
+    if (!base.endsWith('/')) {
+      base += '/'
+    }
+
+    return base + `${driveId}/${type}/${variant}/${name}`
+  }
+
+  /**
+   * Write blobs for provided variants of a file
+   * @param {{ original: string, preview?: string, thumbnail?: string }} filepaths
+   * @param {Metadata} metadata
+   * @returns {Promise<{ driveId: string, name: string, type: 'photo' | 'video' | 'audio', hash: string }>}
+   */
+  async create(filepaths, metadata) {
+    const { original, preview, thumbnail } = filepaths
+    const { mimeType } = metadata
+    const type = getType(mimeType)
+    const name = randomBytes(8).toString('hex')
+    const hash = createHash('sha256')
+
+    const ws = this.#blobStore.createWriteStream(
+      { type, variant: 'original', name },
+      { metadata }
+    )
+    const writePromises = [
+      pipeline(fs.createReadStream(original), hashTransform(hash), ws),
+    ]
+
+    if (preview) {
+      const ws = this.#blobStore.createWriteStream(
+        { type, variant: 'preview', name },
+        { metadata }
+      )
+      writePromises.push(pipeline(fs.createReadStream(preview), ws))
+    }
+
+    if (thumbnail) {
+      const ws = this.#blobStore.createWriteStream(
+        { type, variant: 'thumbnail', name },
+        { metadata }
+      )
+      writePromises.push(pipeline(fs.createReadStream(thumbnail), ws))
+    }
+
+    await Promise.all(writePromises)
+
+    return {
+      driveId: this.#blobStore.writerDriveId,
+      name,
+      type,
+      hash: hash.digest('hex'),
+    }
+  }
+}
+
+/**
+ * @param {import('node:crypto').Hash} hash
+ */
+function hashTransform(hash) {
+  return new Transform({
+    transform: (data, cb) => {
+      hash.update(data)
+      cb(null, data)
+    },
+  })
+}
+
+/**
+ * @param {string} mimeType
+ * @returns {BlobType}
+ */
+function getType(mimeType) {
+  if (mimeType.startsWith('image')) return 'photo'
+  if (mimeType.startsWith('video')) return 'video'
+  if (mimeType.startsWith('audio')) return 'audio'
+
+  throw new Error(`Unsupported mimeType: ${mimeType}`)
+}

package/src/blob-store/index.js

@@ -0,0 +1,325 @@
+import Hyperdrive from 'hyperdrive'
+import b4a from 'b4a'
+import util from 'node:util'
+import { discoveryKey } from 'hypercore-crypto'
+import { TypedEmitter } from 'tiny-typed-emitter'
+import { LiveDownload } from './live-download.js'
+/** @import { BlobId } from '../types.js' */
+
+/** @typedef {TypedEmitter<{ 'add-drive': (drive: import('hyperdrive')) => void }>} InternalDriveEmitter */
+
+// prop = blob type name
+// value = array of blob variants supported for that type
+const SUPPORTED_BLOB_VARIANTS = /** @type {const} */ ({
+  photo: ['original', 'preview', 'thumbnail'],
+  audio: ['original'],
+  video: ['original'],
+})
+
+// Cannot directly export the const assignment above because export does not
+// like the assignment being wrapped in parenthesis, which is necessary to cast
+// the type with JSDoc
+export { SUPPORTED_BLOB_VARIANTS }
+
+class ErrNotFound extends Error {
+  constructor(message = 'NotFound') {
+    super(message)
+    this.code = 'ENOENT'
+  }
+}
+
+export class BlobStore {
+  /** @type {Map<string, Hyperdrive>} Indexed by hex-encoded discovery key */
+  #hyperdrives = new Map()
+  #writer
+  /**
+   * Used to communicate to live download instances when new drives are added
+   * @type {InternalDriveEmitter}
+   */
+  #driveEmitter = new TypedEmitter()
+
+  /**
+   * @param {object} options
+   * @param {import('../core-manager/index.js').CoreManager} options.coreManager
+   */
+  constructor({ coreManager }) {
+    /** @type {undefined | (Hyperdrive & { key: Buffer })} */
+    let writer
+    const corestore = new PretendCorestore({ coreManager })
+    const blobIndexCores = coreManager.getCores('blobIndex')
+    const { key: writerKey } = coreManager.getWriterCore('blobIndex')
+    for (const { key } of blobIndexCores) {
+      // @ts-ignore - we know pretendCorestore is not actually a Corestore
+      const drive = new Hyperdrive(corestore, key)
+      // We use the discovery key to derive the id for a drive
+      this.#hyperdrives.set(getDiscoveryId(key), drive)
+      if (key.equals(writerKey)) {
+        writer = proxyProps(drive, { key: writerKey })
+      }
+    }
+    if (!writer) {
+      throw new Error('Could not find a writer for the blobIndex namespace')
+    }
+    this.#writer = writer
+
+    coreManager.on('add-core', ({ key, namespace }) => {
+      if (namespace !== 'blobIndex') return
+      // We use the discovery key to derive the id for a drive
+      const driveId = getDiscoveryId(key)
+      if (this.#hyperdrives.has(driveId)) return
+      // @ts-ignore - we know pretendCorestore is not actually a Corestore
+      const drive = new Hyperdrive(corestore, key)
+      this.#hyperdrives.set(driveId, drive)
+      this.#driveEmitter.emit('add-drive', drive)
+    })
+  }
+
+  get writerDriveId() {
+    return getDiscoveryId(this.#writer.key)
+  }
+
+  /**
+   * @param {string} driveId hex-encoded discovery key
+   */
+  #getDrive(driveId) {
+    const drive = this.#hyperdrives.get(driveId)
+    if (!drive) throw new Error('Drive not found ' + driveId.slice(0, 7))
+    return drive
+  }
+
+  /**
+   * @param {BlobId} blobId
+   * @param {object} opts
+   * @param {false} [opts.wait=false] Set to `true` to wait for a blob to download, otherwise will throw if blob is not available locally
+   * @param {never} [opts.timeout] Optional timeout to wait for a blob to download
+   */
+  async get({ type, variant, name, driveId }, { wait = false, timeout } = {}) {
+    const drive = this.#getDrive(driveId)
+    const path = makePath({ type, variant, name })
+    const blob = await drive.get(path, { wait, timeout })
+    if (!blob) throw new ErrNotFound()
+    return blob
+  }
+
+  /**
+   * Download blobs from all drives, optionally filtering particular blob types
+   * or blob variants. Download will be 'live' and will continue downloading new
+   * data as it becomes available from any replicating drive.
+   *
+   * If no filter is specified, all blobs will be downloaded. If a filter is
+   * specified, then _only_ blobs that match the filter will be downloaded.
+   *
+   * @param {import('../types.js').BlobFilter} [filter] Filter blob types and/or variants to download. Filter is { [BlobType]: BlobVariants[] }. At least one blob variant must be specified for each blob type.
+   * @param {object} options
+   * @param {AbortSignal} [options.signal] Optional AbortSignal to cancel in-progress download
+   * @returns EventEmitter with `.state` propery, emits `state` with new state when it updates
+   */
+  download(filter, { signal } = {}) {
+    return new LiveDownload(this.#hyperdrives.values(), this.#driveEmitter, {
+      filter,
+      signal,
+    })
+  }
+
+  /**
+   * @param {BlobId} blobId
+   * @param {object} [options]
+   * @param {boolean} [options.wait=false] Set to `true` to wait for a blob to download, otherwise will throw if blob is not available locally
+   * @param {number} [options.timeout] Optional timeout to wait for a blob to download
+   */
+  createReadStream(
+    { type, variant, name, driveId },
+    options = { wait: false }
+  ) {
+    // TODO: Error thrown from this be an emit error on the returned stream?
+    const drive = this.#getDrive(driveId)
+    const path = makePath({ type, variant, name })
+
+    // @ts-ignore - TODO: update @digidem/types to include wait/timeout options
+    return drive.createReadStream(path, options)
+  }
+
+  /**
+   * Optimization for creating the blobs read stream when you have
+   * previously read the entry from Hyperdrive using `drive.entry`
+   * @param {BlobId['driveId']} driveId Hyperdrive drive discovery id
+   * @param {import('hyperdrive').HyperdriveEntry} entry Hyperdrive entry
+   * @param {object} [options]
+   * @param {boolean} [options.wait=false] Set to `true` to wait for a blob to download, otherwise will throw if blob is not available locally
+   */
+  async createEntryReadStream(driveId, entry, options = { wait: false }) {
+    const drive = this.#getDrive(driveId)
+    const blobs = await drive.getBlobs()
+
+    if (!blobs) {
+      throw new Error(
+        'Hyperblobs instance not found for drive ' + driveId.slice(0, 7)
+      )
+    }
+
+    return blobs.createReadStream(entry.value.blob, options)
+  }
+
+  /**
+   * @param {BlobId['driveId']} driveId Hyperdrive drive id
+   * @param {import('hyperdrive').HyperdriveEntry} entry Hyperdrive entry
+   * @param {object} [opts]
+   * @param {number} [opts.length]
+   *
+   * @returns {Promise<Buffer | null>}
+   */
+  async getEntryBlob(driveId, entry, { length } = {}) {
+    const drive = this.#getDrive(driveId)
+    const blobs = await drive.getBlobs()
+
+    if (!blobs) {
+      throw new Error(
+        'Hyperblobs instance not found for drive ' + driveId.slice(0, 7)
+      )
+    }
+
+    return blobs.get(entry.value.blob, { wait: false, start: 0, length })
+  }
+
+  /**
+   *
+   * @param {Omit<BlobId, 'driveId'>} blobId
+   * @param {Buffer} blob
+   * @param {object} [options]
+   * @param {{mimeType: string}} [options.metadata] Metadata to store with the blob
+   * @returns {Promise<string>} discovery key as hex string of hyperdrive where blob is stored
+   */
+  async put({ type, variant, name }, blob, options) {
+    const path = makePath({ type, variant, name })
+    await this.#writer.put(path, blob, options)
+    return this.writerDriveId
+  }
+
+  /**
+   * @param {Omit<BlobId, 'driveId'>} blobId
+   * @param {object} [options]
+   * @param {{mimeType: string}} [options.metadata] Metadata to store with the blob
+   */
+  createWriteStream({ type, variant, name }, options) {
+    const path = makePath({ type, variant, name })
+    const stream = this.#writer.createWriteStream(path, options)
+    return proxyProps(stream, {
+      driveId: this.writerDriveId,
+    })
+  }
+
+  /**
+   * @param {BlobId} blobId
+   * @param {object} [options]
+   * @param {boolean} [options.follow=false] Set to `true` to follow symlinks (16 max or throws an error)
+   * @param {false} [options.wait=false] Set to `true` to wait for a blob to download, otherwise will throw if blob is not available locally
+   * @param {never} [options.timeout] Optional timeout to wait for a blob to download
+   * @returns {Promise<import('hyperdrive').HyperdriveEntry | null>}
+   */
+  async entry(
+    { type, variant, name, driveId },
+    options = { follow: false, wait: false }
+  ) {
+    const drive = this.#hyperdrives.get(driveId)
+    if (!drive) throw new Error('Drive not found ' + driveId.slice(0, 7))
+    const path = makePath({ type, variant, name })
+    const entry = await drive.entry(path, options)
+    return entry
+  }
+
+  /**
+   * @param {BlobId} blobId
+   * @param {object} [options]
+   * @param {boolean} [options.diff=false] Enable to return an object with a `block` property with number of bytes removed
+   * @return {Promise<{ blocks: number } | null>}
+   */
+  async clear({ type, variant, name, driveId }, options = {}) {
+    const path = makePath({ type, variant, name })
+    const drive = this.#getDrive(driveId)
+
+    return drive.clear(path, options)
+  }
+}
+
+/**
+ * @template {object} T
+ * @template {object} U
+ * @param {T} target
+ * @param {U} props
+ * @returns {T & U}
+ */
+function proxyProps(target, props) {
+  // @ts-ignore - too much time to learn how to teach this to Typescript
+  return new Proxy(target, {
+    get(target, prop, receiver) {
+      if (Object.hasOwn(props, prop)) {
+        return Reflect.get(props, prop, receiver)
+      } else {
+        return Reflect.get(target, prop, receiver)
+      }
+    },
+  })
+}
+
+/** @param {Pick<BlobId, 'type' | 'variant' | 'name'>} opts */
+function makePath({ type, variant, name }) {
+  return `/${type}/${variant}/${name}`
+}
+
+/**
+ * Implements the `get()` method as used by hyperdrive-next. It returns the
+ * relevant cores from the Mapeo CoreManager.
+ */
+class PretendCorestore {
+  #coreManager
+  /**
+   * @param {object} options
+   * @param {import('../core-manager/index.js').CoreManager} options.coreManager
+   */
+  constructor({ coreManager }) {
+    this.#coreManager = coreManager
+  }
+
+  /**
+   * @param {Buffer | { publicKey: Buffer } | { name: string }} opts
+   * @returns {import('hypercore')<"binary", Buffer> | undefined}
+   */
+  get(opts) {
+    if (b4a.isBuffer(opts)) {
+      opts = { publicKey: opts }
+    }
+    if ('key' in opts) {
+      // @ts-ignore
+      opts.publicKey = opts.key
+    }
+    if ('publicKey' in opts) {
+      // NB! We should always add blobIndex (Hyperbee) cores to the core manager
+      // before we use them here. We would only reach the addCore path if the
+      // blob core is read from the hyperbee header (before it is added to the
+      // core manager)
+      return (
+        this.#coreManager.getCoreByKey(opts.publicKey) ||
+        this.#coreManager.addCore(opts.publicKey, 'blob').core
+      )
+    } else if (opts.name === 'db') {
+      return this.#coreManager.getWriterCore('blobIndex').core
+    } else if (opts.name.includes('blobs')) {
+      return this.#coreManager.getWriterCore('blob').core
+    } else {
+      throw new Error(
+        'Unsupported corestore.get() with opts ' + util.inspect(opts)
+      )
+    }
+  }
+
+  /** no-op */
+  close() {}
+}
+
+/**
+ * @param {Buffer} key Public key of hypercore
+ * @returns {string} Hex-encoded string of derived discovery key
+ */
+function getDiscoveryId(key) {
+  return discoveryKey(key).toString('hex')
+}