@comapeo/core 1.0.0

package/src/core-manager/remote-bitfield.js

@@ -0,0 +1,416 @@
+// Thanks to:
+// https://github.com/holepunchto/hypercore/blob/c572826/lib/remote-bitfield.js
+// With added JSDoc types
+
+import BigSparseArray from 'big-sparse-array'
+import quickbit from 'quickbit-universal'
+
+export const BITS_PER_PAGE = 32768
+const BYTES_PER_PAGE = BITS_PER_PAGE / 8
+const WORDS_PER_PAGE = BYTES_PER_PAGE / 4
+const BITS_PER_SEGMENT = 2097152
+const BYTES_PER_SEGMENT = BITS_PER_SEGMENT / 8
+const PAGES_PER_SEGMENT = BITS_PER_SEGMENT / BITS_PER_PAGE
+
+class RemoteBitfieldPage {
+  /**
+   *
+   * @param {number} index
+   * @param {Uint32Array} bitfield
+   * @param {RemoteBitfieldSegment} segment
+   */
+  constructor(index, bitfield, segment) {
+    /** @type {typeof index} */
+    this.index = index
+    /** @type {number} */
+    this.offset = index * BYTES_PER_PAGE - segment.offset
+    /** @type {typeof bitfield} */
+    this.bitfield = bitfield
+    /** @type {typeof segment} */
+    this.segment = segment
+
+    segment.add(this)
+  }
+
+  get tree() {
+    return this.segment.tree
+  }
+
+  /**
+   *
+   * @param {number} index
+   * @returns {boolean}
+   */
+  get(index) {
+    return quickbit.get(this.bitfield, index)
+  }
+
+  /**
+   *
+   * @param {number} index
+   * @param {boolean} val
+   */
+  set(index, val) {
+    if (quickbit.set(this.bitfield, index, val)) {
+      this.tree.update(this.offset * 8 + index)
+    }
+  }
+
+  /**
+   *
+   * @param {number} start
+   * @param {number} length
+   * @param {boolean} val
+   */
+  setRange(start, length, val) {
+    quickbit.fill(this.bitfield, val, start, start + length)
+
+    let i = Math.floor(start / 128)
+    const n = i + Math.ceil(length / 128)
+
+    while (i <= n) this.tree.update(this.offset * 8 + i++ * 128)
+  }
+  /**
+   *
+   * @param {boolean} val
+   * @param {number} position
+   */
+  findFirst(val, position) {
+    return quickbit.findFirst(this.bitfield, val, position)
+  }
+  /**
+   * @param {boolean} val
+   * @param {number} position
+   */
+  findLast(val, position) {
+    return quickbit.findLast(this.bitfield, val, position)
+  }
+
+  /**
+   *
+   * @param {number} start
+   * @param {Uint32Array} bitfield
+   */
+  insert(start, bitfield) {
+    this.bitfield.set(bitfield, start / 32)
+    this.segment.refresh()
+  }
+}
+
+class RemoteBitfieldSegment {
+  /**
+   *
+   * @param {number} index
+   */
+  constructor(index) {
+    this.index = index
+    this.offset = index * BYTES_PER_SEGMENT
+    this.tree = /** @type {import('quickbit-universal').SparseIndex} */ (
+      quickbit.Index.from([], BYTES_PER_SEGMENT)
+    )
+    this.pages = new Array(PAGES_PER_SEGMENT)
+    this.pagesLength = 0
+  }
+
+  get chunks() {
+    return this.tree.chunks
+  }
+
+  refresh() {
+    this.tree = /** @type {import('quickbit-universal').SparseIndex} */ (
+      quickbit.Index.from(this.tree.chunks, BYTES_PER_SEGMENT)
+    )
+  }
+
+  /**
+   * @param {RemoteBitfieldPage} page
+   */
+  add(page) {
+    const pageIndex = page.index - this.index * PAGES_PER_SEGMENT
+    if (pageIndex >= this.pagesLength) this.pagesLength = pageIndex + 1
+
+    this.pages[pageIndex] = page
+
+    const chunk = { field: page.bitfield, offset: page.offset }
+
+    this.chunks.push(chunk)
+
+    for (let i = this.chunks.length - 2; i >= 0; i--) {
+      const prev = this.chunks[i]
+      if (prev.offset <= chunk.offset) break
+      this.chunks[i] = chunk
+      this.chunks[i + 1] = prev
+    }
+  }
+
+  /**
+   *
+   * @param {boolean} val
+   * @param {number} position
+   */
+  findFirst(val, position) {
+    position = this.tree.skipFirst(!val, position)
+
+    let j = position & (BITS_PER_PAGE - 1)
+    let i = (position - j) / BITS_PER_PAGE
+
+    if (i >= PAGES_PER_SEGMENT) return -1
+
+    while (i < this.pagesLength) {
+      const p = this.pages[i]
+
+      let index = -1
+
+      if (p) index = p.findFirst(val, j)
+      else if (!val) index = j
+
+      if (index !== -1) return i * BITS_PER_PAGE + index
+
+      j = 0
+      i++
+    }
+
+    return -1
+  }
+
+  /**
+   * @param {boolean} val
+   * @param {number} position
+   */
+  findLast(val, position) {
+    position = this.tree.skipLast(!val, position)
+
+    let j = position & (BITS_PER_PAGE - 1)
+    let i = (position - j) / BITS_PER_PAGE
+
+    if (i >= PAGES_PER_SEGMENT) return -1
+
+    while (i >= 0) {
+      const p = this.pages[i]
+
+      let index = -1
+
+      if (p) index = p.findLast(val, j)
+      else if (!val) index = j
+
+      if (index !== -1) return i * BITS_PER_PAGE + index
+
+      j = BITS_PER_PAGE - 1
+      i--
+    }
+
+    return -1
+  }
+}
+
+export default class RemoteBitfield {
+  constructor() {
+    /** @type {BigSparseArray<RemoteBitfieldPage>} */
+    this._pages = new BigSparseArray()
+    /** @type {BigSparseArray<RemoteBitfieldSegment>} */
+    this._segments = new BigSparseArray()
+    this._maxSegments = 0
+  }
+
+  /**
+   * @param {number} index
+   */
+  get(index) {
+    const j = index & (BITS_PER_PAGE - 1)
+    const i = (index - j) / BITS_PER_PAGE
+
+    const p = this._pages.get(i)
+
+    return p ? p.get(j) : false
+  }
+
+  /**
+   * @param {number} index
+   */
+  getBitfield(index) {
+    const j = index & (BITS_PER_PAGE - 1)
+    const i = (index - j) / BITS_PER_PAGE
+
+    const p = this._pages.get(i)
+    return p || null
+  }
+
+  /**
+   * @param {number} index
+   * @param {boolean} val
+   */
+  set(index, val) {
+    const j = index & (BITS_PER_PAGE - 1)
+    const i = (index - j) / BITS_PER_PAGE
+
+    let p = this._pages.get(i)
+
+    if (!p && val) {
+      const k = Math.floor(i / PAGES_PER_SEGMENT)
+      const s =
+        this._segments.get(k) ||
+        this._segments.set(k, new RemoteBitfieldSegment(k))
+      if (this._maxSegments <= k) this._maxSegments = k + 1
+
+      p = this._pages.set(
+        i,
+        new RemoteBitfieldPage(i, new Uint32Array(WORDS_PER_PAGE), s)
+      )
+    }
+
+    if (p) p.set(j, val)
+  }
+
+  /**
+   * @param {number} start
+   * @param {number} length
+   * @param {boolean} val
+   */
+  setRange(start, length, val) {
+    let j = start & (BITS_PER_PAGE - 1)
+    let i = (start - j) / BITS_PER_PAGE
+
+    while (length > 0) {
+      let p = this._pages.get(i)
+
+      if (!p && val) {
+        const k = Math.floor(i / PAGES_PER_SEGMENT)
+        const s =
+          this._segments.get(k) ||
+          this._segments.set(k, new RemoteBitfieldSegment(k))
+        if (this._maxSegments <= k) this._maxSegments = k + 1
+
+        p = this._pages.set(
+          i,
+          new RemoteBitfieldPage(i, new Uint32Array(WORDS_PER_PAGE), s)
+        )
+      }
+
+      const end = Math.min(j + length, BITS_PER_PAGE)
+      const range = end - j
+
+      if (p) p.setRange(j, range, val)
+
+      j = 0
+      i++
+      length -= range
+    }
+  }
+
+  /**
+   * @param {boolean} val
+   * @param {number} position
+   */
+  findFirst(val, position) {
+    let j = position & (BITS_PER_SEGMENT - 1)
+    let i = (position - j) / BITS_PER_SEGMENT
+
+    while (i < this._maxSegments) {
+      const s = this._segments.get(i)
+
+      let index = -1
+
+      if (s) index = s.findFirst(val, j)
+      else if (!val) index = j
+
+      if (index !== -1) return i * BITS_PER_SEGMENT + index
+
+      j = 0
+      i++
+    }
+
+    return val ? -1 : position
+  }
+  /**
+   * @param {number} position
+   */
+  firstSet(position) {
+    return this.findFirst(true, position)
+  }
+  /**
+   * @param {number} position
+   */
+  firstUnset(position) {
+    return this.findFirst(false, position)
+  }
+  /**
+   * @param {boolean} val
+   * @param {number} position
+   */
+  findLast(val, position) {
+    let j = position & (BITS_PER_SEGMENT - 1)
+    let i = (position - j) / BITS_PER_SEGMENT
+
+    while (i >= 0) {
+      const s = this._segments.get(i)
+
+      let index = -1
+
+      if (s) index = s.findLast(val, j)
+      else if (!val) index = j
+
+      if (index !== -1) return i * BITS_PER_SEGMENT + index
+
+      j = BITS_PER_SEGMENT - 1
+      i--
+    }
+
+    return -1
+  }
+
+  /**
+   * @param {number} position
+   */
+  lastSet(position) {
+    return this.findLast(true, position)
+  }
+  /**
+   * @param {number} position
+   */
+  lastUnset(position) {
+    return this.findLast(false, position)
+  }
+  /**
+   * @param {number} start
+   * @param {Uint32Array} bitfield
+   * @returns {boolean}
+   */
+  insert(start, bitfield) {
+    if (start % 32 !== 0) return false
+
+    let length = bitfield.byteLength * 8
+
+    let j = start & (BITS_PER_PAGE - 1)
+    let i = (start - j) / BITS_PER_PAGE
+
+    while (length > 0) {
+      let p = this._pages.get(i)
+
+      if (!p) {
+        const k = Math.floor(i / PAGES_PER_SEGMENT)
+        const s =
+          this._segments.get(k) ||
+          this._segments.set(k, new RemoteBitfieldSegment(k))
+        if (this._maxSegments <= k) this._maxSegments = k + 1
+
+        p = this._pages.set(
+          i,
+          new RemoteBitfieldPage(i, new Uint32Array(WORDS_PER_PAGE), s)
+        )
+      }
+
+      const end = Math.min(j + length, BITS_PER_PAGE)
+      const range = end - j
+
+      p.insert(j, bitfield.subarray(0, range / 32))
+
+      bitfield = bitfield.subarray(range / 32)
+
+      j = 0
+      i++
+      length -= range
+    }
+
+    return true
+  }
+}

package/src/core-ownership.js

@@ -0,0 +1,235 @@
+import { verifySignature, sign } from '@mapeo/crypto'
+import { parseVersionId } from '@comapeo/schema'
+import { defaultGetWinner } from '@mapeo/sqlite-indexer'
+import assert from 'node:assert/strict'
+import sodium from 'sodium-universal'
+import {
+  kTable,
+  kSelect,
+  kCreateWithDocId,
+  kDataStore,
+} from './datatype/index.js'
+import { eq, or } from 'drizzle-orm'
+import mapObject from 'map-obj'
+import { discoveryKey } from 'hypercore-crypto'
+import pDefer from 'p-defer'
+import { NAMESPACES } from './constants.js'
+import { TypedEmitter } from 'tiny-typed-emitter'
+/**
+ * @import {
+ *   CoreOwnershipWithSignatures,
+ *   CoreOwnershipWithSignaturesValue,
+ *   KeyPair,
+ *   Namespace
+ * } from './types.js'
+ */
+
+/**
+ * @typedef {object} CoreOwnershipEvents
+ * @property {(docIds: Set<string>) => void} update Emitted when new coreOwnership records are indexed
+ */
+
+/**
+ * @extends {TypedEmitter<CoreOwnershipEvents>}
+ */
+export class CoreOwnership extends TypedEmitter {
+  #dataType
+  #ownershipWriteDone
+  /**
+   *
+   * @param {object} opts
+   * @param {import('./datatype/index.js').DataType<
+   *   import('./datastore/index.js').DataStore<'auth'>,
+   *   typeof import('./schema/project.js').coreOwnershipTable,
+   *   'coreOwnership',
+   *   import('@comapeo/schema').CoreOwnership,
+   *   import('@comapeo/schema').CoreOwnershipValue
+   * >} opts.dataType
+   * @param {Record<Namespace, KeyPair>} opts.coreKeypairs
+   * @param {KeyPair} opts.identityKeypair
+   */
+  constructor({ dataType, coreKeypairs, identityKeypair }) {
+    super()
+    this.#dataType = dataType
+    const authWriterCore = dataType[kDataStore].writerCore
+    const deferred = pDefer()
+    this.#ownershipWriteDone = deferred.promise
+
+    const writeOwnership = () => {
+      if (authWriterCore.length > 0) {
+        deferred.resolve()
+        return
+      }
+      this.#writeOwnership(identityKeypair, coreKeypairs)
+        .then(deferred.resolve)
+        .catch(deferred.reject)
+    }
+    // @ts-ignore - opened missing from types
+    if (authWriterCore.opened) {
+      writeOwnership()
+    } else {
+      authWriterCore.once('ready', writeOwnership)
+    }
+
+    dataType[kDataStore].on('coreOwnership', this.emit.bind(this, 'update'))
+  }
+
+  /**
+   * @param {string} coreId
+   * @returns {Promise<string>} deviceId of device that owns the core
+   */
+  async getOwner(coreId) {
+    await this.#ownershipWriteDone
+    const table = this.#dataType[kTable]
+    const expressions = []
+    for (const namespace of NAMESPACES) {
+      expressions.push(eq(table[`${namespace}CoreId`], coreId))
+    }
+    // prettier-ignore
+    const result = (await this.#dataType[kSelect]())
+      .where(or.apply(null, expressions))
+      .get()
+    if (!result) {
+      throw new Error('NotFound')
+    }
+    return result.docId
+  }
+
+  /**
+   *
+   * @param {string} deviceId
+   * @param {Namespace} namespace
+   * @returns {Promise<string>} coreId of core belonging to `deviceId` for `namespace`
+   */
+  async getCoreId(deviceId, namespace) {
+    const result = await this.get(deviceId)
+    return result[`${namespace}CoreId`]
+  }
+
+  /**
+   * Get capabilities for a given deviceId
+   *
+   * @param {string} deviceId
+   */
+  async get(deviceId) {
+    await this.#ownershipWriteDone
+    return this.#dataType.getByDocId(deviceId)
+  }
+
+  async getAll() {
+    await this.#ownershipWriteDone
+    return this.#dataType.getMany()
+  }
+
+  /**
+   *
+   * @param {KeyPair} identityKeypair
+   * @param {Record<Namespace, KeyPair>} coreKeypairs
+   */
+  async #writeOwnership(identityKeypair, coreKeypairs) {
+    /** @type {CoreOwnershipWithSignaturesValue} */
+    const docValue = {
+      schemaName: 'coreOwnership',
+      ...mapObject(coreKeypairs, (key, value) => {
+        return [`${key}CoreId`, value.publicKey.toString('hex')]
+      }),
+      identitySignature: sign(
+        identityKeypair.publicKey,
+        identityKeypair.secretKey
+      ),
+      coreSignatures: mapObject(coreKeypairs, (key, value) => {
+        return [key, sign(value.publicKey, value.secretKey)]
+      }),
+    }
+    const docId = identityKeypair.publicKey.toString('hex')
+    await this.#dataType[kCreateWithDocId](docId, docValue)
+  }
+}
+
+/**
+ * - Validate that the doc is written to the core identified by doc.authCoreId
+ * - Verify the signatures
+ * - Remove the signatures (we don't add them to the indexer)
+ * - Set doc.links to an empty array - this forces the indexer to treat every
+ *   document as a fork, so getWinner is called for every doc, which resolves to
+ *   the doc with the lowest index (e.g. the first)
+ *
+ * @param {CoreOwnershipWithSignatures} doc
+ * @param {import('@comapeo/schema').VersionIdObject} version
+ * @returns {import('@comapeo/schema').CoreOwnership}
+ */
+export function mapAndValidateCoreOwnership(doc, { coreDiscoveryKey }) {
+  if (
+    !coreDiscoveryKey.equals(discoveryKey(Buffer.from(doc.authCoreId, 'hex')))
+  ) {
+    throw new Error('Invalid coreOwnership record: mismatched authCoreId')
+  }
+  if (!verifyCoreOwnership(doc)) {
+    throw new Error('Invalid coreOwnership record: signatures are invalid')
+  }
+  // eslint-disable-next-line no-unused-vars
+  const { identitySignature, coreSignatures, ...docWithoutSignatures } = doc
+  docWithoutSignatures.links = []
+  return docWithoutSignatures
+}
+
+/**
+ * Verify the signatures of a coreOwnership record, which verify that the device
+ * with the identityKey matching the docIds does own (e.g. can write to) cores
+ * with the given core IDs
+ *
+ * @param {CoreOwnershipWithSignatures} doc
+ * @returns {boolean}
+ */
+function verifyCoreOwnership(doc) {
+  const { coreSignatures, identitySignature } = doc
+  for (const namespace of NAMESPACES) {
+    const signature = coreSignatures[namespace]
+    const coreKey = Buffer.from(doc[`${namespace}CoreId`], 'hex')
+    assert.equal(
+      signature.length,
+      sodium.crypto_sign_BYTES,
+      'Invalid core ownership signature'
+    )
+    assert.equal(
+      coreKey.length,
+      sodium.crypto_sign_PUBLICKEYBYTES,
+      'Invalid core ownership coreId'
+    )
+    const isValidSignature = verifySignature(coreKey, signature, coreKey)
+    if (!isValidSignature) return false
+  }
+  const identityPublicKey = Buffer.from(doc.docId, 'hex')
+  assert.equal(identitySignature.length, sodium.crypto_sign_BYTES)
+  assert.equal(identityPublicKey.length, sodium.crypto_sign_PUBLICKEYBYTES)
+  const isValidIdentitySignature = verifySignature(
+    identityPublicKey,
+    identitySignature,
+    identityPublicKey
+  )
+  if (!isValidIdentitySignature) return false
+  return true
+}
+
+/**
+ * For coreOwnership records, we only trust the first record written to the core.
+ *
+ * @type {NonNullable<ConstructorParameters<typeof import('./index-writer/index.js').IndexWriter>[0]['getWinner']>}
+ */
+export function getWinner(docA, docB) {
+  if (
+    'schemaName' in docA &&
+    docA.schemaName === 'coreOwnership' &&
+    'schemaName' in docB &&
+    docB.schemaName === 'coreOwnership'
+  ) {
+    // Assumes docA and docB have same coreKey, so we choose the first one
+    // written to the core
+    const docAindex = parseVersionId(docA.versionId).index
+    const docBindex = parseVersionId(docB.versionId).index
+    if (docAindex < docBindex) return docA
+    return docB
+  } else {
+    return defaultGetWinner(docA, docB)
+  }
+}

package/src/datastore/README.md

@@ -0,0 +1,46 @@
+# DataStore
+
+> Manage reading cores for indexing, and reading and writing documents to cores.
+
+## Purpose
+
+The `DataStore` class is an API over a CoreManager namespace, responsible for reading blocks for indexing from all cores in a namespace; writing new documents to the namespace writer core, and reading existing documents from any core in the namespace based on the `versionId`. `DataStore` does not write documents to an index, it only reads them for indexing - it will call the `batch()` constructor option with entries that are read from cores in the namespace that the datastore manages. Writes will only resolve once `batch()` resolves (e.g. once a document has been written to the SQLite index tables).
+
+## Usage
+
+The `DataStore` class is used internally by the [`DataType`](../datatype/) class.
+
+An example of `DataStore` usage taken from the [datastore tests](../../tests/datastore.js):
+
+```js
+const datastore = new DataStore({
+  coreManager,
+  batch: async (entries) => {
+    // Process entries here using an indexer...
+  },
+  namespace: 'data',
+})
+
+/** @type {MapeoDoc} */
+const newObservation = await datastore.write(observationValue)
+/** @type {MapeoDoc} */
+const existingObservation = await datastore.read(versionId)
+
+datastore.on('index-state', ({ current, remaining, entriesPerSecond }) => {
+  if (current === 'idle') {
+    // indexing done for now
+  } else if (current === 'indexing') {
+    // show state to user that indexing is happening
+  }
+})
+
+const { current, remaining, entriesPerSecond } = datastore.getIndexState()
+```
+
+## API docs
+
+TODO!
+
+## Tests
+
+Tests for this module are in [tests/datastore.js](../../tests/datastore.js)