@comapeo/core 1.0.0

package/src/datastore/index.js

@@ -0,0 +1,234 @@
+import { TypedEmitter } from 'tiny-typed-emitter'
+import { encode, decode, getVersionId, parseVersionId } from '@comapeo/schema'
+import MultiCoreIndexer from 'multi-core-indexer'
+import pDefer from 'p-defer'
+import { discoveryKey } from 'hypercore-crypto'
+import { NAMESPACE_SCHEMAS } from '../constants.js'
+import { createMap } from '../utils.js'
+/** @import { MapeoDoc } from '@comapeo/schema' */
+
+/**
+ * For each schemaName in this dataStore, emits an array of docIds that have
+ * been indexed whenever indexing finishes (becomes idle). `projectSettings`
+ * currently not supported.
+ *
+ * @template {MapeoDoc['schemaName']} TSchemaName
+ * @typedef {{
+ *   [S in Exclude<TSchemaName, 'projectSettings'>]: (docIds: Set<string>) => void
+ * }} DataStoreEvents
+ */
+/**
+ * @template T
+ * @template {keyof any} K
+ * @typedef {T extends any ? Omit<T, K> : never} OmitUnion
+ */
+
+/**
+ * @typedef {typeof NAMESPACE_SCHEMAS} NamespaceSchemas
+ */
+
+/**
+ * @template {keyof NamespaceSchemas} [TNamespace=keyof NamespaceSchemas]
+ * @template {NamespaceSchemas[TNamespace][number]} [TSchemaName=NamespaceSchemas[TNamespace][number]]
+ * @extends {TypedEmitter<DataStoreEvents<TSchemaName>>}
+ */
+export class DataStore extends TypedEmitter {
+  #coreManager
+  #namespace
+  #batch
+  #writerCore
+  #coreIndexer
+  /** @type {Map<string, import('p-defer').DeferredPromise<void>>} */
+  #pendingIndex = new Map()
+  /** @type {Set<import('p-defer').DeferredPromise<void>['promise']>} */
+  #pendingAppends = new Set()
+  /** @type {Record<MapeoDoc['schemaName'], Set<string>>} */
+  #pendingEmits
+
+  /**
+   * @param {object} opts
+   * @param {import('../core-manager/index.js').CoreManager} opts.coreManager
+   * @param {TNamespace} opts.namespace
+   * @param {(entries: MultiCoreIndexer.Entry<'binary'>[]) => Promise<import('../index-writer/index.js').IndexedDocIds>} opts.batch
+   * @param {MultiCoreIndexer.StorageParam} opts.storage
+   */
+  constructor({ coreManager, namespace, batch, storage }) {
+    super()
+    this.#coreManager = coreManager
+    this.#namespace = namespace
+    this.#batch = batch
+    this.#pendingEmits = createMap(
+      NAMESPACE_SCHEMAS[namespace],
+      () => new Set()
+    )
+    this.#writerCore = coreManager.getWriterCore(namespace).core
+    const cores = coreManager.getCores(namespace).map((cr) => cr.core)
+    this.#coreIndexer = new MultiCoreIndexer(cores, {
+      storage,
+      batch: (entries) => this.#handleEntries(entries),
+    })
+    coreManager.on('add-core', (coreRecord) => {
+      if (coreRecord.namespace !== namespace) return
+      this.#coreIndexer.addCore(coreRecord.core)
+    })
+    this.#coreIndexer.on('idle', this.#handleIndexerIdle)
+  }
+
+  get indexer() {
+    return this.#coreIndexer
+  }
+
+  get namespace() {
+    return this.#namespace
+  }
+
+  get schemas() {
+    // Return a shallow copy (slice(0)) to avoid mutation bugs
+    return NAMESPACE_SCHEMAS[this.#namespace].slice(0)
+  }
+
+  get writerCore() {
+    return this.#writerCore
+  }
+
+  getIndexState() {
+    return this.#coreIndexer.state
+  }
+
+  /**
+   *
+   * @param {MultiCoreIndexer.Entry<'binary'>[]} entries
+   */
+  async #handleEntries(entries) {
+    const indexed = await this.#batch(entries)
+    await Promise.all(this.#pendingAppends)
+    // Writes to the writerCore need to wait until the entry is indexed before
+    // returning, so we check if any incoming entry has a pending promise
+    for (const entry of entries) {
+      if (!entry.key.equals(this.#writerCore.key)) continue
+      const versionId = getVersionId({
+        coreDiscoveryKey: discoveryKey(entry.key),
+        index: entry.index,
+      })
+      const pending = this.#pendingIndex.get(versionId)
+      if (!pending) continue
+      pending.resolve()
+    }
+    for (const schemaName of this.schemas) {
+      // Unsupported initially
+      if (schemaName === 'projectSettings') continue
+      const docIds = indexed[schemaName]
+      if (!docIds) continue
+      for (const docId of docIds) {
+        // We'll only emit these once the indexer is idle - a particular docId
+        // could have many updates, and we only emit it once after indexing
+        this.#pendingEmits[schemaName].add(docId)
+      }
+    }
+  }
+
+  /**
+   * UNSAFE: Does not check links: [] refer to a valid doc - should only be used
+   * internally.
+   *
+   * Write a doc, must be one of the schema types supported by the namespace of
+   * this DataStore.
+   * @template {Extract<Parameters<encode>[0], { schemaName: TSchemaName }>} TDoc
+   * @param {TDoc} doc
+   * @returns {Promise<Extract<MapeoDoc, TDoc>>}
+   */
+  async write(doc) {
+    // @ts-ignore
+    if (!NAMESPACE_SCHEMAS[this.#namespace].includes(doc.schemaName)) {
+      throw new Error(
+        `Schema '${doc.schemaName}' is not allowed in namespace '${
+          this.#namespace
+        }'`
+      )
+    }
+    const block = encode(doc)
+    // The indexer batch can sometimes complete before the append below
+    // resolves, so in the batch function we await any pending appends. We can't
+    // know the versionId before the append, because docs can be written in the
+    // same tick, so we can't know their index before append resolves.
+    const deferredAppend = pDefer()
+    this.#pendingAppends.add(deferredAppend.promise)
+    const { length } = await this.#writerCore.append(block)
+    deferredAppend.resolve()
+    this.#pendingAppends.delete(deferredAppend.promise)
+
+    const index = length - 1
+    const coreDiscoveryKey = this.#writerCore.discoveryKey
+    if (!coreDiscoveryKey) {
+      throw new Error('Writer core is not ready')
+    }
+    const versionId = getVersionId({ coreDiscoveryKey, index })
+    /** @type {import('p-defer').DeferredPromise<void>} */
+    const deferred = pDefer()
+    this.#pendingIndex.set(versionId, deferred)
+    await deferred.promise
+
+    return /** @type {Extract<MapeoDoc, TDoc>} */ (
+      decode(block, { coreDiscoveryKey, index })
+    )
+  }
+
+  /**
+   *
+   * @param {string} versionId
+   * @returns {Promise<MapeoDoc>}
+   */
+  async read(versionId) {
+    const { coreDiscoveryKey, index } = parseVersionId(versionId)
+    const coreRecord = this.#coreManager.getCoreByDiscoveryKey(coreDiscoveryKey)
+    if (!coreRecord) throw new Error('Invalid versionId')
+    const block = await coreRecord.core.get(index, { wait: false })
+    if (!block) throw new Error('Not Found')
+    return decode(block, { coreDiscoveryKey, index })
+  }
+
+  /** @param {Buffer} buf */
+  async writeRaw(buf) {
+    const { length } = await this.#writerCore.append(buf)
+    const index = length - 1
+    const coreDiscoveryKey = this.#writerCore.discoveryKey
+    if (!coreDiscoveryKey) {
+      throw new Error('Writer core is not ready')
+    }
+    const versionId = getVersionId({ coreDiscoveryKey, index })
+    return versionId
+  }
+
+  /** @param {string} versionId */
+  async readRaw(versionId) {
+    const { coreDiscoveryKey, index } = parseVersionId(versionId)
+    const coreRecord = this.#coreManager.getCoreByDiscoveryKey(coreDiscoveryKey)
+    if (!coreRecord) throw new Error('core not found')
+    const block = await coreRecord.core.get(index, { wait: false })
+    if (!block) throw new Error('Not Found')
+    return block
+  }
+
+  async close() {
+    await this.#coreIndexer.close()
+  }
+
+  /**
+   * Unlink all index files. This should only be called after `close()` has resolved.
+   */
+  async unlink() {
+    await this.#coreIndexer.unlink()
+  }
+
+  #handleIndexerIdle = () => {
+    for (const eventName of this.eventNames()) {
+      if (!(eventName in this.#pendingEmits)) continue
+      const docIds = this.#pendingEmits[eventName]
+      if (!docIds.size) continue
+      // @ts-ignore - I'm pretty sure TS is just not smart enough here, it's not me!
+      this.emit(eventName, docIds)
+    }
+    // Make sure we reset this, otherwise we'd get a memory leak of this growing without bounds.
+    this.#pendingEmits = createMap(this.schemas, () => new Set())
+  }
+}

package/src/datatype/README.md

@@ -0,0 +1,33 @@
+# DataType
+
+> Create, read, update, delete, and query data.
+
+## Purpose
+
+The `DataType` class implements CRUD methods for a particular Mapeo data type. Reads and queries are done from the SQLite indexes / materialized views. Historical data read from a `versionId` is read from the associated `DataStore`.
+
+## Usage
+
+`DataType` is exposed directly on the client API for data types that can be read/written directly by the client. It serves as an abstraction for reading indexed data and creating/updating documents for all data types stored in Mapeo core namespaces.
+
+A usage example:
+
+```js
+const dataType = new DataType({
+  table: observationTable, // Drizzle table schema definition
+  db, // Drizzle instance
+  dataStore, // DataStore instance
+})
+
+const observation = await observation.getByDocId(id)
+const updated = await observation.update(observation.versionId, newValue)
+const allObservations = await observation.getMany()
+```
+
+## API docs
+
+TODO!
+
+## Tests
+
+Tests for this module are in [tests/datatype.js](../../tests/datatype.js)

package/src/datatype/index.d.ts

@@ -0,0 +1,108 @@
+// Typescript was incorrectly compiling declaration files for this, so the
+// declaration for DataType is written manually below, and copied into the
+// `dist` folder at build-time. The types are checked in `test-types/data-types.ts`
+
+import { type MapeoDoc, type MapeoValue } from '@comapeo/schema'
+import {
+  type MapeoDocMap,
+  type MapeoValueMap,
+  type CoreOwnershipWithSignaturesValue,
+} from '../types.js'
+import { type BetterSQLite3Database } from 'drizzle-orm/better-sqlite3'
+import { SQLiteSelectBase } from 'drizzle-orm/sqlite-core'
+import { RunResult } from 'better-sqlite3'
+import type Hypercore from 'hypercore'
+import { TypedEmitter } from 'tiny-typed-emitter'
+import TranslationApi from '../translation-api.js'
+
+type MapeoDocTableName = `${MapeoDoc['schemaName']}Table`
+type GetMapeoDocTables<T> = T[keyof T & MapeoDocTableName]
+/** Union of Drizzle schema tables that correspond to MapeoDoc types (e.g. excluding backlink tables and other utility tables) */
+type MapeoDocTables =
+  | GetMapeoDocTables<typeof import('../schema/project.js')>
+  | GetMapeoDocTables<typeof import('../schema/client.js')>
+type MapeoDocTablesMap = {
+  [K in MapeoDocTables['_']['name']]: Extract<
+    MapeoDocTables,
+    { _: { name: K } }
+  >
+}
+export interface DataTypeEvents<TDoc extends MapeoDoc> {
+  'updated-docs': (docs: TDoc[]) => void
+}
+
+export const kCreateWithDocId: unique symbol
+export const kSelect: unique symbol
+export const kTable: unique symbol
+export const kDataStore: unique symbol
+
+type OmitUnion<T, K extends keyof any> = T extends any ? Omit<T, K> : never
+type ExcludeSchema<
+  T extends MapeoValue,
+  S extends MapeoValue['schemaName']
+> = Exclude<T, { schemaName: S }>
+
+export class DataType<
+  TDataStore extends import('../datastore/index.js').DataStore,
+  TTable extends MapeoDocTables,
+  TSchemaName extends TTable['_']['name'],
+  TDoc extends MapeoDocMap[TSchemaName],
+  TValue extends MapeoValueMap[TSchemaName]
+> extends TypedEmitter<DataTypeEvents<TDoc>> {
+  constructor({
+    dataStore,
+    table,
+    db,
+    getTranslations,
+  }: {
+    table: TTable
+    dataStore: TDataStore
+    db: import('drizzle-orm/better-sqlite3').BetterSQLite3Database
+    getTranslations: TranslationApi['get']
+  })
+
+  get [kTable](): TTable
+
+  get [kDataStore](): TDataStore
+
+  get schemaName(): TSchemaName
+
+  get namespace(): TDataStore.namespace
+
+  [kCreateWithDocId](
+    docId: string,
+    value:
+      | ExcludeSchema<TValue, 'coreOwnership'>
+      | CoreOwnershipWithSignaturesValue
+  ): Promise<TDoc & { forks: string[] }>
+
+  [kSelect](): Promise<SQLiteSelectBase<TTable, 'sync', RunResult>>
+
+  create<
+    T extends import('type-fest').Exact<
+      ExcludeSchema<TValue, 'coreOwnership'>,
+      T
+    >
+  >(value: T): Promise<TDoc & { forks: string[] }>
+
+  getByDocId(
+    docId: string,
+    opts?: { lang?: string }
+  ): Promise<TDoc & { forks: string[] }>
+
+  getByVersionId(versionId: string, opts?: { lang?: string }): Promise<TDoc>
+
+  getMany(opts?: {
+    includeDeleted?: boolean
+    lang?: string
+  }): Promise<Array<TDoc & { forks: string[] }>>
+
+  update<
+    T extends import('type-fest').Exact<
+      ExcludeSchema<TValue, 'coreOwnership'>,
+      T
+    >
+  >(versionId: string | string[], value: T): Promise<TDoc & { forks: string[] }>
+
+  delete(docId: string): Promise<TDoc & { forks: string[] }>
+}