@comapeo/core 1.0.0

package/src/icon-api.js

@@ -0,0 +1,282 @@
+/** @import { PresetValue, IconValue } from '@comapeo/schema' */
+
+export const kGetIconBlob = Symbol('getIcon')
+
+/** @typedef {PresetValue['iconRef']} IconRef */
+/** @typedef {IconValue['variants']} IconVariants */
+/** @typedef {IconVariants[number]} IconVariant */
+
+/**
+ * @typedef {Exclude<IconVariant['size'], 'size_unspecified'>} ValidSizes
+ */
+
+/**
+ * @typedef {Object} BitmapOpts
+ * @property {Extract<IconVariant['mimeType'], 'image/png'>} mimeType
+ * @property {Extract<IconVariant, {mimeType: 'image/png'}>['pixelDensity']} pixelDensity
+ * @property {ValidSizes} size
+ *
+ * @typedef {Object} SvgOpts
+ * @property {Extract<IconVariant['mimeType'], 'image/svg+xml'>} mimeType
+ * @property {ValidSizes} size
+ */
+
+/** @type {{ [mime in IconVariant['mimeType']]: string }} */
+const MIME_TO_EXTENSION = {
+  'image/png': '.png',
+  'image/svg+xml': '.svg',
+}
+
+export class IconApi {
+  #dataType
+  #dataStore
+  #getMediaBaseUrl
+
+  /**
+   * @param {Object} opts
+   * @param {import('./datatype/index.js').DataType<
+   *   import('./datastore/index.js').DataStore<'config'>,
+   *   typeof import('./schema/project.js').iconTable,
+   *   'icon',
+   *   import('@comapeo/schema').Icon,
+   *   import('@comapeo/schema').IconValue
+   * >} opts.iconDataType
+   * @param {import('./datastore/index.js').DataStore<'config'>} opts.iconDataStore
+   * @param {() => Promise<string>} opts.getMediaBaseUrl
+   */
+  constructor({ iconDataType, iconDataStore, getMediaBaseUrl }) {
+    this.#dataType = iconDataType
+    this.#dataStore = iconDataStore
+    this.#getMediaBaseUrl = getMediaBaseUrl
+  }
+
+  /**
+   * @param {object} icon
+   * @param {import('@comapeo/schema').IconValue['name']} icon.name
+   * @param {Array<(BitmapOpts | SvgOpts) & { blob: Buffer }>} icon.variants
+   *
+   * @returns {Promise<import('@comapeo/schema').Icon>}
+   */
+  async create(icon) {
+    if (icon.variants.length < 1) {
+      throw new Error('empty variants array')
+    }
+
+    const savedVariants = await Promise.all(
+      icon.variants.map(async ({ blob, ...variant }) => {
+        const blobVersionId = await this.#dataStore.writeRaw(blob)
+        return { ...variant, blobVersionId }
+      })
+    )
+
+    return await this.#dataType.create({
+      schemaName: 'icon',
+      name: icon.name,
+      variants: savedVariants,
+    })
+  }
+
+  /**
+   * @param {string} iconId
+   * @param {BitmapOpts | SvgOpts} opts
+   *
+   * @returns {Promise<Buffer>}
+   */
+  async [kGetIconBlob](iconId, opts) {
+    const iconRecord = await this.#dataType.getByDocId(iconId)
+    const iconVariant = getBestVariant(iconRecord.variants, opts)
+    const blob = await this.#dataStore.readRaw(iconVariant.blobVersionId)
+    return blob
+  }
+
+  /**
+   * @param {string} iconId
+   * @param {BitmapOpts | SvgOpts} opts
+   *
+   * @returns {Promise<string>}
+   */
+  async getIconUrl(iconId, opts) {
+    let base = await this.#getMediaBaseUrl()
+
+    if (!base.endsWith('/')) {
+      base += '/'
+    }
+
+    const mimeExtension = MIME_TO_EXTENSION[opts.mimeType]
+
+    const pixelDensity =
+      opts.mimeType === 'image/svg+xml' ||
+      // if the pixel density is 1, we can omit the density suffix in the resulting url
+      // and assume the pixel density is 1 for applicable mime types when using the url
+      opts.pixelDensity === 1
+        ? undefined
+        : opts.pixelDensity
+
+    return (
+      base +
+      constructIconPath({
+        pixelDensity,
+        size: opts.size,
+        extension: mimeExtension,
+        iconId,
+      })
+    )
+  }
+}
+
+/**
+ * @type {Record<IconVariant['size'], number>}
+ */
+const SIZE_AS_NUMERIC = {
+  // NOTE: when size is unspecified, we fallback to medium
+  size_unspecified: 2,
+  small: 1,
+  medium: 2,
+  large: 3,
+}
+
+/**
+ * Given a list of icon variants returns the variant that most closely matches the desired parameters.
+ * Rules, in order of precedence:
+ *
+ * 1. Matching mime type (throw if no matches)
+ * 2. Matching size. If no exact match:
+ *     1. If smaller ones exist, prefer closest smaller size.
+ *     2. Otherwise prefer closest larger size.
+ * 3. Matching pixel density (when asking for PNGs). If no exact match:
+ *     1. If smaller ones exist, prefer closest smaller density.
+ *     2. Otherwise prefer closest larger density.
+ *
+ * @param {IconVariants} variants
+ * @param {BitmapOpts | SvgOpts} opts
+ */
+export function getBestVariant(variants, opts) {
+  const { size: wantedSize, mimeType: wantedMimeType } = opts
+  /** @type {BitmapOpts['pixelDensity']} */
+  let wantedPixelDensity
+  if (opts.mimeType === 'image/png') {
+    wantedPixelDensity = opts.pixelDensity
+  }
+  if (variants.length === 0) {
+    throw new Error('No variants exist')
+  }
+
+  const matchingMime = variants.filter((v) => v.mimeType === wantedMimeType)
+
+  if (matchingMime.length === 0) {
+    throw new Error(
+      `No variants with desired mime type ${wantedMimeType} exist`
+    )
+  }
+  const wantedSizeNum = SIZE_AS_NUMERIC[wantedSize]
+
+  // Sort the relevant variants based on the desired size and pixel density, using the rules of the preference.
+  // Sorted from closest match to furthest match.
+  matchingMime.sort((a, b) => {
+    const aSizeNum = SIZE_AS_NUMERIC[a.size]
+    const bSizeNum = SIZE_AS_NUMERIC[b.size]
+
+    const aSizeDiff = aSizeNum - wantedSizeNum
+    const bSizeDiff = bSizeNum - wantedSizeNum
+
+    // Both variants match desired size, use pixel density (when png) to determine preferred match
+    if (aSizeDiff === 0 && bSizeDiff === 0) {
+      // What to do if asking for an svg and both (a and b) are svgs and have the same size, what criteria do we use?
+      // For now, we don't change sort order
+      if (wantedMimeType === 'image/svg+xml') {
+        return 0
+      } else if (
+        wantedMimeType === 'image/png' &&
+        a.mimeType === 'image/png' &&
+        b.mimeType === 'image/png'
+      ) {
+        return determineSortValue(
+          wantedPixelDensity,
+          a.pixelDensity,
+          b.pixelDensity
+        )
+      }
+    }
+
+    return determineSortValue(wantedSizeNum, aSizeNum, bSizeNum)
+  })
+
+  // Closest match will be first element
+  return matchingMime[0]
+}
+
+/**
+ * Determines a sort value based on the order of precedence outlined below. Winning value moves closer to front.
+ *
+ * 1. Exactly match `target`
+ * 2. Closest value smaller than `target`
+ * 3. Closest value larger than `target`
+ *
+ * @param {number} target
+ * @param {number} a
+ * @param {number} b
+ *
+ * @returns {-1 | 0 | 1}
+ */
+function determineSortValue(target, a, b) {
+  const aDiff = a - target
+  const bDiff = b - target
+
+  // Both match exactly, don't change sort order
+  if (aDiff === 0 && bDiff === 0) {
+    return 0
+  }
+
+  // a matches but b doesn't, prefer a
+  if (aDiff === 0 && bDiff !== 0) {
+    return -1
+  }
+
+  // b matches but a doesn't, prefer b
+  if (bDiff === 0 && aDiff !== 0) {
+    return 1
+  }
+
+  // Both are larger than desired, prefer smaller of the two
+  if (aDiff > 0 && bDiff > 0) {
+    return a < b ? -1 : 1
+  }
+
+  // Both are smaller than desired, prefer larger of the two
+  if (aDiff < 0 && bDiff < 0) {
+    return a < b ? 1 : -1
+  }
+
+  // Mix of smaller and larger than desired, prefer smaller of the two
+  return a < b ? -1 : 1
+}
+
+/**
+ * General purpose path builder for an icon
+ *
+ * @param {object} opts
+ * @param {string} opts.iconId
+ * @param {string} opts.size
+ * @param {number} [opts.pixelDensity]
+ * @param {string} opts.extension
+ *
+ * @returns {string}
+ */
+export function constructIconPath({ size, pixelDensity, iconId, extension }) {
+  if (iconId.length === 0 || size.length === 0 || extension.length === 0) {
+    throw new Error('iconId, size, and extension cannot be empty strings')
+  }
+
+  let result = `${iconId}/${size}`
+
+  if (typeof pixelDensity === 'number') {
+    if (pixelDensity < 1) {
+      throw new Error('pixelDensity must be a positive number')
+    }
+    result += `@${pixelDensity}x`
+  }
+
+  result += extension.startsWith('.') ? extension : '.' + extension
+
+  return result
+}

package/src/index-writer/README.md

@@ -0,0 +1,38 @@
+# IndexWriter
+
+> Index documents by `docId` in a SQLite database
+
+## Purpose
+
+The `IndexWriter` class resolves the DAG (Directed Acyclic Graph) for each `docId` to create a materialized view in SQLite of the current "head" for each `docId`. A single `IndexWriter` is responsible for decoding hypercore entries and writing them to the appropriate table (one table for each Mapeo data type). Unknown entries / data types are ignored.
+
+## Usage
+
+`IndexWriter` is used by `DataStore` instances for writing entries that are read from the cores in the namespace managed by each `DataStore`. It exports a single method `batch(entries)` where an entry is:
+
+```ts
+type Entry = {
+  block: Buffer // raw data entry read from a hypercore
+  key: Buffer // public key of the hypercore where the block was read from
+  index: number // index of the block in the hypercore
+}
+```
+
+A usage example:
+
+```js
+const indexWriter = new IndexWriter({
+  tables, // Array of Drizzle table schema definitions to index
+  sqlite, // better-sqlite3 Database instance
+})
+
+indexWriter.batch(entries)
+```
+
+## API docs
+
+TODO!
+
+## Tests
+
+There are no unit tests for the IndexWriter, tests coverage will be from end-to-end / integration tests.

package/src/index-writer/index.js

@@ -0,0 +1,124 @@
+import { decode } from '@comapeo/schema'
+import SqliteIndexer from '@mapeo/sqlite-indexer'
+import { getTableConfig } from 'drizzle-orm/sqlite-core'
+import { getBacklinkTableName } from '../schema/utils.js'
+import { discoveryKey } from 'hypercore-crypto'
+import { Logger } from '../logger.js'
+/** @import { MapeoDoc, VersionIdObject } from '@comapeo/schema' */
+/** @import { MapeoDocTables } from '../datatype/index.js' */
+
+/**
+ * @typedef {{ [K in MapeoDoc['schemaName']]?: string[] }} IndexedDocIds
+ */
+/**
+ * @typedef {ReturnType<typeof decode>} MapeoDocInternal
+ */
+
+/**
+ * @template {MapeoDocTables} [TTables=MapeoDocTables]
+ */
+export class IndexWriter {
+  /**
+   * @internal
+   * @typedef {TTables['_']['name']} SchemaName
+   */
+
+  /** @type {Map<SchemaName, SqliteIndexer>} */
+  #indexers = new Map()
+  #mapDoc
+  #l
+  /**
+   *
+   * @param {object} opts
+   * @param {import('better-sqlite3').Database} opts.sqlite
+   * @param {TTables[]} opts.tables
+   * @param {(doc: MapeoDocInternal, version: VersionIdObject) => MapeoDoc} [opts.mapDoc] optionally transform a document prior to indexing. Can also validate, if an error is thrown then the document will not be indexed
+   * @param {typeof import('@mapeo/sqlite-indexer').defaultGetWinner} [opts.getWinner] custom function to determine the "winner" of two forked documents. Defaults to choosing the document with the most recent `updatedAt`
+   * @param {Logger} [opts.logger]
+   */
+  constructor({ tables, sqlite, mapDoc = (d) => d, getWinner, logger }) {
+    this.#l = Logger.create('indexWriter', logger)
+    this.#mapDoc = mapDoc
+    for (const table of tables) {
+      const config = getTableConfig(table)
+      const schemaName = /** @type {(typeof table)['_']['name']} */ (
+        config.name
+      )
+      const indexer = new SqliteIndexer(sqlite, {
+        docTableName: config.name,
+        backlinkTableName: getBacklinkTableName(config.name),
+        getWinner,
+      })
+      this.#indexers.set(schemaName, indexer)
+    }
+  }
+
+  /**
+   * @returns {Iterable<SchemaName>}
+   */
+  get schemas() {
+    return this.#indexers.keys()
+  }
+
+  /**
+   * @param {import('multi-core-indexer').Entry[]} entries
+   * @returns {Promise<IndexedDocIds>} map of indexed docIds by schemaName
+   */
+  async batch(entries) {
+    // sqlite-indexer is _significantly_ faster when batching even <10 at a
+    // time, so best to queue docs here before calling sliteIndexer.batch()
+    /** @type {Record<string, MapeoDoc[]>} */
+    const queued = {}
+    /** @type {IndexedDocIds} */
+    const indexed = {}
+    for (const { block, key, index } of entries) {
+      /** @type {MapeoDoc} */ let doc
+      try {
+        const version = { coreDiscoveryKey: discoveryKey(key), index }
+        doc = this.#mapDoc(decode(block, version), version)
+      } catch (e) {
+        this.#l.log('Could not decode entry %d of %h', index, key)
+        // Unknown or invalid entry - silently ignore
+        continue
+      }
+      // Don't have an indexer for this type - silently ignore
+      if (!this.#indexers.has(doc.schemaName)) continue
+      if (queued[doc.schemaName]) {
+        queued[doc.schemaName].push(doc)
+        // @ts-expect-error - we know this is defined, TS doesn't
+        indexed[doc.schemaName].push(doc.docId)
+      } else {
+        queued[doc.schemaName] = [doc]
+        indexed[doc.schemaName] = [doc.docId]
+      }
+    }
+    for (const [schemaName, docs] of Object.entries(queued)) {
+      // @ts-expect-error
+      const indexer = this.#indexers.get(schemaName)
+      if (!indexer) continue // Won't happen, but TS doesn't know that
+      indexer.batch(docs)
+      if (this.#l.enabled) {
+        for (const doc of docs) {
+          this.#l.log(
+            'Indexed %s %S @ %S',
+            doc.schemaName,
+            doc.docId,
+            doc.versionId
+          )
+        }
+      }
+    }
+    return indexed
+  }
+
+  /**
+   * @param {SchemaName} schemaName
+   */
+  deleteSchema(schemaName) {
+    const indexer = this.#indexers.get(schemaName)
+    if (!indexer) {
+      throw new Error(`IndexWriter doesn't know a schema named "${schemaName}"`)
+    }
+    indexer.deleteAll()
+  }
+}

package/src/index.js

@@ -0,0 +1,16 @@
+import {
+  CREATOR_ROLE_ID,
+  COORDINATOR_ROLE_ID,
+  MEMBER_ROLE_ID,
+} from './roles.js'
+export { plugin as MapeoStaticMapsFastifyPlugin } from './fastify-plugins/maps/static-maps.js'
+export { plugin as MapeoOfflineFallbackMapFastifyPlugin } from './fastify-plugins/maps/offline-fallback-map.js'
+export { plugin as MapeoMapsFastifyPlugin } from './fastify-plugins/maps/index.js'
+export { FastifyController } from './fastify-controller.js'
+export { MapeoManager } from './mapeo-manager.js'
+
+export const roles = /** @type {const} */ ({
+  CREATOR_ROLE_ID,
+  COORDINATOR_ROLE_ID,
+  MEMBER_ROLE_ID,
+})