@comapeo/core 3.1.2 → 4.0.0

package/drizzle/project/meta/_journal.json

@@ -15,6 +15,13 @@
       "when": 1729783892753,
       "tag": "0001_medical_wendell_rand",
       "breakpoints": true
+    },
+    {
+      "idx": 2,
+      "version": "5",
+      "when": 1749657174497,
+      "tag": "0002_cooing_princess_powerful",
+      "breakpoints": true
     }
   ]
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@comapeo/core",
-  "version": "3.1.2",
+  "version": "4.0.0",
   "description": "Offline p2p mapping library",
   "main": "src/index.js",
   "types": "dist/index.d.ts",
@@ -112,12 +112,13 @@
     "@comapeo/core2.0.1": "npm:@comapeo/core@2.0.1",
     "@comapeo/ipc": "^2.1.0",
     "@mapeo/default-config": "5.0.0",
-    "@mapeo/mock-data": "^2.1.5",
+    "@mapeo/mock-data": "^5.0.0",
     "@sinonjs/fake-timers": "^10.0.2",
     "@types/b4a": "^1.6.0",
     "@types/bogon": "^1.0.2",
     "@types/compact-encoding": "^2.15.0",
     "@types/debug": "^4.1.8",
+    "@types/geojson": "^7946.0.16",
     "@types/json-schema": "^7.0.11",
     "@types/json-stable-stringify": "^1.0.36",
     "@types/nanobench": "^3.0.0",
@@ -136,12 +137,14 @@
     "drizzle-kit": "^0.20.14",
     "eslint": "^8.57.0",
     "execa": "^9.5.1",
+    "filter-obj": "^6.1.0",
     "husky": "^8.0.0",
     "iterpal": "^0.4.0",
     "lint-staged": "^14.0.1",
     "mapeo-offline-map": "^2.0.0",
     "math-random-seed": "^2.0.0",
     "nanobench": "^3.0.0",
+    "node-stream-zip": "^1.15.0",
     "npm-run-all": "^4.1.5",
     "prettier": "^2.8.8",
     "random-access-file": "^4.0.7",
@@ -159,7 +162,7 @@
   },
   "dependencies": {
     "@comapeo/fallback-smp": "^1.0.0",
-    "@comapeo/schema": "1.5.0",
+    "@comapeo/schema": "2.0.0",
     "@digidem/types": "^2.3.0",
     "@fastify/error": "^3.4.1",
     "@fastify/type-provider-typebox": "^4.1.0",
@@ -188,7 +191,7 @@
     "json-stable-stringify": "^1.1.1",
     "magic-bytes.js": "^1.10.0",
     "map-obj": "^5.0.2",
-    "mime": "^4.0.3",
+    "mime": "^4.0.7",
     "multi-core-indexer": "^1.0.0",
     "p-defer": "^4.0.0",
     "p-event": "^6.0.1",
@@ -210,6 +213,7 @@
     "varint": "^6.0.0",
     "ws": "^8.18.0",
     "xstate": "^5.19.2",
-    "yauzl-promise": "^4.0.0"
+    "yauzl-promise": "^4.0.0",
+    "zip-stream-promise": "^1.0.2"
   }
 }

package/src/blob-api.js

@@ -4,35 +4,9 @@ import { Transform, pipelinePromise as pipeline } from 'streamx'
 import { createHash, randomBytes } from 'node:crypto'
 /** @import { BlobId, BlobType } from './types.js' */
 
-/**
- * Location coordinate data. Based on [Expo's `LocationObjectCoords`][0].
- * [0]: https://docs.expo.dev/versions/latest/sdk/location/#locationobjectcoords
- *
- * @typedef {object} LocationObjectCoords
- * @prop {number | null} accuracy
- * @prop {number | null} altitude
- * @prop {number | null} altitudeAccuracy
- * @prop {number | null} heading
- * @prop {number} latitude
- * @prop {number} longitude
- * @prop {number | null} speed
- */
-
-/**
- * Location metadata for a blob. Based on [Expo's `LocationObject`][0].
- * [0]: https://docs.expo.dev/versions/latest/sdk/location/#locationobject
- *
- * @typedef {object} LocationObject
- * @prop {LocationObjectCoords} coords
- * @prop {boolean} [mocked]
- * @prop {number} timestamp
- */
-
 /**
  * @typedef {object} Metadata
  * @prop {string} mimeType
- * @prop {number} timestamp
- * @prop {LocationObject} [location]
  */
 
 export class BlobApi {

package/src/mapeo-project.js

@@ -4,6 +4,11 @@ import { decodeBlockPrefix, decode, parseVersionId } from '@comapeo/schema'
 import { drizzle } from 'drizzle-orm/better-sqlite3'
 import { discoveryKey } from 'hypercore-crypto'
 import { TypedEmitter } from 'tiny-typed-emitter'
+import ZipArchive from 'zip-stream-promise'
+import * as b4a from 'b4a'
+import mime from 'mime/lite'
+// @ts-expect-error
+import { Readable, pipelinePromise } from 'streamx'
 
 import { NAMESPACES, NAMESPACE_SCHEMAS } from './constants.js'
 import { CoreManager } from './core-manager/index.js'
@@ -38,6 +43,7 @@ import {
 } from './roles.js'
 import {
   assert,
+  buildBlobId,
   ExhaustivenessError,
   getDeviceId,
   projectKeyToId,
@@ -58,11 +64,16 @@ import { readConfig } from './config-import.js'
 import TranslationApi from './translation-api.js'
 import { NotFoundError, nullIfNotFound } from './errors.js'
 import { WebSocket } from 'ws'
-/** @import { ProjectSettingsValue } from '@comapeo/schema' */
-/** @import { CoreStorage, BlobFilter, BlobStoreEntriesStream, KeyPair, Namespace, ReplicationStream, GenericBlobFilter } from './types.js' */
-
+import { createWriteStream } from 'fs'
+/** @import { ProjectSettingsValue, Observation, Track } from '@comapeo/schema' */
+/** @import { Attachment, CoreStorage, BlobFilter, BlobId, BlobStoreEntriesStream, KeyPair, Namespace, ReplicationStream, GenericBlobFilter, MapeoValueMap, MapeoDocMap } from './types.js' */
 /** @typedef {Omit<ProjectSettingsValue, 'schemaName'>} EditableProjectSettings */
 /** @typedef {ProjectSettingsValue['configMetadata']} ConfigMetadata */
+/** @typedef {Map<string,Attachment>} SeenAttachments*/
+/** @typedef {object} BlobRef
+ * @prop {string|undefined} mimeType
+ * @prop {BlobId} blobId
+ */
 
 const CORESTORE_STORAGE_FOLDER_NAME = 'corestore'
 const INDEXER_STORAGE_FOLDER_NAME = 'indexer'
@@ -76,9 +87,13 @@ export const kProjectLeave = Symbol('leave project')
 export const kClearDataIfLeft = Symbol('clear data if left project')
 export const kSetIsArchiveDevice = Symbol('set isArchiveDevice')
 export const kIsArchiveDevice = Symbol('isArchiveDevice (temp - test only)')
+export const kGeoJSONFileName = Symbol('geoJSONFileName')
 
 const EMPTY_PROJECT_SETTINGS = Object.freeze({})
 
+/** @type BlobId['variant'][]*/
+const VARIANT_EXPORT_ORDER = ['original', 'preview', 'thumbnail']
+
 /**
  * @extends {TypedEmitter<{ close: () => void }>}
  */
@@ -749,6 +764,450 @@ export class MapeoProject extends TypedEmitter {
     return this.#iconApi
   }
 
+  /**
+   * @param {Iterable<Observation>} observations
+   * @param {Object} options
+   * @param {Set<string>} [options.seenObservations=new Set()]
+   * @param {SeenAttachments} [options.seenAttachments]
+   * @returns {AsyncIterable<Buffer | Uint8Array>}
+   */
+  async *#exportObservations(
+    observations,
+    { seenObservations = new Set(), seenAttachments = new Map() }
+  ) {
+    let first = true
+    for (const observation of observations) {
+      const { lat, lon, docId } = observation
+      if (seenObservations.has(docId)) {
+        continue
+      }
+      seenObservations.add(docId)
+      for (const attachment of observation.attachments) {
+        const { hash } = attachment
+        if (!seenAttachments.has(hash)) {
+          seenAttachments.set(hash, attachment)
+        }
+      }
+
+      const metadataCoords = observation.metadata?.position?.coords
+      const altitude = metadataCoords?.altitude
+
+      /** @type {[number, number] | [number, number, number] | null} */
+      let coordinates = null
+
+      // Prioritize using the observation's `lat` and `lon` fields
+      if (typeof lat === 'number' && typeof lon === 'number') {
+        coordinates =
+          typeof altitude === 'number' ? [lon, lat, altitude] : [lon, lat]
+      } else {
+        // Fall back to using the observation metadata's position if possible
+        if (
+          typeof metadataCoords?.latitude === 'number' &&
+          typeof metadataCoords?.longitude === 'number'
+        ) {
+          coordinates =
+            typeof altitude === 'number'
+              ? [metadataCoords.longitude, metadataCoords.latitude, altitude]
+              : [metadataCoords.longitude, metadataCoords.latitude]
+        }
+      }
+
+      /** @type {import('geojson').Feature<import('geojson').Point | null>} */
+      const feature = {
+        type: 'Feature',
+        properties: observation,
+        geometry: coordinates
+          ? {
+              type: 'Point',
+              coordinates,
+            }
+          : null,
+      }
+      const comma = first ? '' : ','
+      first = false
+      yield b4a.from(`${comma}\n      ` + JSON.stringify(feature))
+    }
+  }
+
+  /**
+   * @param {Iterable<Track>} tracks
+   * @param {Object} options
+   * @param {Set<string>} [options.seenObservations=new Set()]
+   * @param {SeenAttachments} [options.seenAttachments]
+   * @param {string} [options.lang]
+   * @returns {AsyncIterable<Buffer | Uint8Array>}
+   */
+  async *#exportTracks(
+    tracks,
+    { lang, seenObservations = new Set(), seenAttachments = new Map() } = {}
+  ) {
+    let first = true
+    for (const track of tracks) {
+      const { observationRefs } = track
+
+      const observations = await Promise.all(
+        observationRefs.map(({ docId }) =>
+          this.#dataTypes.observation.getByDocId(docId, { lang })
+        )
+      )
+
+      const coordinates = track.locations.map(
+        ({ coords: { longitude, latitude, altitude } }) => [
+          longitude,
+          latitude,
+          altitude,
+        ]
+      )
+      const comma = first ? '' : ','
+      first = false
+      yield b4a.from(
+        `${comma}\n      ` +
+          JSON.stringify({
+            type: 'Feature',
+            properties: track,
+            geometry: {
+              type: 'LineString',
+              coordinates,
+            },
+          }) +
+          '\n'
+      )
+
+      let firstObs = true
+      for await (const chunk of this.#exportObservations(observations, {
+        seenObservations,
+        seenAttachments,
+      })) {
+        if (firstObs) {
+          yield b4a.from(',')
+          firstObs = false
+        }
+        yield chunk
+      }
+    }
+  }
+
+  /**
+   * @param {Object} [options={}]
+   * @param {boolean} [options.observations=true] Whether observations should be exported
+   * @param {boolean} [options.tracks=true] Whether all tracks and their observations should be exported
+   * @param {SeenAttachments} [options.seenAttachments]
+   * @param {string} [options.lang]
+   * @returns {AsyncIterable<Buffer | Uint8Array>}
+   */
+  async *#exportGeoJSONIterator({
+    observations = true,
+    tracks = true,
+    lang,
+    seenAttachments = new Map(),
+  } = {}) {
+    yield b4a.from(`{
+  "type": "FeatureCollection",
+  "features": [
+`)
+
+    const seenObservations = new Set()
+
+    let hadTracks = false
+    if (tracks) {
+      const rows = await this.#dataTypes.track.getMany({ lang })
+      for await (const chunk of this.#exportTracks(rows, {
+        lang,
+        seenObservations,
+        seenAttachments,
+      })) {
+        hadTracks = true
+        yield chunk
+      }
+    }
+
+    if (observations) {
+      if (hadTracks) {
+        yield b4a.from(',')
+      }
+      const rows = await this.#dataTypes.observation.getMany({ lang })
+      yield* this.#exportObservations(rows, {
+        seenObservations,
+        seenAttachments,
+      })
+    }
+
+    yield b4a.from(`
+  ]
+}
+`)
+  }
+
+  /**
+   * Export observations and or tracks as a stream of GeoJSON data
+   * @param {Object} [options={}]
+   * @param {boolean} [options.observations=true] Whether observations should be exported
+   * @param {boolean} [options.tracks=true] Whether all tracks and their observations should be exported
+   * @param {SeenAttachments} [options.seenAttachments]
+   * @param {string} [options.lang]
+   * @returns {Readable<Buffer | Uint8Array>}
+   */
+  #exportGeoJSONStream({
+    observations = true,
+    tracks = true,
+    lang,
+    seenAttachments = new Map(),
+  } = {}) {
+    // Format based on https://doc.arcgis.com/en/arcgis-online/reference/geojson.htm
+
+    return Readable.from(
+      this.#exportGeoJSONIterator({
+        observations,
+        tracks,
+        lang,
+        seenAttachments,
+      })
+    )
+  }
+
+  /**
+   *
+   * @param {string} type Type of export this will be
+   * @returns {Promise<string>}
+   */
+  async #exportPrefix(type = '') {
+    const name = await this.#getProjectName()
+    const date = new Date()
+    const dateSection = date
+      .toLocaleDateString('nu', {
+        year: 'numeric',
+        month: 'numeric',
+        day: 'numeric',
+      })
+      .replaceAll('/', '_')
+      .replaceAll('-', '_')
+    return `CoMapeo_${name}_${type}_${dateSection}`
+  }
+
+  /**
+   * @param {boolean} observations
+   * @param {boolean} tracks
+   * @returns {Promise<string>}
+   */
+  async [kGeoJSONFileName](observations, tracks) {
+    let exportType = ''
+    if (observations) exportType += 'Obsvns'
+    if (tracks) {
+      if (observations) exportType += '_'
+      exportType += 'Tracks'
+    }
+    const prefix = await this.#exportPrefix(exportType)
+
+    return prefix + '.geojson'
+  }
+
+  /**
+   * @param {boolean} observations
+   * @param {boolean} tracks
+   * @returns {Promise<string>}
+   */
+  async #zipFileName(observations, tracks) {
+    let exportType = ''
+    if (observations) exportType += 'Obsvns'
+    if (tracks) {
+      if (observations) exportType += '_'
+      exportType += 'Tracks'
+    }
+    const prefix = await this.#exportPrefix(exportType)
+
+    return prefix + '.zip'
+  }
+
+  /**
+   * Export observations and or tracks as a GeoJSON file
+   * @param {string} exportFolder Path to save the file. The file name is auto-generated
+   * @param {Object} [options={}]
+   * @param {boolean} [options.observations=true] Whether observations should be exported
+   * @param {boolean} [options.tracks=true] Whether all tracks and their observations should be exported
+   * @param {string} [options.lang]
+   * @returns {Promise<string>} The full path that the file was exported at
+   */
+  async exportGeoJSONFile(
+    exportFolder,
+    { observations = true, tracks = true, lang } = {}
+  ) {
+    const fileName = await this[kGeoJSONFileName](observations, tracks)
+    const filePath = path.join(exportFolder, fileName)
+    const source = this.#exportGeoJSONStream({ observations, tracks, lang })
+    const sink = createWriteStream(filePath)
+    await pipelinePromise(source, sink)
+
+    return filePath
+  }
+
+  /**
+   * @param {Attachment} attachment
+   * @returns {Promise<null | BlobRef>}
+   */
+  async #tryGetAttachmentBlob(attachment) {
+    // Audio must not have variants
+    for (const variant of VARIANT_EXPORT_ORDER) {
+      try {
+        const blobId = buildBlobId(attachment, variant)
+        const entry = await this.#blobStore.entry(blobId)
+        if (!entry) continue
+        const metadata = entry.value.metadata
+        if (!metadata || typeof metadata !== 'object') continue
+        let mimeType = undefined
+        if ('mimeType' in metadata) {
+          if (typeof metadata.mimeType === 'string') {
+            mimeType = metadata.mimeType
+          } else {
+            this.#l.log('Invalid type for mimeType in blob', blobId, entry)
+            continue
+          }
+        }
+        return { blobId, mimeType }
+      } catch (e) {
+        if (!(e instanceof Error)) throw e
+        this.#l.log(
+          'Error loading blob id for attachment',
+          attachment,
+          variant,
+          e.message
+        )
+        continue
+      }
+    }
+
+    return null
+  }
+
+  /**
+   * @param {ZipArchive} archive
+   * @param {Object} [options={}]
+   * @param {boolean} [options.observations=true] Whether observations should be exported
+   * @param {boolean} [options.tracks=true] Whether all tracks and their observations should be exported
+   * @param {boolean} [options.attachments=true] Whether all attachments for observations should be exported
+   * @param {string} [options.lang]
+   * @returns {Promise<void>}
+   */
+  async #exportToArchive(
+    archive,
+    { observations = true, tracks = true, attachments = true, lang } = {}
+  ) {
+    // GeoJSON
+    const geoJSONFileName = await this[kGeoJSONFileName](observations, tracks)
+    const seenAttachments = new Map()
+    const geoJSONStream = this.#exportGeoJSONStream({
+      observations,
+      tracks,
+      lang,
+      seenAttachments,
+    })
+
+    // @ts-expect-error
+    await archive.entry(geoJSONStream, { name: geoJSONFileName })
+
+    const missingAttachments = []
+    // Attachments
+    if (attachments) {
+      const mediaFolder = (await this.#exportPrefix('Media')) + '/'
+      for (const attachment of seenAttachments.values()) {
+        const ref = await this.#tryGetAttachmentBlob(attachment)
+        if (ref === null) {
+          missingAttachments.push(attachment)
+          continue
+        }
+
+        const { blobId, mimeType } = ref
+        let extensionString = ''
+        if (mimeType) {
+          const extension = mime.getExtension(mimeType)
+          if (extension) {
+            extensionString = '.' + extension
+          } else {
+            this.#l.log('Got unknown mime type in attachment blob', attachment)
+          }
+        }
+
+        const stream = this.#blobStore.createReadStream(blobId)
+        const name = path.posix.join(
+          mediaFolder,
+          blobId.variant,
+          `${attachment.name}${extensionString}`
+        )
+
+        // @ts-expect-error
+        await archive.entry(stream, { name })
+      }
+
+      if (missingAttachments.length) {
+        this.#l.log(`Found missing attachments during export`)
+        const missingContent = missingAttachments
+          .map((attachment) => JSON.stringify(attachment))
+          .join('\n')
+
+        await archive.entry(missingContent, {
+          name: mediaFolder + 'missing.ndjson',
+        })
+      }
+    }
+    // Finalize
+    archive.finalize()
+  }
+
+  /**
+   * Export observations, tracks, and or attachments as a zip file stream.
+   * @param {Object} [options={}]
+   * @param {boolean} [options.observations=true] Whether observations should be exported
+   * @param {boolean} [options.tracks=true] Whether all tracks and their observations should be exported
+   * @param {boolean} [options.attachments=true] Whether all attachments for observations should be exported
+   * @param {string} [options.lang]
+   * @returns {Readable<Buffer | Uint8Array>}
+   */
+  #exportZipStream({
+    observations = true,
+    tracks = true,
+    attachments = true,
+    lang,
+  } = {}) {
+    const archive = new ZipArchive()
+
+    this.#exportToArchive(archive, {
+      observations,
+      tracks,
+      attachments,
+      lang,
+    }).catch((e) => archive.emit('error', e))
+
+    // @ts-expect-error
+    return archive
+  }
+
+  /**
+   * Export observations, tracks, and or attachments as a zip file.
+   * @param {string} exportFolder Path to save the file. The file name is auto-generated
+   * @param {Object} [options={}]
+   * @param {boolean} [options.observations=true] Whether observations should be exported
+   * @param {boolean} [options.tracks=true] Whether all tracks and their observations should be exported
+   * @param {boolean} [options.attachments=true] Whether all attachments for observations should be exported
+   * @param {string} [options.lang]
+   * @returns {Promise<string>} The full path that the file was exported at
+   */
+  async exportZipFile(
+    exportFolder,
+    { observations = true, tracks = true, attachments = true, lang } = {}
+  ) {
+    const fileName = await this.#zipFileName(observations, tracks)
+    const filePath = path.join(exportFolder, fileName)
+    const source = this.#exportZipStream({
+      observations,
+      tracks,
+      attachments,
+      lang,
+    })
+    const sink = createWriteStream(filePath)
+    await pipelinePromise(source, sink)
+
+    return filePath
+  }
+
   /**
    * @returns {Promise<void>}
    */

package/src/types.ts

@@ -6,7 +6,13 @@ import type {
   SetOptional,
 } from 'type-fest'
 import { SUPPORTED_BLOB_VARIANTS } from './blob-store/index.js'
-import { MapeoCommon, MapeoDoc, MapeoValue, decode } from '@comapeo/schema'
+import {
+  MapeoCommon,
+  MapeoDoc,
+  MapeoValue,
+  Observation,
+  decode,
+} from '@comapeo/schema'
 import type BigSparseArray from 'big-sparse-array'
 import type Protomux from 'protomux'
 import type NoiseStream from '@hyperswarm/secret-stream'
@@ -156,6 +162,8 @@ export type BlobStoreEntriesStream = Readable & {
   >
 }
 
+export type Attachment = Observation['attachments'][number]
+
 export type StringToTaggedUnion<T extends string> = {
   [K in T]: {
     type: K

package/src/utils.js

@@ -4,6 +4,8 @@ import { createHash } from 'node:crypto'
 import stableStringify from 'json-stable-stringify'
 import { omit } from './lib/omit.js'
 
+/** @import {Attachment, BlobId} from "./types.js" */
+
 const PROJECT_INVITE_ID_SALT = Buffer.from('mapeo project invite id', 'ascii')
 
 /**
@@ -185,3 +187,35 @@ export function hashObject(obj) {
     .digest()
     .toString('hex')
 }
+
+/**
+ * Convert attachments to BlobIds for use in the BlobStore, adapted from comapeo-mobile
+ * @param {Attachment} attachment
+ * @param {'original' | 'thumbnail' | 'preview'} requestedVariant
+ * @returns {BlobId}
+ */
+export function buildBlobId(attachment, requestedVariant) {
+  if (
+    attachment.type !== 'photo' &&
+    attachment.type !== 'audio' &&
+    attachment.type !== 'video'
+  ) {
+    throw new Error(`Cannot fetch URL for attachment type "${attachment.type}"`)
+  }
+
+  if (attachment.type === 'photo') {
+    return {
+      type: 'photo',
+      variant: requestedVariant,
+      name: attachment.name,
+      driveId: attachment.driveDiscoveryId,
+    }
+  }
+
+  return {
+    type: attachment.type,
+    variant: 'original',
+    name: attachment.name,
+    driveId: attachment.driveDiscoveryId,
+  }
+}