@comapeo/core 3.2.0 → 4.0.0

package/drizzle/project/meta/_journal.json

@@ -15,6 +15,13 @@
       "when": 1729783892753,
       "tag": "0001_medical_wendell_rand",
       "breakpoints": true
+    },
+    {
+      "idx": 2,
+      "version": "5",
+      "when": 1749657174497,
+      "tag": "0002_cooing_princess_powerful",
+      "breakpoints": true
     }
   ]
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@comapeo/core",
-  "version": "3.2.0",
+  "version": "4.0.0",
   "description": "Offline p2p mapping library",
   "main": "src/index.js",
   "types": "dist/index.d.ts",
@@ -112,12 +112,13 @@
     "@comapeo/core2.0.1": "npm:@comapeo/core@2.0.1",
     "@comapeo/ipc": "^2.1.0",
     "@mapeo/default-config": "5.0.0",
-    "@mapeo/mock-data": "^3.0.0",
+    "@mapeo/mock-data": "^5.0.0",
     "@sinonjs/fake-timers": "^10.0.2",
     "@types/b4a": "^1.6.0",
     "@types/bogon": "^1.0.2",
     "@types/compact-encoding": "^2.15.0",
     "@types/debug": "^4.1.8",
+    "@types/geojson": "^7946.0.16",
     "@types/json-schema": "^7.0.11",
     "@types/json-stable-stringify": "^1.0.36",
     "@types/nanobench": "^3.0.0",
@@ -136,6 +137,7 @@
     "drizzle-kit": "^0.20.14",
     "eslint": "^8.57.0",
     "execa": "^9.5.1",
+    "filter-obj": "^6.1.0",
     "husky": "^8.0.0",
     "iterpal": "^0.4.0",
     "lint-staged": "^14.0.1",
@@ -160,7 +162,7 @@
   },
   "dependencies": {
     "@comapeo/fallback-smp": "^1.0.0",
-    "@comapeo/schema": "1.6.0",
+    "@comapeo/schema": "2.0.0",
     "@digidem/types": "^2.3.0",
     "@fastify/error": "^3.4.1",
     "@fastify/type-provider-typebox": "^4.1.0",
@@ -189,7 +191,7 @@
     "json-stable-stringify": "^1.1.1",
     "magic-bytes.js": "^1.10.0",
     "map-obj": "^5.0.2",
-    "mime": "^4.0.3",
+    "mime": "^4.0.7",
     "multi-core-indexer": "^1.0.0",
     "p-defer": "^4.0.0",
     "p-event": "^6.0.1",

package/src/blob-api.js

@@ -4,35 +4,9 @@ import { Transform, pipelinePromise as pipeline } from 'streamx'
 import { createHash, randomBytes } from 'node:crypto'
 /** @import { BlobId, BlobType } from './types.js' */
 
-/**
- * Location coordinate data. Based on [Expo's `LocationObjectCoords`][0].
- * [0]: https://docs.expo.dev/versions/latest/sdk/location/#locationobjectcoords
- *
- * @typedef {object} LocationObjectCoords
- * @prop {number | null} accuracy
- * @prop {number | null} altitude
- * @prop {number | null} altitudeAccuracy
- * @prop {number | null} heading
- * @prop {number} latitude
- * @prop {number} longitude
- * @prop {number | null} speed
- */
-
-/**
- * Location metadata for a blob. Based on [Expo's `LocationObject`][0].
- * [0]: https://docs.expo.dev/versions/latest/sdk/location/#locationobject
- *
- * @typedef {object} LocationObject
- * @prop {LocationObjectCoords} coords
- * @prop {boolean} [mocked]
- * @prop {number} timestamp
- */
-
 /**
  * @typedef {object} Metadata
  * @prop {string} mimeType
- * @prop {number} timestamp
- * @prop {LocationObject} [location]
  */
 
 export class BlobApi {

package/src/mapeo-project.js

@@ -6,6 +6,7 @@ import { discoveryKey } from 'hypercore-crypto'
 import { TypedEmitter } from 'tiny-typed-emitter'
 import ZipArchive from 'zip-stream-promise'
 import * as b4a from 'b4a'
+import mime from 'mime/lite'
 // @ts-expect-error
 import { Readable, pipelinePromise } from 'streamx'
 
@@ -69,6 +70,10 @@ import { createWriteStream } from 'fs'
 /** @typedef {Omit<ProjectSettingsValue, 'schemaName'>} EditableProjectSettings */
 /** @typedef {ProjectSettingsValue['configMetadata']} ConfigMetadata */
 /** @typedef {Map<string,Attachment>} SeenAttachments*/
+/** @typedef {object} BlobRef
+ * @prop {string|undefined} mimeType
+ * @prop {BlobId} blobId
+ */
 
 const CORESTORE_STORAGE_FOLDER_NAME = 'corestore'
 const INDEXER_STORAGE_FOLDER_NAME = 'indexer'
@@ -83,8 +88,6 @@ export const kClearDataIfLeft = Symbol('clear data if left project')
 export const kSetIsArchiveDevice = Symbol('set isArchiveDevice')
 export const kIsArchiveDevice = Symbol('isArchiveDevice (temp - test only)')
 export const kGeoJSONFileName = Symbol('geoJSONFileName')
-export const kExportGeoJSONStream = Symbol('exportGeoJSONStream')
-export const kExportZipStream = Symbol('exportZipStream')
 
 const EMPTY_PROJECT_SETTINGS = Object.freeze({})
 
@@ -786,40 +789,43 @@ export class MapeoProject extends TypedEmitter {
         }
       }
 
-      let latitude = lat
-      let longitude = lon
-      let altitude = null
-      const position = observation?.metadata?.position?.coords
-      if (position) {
-        latitude = position.latitude
-        longitude = position.longitude
-        if (position.altitude !== undefined) {
-          altitude = position.altitude
+      const metadataCoords = observation.metadata?.position?.coords
+      const altitude = metadataCoords?.altitude
+
+      /** @type {[number, number] | [number, number, number] | null} */
+      let coordinates = null
+
+      // Prioritize using the observation's `lat` and `lon` fields
+      if (typeof lat === 'number' && typeof lon === 'number') {
+        coordinates =
+          typeof altitude === 'number' ? [lon, lat, altitude] : [lon, lat]
+      } else {
+        // Fall back to using the observation metadata's position if possible
+        if (
+          typeof metadataCoords?.latitude === 'number' &&
+          typeof metadataCoords?.longitude === 'number'
+        ) {
+          coordinates =
+            typeof altitude === 'number'
+              ? [metadataCoords.longitude, metadataCoords.latitude, altitude]
+              : [metadataCoords.longitude, metadataCoords.latitude]
         }
       }
 
-      const coordinates = [longitude, latitude]
-      if (typeof altitude === 'number') {
-        coordinates.push(altitude)
+      /** @type {import('geojson').Feature<import('geojson').Point | null>} */
+      const feature = {
+        type: 'Feature',
+        properties: observation,
+        geometry: coordinates
+          ? {
+              type: 'Point',
+              coordinates,
+            }
+          : null,
       }
-      const hasLatLon =
-        typeof longitude === 'number' && typeof latitude === 'number'
-      const geometry = hasLatLon
-        ? {
-            type: 'Point',
-            coordinates,
-          }
-        : null
       const comma = first ? '' : ','
       first = false
-      yield b4a.from(
-        `${comma}\n      ` +
-          JSON.stringify({
-            type: 'Feature',
-            properties: observation,
-            geometry,
-          })
-      )
+      yield b4a.from(`${comma}\n      ` + JSON.stringify(feature))
     }
   }
 
@@ -941,7 +947,7 @@ export class MapeoProject extends TypedEmitter {
    * @param {string} [options.lang]
    * @returns {Readable<Buffer | Uint8Array>}
    */
-  [kExportGeoJSONStream]({
+  #exportGeoJSONStream({
     observations = true,
     tracks = true,
     lang,
@@ -1027,7 +1033,7 @@ export class MapeoProject extends TypedEmitter {
   ) {
     const fileName = await this[kGeoJSONFileName](observations, tracks)
     const filePath = path.join(exportFolder, fileName)
-    const source = this[kExportGeoJSONStream]({ observations, tracks, lang })
+    const source = this.#exportGeoJSONStream({ observations, tracks, lang })
     const sink = createWriteStream(filePath)
     await pipelinePromise(source, sink)
 
@@ -1036,15 +1042,37 @@ export class MapeoProject extends TypedEmitter {
 
   /**
    * @param {Attachment} attachment
-   * @returns {Promise<null | BlobId>}
+   * @returns {Promise<null | BlobRef>}
    */
-  async #tryGetBlobId(attachment) {
+  async #tryGetAttachmentBlob(attachment) {
     // Audio must not have variants
     for (const variant of VARIANT_EXPORT_ORDER) {
-      const blobId = buildBlobId(attachment, variant)
-      const entry = await this.#blobStore.entry(blobId)
-      if (!entry) continue
-      return blobId
+      try {
+        const blobId = buildBlobId(attachment, variant)
+        const entry = await this.#blobStore.entry(blobId)
+        if (!entry) continue
+        const metadata = entry.value.metadata
+        if (!metadata || typeof metadata !== 'object') continue
+        let mimeType = undefined
+        if ('mimeType' in metadata) {
+          if (typeof metadata.mimeType === 'string') {
+            mimeType = metadata.mimeType
+          } else {
+            this.#l.log('Invalid type for mimeType in blob', blobId, entry)
+            continue
+          }
+        }
+        return { blobId, mimeType }
+      } catch (e) {
+        if (!(e instanceof Error)) throw e
+        this.#l.log(
+          'Error loading blob id for attachment',
+          attachment,
+          variant,
+          e.message
+        )
+        continue
+      }
     }
 
     return null
@@ -1066,7 +1094,7 @@ export class MapeoProject extends TypedEmitter {
     // GeoJSON
     const geoJSONFileName = await this[kGeoJSONFileName](observations, tracks)
     const seenAttachments = new Map()
-    const geoJSONStream = this[kExportGeoJSONStream]({
+    const geoJSONStream = this.#exportGeoJSONStream({
       observations,
       tracks,
       lang,
@@ -1079,16 +1107,31 @@ export class MapeoProject extends TypedEmitter {
     const missingAttachments = []
     // Attachments
     if (attachments) {
-      const mediaFolder = this.#exportPrefix('Media') + '/'
+      const mediaFolder = (await this.#exportPrefix('Media')) + '/'
       for (const attachment of seenAttachments.values()) {
-        const blobId = await this.#tryGetBlobId(attachment)
-        if (blobId === null) {
+        const ref = await this.#tryGetAttachmentBlob(attachment)
+        if (ref === null) {
           missingAttachments.push(attachment)
           continue
         }
 
+        const { blobId, mimeType } = ref
+        let extensionString = ''
+        if (mimeType) {
+          const extension = mime.getExtension(mimeType)
+          if (extension) {
+            extensionString = '.' + extension
+          } else {
+            this.#l.log('Got unknown mime type in attachment blob', attachment)
+          }
+        }
+
         const stream = this.#blobStore.createReadStream(blobId)
-        const name = mediaFolder + blobId.variant + '/' + attachment.name
+        const name = path.posix.join(
+          mediaFolder,
+          blobId.variant,
+          `${attachment.name}${extensionString}`
+        )
 
         // @ts-expect-error
         await archive.entry(stream, { name })
@@ -1118,7 +1161,7 @@ export class MapeoProject extends TypedEmitter {
    * @param {string} [options.lang]
    * @returns {Readable<Buffer | Uint8Array>}
    */
-  [kExportZipStream]({
+  #exportZipStream({
     observations = true,
     tracks = true,
     attachments = true,
@@ -1153,7 +1196,7 @@ export class MapeoProject extends TypedEmitter {
   ) {
     const fileName = await this.#zipFileName(observations, tracks)
     const filePath = path.join(exportFolder, fileName)
-    const source = this[kExportZipStream]({
+    const source = this.#exportZipStream({
       observations,
       tracks,
       attachments,

package/src/types.ts

@@ -162,7 +162,7 @@ export type BlobStoreEntriesStream = Readable & {
   >
 }
 
-export type Attachment = Observation['attachments'][0]
+export type Attachment = Observation['attachments'][number]
 
 export type StringToTaggedUnion<T extends string> = {
   [K in T]: {