@comapeo/core 3.2.0 → 4.1.0

package/drizzle/project/meta/_journal.json

@@ -15,6 +15,13 @@
       "when": 1729783892753,
       "tag": "0001_medical_wendell_rand",
       "breakpoints": true
+    },
+    {
+      "idx": 2,
+      "version": "5",
+      "when": 1749657174497,
+      "tag": "0002_cooing_princess_powerful",
+      "breakpoints": true
     }
   ]
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@comapeo/core",
-  "version": "3.2.0",
+  "version": "4.1.0",
   "description": "Offline p2p mapping library",
   "main": "src/index.js",
   "types": "dist/index.d.ts",
@@ -112,12 +112,13 @@
     "@comapeo/core2.0.1": "npm:@comapeo/core@2.0.1",
     "@comapeo/ipc": "^2.1.0",
     "@mapeo/default-config": "5.0.0",
-    "@mapeo/mock-data": "^3.0.0",
+    "@mapeo/mock-data": "^5.0.0",
     "@sinonjs/fake-timers": "^10.0.2",
     "@types/b4a": "^1.6.0",
     "@types/bogon": "^1.0.2",
     "@types/compact-encoding": "^2.15.0",
     "@types/debug": "^4.1.8",
+    "@types/geojson": "^7946.0.16",
     "@types/json-schema": "^7.0.11",
     "@types/json-stable-stringify": "^1.0.36",
     "@types/nanobench": "^3.0.0",
@@ -136,6 +137,7 @@
     "drizzle-kit": "^0.20.14",
     "eslint": "^8.57.0",
     "execa": "^9.5.1",
+    "filter-obj": "^6.1.0",
     "husky": "^8.0.0",
     "iterpal": "^0.4.0",
     "lint-staged": "^14.0.1",
@@ -160,7 +162,7 @@
   },
   "dependencies": {
     "@comapeo/fallback-smp": "^1.0.0",
-    "@comapeo/schema": "1.6.0",
+    "@comapeo/schema": "2.0.0",
     "@digidem/types": "^2.3.0",
     "@fastify/error": "^3.4.1",
     "@fastify/type-provider-typebox": "^4.1.0",
@@ -189,7 +191,7 @@
     "json-stable-stringify": "^1.1.1",
     "magic-bytes.js": "^1.10.0",
     "map-obj": "^5.0.2",
-    "mime": "^4.0.3",
+    "mime": "^4.0.7",
     "multi-core-indexer": "^1.0.0",
     "p-defer": "^4.0.0",
     "p-event": "^6.0.1",

package/src/blob-api.js

@@ -4,35 +4,9 @@ import { Transform, pipelinePromise as pipeline } from 'streamx'
 import { createHash, randomBytes } from 'node:crypto'
 /** @import { BlobId, BlobType } from './types.js' */
 
-/**
- * Location coordinate data. Based on [Expo's `LocationObjectCoords`][0].
- * [0]: https://docs.expo.dev/versions/latest/sdk/location/#locationobjectcoords
- *
- * @typedef {object} LocationObjectCoords
- * @prop {number | null} accuracy
- * @prop {number | null} altitude
- * @prop {number | null} altitudeAccuracy
- * @prop {number | null} heading
- * @prop {number} latitude
- * @prop {number} longitude
- * @prop {number | null} speed
- */
-
-/**
- * Location metadata for a blob. Based on [Expo's `LocationObject`][0].
- * [0]: https://docs.expo.dev/versions/latest/sdk/location/#locationobject
- *
- * @typedef {object} LocationObject
- * @prop {LocationObjectCoords} coords
- * @prop {boolean} [mocked]
- * @prop {number} timestamp
- */
-
 /**
  * @typedef {object} Metadata
  * @prop {string} mimeType
- * @prop {number} timestamp
- * @prop {LocationObject} [location]
  */
 
 export class BlobApi {

package/src/constants.js

@@ -35,3 +35,6 @@ export const SUPPORTED_CONFIG_VERSION = 1
 
 // WARNING: This value is persisted. Be careful when changing it.
 export const DRIZZLE_MIGRATIONS_TABLE = '__drizzle_migrations'
+
+// Oldest possible time, ensure it gets overwritten with any updates
+export const UNIX_EPOCH_DATE = new Date(0).toISOString()

package/src/mapeo-manager.js

@@ -38,6 +38,7 @@ import {
   projectKeyToProjectInviteId,
   projectKeyToPublicId,
 } from './utils.js'
+import { UNIX_EPOCH_DATE } from './constants.js'
 import { openedNoiseSecretStream } from './lib/noise-secret-stream-helpers.js'
 import { omit } from './lib/omit.js'
 import { RandomAccessFilePool } from './core-manager/random-access-file-pool.js'
@@ -563,11 +564,13 @@ export class MapeoManager extends TypedEmitter {
         (p) => p.projectId === projectId
       )
 
+      if (!existingProject) continue
+
       result.push(
         deNullify({
           projectId: projectPublicId,
-          createdAt: existingProject?.createdAt,
-          updatedAt: existingProject?.updatedAt,
+          createdAt: ignoreUnixDate(existingProject?.createdAt),
+          updatedAt: ignoreUnixDate(existingProject?.updatedAt),
           name: existingProject?.name || projectInfo.name,
           projectColor:
             existingProject?.projectColor || projectInfo.projectColor,
@@ -658,6 +661,23 @@ export class MapeoManager extends TypedEmitter {
         )
       }
 
+      /** @type {import('drizzle-orm').InferInsertModel<typeof projectSettingsTable>} */
+      const settingsDoc = {
+        schemaName: 'projectSettings',
+        docId: projectId,
+        versionId: 'unknown',
+        originalVersionId: 'unknown',
+        createdAt: UNIX_EPOCH_DATE,
+        updatedAt: UNIX_EPOCH_DATE,
+        deleted: false,
+        links: [],
+        forks: [],
+        name: projectName,
+        projectDescription,
+      }
+
+      await this.#db.insert(projectSettingsTable).values([settingsDoc])
+
       // 5. Wait for initial project sync
       if (waitForSync) {
         await this.#waitForInitialSync(project)
@@ -692,17 +712,15 @@ export class MapeoManager extends TypedEmitter {
    * @returns {Promise<boolean>}
    */
   async #waitForInitialSync(project, { timeoutMs = 5000 } = {}) {
-    const [ownRole, projectSettings] = await Promise.all([
+    const [ownRole, isProjectSettingsSynced] = await Promise.all([
       project.$getOwnRole(),
-      project.$getProjectSettings(),
+      project.$hasSyncedProjectSettings(),
     ])
     const {
       auth: { localState: authState },
       config: { localState: configState },
     } = project.$sync[kSyncState].getState()
     const isRoleSynced = ownRole !== Roles.NO_ROLE
-    const isProjectSettingsSynced =
-      projectSettings !== MapeoProject.EMPTY_PROJECT_SETTINGS
     // Assumes every project that someone is invited to has at least one record
     // in the auth store - the row record for the invited device
     const isAuthSynced = authState.want === 0 && authState.have > 0
@@ -721,7 +739,6 @@ export class MapeoManager extends TypedEmitter {
       this.#l.log(
         'Pending initial sync: role %s, projectSettings %o, auth %o, config %o',
         isRoleSynced,
-        isProjectSettingsSynced,
         isAuthSynced,
         isConfigSynced
       )
@@ -1012,3 +1029,13 @@ function validateProjectKeys(projectKeys) {
 function hasSavedDeviceInfo(partialDeviceInfo) {
   return Boolean(partialDeviceInfo.name)
 }
+
+/**
+ * @param {string|undefined} date
+ * @returns {string|null}
+ */
+function ignoreUnixDate(date) {
+  if (date === UNIX_EPOCH_DATE) return null
+  if (date === undefined) return null
+  return date
+}

package/src/mapeo-project.js

@@ -6,10 +6,11 @@ import { discoveryKey } from 'hypercore-crypto'
 import { TypedEmitter } from 'tiny-typed-emitter'
 import ZipArchive from 'zip-stream-promise'
 import * as b4a from 'b4a'
+import mime from 'mime/lite'
 // @ts-expect-error
 import { Readable, pipelinePromise } from 'streamx'
 
-import { NAMESPACES, NAMESPACE_SCHEMAS } from './constants.js'
+import { NAMESPACES, NAMESPACE_SCHEMAS, UNIX_EPOCH_DATE } from './constants.js'
 import { CoreManager } from './core-manager/index.js'
 import { DataStore } from './datastore/index.js'
 import { DataType, kCreateWithDocId } from './datatype/index.js'
@@ -69,6 +70,10 @@ import { createWriteStream } from 'fs'
 /** @typedef {Omit<ProjectSettingsValue, 'schemaName'>} EditableProjectSettings */
 /** @typedef {ProjectSettingsValue['configMetadata']} ConfigMetadata */
 /** @typedef {Map<string,Attachment>} SeenAttachments*/
+/** @typedef {object} BlobRef
+ * @prop {string|undefined} mimeType
+ * @prop {BlobId} blobId
+ */
 
 const CORESTORE_STORAGE_FOLDER_NAME = 'corestore'
 const INDEXER_STORAGE_FOLDER_NAME = 'indexer'
@@ -83,8 +88,6 @@ export const kClearDataIfLeft = Symbol('clear data if left project')
 export const kSetIsArchiveDevice = Symbol('set isArchiveDevice')
 export const kIsArchiveDevice = Symbol('isArchiveDevice (temp - test only)')
 export const kGeoJSONFileName = Symbol('geoJSONFileName')
-export const kExportGeoJSONStream = Symbol('exportGeoJSONStream')
-export const kExportZipStream = Symbol('exportZipStream')
 
 const EMPTY_PROJECT_SETTINGS = Object.freeze({})
 
@@ -658,6 +661,21 @@ export class MapeoProject extends TypedEmitter {
     }
   }
 
+  /**
+   * @returns {Promise<boolean>}
+   */
+  async $hasSyncedProjectSettings() {
+    try {
+      const settings = await this.#dataTypes.projectSettings.getByDocId(
+        this.#projectId
+      )
+
+      return settings.createdAt !== UNIX_EPOCH_DATE
+    } catch (e) {
+      return false
+    }
+  }
+
   /**
    * @returns {Promise<undefined | string>}
    */
@@ -786,40 +804,43 @@ export class MapeoProject extends TypedEmitter {
         }
       }
 
-      let latitude = lat
-      let longitude = lon
-      let altitude = null
-      const position = observation?.metadata?.position?.coords
-      if (position) {
-        latitude = position.latitude
-        longitude = position.longitude
-        if (position.altitude !== undefined) {
-          altitude = position.altitude
+      const metadataCoords = observation.metadata?.position?.coords
+      const altitude = metadataCoords?.altitude
+
+      /** @type {[number, number] | [number, number, number] | null} */
+      let coordinates = null
+
+      // Prioritize using the observation's `lat` and `lon` fields
+      if (typeof lat === 'number' && typeof lon === 'number') {
+        coordinates =
+          typeof altitude === 'number' ? [lon, lat, altitude] : [lon, lat]
+      } else {
+        // Fall back to using the observation metadata's position if possible
+        if (
+          typeof metadataCoords?.latitude === 'number' &&
+          typeof metadataCoords?.longitude === 'number'
+        ) {
+          coordinates =
+            typeof altitude === 'number'
+              ? [metadataCoords.longitude, metadataCoords.latitude, altitude]
+              : [metadataCoords.longitude, metadataCoords.latitude]
         }
       }
 
-      const coordinates = [longitude, latitude]
-      if (typeof altitude === 'number') {
-        coordinates.push(altitude)
+      /** @type {import('geojson').Feature<import('geojson').Point | null>} */
+      const feature = {
+        type: 'Feature',
+        properties: observation,
+        geometry: coordinates
+          ? {
+              type: 'Point',
+              coordinates,
+            }
+          : null,
       }
-      const hasLatLon =
-        typeof longitude === 'number' && typeof latitude === 'number'
-      const geometry = hasLatLon
-        ? {
-            type: 'Point',
-            coordinates,
-          }
-        : null
       const comma = first ? '' : ','
       first = false
-      yield b4a.from(
-        `${comma}\n      ` +
-          JSON.stringify({
-            type: 'Feature',
-            properties: observation,
-            geometry,
-          })
-      )
+      yield b4a.from(`${comma}\n      ` + JSON.stringify(feature))
     }
   }
 
@@ -941,7 +962,7 @@ export class MapeoProject extends TypedEmitter {
    * @param {string} [options.lang]
    * @returns {Readable<Buffer | Uint8Array>}
    */
-  [kExportGeoJSONStream]({
+  #exportGeoJSONStream({
     observations = true,
     tracks = true,
     lang,
@@ -1027,7 +1048,7 @@ export class MapeoProject extends TypedEmitter {
   ) {
     const fileName = await this[kGeoJSONFileName](observations, tracks)
     const filePath = path.join(exportFolder, fileName)
-    const source = this[kExportGeoJSONStream]({ observations, tracks, lang })
+    const source = this.#exportGeoJSONStream({ observations, tracks, lang })
     const sink = createWriteStream(filePath)
     await pipelinePromise(source, sink)
 
@@ -1036,15 +1057,37 @@ export class MapeoProject extends TypedEmitter {
 
   /**
    * @param {Attachment} attachment
-   * @returns {Promise<null | BlobId>}
+   * @returns {Promise<null | BlobRef>}
    */
-  async #tryGetBlobId(attachment) {
+  async #tryGetAttachmentBlob(attachment) {
     // Audio must not have variants
     for (const variant of VARIANT_EXPORT_ORDER) {
-      const blobId = buildBlobId(attachment, variant)
-      const entry = await this.#blobStore.entry(blobId)
-      if (!entry) continue
-      return blobId
+      try {
+        const blobId = buildBlobId(attachment, variant)
+        const entry = await this.#blobStore.entry(blobId)
+        if (!entry) continue
+        const metadata = entry.value.metadata
+        if (!metadata || typeof metadata !== 'object') continue
+        let mimeType = undefined
+        if ('mimeType' in metadata) {
+          if (typeof metadata.mimeType === 'string') {
+            mimeType = metadata.mimeType
+          } else {
+            this.#l.log('Invalid type for mimeType in blob', blobId, entry)
+            continue
+          }
+        }
+        return { blobId, mimeType }
+      } catch (e) {
+        if (!(e instanceof Error)) throw e
+        this.#l.log(
+          'Error loading blob id for attachment',
+          attachment,
+          variant,
+          e.message
+        )
+        continue
+      }
     }
 
     return null
@@ -1066,7 +1109,7 @@ export class MapeoProject extends TypedEmitter {
     // GeoJSON
     const geoJSONFileName = await this[kGeoJSONFileName](observations, tracks)
     const seenAttachments = new Map()
-    const geoJSONStream = this[kExportGeoJSONStream]({
+    const geoJSONStream = this.#exportGeoJSONStream({
       observations,
       tracks,
       lang,
@@ -1079,16 +1122,31 @@ export class MapeoProject extends TypedEmitter {
     const missingAttachments = []
     // Attachments
     if (attachments) {
-      const mediaFolder = this.#exportPrefix('Media') + '/'
+      const mediaFolder = (await this.#exportPrefix('Media')) + '/'
       for (const attachment of seenAttachments.values()) {
-        const blobId = await this.#tryGetBlobId(attachment)
-        if (blobId === null) {
+        const ref = await this.#tryGetAttachmentBlob(attachment)
+        if (ref === null) {
           missingAttachments.push(attachment)
           continue
         }
 
+        const { blobId, mimeType } = ref
+        let extensionString = ''
+        if (mimeType) {
+          const extension = mime.getExtension(mimeType)
+          if (extension) {
+            extensionString = '.' + extension
+          } else {
+            this.#l.log('Got unknown mime type in attachment blob', attachment)
+          }
+        }
+
         const stream = this.#blobStore.createReadStream(blobId)
-        const name = mediaFolder + blobId.variant + '/' + attachment.name
+        const name = path.posix.join(
+          mediaFolder,
+          blobId.variant,
+          `${attachment.name}${extensionString}`
+        )
 
         // @ts-expect-error
         await archive.entry(stream, { name })
@@ -1118,7 +1176,7 @@ export class MapeoProject extends TypedEmitter {
    * @param {string} [options.lang]
    * @returns {Readable<Buffer | Uint8Array>}
    */
-  [kExportZipStream]({
+  #exportZipStream({
     observations = true,
     tracks = true,
     attachments = true,
@@ -1153,7 +1211,7 @@ export class MapeoProject extends TypedEmitter {
   ) {
     const fileName = await this.#zipFileName(observations, tracks)
     const filePath = path.join(exportFolder, fileName)
-    const source = this[kExportZipStream]({
+    const source = this.#exportZipStream({
       observations,
       tracks,
       attachments,

package/src/member-api.js

@@ -472,6 +472,7 @@ export class MemberApi extends TypedEmitter {
       )
     }
 
+    const onClosePromise = pEvent(websocket, 'close')
     const onErrorPromise = pEvent(websocket, 'error')
 
     const replicationStream = this.#getReplicationStream()
@@ -494,7 +495,6 @@ export class MemberApi extends TypedEmitter {
       websocket.close()
       throw errorEvent.error
     } else {
-      const onClosePromise = pEvent(websocket, 'close')
       onErrorPromise.cancel()
       websocket.close()
       await onClosePromise

package/src/types.ts

@@ -162,7 +162,7 @@ export type BlobStoreEntriesStream = Readable & {
   >
 }
 
-export type Attachment = Observation['attachments'][0]
+export type Attachment = Observation['attachments'][number]
 
 export type StringToTaggedUnion<T extends string> = {
   [K in T]: {