@comapeo/core 4.4.0 → 5.0.0

package/src/mapeo-project.js

@@ -2,7 +2,7 @@ import path from 'path'
 import Database from 'better-sqlite3'
 import { decodeBlockPrefix, decode, parseVersionId } from '@comapeo/schema'
 import { drizzle } from 'drizzle-orm/better-sqlite3'
-import { sql, count } from 'drizzle-orm'
+import { sql, count, eq } from 'drizzle-orm'
 import { discoveryKey } from 'hypercore-crypto'
 import { TypedEmitter } from 'tiny-typed-emitter'
 import ZipArchive from 'zip-stream-promise'
@@ -11,7 +11,7 @@ import mime from 'mime/lite'
 // @ts-expect-error
 import { Readable, pipelinePromise } from 'streamx'
 
-import { NAMESPACES, NAMESPACE_SCHEMAS, UNIX_EPOCH_DATE } from './constants.js'
+import { NAMESPACES, NAMESPACE_SCHEMAS } from './constants.js'
 import { CoreManager } from './core-manager/index.js'
 import { DataStore } from './datastore/index.js'
 import { DataType, kCreateWithDocId } from './datatype/index.js'
@@ -61,11 +61,12 @@ import {
 } from './sync/sync-api.js'
 import { Logger } from './logger.js'
 import { IconApi } from './icon-api.js'
-import { readConfig } from './config-import.js'
+import { importCategories } from './import-categories.js'
 import TranslationApi from './translation-api.js'
 import { NotFoundError, nullIfNotFound } from './errors.js'
 import { WebSocket } from 'ws'
 import { createWriteStream } from 'fs'
+import ensureError from 'ensure-error'
 /** @import { ProjectSettingsValue, Observation, Track } from '@comapeo/schema' */
 /** @import { Attachment, CoreStorage, BlobFilter, BlobId, BlobStoreEntriesStream, KeyPair, Namespace, ReplicationStream, GenericBlobFilter, MapeoValueMap, MapeoDocMap } from './types.js' */
 /** @typedef {Omit<ProjectSettingsValue, 'schemaName'>} EditableProjectSettings */
@@ -97,12 +98,12 @@ export const kBlobStore = Symbol('blobStore')
 export const kProjectReplicate = Symbol('replicate project')
 export const kDataTypes = Symbol('dataTypes')
 export const kProjectLeave = Symbol('leave project')
-export const kClearDataIfLeft = Symbol('clear data if left project')
+export const kClearData = Symbol('clear project data')
 export const kSetIsArchiveDevice = Symbol('set isArchiveDevice')
 export const kIsArchiveDevice = Symbol('isArchiveDevice (temp - test only)')
 export const kGeoJSONFileName = Symbol('geoJSONFileName')
 
-const EMPTY_PROJECT_SETTINGS = Object.freeze({})
+const EMPTY_PROJECT_SETTINGS = Object.freeze({ sendStats: false })
 
 /** @type BlobId['variant'][]*/
 const VARIANT_EXPORT_ORDER = ['original', 'preview', 'thumbnail']
@@ -130,9 +131,10 @@ export class MapeoProject extends TypedEmitter {
   #translationApi
   #l
   /** @type {Boolean} this avoids loading multiple configs in parallel */
-  #loadingConfig
+  #importingCategories
 
   static EMPTY_PROJECT_SETTINGS = EMPTY_PROJECT_SETTINGS
+  #getFallbackProjectInfo
 
   /**
    * @param {Object} opts
@@ -149,6 +151,7 @@ export class MapeoProject extends TypedEmitter {
    * @param {(url: string) => WebSocket} [opts.makeWebsocket]
    * @param {import('./local-peers.js').LocalPeers} opts.localPeers
    * @param {boolean} opts.isArchiveDevice Whether this device is an archive device
+   * @param {() => import('./schema/client.js').ProjectInfo | undefined} opts.getFallbackProjectInfo
    * @param {Logger} [opts.logger]
    *
    */
@@ -167,13 +170,15 @@ export class MapeoProject extends TypedEmitter {
     localPeers,
     logger,
     isArchiveDevice,
+    getFallbackProjectInfo,
   }) {
     super()
 
     this.#l = Logger.create('project', logger)
     this.#deviceId = getDeviceId(keyManager)
     this.#projectKey = projectKey
-    this.#loadingConfig = false
+    this.#importingCategories = false
+    this.#getFallbackProjectInfo = getFallbackProjectInfo
 
     const getReplicationStream = this[kProjectReplicate].bind(this, true)
 
@@ -216,6 +221,11 @@ export class MapeoProject extends TypedEmitter {
 
     if (reindex) {
       for (const table of indexedTables) db.delete(table).run()
+
+      sharedDb
+        .delete(projectSettingsTable)
+        .where(eq(projectSettingsTable.docId, this.#projectId))
+        .run()
     }
 
     ///////// 3. Setup random-access-storage functions
@@ -679,7 +689,12 @@ export class MapeoProject extends TypedEmitter {
         await this.#dataTypes.projectSettings.getByDocId(this.#projectId)
       )
     } catch (e) {
-      return /** @type {EditableProjectSettings} */ (EMPTY_PROJECT_SETTINGS)
+      // if (e instanceof Error && e.name !== 'NotFoundError') throw e
+      // If the project has not completed an initial sync, project settings will
+      // not be available, so use fallback project info which is set from the
+      // invite that was used to join the project.
+      const fallbackInfo = this.#getFallbackProjectInfo()
+      return fallbackInfo || EMPTY_PROJECT_SETTINGS
     }
   }
 
@@ -688,11 +703,9 @@ export class MapeoProject extends TypedEmitter {
    */
   async $hasSyncedProjectSettings() {
     try {
-      const settings = await this.#dataTypes.projectSettings.getByDocId(
-        this.#projectId
-      )
-
-      return settings.createdAt !== UNIX_EPOCH_DATE
+      // Should error if we haven't synced before
+      await this.#dataTypes.projectSettings.getByDocId(this.#projectId)
+      return true
     } catch (e) {
       return false
     }
@@ -1302,19 +1315,14 @@ export class MapeoProject extends TypedEmitter {
 
     await this.#roles.assignRole(this.#deviceId, LEFT_ROLE_ID)
 
-    await this[kClearDataIfLeft]()
+    await this[kClearData]()
   }
 
   /**
-   * Clear data if we've left the project. No-op if you're still in the project.
+   * Clear synced data, but keep auth data and own data
    * @returns {Promise<void>}
    */
-  async [kClearDataIfLeft]() {
-    const role = await this.$getOwnRole()
-    if (role.roleId !== LEFT_ROLE_ID) {
-      return
-    }
-
+  async [kClearData]() {
     const namespacesWithoutAuth =
       /** @satisfies {Exclude<Namespace, 'auth'>[]} */ ([
         'config',
@@ -1345,172 +1353,44 @@ export class MapeoProject extends TypedEmitter {
     }
   }
 
-  /** @param {Object} opts
-   *  @param {string} opts.configPath
-   *  @returns {Promise<Error[]>}
+  /**
+   * @deprecated
+   * @param {object} opts
+   * @param {string} opts.configPath
+   * @returns {Promise<Error[]>}
    */
   async importConfig({ configPath }) {
+    try {
+      await this.$importCategories({ filePath: configPath })
+      return []
+    } catch (e) {
+      return [ensureError(e)]
+    }
+  }
+
+  /**
+   * @param {object} opts
+   * @param {string} opts.filePath
+   * @returns {Promise<void>}
+   */
+  async $importCategories({ filePath }) {
     assert(
-      !this.#loadingConfig,
-      'Cannot run multiple config imports at the same time'
+      !this.#importingCategories,
+      'Cannot run multiple category imports at the same time'
     )
-    this.#loadingConfig = true
+    this.#importingCategories = true
 
     try {
-      // check for already present fields and presets and delete them if exist
-      const presetsToDelete = await grabDocsToDelete(this.preset)
-      const fieldsToDelete = await grabDocsToDelete(this.field)
-      // delete only translations that refer to deleted fields and presets
-      const translationsToDelete = await grabTranslationsToDelete({
-        logger: this.#l,
-        translation: this.$translation.dataType,
-        preset: this.preset,
-        field: this.field,
-      })
-
-      const config = await readConfig(configPath)
-      /** @type {Map<string, import('./icon-api.js').IconRef>} */
-      const iconNameToRef = new Map()
-      /** @type {Map<string, import('@comapeo/schema').PresetValue['fieldRefs'][1]>} */
-      const fieldNameToRef = new Map()
-      /** @type {Map<string,import('@comapeo/schema').TranslationValue['docRef']>} */
-      const presetNameToRef = new Map()
-
-      // Do this in serial not parallel to avoid memory issues (avoid keeping all icon buffers in memory)
-      for await (const icon of config.icons()) {
-        const { docId, versionId } = await this.#iconApi.create(icon)
-        iconNameToRef.set(icon.name, { docId, versionId })
-      }
-
-      // Ok to create fields and presets in parallel
-      const fieldPromises = []
-      for (const { name, value } of config.fields()) {
-        fieldPromises.push(
-          this.#dataTypes.field.create(value).then(({ docId, versionId }) => {
-            fieldNameToRef.set(name, { docId, versionId })
-          })
-        )
-      }
-      await Promise.all(fieldPromises)
-
-      const presetsWithRefs = []
-      for (const { fieldNames, iconName, value, name } of config.presets()) {
-        const fieldRefs = fieldNames.map((fieldName) => {
-          const fieldRef = fieldNameToRef.get(fieldName)
-          if (!fieldRef) {
-            throw new NotFoundError(
-              `field ${fieldName} not found (referenced by preset ${value.name})})`
-            )
-          }
-          return fieldRef
-        })
-
-        if (!iconName) {
-          throw new Error(`preset ${value.name} is missing an icon name`)
-        }
-        const iconRef = iconNameToRef.get(iconName)
-        if (!iconRef) {
-          throw new NotFoundError(
-            `icon ${iconName} not found (referenced by preset ${value.name})`
-          )
-        }
-
-        presetsWithRefs.push({
-          preset: {
-            ...value,
-            iconRef,
-            fieldRefs,
-          },
-          name,
-        })
-      }
-
-      const presetPromises = []
-      for (const { preset, name } of presetsWithRefs) {
-        presetPromises.push(
-          this.preset.create(preset).then(({ docId, versionId }) => {
-            presetNameToRef.set(name, { docId, versionId })
-          })
-        )
-      }
-
-      await Promise.all(presetPromises)
-
-      const translationPromises = []
-      for (const { name, value } of config.translations()) {
-        let docRef
-        if (value.docRefType === 'field') {
-          docRef = { ...fieldNameToRef.get(name) }
-        } else if (value.docRefType === 'preset') {
-          docRef = { ...presetNameToRef.get(name) }
-        } else {
-          throw new Error(`invalid docRefType ${value.docRefType}`)
-        }
-        if (docRef.docId && docRef.versionId) {
-          translationPromises.push(
-            this.$translation.put({
-              ...value,
-              docRef: {
-                docId: docRef.docId,
-                versionId: docRef.versionId,
-              },
-            })
-          )
-        } else {
-          throw new NotFoundError(
-            `docRef for ${value.docRefType} with name ${name} not found`
-          )
-        }
-      }
-      await Promise.all(translationPromises)
-
-      // close the zip handles after we know we won't be needing them anymore
-      await config.close()
-      const presetIds = [...presetNameToRef.values()].map((val) => val.docId)
-
-      await this.$setProjectSettings({
-        defaultPresets: {
-          point: presetIds,
-          line: [],
-          area: [],
-          vertex: [],
-          relation: [],
-        },
-        configMetadata: config.metadata,
-      })
-
-      const deletePresetsPromise = Promise.all(
-        presetsToDelete.map(async (docId) => {
-          const { deleted } = await this.preset.getByDocId(docId)
-          if (!deleted) await this.preset.delete(docId)
-        })
-      )
-      const deleteFieldsPromise = Promise.all(
-        fieldsToDelete.map(async (docId) => {
-          const { deleted } = await this.field.getByDocId(docId)
-          if (!deleted) await this.field.delete(docId)
-        })
-      )
-      const deleteTranslationsPromise = Promise.all(
-        [...translationsToDelete].map(async (docId) => {
-          const { deleted } = await this.$translation.dataType.getByDocId(docId)
-          if (!deleted) await this.$translation.dataType.delete(docId)
-        })
-      )
-      await Promise.all([
-        deletePresetsPromise,
-        deleteFieldsPromise,
-        deleteTranslationsPromise,
-      ])
-      this.#loadingConfig = false
-      return config.warnings
+      await importCategories(this, { filePath, logger: this.#l })
     } catch (e) {
       this.#l.log('error loading config', e)
-      this.#loadingConfig = false
-      return /** @type Error[] */ []
+      throw e
+    } finally {
+      this.#importingCategories = false
     }
   }
 }
+
 /**
  * @param {import("@comapeo/schema").ProjectSettings & { forks: string[] }} projectDoc
  * @returns {EditableProjectSettings}
@@ -1519,48 +1399,6 @@ function extractEditableProjectSettings(projectDoc) {
   return omit(valueOf(projectDoc), ['schemaName'])
 }
 
-/**
- @param {MapeoProject['field'] | MapeoProject['preset']} dataType
- @returns {Promise<String[]>}
- */
-async function grabDocsToDelete(dataType) {
-  const toDelete = []
-  for (const { docId } of await dataType.getMany()) {
-    toDelete.push(docId)
-  }
-  return toDelete
-}
-
-/**
- * @param {Object} opts
- * @param {Logger} opts.logger
- * @param {MapeoProject['$translation']['dataType']} opts.translation
- * @param {MapeoProject['preset']} opts.preset
- * @param {MapeoProject['field']} opts.field
- * @returns {Promise<Set<String>>}
- */
-async function grabTranslationsToDelete(opts) {
-  /** @type {Set<String>} */
-  const toDelete = new Set()
-  const translations = await opts.translation.getMany()
-  await Promise.all(
-    translations.map(async ({ docRefType, docRef, docId }) => {
-      if (docRefType === 'field' || docRefType === 'preset') {
-        let doc
-        try {
-          doc = await opts[docRefType].getByVersionId(docRef.versionId)
-        } catch (e) {
-          opts.logger.log(`referred ${docRef.versionId} is not found`)
-        }
-        if (doc) {
-          toDelete.add(docId)
-        }
-      }
-    })
-  )
-  return toDelete
-}
-
 /**
  * Return a map of namespace -> core keypair
  *

package/src/roles.js

@@ -148,7 +148,7 @@ export const NO_ROLE = {
   roleAssignment: [],
   sync: {
     auth: 'allowed',
-    config: 'allowed',
+    config: 'blocked',
     data: 'blocked',
     blobIndex: 'blocked',
     blob: 'blocked',

package/src/schema/client.js

@@ -3,55 +3,49 @@
 // device
 import { blob, sqliteTable, text, int } from 'drizzle-orm/sqlite-core'
 import { dereferencedDocSchemas as schemas } from '@comapeo/schema'
-import { jsonSchemaToDrizzleColumns as toColumns } from './schema-to-drizzle.js'
-import { backlinkTable, customJson } from './utils.js'
+import {
+  comapeoSchemaToDrizzleTable as toDrizzle,
+  backlinkTable,
+} from './comapeo-to-drizzle.js'
 
 /**
  * @import { ProjectSettings } from '@comapeo/schema'
+ * @import { $Type } from 'drizzle-orm'
+ * @import { SQLiteTextJsonBuilder } from 'drizzle-orm/sqlite-core'
  *
  * @internal
- * @typedef {Pick<ProjectSettings, 'name' | 'projectColor' | 'projectDescription'>} ProjectInfo
+ * @typedef {Pick<ProjectSettings, 'name' | 'projectColor' | 'projectDescription' | 'sendStats'>} ProjectInfo
  */
 
-const projectInfoColumn =
-  /** @type {ReturnType<typeof import('drizzle-orm/sqlite-core').customType<{data: ProjectInfo}>>} */ (
-    customJson
-  )
-
 /** @type {ProjectInfo} */
-const PROJECT_INFO_DEFAULT_VALUE = {}
+const PROJECT_INFO_DEFAULT_VALUE = { sendStats: false }
 
-export const projectSettingsTable = sqliteTable(
-  'projectSettings',
-  toColumns(schemas.projectSettings)
-)
-export const projectBacklinkTable = backlinkTable(projectSettingsTable)
+export const projectSettingsTable = toDrizzle(schemas.projectSettings)
+export const projectBacklinkTable = backlinkTable('projectSettings')
 export const projectKeysTable = sqliteTable('projectKeys', {
   projectId: text('projectId').notNull().primaryKey(),
   projectPublicId: text('projectPublicId').notNull(),
-  projectInviteId: blob('projectInviteId').notNull(),
+  projectInviteId: blob('projectInviteId', { mode: 'buffer' }).notNull(),
   keysCipher: blob('keysCipher', { mode: 'buffer' }).notNull(),
-  projectInfo: projectInfoColumn('projectInfo')
-    .default(
-      // TODO: There's a bug in Drizzle where the default value does not get transformed by the custom type
-      // @ts-expect-error
-      JSON.stringify(PROJECT_INFO_DEFAULT_VALUE)
-    )
-    .notNull(),
+  projectInfo:
+    /** @type {$Type<SQLiteTextJsonBuilder, ProjectInfo>} */
+    (text('projectInfo', { mode: 'json' }))
+      .default(PROJECT_INFO_DEFAULT_VALUE)
+      .notNull(),
+  hasLeftProject: int('hasLeftProject', { mode: 'boolean' })
+    .notNull()
+    .default(false),
 })
 
 /**
  * @typedef {Omit<import('@comapeo/schema').DeviceInfoValue, 'schemaName'>} DeviceInfoParam
  */
 
-const deviceInfoColumn =
-  /** @type {ReturnType<typeof import('drizzle-orm/sqlite-core').customType<{data: DeviceInfoParam }>>} */ (
-    customJson
-  )
-
 // This table only ever has one row in it.
 export const deviceSettingsTable = sqliteTable('deviceSettings', {
   deviceId: text('deviceId').notNull().unique(),
-  deviceInfo: deviceInfoColumn('deviceInfo'),
+  deviceInfo:
+    /** @type {$Type<SQLiteTextJsonBuilder, DeviceInfoParam>} */
+    (text('deviceInfo', { mode: 'json' })),
   isArchiveDevice: int('isArchiveDevice', { mode: 'boolean' }),
 })

package/src/schema/comapeo-to-drizzle.js

@@ -0,0 +1,57 @@
+import { text, sqliteTable } from 'drizzle-orm/sqlite-core'
+import { jsonSchemaToDrizzleSqliteTable } from './json-schema-to-drizzle.js'
+
+export const BACKLINK_TABLE_POSTFIX = '_backlink'
+
+/**
+ * @import { JsonSchemaToDrizzleSqliteTable } from './types.js'
+ * @import { SQLiteTextJsonBuilder } from 'drizzle-orm/sqlite-core'
+ * @import { $Type } from 'drizzle-orm'
+ * @import { MapeoDocMap } from '../types.js'
+ */
+
+/**
+ * @typedef {typeof import('@comapeo/schema').dereferencedDocSchemas} ComapeoSchemaMap
+ * @typedef {{ forks: $Type<SQLiteTextJsonBuilder, string[]> }} AdditionalColumns
+ */
+
+/**
+ * @template {ComapeoSchemaMap[keyof ComapeoSchemaMap]} TSchema
+ * @param {TSchema} schema
+ * @returns {JsonSchemaToDrizzleSqliteTable<
+ *   MapeoDocMap[TSchema['properties']['schemaName']['const']],
+ *   TSchema,
+ *   TSchema['properties']['schemaName']['const'],
+ *   AdditionalColumns,
+ *   'docId'
+ * >}
+ */
+export function comapeoSchemaToDrizzleTable(schema) {
+  return jsonSchemaToDrizzleSqliteTable(
+    schema.properties.schemaName.const,
+    schema,
+    {
+      additionalColumns: { forks: text('forks', { mode: 'json' }).notNull() },
+      primaryKey: 'docId',
+    }
+  )
+}
+
+/**
+ * Table for storing backlinks, used for indexing. There needs to be one for
+ * each indexed document type, with a specific name `<datatype>_backlink`
+ *
+ * @param {import('@comapeo/schema').MapeoDoc['schemaName']} schemaName
+ */
+export function backlinkTable(schemaName) {
+  return sqliteTable(getBacklinkTableName(schemaName), {
+    versionId: text('versionId').notNull().primaryKey(),
+  })
+}
+
+/**
+ * @param {string} tableName
+ */
+export function getBacklinkTableName(tableName) {
+  return tableName + BACKLINK_TABLE_POSTFIX
+}

package/src/schema/{schema-to-drizzle.js → json-schema-to-drizzle.js}

@@ -1,24 +1,24 @@
-import { text, integer, real } from 'drizzle-orm/sqlite-core'
+import { text, integer, real, sqliteTable } from 'drizzle-orm/sqlite-core'
 import { ExhaustivenessError } from '../utils.js'
-import { customJson } from './utils.js'
-/** @import { MapeoDoc } from '@comapeo/schema' */
-/** @import { MapeoDocMap } from '../types.js' */
 
 /**
-Convert a JSONSchema definition to a Drizzle Columns Map (the parameter for
-`sqliteTable()`).
-
-**NOTE**: The return of this function is _not_ type-checked (it is coerced with
-`as`, because it's not possible to type-check what this function is doing), but
-the return type _should_ be correct when using this function.
-@template {import('./types.js').JSONSchema7WithProps} TSchema
-NB: The inline typescript checker often marks this next line as an error, but this seems to be a bug with JSDoc parsing - running `tsc` does not show this as an error.
-@template {import('type-fest').Get<TSchema, 'properties.schemaName.const'>} TSchemaName
-@template {TSchemaName extends MapeoDoc['schemaName'] ? MapeoDocMap[TSchemaName] : any} TObjectType
-@param {TSchema} schema
-@returns {import('./types.js').SchemaToDrizzleColumns<TSchema, TObjectType>}
+ * @template {{ [ K in keyof TSchema['properties'] ]?: any }} TObjectType
+ * @template {import('./types.js').JSONSchema7Object} TSchema
+ * @template {string} TTableName
+ * @template {Record<string, import('drizzle-orm').ColumnBuilderBase>} TColumnsMap
+ * @template {keyof TSchema['properties']} TPrimaryKey
+ * @param {TTableName} tableName
+ * @param {TSchema} schema
+ * @param {object} [opts]
+ * @param {TColumnsMap} [opts.additionalColumns]
+ * @param {TPrimaryKey} [opts.primaryKey] - Column name to use as primary key, if not specified in schema
+ * @returns {import('./types.js').JsonSchemaToDrizzleSqliteTable<TObjectType, TSchema, TTableName, TColumnsMap, TPrimaryKey>}
  */
-export function jsonSchemaToDrizzleColumns(schema) {
+export function jsonSchemaToDrizzleSqliteTable(
+  tableName,
+  schema,
+  { additionalColumns, primaryKey } = {}
+) {
   if (schema.type !== 'object' || !schema.properties) {
     throw new Error('Cannot process JSONSchema as SQL table')
   }
@@ -46,14 +46,11 @@ export function jsonSchemaToDrizzleColumns(schema) {
           ? /** @type {[typeof value.const]} */ ([value.const])
           : undefined
         columns[key] = text(key, { enum: enumValue })
-        if (key === 'docId') {
-          columns[key] = columns[key].primaryKey()
-        }
         break
       }
       case 'array':
       case 'object':
-        columns[key] = customJson(key)
+        columns[key] = text(key, { mode: 'json' })
         break
       case 'null':
         // Skip handling this right now
@@ -69,10 +66,13 @@ export function jsonSchemaToDrizzleColumns(schema) {
         columns[key] = columns[key].default(defaultValue)
       }
     }
+    if (key === primaryKey) {
+      columns[key] = columns[key].primaryKey()
+    }
   }
-  // Not yet in @comapeo/schema
-  columns.forks = customJson('forks').notNull()
-  return /** @type {any} */ (columns)
+  return /** @type {any} */ (
+    sqliteTable(tableName, { ...columns, ...additionalColumns })
+  )
 }
 
 /**
@@ -85,7 +85,7 @@ function getDefault(value) {
 }
 
 /**
- * @param {import('./types.js').JSONSchema7WithProps} schema
+ * @param {import('./types.js').JSONSchema7Object} schema
  * @param {string} key
  * @returns {boolean}
  */