@comapeo/core 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (186) hide show
  1. package/LICENSE.md +9 -0
  2. package/README.md +31 -0
  3. package/dist/blob-api.d.ts +92 -0
  4. package/dist/blob-api.d.ts.map +1 -0
  5. package/dist/blob-store/index.d.ts +163 -0
  6. package/dist/blob-store/index.d.ts.map +1 -0
  7. package/dist/blob-store/live-download.d.ts +107 -0
  8. package/dist/blob-store/live-download.d.ts.map +1 -0
  9. package/dist/config-import.d.ts +74 -0
  10. package/dist/config-import.d.ts.map +1 -0
  11. package/dist/constants.d.ts +14 -0
  12. package/dist/constants.d.ts.map +1 -0
  13. package/dist/core-manager/bitfield-rle.d.ts +25 -0
  14. package/dist/core-manager/bitfield-rle.d.ts.map +1 -0
  15. package/dist/core-manager/core-index.d.ts +56 -0
  16. package/dist/core-manager/core-index.d.ts.map +1 -0
  17. package/dist/core-manager/index.d.ts +125 -0
  18. package/dist/core-manager/index.d.ts.map +1 -0
  19. package/dist/core-manager/random-access-file-pool.d.ts +17 -0
  20. package/dist/core-manager/random-access-file-pool.d.ts.map +1 -0
  21. package/dist/core-manager/remote-bitfield.d.ts +146 -0
  22. package/dist/core-manager/remote-bitfield.d.ts.map +1 -0
  23. package/dist/core-ownership.d.ts +112 -0
  24. package/dist/core-ownership.d.ts.map +1 -0
  25. package/dist/datastore/index.d.ts +91 -0
  26. package/dist/datastore/index.d.ts.map +1 -0
  27. package/dist/datatype/index.d.ts +108 -0
  28. package/dist/discovery/local-discovery.d.ts +64 -0
  29. package/dist/discovery/local-discovery.d.ts.map +1 -0
  30. package/dist/errors.d.ts +4 -0
  31. package/dist/errors.d.ts.map +1 -0
  32. package/dist/fastify-controller.d.ts +27 -0
  33. package/dist/fastify-controller.d.ts.map +1 -0
  34. package/dist/fastify-plugins/blobs.d.ts +6 -0
  35. package/dist/fastify-plugins/blobs.d.ts.map +1 -0
  36. package/dist/fastify-plugins/constants.d.ts +3 -0
  37. package/dist/fastify-plugins/constants.d.ts.map +1 -0
  38. package/dist/fastify-plugins/icons.d.ts +6 -0
  39. package/dist/fastify-plugins/icons.d.ts.map +1 -0
  40. package/dist/fastify-plugins/maps/index.d.ts +11 -0
  41. package/dist/fastify-plugins/maps/index.d.ts.map +1 -0
  42. package/dist/fastify-plugins/maps/offline-fallback-map.d.ts +12 -0
  43. package/dist/fastify-plugins/maps/offline-fallback-map.d.ts.map +1 -0
  44. package/dist/fastify-plugins/maps/static-maps.d.ts +11 -0
  45. package/dist/fastify-plugins/maps/static-maps.d.ts.map +1 -0
  46. package/dist/fastify-plugins/utils.d.ts +23 -0
  47. package/dist/fastify-plugins/utils.d.ts.map +1 -0
  48. package/dist/generated/extensions.d.ts +44 -0
  49. package/dist/generated/extensions.d.ts.map +1 -0
  50. package/dist/generated/keys.d.ts +36 -0
  51. package/dist/generated/keys.d.ts.map +1 -0
  52. package/dist/generated/rpc.d.ts +87 -0
  53. package/dist/generated/rpc.d.ts.map +1 -0
  54. package/dist/icon-api.d.ts +109 -0
  55. package/dist/icon-api.d.ts.map +1 -0
  56. package/dist/index-writer/index.d.ts +51 -0
  57. package/dist/index-writer/index.d.ts.map +1 -0
  58. package/dist/index.d.ts +14 -0
  59. package/dist/index.d.ts.map +1 -0
  60. package/dist/invite-api.d.ts +70 -0
  61. package/dist/invite-api.d.ts.map +1 -0
  62. package/dist/lib/hashmap.d.ts +62 -0
  63. package/dist/lib/hashmap.d.ts.map +1 -0
  64. package/dist/lib/hypercore-helpers.d.ts +6 -0
  65. package/dist/lib/hypercore-helpers.d.ts.map +1 -0
  66. package/dist/lib/noise-secret-stream-helpers.d.ts +45 -0
  67. package/dist/lib/noise-secret-stream-helpers.d.ts.map +1 -0
  68. package/dist/lib/ponyfills.d.ts +10 -0
  69. package/dist/lib/ponyfills.d.ts.map +1 -0
  70. package/dist/lib/string.d.ts +2 -0
  71. package/dist/lib/string.d.ts.map +1 -0
  72. package/dist/lib/timing-safe-equal.d.ts +15 -0
  73. package/dist/lib/timing-safe-equal.d.ts.map +1 -0
  74. package/dist/local-peers.d.ts +151 -0
  75. package/dist/local-peers.d.ts.map +1 -0
  76. package/dist/logger.d.ts +32 -0
  77. package/dist/logger.d.ts.map +1 -0
  78. package/dist/mapeo-manager.d.ts +178 -0
  79. package/dist/mapeo-manager.d.ts.map +1 -0
  80. package/dist/mapeo-project.d.ts +3233 -0
  81. package/dist/mapeo-project.d.ts.map +1 -0
  82. package/dist/member-api.d.ts +114 -0
  83. package/dist/member-api.d.ts.map +1 -0
  84. package/dist/roles.d.ts +157 -0
  85. package/dist/roles.d.ts.map +1 -0
  86. package/dist/schema/client.d.ts +284 -0
  87. package/dist/schema/client.d.ts.map +1 -0
  88. package/dist/schema/project.d.ts +1812 -0
  89. package/dist/schema/project.d.ts.map +1 -0
  90. package/dist/schema/schema-to-drizzle.d.ts +20 -0
  91. package/dist/schema/schema-to-drizzle.d.ts.map +1 -0
  92. package/dist/schema/types.d.ts +98 -0
  93. package/dist/schema/types.d.ts.map +1 -0
  94. package/dist/schema/utils.d.ts +55 -0
  95. package/dist/schema/utils.d.ts.map +1 -0
  96. package/dist/sync/core-sync-state.d.ts +252 -0
  97. package/dist/sync/core-sync-state.d.ts.map +1 -0
  98. package/dist/sync/namespace-sync-state.d.ts +47 -0
  99. package/dist/sync/namespace-sync-state.d.ts.map +1 -0
  100. package/dist/sync/peer-sync-controller.d.ts +44 -0
  101. package/dist/sync/peer-sync-controller.d.ts.map +1 -0
  102. package/dist/sync/sync-api.d.ts +158 -0
  103. package/dist/sync/sync-api.d.ts.map +1 -0
  104. package/dist/sync/sync-state.d.ts +40 -0
  105. package/dist/sync/sync-state.d.ts.map +1 -0
  106. package/dist/translation-api.d.ts +288 -0
  107. package/dist/translation-api.d.ts.map +1 -0
  108. package/dist/types.d.ts +115 -0
  109. package/dist/types.d.ts.map +1 -0
  110. package/dist/utils.d.ts +115 -0
  111. package/dist/utils.d.ts.map +1 -0
  112. package/dist/utils_types.d.ts +14 -0
  113. package/drizzle/client/0000_bumpy_carnage.sql +33 -0
  114. package/drizzle/client/meta/0000_snapshot.json +199 -0
  115. package/drizzle/client/meta/_journal.json +13 -0
  116. package/drizzle/project/0000_spooky_lady_ursula.sql +192 -0
  117. package/drizzle/project/meta/0000_snapshot.json +1137 -0
  118. package/drizzle/project/meta/_journal.json +13 -0
  119. package/package.json +202 -0
  120. package/src/blob-api.js +139 -0
  121. package/src/blob-store/index.js +325 -0
  122. package/src/blob-store/live-download.js +373 -0
  123. package/src/config-import.js +604 -0
  124. package/src/constants.js +34 -0
  125. package/src/core-manager/bitfield-rle.js +235 -0
  126. package/src/core-manager/core-index.js +87 -0
  127. package/src/core-manager/index.js +504 -0
  128. package/src/core-manager/random-access-file-pool.js +30 -0
  129. package/src/core-manager/remote-bitfield.js +416 -0
  130. package/src/core-ownership.js +235 -0
  131. package/src/datastore/README.md +46 -0
  132. package/src/datastore/index.js +234 -0
  133. package/src/datatype/README.md +33 -0
  134. package/src/datatype/index.d.ts +108 -0
  135. package/src/datatype/index.js +358 -0
  136. package/src/discovery/local-discovery.js +303 -0
  137. package/src/errors.js +5 -0
  138. package/src/fastify-controller.js +84 -0
  139. package/src/fastify-plugins/blobs.js +139 -0
  140. package/src/fastify-plugins/constants.js +5 -0
  141. package/src/fastify-plugins/icons.js +158 -0
  142. package/src/fastify-plugins/maps/index.js +173 -0
  143. package/src/fastify-plugins/maps/offline-fallback-map.js +114 -0
  144. package/src/fastify-plugins/maps/static-maps.js +271 -0
  145. package/src/fastify-plugins/utils.js +52 -0
  146. package/src/generated/README.md +3 -0
  147. package/src/generated/extensions.d.ts +44 -0
  148. package/src/generated/extensions.js +196 -0
  149. package/src/generated/extensions.ts +237 -0
  150. package/src/generated/keys.d.ts +36 -0
  151. package/src/generated/keys.js +148 -0
  152. package/src/generated/keys.ts +185 -0
  153. package/src/generated/rpc.d.ts +87 -0
  154. package/src/generated/rpc.js +389 -0
  155. package/src/generated/rpc.ts +463 -0
  156. package/src/icon-api.js +282 -0
  157. package/src/index-writer/README.md +38 -0
  158. package/src/index-writer/index.js +124 -0
  159. package/src/index.js +16 -0
  160. package/src/invite-api.js +450 -0
  161. package/src/lib/hashmap.js +91 -0
  162. package/src/lib/hypercore-helpers.js +18 -0
  163. package/src/lib/noise-secret-stream-helpers.js +37 -0
  164. package/src/lib/ponyfills.js +25 -0
  165. package/src/lib/string.js +7 -0
  166. package/src/lib/timing-safe-equal.js +34 -0
  167. package/src/local-peers.js +737 -0
  168. package/src/logger.js +99 -0
  169. package/src/mapeo-manager.js +914 -0
  170. package/src/mapeo-project.js +980 -0
  171. package/src/member-api.js +319 -0
  172. package/src/roles.js +412 -0
  173. package/src/schema/client.js +55 -0
  174. package/src/schema/project.js +44 -0
  175. package/src/schema/schema-to-drizzle.js +118 -0
  176. package/src/schema/types.ts +153 -0
  177. package/src/schema/utils.js +51 -0
  178. package/src/sync/core-sync-state.js +440 -0
  179. package/src/sync/namespace-sync-state.js +193 -0
  180. package/src/sync/peer-sync-controller.js +332 -0
  181. package/src/sync/sync-api.js +588 -0
  182. package/src/sync/sync-state.js +63 -0
  183. package/src/translation-api.js +141 -0
  184. package/src/types.ts +149 -0
  185. package/src/utils.js +210 -0
  186. package/src/utils_types.d.ts +14 -0
@@ -0,0 +1,914 @@
1
+ import { randomBytes } from 'crypto'
2
+ import path from 'path'
3
+ import { KeyManager } from '@mapeo/crypto'
4
+ import Database from 'better-sqlite3'
5
+ import { eq } from 'drizzle-orm'
6
+ import { drizzle } from 'drizzle-orm/better-sqlite3'
7
+ import { migrate } from 'drizzle-orm/better-sqlite3/migrator'
8
+ import Hypercore from 'hypercore'
9
+ import { TypedEmitter } from 'tiny-typed-emitter'
10
+ import pTimeout from 'p-timeout'
11
+
12
+ import { IndexWriter } from './index-writer/index.js'
13
+ import {
14
+ MapeoProject,
15
+ kBlobStore,
16
+ kClearDataIfLeft,
17
+ kProjectLeave,
18
+ kSetOwnDeviceInfo,
19
+ } from './mapeo-project.js'
20
+ import {
21
+ localDeviceInfoTable,
22
+ projectKeysTable,
23
+ projectSettingsTable,
24
+ } from './schema/client.js'
25
+ import { ProjectKeys } from './generated/keys.js'
26
+ import {
27
+ deNullify,
28
+ getDeviceId,
29
+ keyToId,
30
+ projectIdToNonce,
31
+ projectKeyToId,
32
+ projectKeyToProjectInviteId,
33
+ projectKeyToPublicId,
34
+ } from './utils.js'
35
+ import { openedNoiseSecretStream } from './lib/noise-secret-stream-helpers.js'
36
+ import { RandomAccessFilePool } from './core-manager/random-access-file-pool.js'
37
+ import BlobServerPlugin from './fastify-plugins/blobs.js'
38
+ import IconServerPlugin from './fastify-plugins/icons.js'
39
+ import { getFastifyServerAddress } from './fastify-plugins/utils.js'
40
+ import { LocalPeers } from './local-peers.js'
41
+ import { InviteApi } from './invite-api.js'
42
+ import { LocalDiscovery } from './discovery/local-discovery.js'
43
+ import { Roles } from './roles.js'
44
+ import NoiseSecretStream from '@hyperswarm/secret-stream'
45
+ import { Logger } from './logger.js'
46
+ import {
47
+ kSyncState,
48
+ kRequestFullStop,
49
+ kRescindFullStopRequest,
50
+ } from './sync/sync-api.js'
51
+ /** @import { ProjectSettingsValue as ProjectValue } from '@comapeo/schema' */
52
+ /** @import { SetNonNullable } from 'type-fest' */
53
+ /** @import { CoreStorage, Namespace } from './types.js' */
54
+ /** @import { DeviceInfoParam } from './schema/client.js' */
55
+ /** @import { OpenedNoiseStream } from './lib/noise-secret-stream-helpers.js' */
56
+
57
+ /** @typedef {SetNonNullable<ProjectKeys, 'encryptionKeys'>} ValidatedProjectKeys */
58
+
59
+ const CLIENT_SQLITE_FILE_NAME = 'client.db'
60
+
61
+ // Max file descriptors that RandomAccessFile should use for hypercore storage
62
+ // and index bitfield persistence (used by MultiCoreIndexer). Android has a
63
+ // limit of 1024 per process, so choosing 768 to leave 256 descriptors free for
64
+ // other things e.g. SQLite and other parts of the app.
65
+ const MAX_FILE_DESCRIPTORS = 768
66
+
67
+ // Prefix names for routes registered with http server
68
+ const BLOBS_PREFIX = 'blobs'
69
+ const ICONS_PREFIX = 'icons'
70
+
71
+ export const kRPC = Symbol('rpc')
72
+ export const kManagerReplicate = Symbol('replicate manager')
73
+
74
+ /**
75
+ * @typedef {Omit<import('./local-peers.js').PeerInfo, 'protomux'>} PublicPeerInfo
76
+ */
77
+
78
+ /**
79
+ * @typedef {object} MapeoManagerEvents
80
+ * @property {(peers: PublicPeerInfo[]) => void} local-peers Emitted when the list of connected peers changes (new ones added, or connection status changes)
81
+ */
82
+
83
+ /**
84
+ * @extends {TypedEmitter<MapeoManagerEvents>}
85
+ */
86
+ export class MapeoManager extends TypedEmitter {
87
+ #keyManager
88
+ #projectSettingsIndexWriter
89
+ #db
90
+ // Maps project public id -> project instance
91
+ /** @type {Map<string, MapeoProject>} */
92
+ #activeProjects
93
+ /** @type {CoreStorage} */
94
+ #coreStorage
95
+ #dbFolder
96
+ /** @type {string} */
97
+ #projectMigrationsFolder
98
+ #deviceId
99
+ #localPeers
100
+ #invite
101
+ #fastify
102
+ #localDiscovery
103
+ #loggerBase
104
+ #l
105
+ #defaultConfigPath
106
+
107
+ /**
108
+ * @param {Object} opts
109
+ * @param {Buffer} opts.rootKey 16-bytes of random data that uniquely identify the device, used to derive a 32-byte master key, which is used to derive all the keypairs used for Mapeo
110
+ * @param {string} opts.dbFolder Folder for sqlite Dbs. Folder must exist. Use ':memory:' to store everything in-memory
111
+ * @param {string} opts.projectMigrationsFolder path for drizzle migrations folder for project database
112
+ * @param {string} opts.clientMigrationsFolder path for drizzle migrations folder for client database
113
+ * @param {string | CoreStorage} opts.coreStorage Folder for hypercore storage or a function that returns a RandomAccessStorage instance
114
+ * @param {import('fastify').FastifyInstance} opts.fastify Fastify server instance
115
+ * @param {String} [opts.defaultConfigPath]
116
+ */
117
+ constructor({
118
+ rootKey,
119
+ dbFolder,
120
+ projectMigrationsFolder,
121
+ clientMigrationsFolder,
122
+ coreStorage,
123
+ fastify,
124
+ defaultConfigPath,
125
+ }) {
126
+ super()
127
+ this.#keyManager = new KeyManager(rootKey)
128
+ this.#deviceId = getDeviceId(this.#keyManager)
129
+ this.#defaultConfigPath = defaultConfigPath
130
+ const logger = (this.#loggerBase = new Logger({ deviceId: this.#deviceId }))
131
+ this.#l = Logger.create('manager', logger)
132
+ this.#dbFolder = dbFolder
133
+ this.#projectMigrationsFolder = projectMigrationsFolder
134
+ const sqlite = new Database(
135
+ dbFolder === ':memory:'
136
+ ? ':memory:'
137
+ : path.join(dbFolder, CLIENT_SQLITE_FILE_NAME)
138
+ )
139
+ this.#db = drizzle(sqlite)
140
+ migrate(this.#db, { migrationsFolder: clientMigrationsFolder })
141
+
142
+ this.#localPeers = new LocalPeers({ logger })
143
+ this.#localPeers.on('peers', (peers) => {
144
+ this.emit('local-peers', omitPeerProtomux(peers))
145
+ })
146
+ this.#localPeers.on('discovery-key', (dk) => {
147
+ if (this.#activeProjects.size === 0) {
148
+ this.#l.log('Received dk %h but no active projects', dk)
149
+ }
150
+ })
151
+
152
+ this.#projectSettingsIndexWriter = new IndexWriter({
153
+ tables: [projectSettingsTable],
154
+ sqlite,
155
+ logger,
156
+ })
157
+ this.#activeProjects = new Map()
158
+
159
+ this.#invite = new InviteApi({
160
+ rpc: this.#localPeers,
161
+ queries: {
162
+ getProjectByInviteId: (projectInviteId) =>
163
+ this.#db
164
+ .select()
165
+ .from(projectKeysTable)
166
+ .where(eq(projectKeysTable.projectInviteId, projectInviteId))
167
+ .get(),
168
+ addProject: this.addProject,
169
+ },
170
+ logger,
171
+ })
172
+
173
+ if (typeof coreStorage === 'string') {
174
+ const pool = new RandomAccessFilePool(MAX_FILE_DESCRIPTORS)
175
+ // @ts-expect-error
176
+ this.#coreStorage = Hypercore.defaultStorage(coreStorage, { pool })
177
+ } else {
178
+ this.#coreStorage = coreStorage
179
+ }
180
+
181
+ this.#fastify = fastify
182
+ this.#fastify.register(BlobServerPlugin, {
183
+ prefix: BLOBS_PREFIX,
184
+ getBlobStore: async (projectPublicId) => {
185
+ const project = await this.getProject(projectPublicId)
186
+ return project[kBlobStore]
187
+ },
188
+ })
189
+ this.#fastify.register(IconServerPlugin, {
190
+ prefix: ICONS_PREFIX,
191
+ getProject: this.getProject.bind(this),
192
+ })
193
+
194
+ this.#localDiscovery = new LocalDiscovery({
195
+ identityKeypair: this.#keyManager.getIdentityKeypair(),
196
+ logger,
197
+ })
198
+ this.#localDiscovery.on('connection', this.#replicate.bind(this))
199
+ }
200
+
201
+ /**
202
+ * MapeoRPC instance, used for tests
203
+ */
204
+ get [kRPC]() {
205
+ return this.#localPeers
206
+ }
207
+
208
+ get deviceId() {
209
+ return this.#deviceId
210
+ }
211
+
212
+ /**
213
+ * Create a Mapeo replication stream. This replication connects the Mapeo RPC
214
+ * channel and allows invites. All active projects will sync automatically to
215
+ * this replication stream. Only use for local (trusted) connections, because
216
+ * the RPC channel key is public. To sync a specific project without
217
+ * connecting RPC, use project[kProjectReplication].
218
+ *
219
+ * @param {boolean} isInitiator
220
+ */
221
+ [kManagerReplicate](isInitiator) {
222
+ const noiseStream = new NoiseSecretStream(isInitiator, undefined, {
223
+ keyPair: this.#keyManager.getIdentityKeypair(),
224
+ })
225
+ return this.#replicate(noiseStream)
226
+ }
227
+
228
+ /**
229
+ * @param {'blobs' | 'icons' | 'maps'} mediaType
230
+ * @returns {Promise<string>}
231
+ */
232
+ async #getMediaBaseUrl(mediaType) {
233
+ /** @type {string | null} */
234
+ let prefix = null
235
+
236
+ switch (mediaType) {
237
+ case 'blobs': {
238
+ prefix = BLOBS_PREFIX
239
+ break
240
+ }
241
+ case 'icons': {
242
+ prefix = ICONS_PREFIX
243
+ break
244
+ }
245
+ default: {
246
+ throw new Error(`Unsupported media type ${mediaType}`)
247
+ }
248
+ }
249
+
250
+ const base = await getFastifyServerAddress(this.#fastify.server, {
251
+ timeout: 5000,
252
+ })
253
+
254
+ return base + '/' + prefix
255
+ }
256
+
257
+ /**
258
+ * @param {NoiseSecretStream<any>} noiseStream
259
+ */
260
+ #replicate(noiseStream) {
261
+ const replicationStream = this.#localPeers.connect(noiseStream)
262
+
263
+ openedNoiseSecretStream(noiseStream)
264
+ .then((openedNoiseStream) => {
265
+ if (openedNoiseStream.destroyed) return
266
+
267
+ const deviceInfo = this.getDeviceInfo()
268
+ if (!hasSavedDeviceInfo(deviceInfo)) return
269
+
270
+ const peerId = keyToId(openedNoiseStream.remotePublicKey)
271
+
272
+ return this.#localPeers.sendDeviceInfo(peerId, deviceInfo)
273
+ })
274
+ .catch((e) => {
275
+ // Ignore error but log
276
+ this.#l.log(
277
+ 'Failed to send device info to peer %h',
278
+ noiseStream.remotePublicKey,
279
+ e
280
+ )
281
+ })
282
+
283
+ return replicationStream
284
+ }
285
+
286
+ /**
287
+ * @param {Buffer} keysCipher
288
+ * @param {string} projectId
289
+ * @returns {ProjectKeys}
290
+ */
291
+ #decodeProjectKeysCipher(keysCipher, projectId) {
292
+ const nonce = projectIdToNonce(projectId)
293
+ return ProjectKeys.decode(
294
+ this.#keyManager.decryptLocalMessage(keysCipher, nonce)
295
+ )
296
+ }
297
+
298
+ /**
299
+ * @param {string} projectId
300
+ * @returns {Pick<ConstructorParameters<typeof MapeoProject>[0], 'dbPath' | 'coreStorage'>}
301
+ */
302
+ #projectStorage(projectId) {
303
+ return {
304
+ dbPath:
305
+ this.#dbFolder === ':memory:'
306
+ ? ':memory:'
307
+ : path.join(this.#dbFolder, projectId + '.db'),
308
+ coreStorage: (name) => this.#coreStorage(path.join(projectId, name)),
309
+ }
310
+ }
311
+
312
+ /**
313
+ * @param {Object} opts
314
+ * @param {string} opts.projectId
315
+ * @param {string} opts.projectPublicId
316
+ * @param {Readonly<Buffer>} opts.projectInviteId
317
+ * @param {ProjectKeys} opts.projectKeys
318
+ * @param {Readonly<{ name?: string }>} [opts.projectInfo]
319
+ */
320
+ #saveToProjectKeysTable({
321
+ projectId,
322
+ projectPublicId,
323
+ projectInviteId,
324
+ projectKeys,
325
+ projectInfo,
326
+ }) {
327
+ const encoded = ProjectKeys.encode(projectKeys).finish()
328
+ const nonce = projectIdToNonce(projectId)
329
+
330
+ const keysCipher = this.#keyManager.encryptLocalMessage(
331
+ Buffer.from(encoded.buffer, encoded.byteOffset, encoded.byteLength),
332
+ nonce
333
+ )
334
+
335
+ this.#db
336
+ .insert(projectKeysTable)
337
+ .values({
338
+ projectId,
339
+ projectPublicId,
340
+ projectInviteId,
341
+ keysCipher,
342
+ projectInfo,
343
+ })
344
+ .onConflictDoUpdate({
345
+ target: projectKeysTable.projectId,
346
+ set: { projectPublicId, projectInviteId, keysCipher, projectInfo },
347
+ })
348
+ .run()
349
+ }
350
+
351
+ /**
352
+ * Create a new project.
353
+ * @param {(
354
+ * import('type-fest').Simplify<(
355
+ * Partial<Pick<ProjectValue, 'name'>> &
356
+ * { configPath?: string }
357
+ * )>
358
+ * )} [options]
359
+ * @returns {Promise<string>} Project public id
360
+ */
361
+ async createProject({ name, configPath = this.#defaultConfigPath } = {}) {
362
+ // 1. Create project keypair
363
+ const projectKeypair = KeyManager.generateProjectKeypair()
364
+
365
+ // 2. Create namespace encryption keys
366
+ /** @type {Record<Namespace, Buffer>} */
367
+ const encryptionKeys = {
368
+ auth: randomBytes(32),
369
+ blob: randomBytes(32),
370
+ blobIndex: randomBytes(32),
371
+ config: randomBytes(32),
372
+ data: randomBytes(32),
373
+ }
374
+
375
+ // 3. Save keys to client db projectKeys table
376
+ /** @type {ProjectKeys} */
377
+ const keys = {
378
+ projectKey: projectKeypair.publicKey,
379
+ projectSecretKey: projectKeypair.secretKey,
380
+ encryptionKeys,
381
+ }
382
+
383
+ const projectId = projectKeyToId(keys.projectKey)
384
+ const projectPublicId = projectKeyToPublicId(keys.projectKey)
385
+ const projectInviteId = projectKeyToProjectInviteId(keys.projectKey)
386
+
387
+ this.#saveToProjectKeysTable({
388
+ projectId,
389
+ projectPublicId,
390
+ projectInviteId,
391
+ projectKeys: keys,
392
+ })
393
+
394
+ // 4. Create MapeoProject instance
395
+ const project = await this.#createProjectInstance({
396
+ encryptionKeys,
397
+ projectKey: projectKeypair.publicKey,
398
+ projectSecretKey: projectKeypair.secretKey,
399
+ })
400
+
401
+ project.once('close', () => {
402
+ this.#activeProjects.delete(projectPublicId)
403
+ })
404
+
405
+ // 5. Write project settings to project instance
406
+ await project.$setProjectSettings({ name })
407
+
408
+ // 6. Write device info into project
409
+ const deviceInfo = this.getDeviceInfo()
410
+ if (hasSavedDeviceInfo(deviceInfo)) {
411
+ await project[kSetOwnDeviceInfo](deviceInfo)
412
+ }
413
+
414
+ // TODO: Close the project instance instead of keeping it around
415
+ this.#activeProjects.set(projectPublicId, project)
416
+
417
+ // 7. Load config, if relevant
418
+ // TODO: see how to expose warnings to frontend
419
+ /* eslint-disable no-unused-vars */
420
+ let warnings
421
+ if (configPath) {
422
+ warnings = await project.importConfig({ configPath })
423
+ }
424
+
425
+ this.#l.log(
426
+ 'created project %h, public id: %S',
427
+ projectKeypair.publicKey,
428
+ projectPublicId
429
+ )
430
+
431
+ // 7. Return project public id
432
+ return projectPublicId
433
+ }
434
+
435
+ /**
436
+ * @param {string} projectPublicId
437
+ * @returns {Promise<MapeoProject>}
438
+ */
439
+ async getProject(projectPublicId) {
440
+ // 1. Check for existing active project
441
+ const activeProject = this.#activeProjects.get(projectPublicId)
442
+
443
+ if (activeProject) return activeProject
444
+
445
+ // 2. Create project instance
446
+ const projectKeysTableResult = this.#db
447
+ .select({
448
+ projectId: projectKeysTable.projectId,
449
+ keysCipher: projectKeysTable.keysCipher,
450
+ })
451
+ .from(projectKeysTable)
452
+ .where(eq(projectKeysTable.projectPublicId, projectPublicId))
453
+ .get()
454
+
455
+ if (!projectKeysTableResult) {
456
+ throw new Error(`NotFound: project ID ${projectPublicId} not found`)
457
+ }
458
+
459
+ const { projectId } = projectKeysTableResult
460
+
461
+ const projectKeys = this.#decodeProjectKeysCipher(
462
+ projectKeysTableResult.keysCipher,
463
+ projectId
464
+ )
465
+
466
+ const project = await this.#createProjectInstance(projectKeys)
467
+
468
+ project.once('close', () => {
469
+ this.#activeProjects.delete(projectPublicId)
470
+ })
471
+
472
+ // 3. Keep track of project instance as we know it's a properly existing project
473
+ this.#activeProjects.set(projectPublicId, project)
474
+
475
+ return project
476
+ }
477
+
478
+ /** @param {ProjectKeys} projectKeys */
479
+ async #createProjectInstance(projectKeys) {
480
+ validateProjectKeys(projectKeys)
481
+ const projectId = keyToId(projectKeys.projectKey)
482
+ const project = new MapeoProject({
483
+ ...this.#projectStorage(projectId),
484
+ ...projectKeys,
485
+ projectMigrationsFolder: this.#projectMigrationsFolder,
486
+ keyManager: this.#keyManager,
487
+ sharedDb: this.#db,
488
+ sharedIndexWriter: this.#projectSettingsIndexWriter,
489
+ localPeers: this.#localPeers,
490
+ logger: this.#loggerBase,
491
+ getMediaBaseUrl: this.#getMediaBaseUrl.bind(this),
492
+ })
493
+ await project[kClearDataIfLeft]()
494
+ return project
495
+ }
496
+
497
+ /**
498
+ * @returns {Promise<Array<Pick<ProjectValue, 'name'> & { projectId: string, createdAt?: string, updatedAt?: string}>>}
499
+ */
500
+ async listProjects() {
501
+ // We use the project keys table as the source of truth for projects that exist
502
+ // because we will always update this table when doing a create or add
503
+ // whereas the project table will only have projects that have been created, or added + synced
504
+ const allProjectKeysResult = this.#db
505
+ .select({
506
+ projectId: projectKeysTable.projectId,
507
+ projectPublicId: projectKeysTable.projectPublicId,
508
+ projectInfo: projectKeysTable.projectInfo,
509
+ })
510
+ .from(projectKeysTable)
511
+ .all()
512
+
513
+ const allProjectsResult = this.#db
514
+ .select({
515
+ projectId: projectSettingsTable.docId,
516
+ createdAt: projectSettingsTable.createdAt,
517
+ updatedAt: projectSettingsTable.updatedAt,
518
+ name: projectSettingsTable.name,
519
+ })
520
+ .from(projectSettingsTable)
521
+ .all()
522
+
523
+ /** @type {Array<Pick<ProjectValue, 'name'> & { projectId: string, createdAt?: string, updatedAt?: string, createdBy?: string }>} */
524
+ const result = []
525
+
526
+ for (const {
527
+ projectId,
528
+ projectPublicId,
529
+ projectInfo,
530
+ } of allProjectKeysResult) {
531
+ const existingProject = allProjectsResult.find(
532
+ (p) => p.projectId === projectId
533
+ )
534
+
535
+ result.push(
536
+ deNullify({
537
+ projectId: projectPublicId,
538
+ createdAt: existingProject?.createdAt,
539
+ updatedAt: existingProject?.updatedAt,
540
+ name: existingProject?.name || projectInfo.name,
541
+ })
542
+ )
543
+ }
544
+
545
+ return result
546
+ }
547
+
548
+ /**
549
+ * Add a project to this device. After adding a project the client should
550
+ * await `project.$waitForInitialSync()` to ensure that the device has
551
+ * downloaded their proof of project membership and the project config.
552
+ *
553
+ * @param {Pick<import('./generated/rpc.js').ProjectJoinDetails, 'projectKey' | 'encryptionKeys'> & { projectName: string }} projectJoinDetails
554
+ * @param {{ waitForSync?: boolean }} [opts] For internal use in tests, set opts.waitForSync = false to not wait for sync during addProject()
555
+ * @returns {Promise<string>}
556
+ */
557
+ addProject = async (
558
+ { projectKey, encryptionKeys, projectName },
559
+ { waitForSync = true } = {}
560
+ ) => {
561
+ const projectPublicId = projectKeyToPublicId(projectKey)
562
+
563
+ // 1. Check for an active project
564
+ const activeProject = this.#activeProjects.get(projectPublicId)
565
+
566
+ if (activeProject) {
567
+ throw new Error(`Project with ID ${projectPublicId} already exists`)
568
+ }
569
+
570
+ // 2. Check if the project exists in the project keys table
571
+ // If it does, that means the project has already been either created or added before
572
+ const projectId = projectKeyToId(projectKey)
573
+ const projectInviteId = projectKeyToProjectInviteId(projectKey)
574
+
575
+ const projectExists = this.#db
576
+ .select()
577
+ .from(projectKeysTable)
578
+ .where(eq(projectKeysTable.projectId, projectId))
579
+ .get()
580
+
581
+ if (projectExists) {
582
+ throw new Error(`Project with ID ${projectPublicId} already exists`)
583
+ }
584
+
585
+ // No awaits here - need to update table in same tick as the projectExists check
586
+
587
+ // 3. Update the project keys table
588
+ this.#saveToProjectKeysTable({
589
+ projectId,
590
+ projectPublicId,
591
+ projectInviteId,
592
+ projectKeys: {
593
+ projectKey,
594
+ encryptionKeys,
595
+ },
596
+ projectInfo: { name: projectName },
597
+ })
598
+
599
+ // Any errors from here we need to remove project from db because it has not
600
+ // been fully added and synced
601
+ try {
602
+ // 4. Write device info into project
603
+ const project = await this.getProject(projectPublicId)
604
+
605
+ try {
606
+ const deviceInfo = this.getDeviceInfo()
607
+ if (hasSavedDeviceInfo(deviceInfo)) {
608
+ await project[kSetOwnDeviceInfo](deviceInfo)
609
+ }
610
+ } catch (e) {
611
+ // Can ignore an error trying to write device info
612
+ this.#l.log(
613
+ 'ERROR: failed to write project %h deviceInfo %o',
614
+ projectKey,
615
+ e
616
+ )
617
+ }
618
+
619
+ // 5. Wait for initial project sync
620
+ if (waitForSync) {
621
+ await this.#waitForInitialSync(project)
622
+ }
623
+
624
+ this.#activeProjects.set(projectPublicId, project)
625
+ } catch (e) {
626
+ this.#l.log('ERROR: could not add project', e)
627
+ this.#db
628
+ .delete(projectKeysTable)
629
+ .where(eq(projectKeysTable.projectId, projectId))
630
+ .run()
631
+ throw e
632
+ }
633
+ this.#l.log('Added project %h, public ID: %S', projectKey, projectPublicId)
634
+ return projectPublicId
635
+ }
636
+
637
+ /**
638
+ * Sync initial data: the `auth` cores which contain the role messages,
639
+ * and the `config` cores which contain the project name & custom config (if
640
+ * it exists). The API consumer should await this after `client.addProject()`
641
+ * to ensure that the device is fully added to the project.
642
+ *
643
+ * @param {MapeoProject} project
644
+ * @param {object} [opts]
645
+ * @param {number} [opts.timeoutMs=5000] Timeout in milliseconds for max time
646
+ * to wait between sync status updates before giving up. As long as syncing is
647
+ * happening, this will never timeout, but if more than timeoutMs passes
648
+ * without any sync activity, then this will resolve `false` e.g. data has not
649
+ * synced
650
+ * @returns {Promise<boolean>}
651
+ */
652
+ async #waitForInitialSync(project, { timeoutMs = 5000 } = {}) {
653
+ const [ownRole, projectSettings] = await Promise.all([
654
+ project.$getOwnRole(),
655
+ project.$getProjectSettings(),
656
+ ])
657
+ const {
658
+ auth: { localState: authState },
659
+ config: { localState: configState },
660
+ } = project.$sync[kSyncState].getState()
661
+ const isRoleSynced = ownRole !== Roles.NO_ROLE
662
+ const isProjectSettingsSynced =
663
+ projectSettings !== MapeoProject.EMPTY_PROJECT_SETTINGS
664
+ // Assumes every project that someone is invited to has at least one record
665
+ // in the auth store - the row record for the invited device
666
+ const isAuthSynced = authState.want === 0 && authState.have > 0
667
+ // Assumes every project that someone is invited to has at least one record
668
+ // in the config store - defining the name of the project.
669
+ // TODO: Enforce adding a project name in the invite method
670
+ const isConfigSynced = configState.want === 0 && configState.have > 0
671
+ if (
672
+ isRoleSynced &&
673
+ isProjectSettingsSynced &&
674
+ isAuthSynced &&
675
+ isConfigSynced
676
+ ) {
677
+ return true
678
+ }
679
+ return new Promise((resolve, reject) => {
680
+ /** @param {import('./sync/sync-state.js').State} syncState */
681
+ const onSyncState = (syncState) => {
682
+ clearTimeout(timeoutId)
683
+ if (syncState.auth.dataToSync || syncState.config.dataToSync) {
684
+ timeoutId = setTimeout(onTimeout, timeoutMs)
685
+ return
686
+ }
687
+ project.$sync[kSyncState].off('state', onSyncState)
688
+ resolve(this.#waitForInitialSync(project, { timeoutMs }))
689
+ }
690
+ const onTimeout = () => {
691
+ project.$sync[kSyncState].off('state', onSyncState)
692
+ reject(new Error('Sync timeout'))
693
+ }
694
+ let timeoutId = setTimeout(onTimeout, timeoutMs)
695
+ project.$sync[kSyncState].on('state', onSyncState)
696
+ })
697
+ }
698
+
699
+ /**
700
+ * @typedef {Exclude<
701
+ * import('./schema/client.js').DeviceInfoParam['deviceType'],
702
+ * 'selfHostedServer'>} RPCDeviceType
703
+ */
704
+
705
+ /**
706
+ * @template {import('type-fest').Exact<
707
+ * import('./schema/client.js').DeviceInfoParam & {deviceType?: RPCDeviceType}, T>} T
708
+ * @param {T} deviceInfo
709
+ */
710
+ async setDeviceInfo(deviceInfo) {
711
+ const values = { deviceId: this.#deviceId, deviceInfo }
712
+ this.#db
713
+ .insert(localDeviceInfoTable)
714
+ .values(values)
715
+ .onConflictDoUpdate({
716
+ target: localDeviceInfoTable.deviceId,
717
+ set: values,
718
+ })
719
+ .run()
720
+
721
+ const listedProjects = await this.listProjects()
722
+ await Promise.all(
723
+ listedProjects.map(async ({ projectId }) => {
724
+ const project = await this.getProject(projectId)
725
+ await project[kSetOwnDeviceInfo](deviceInfo)
726
+ })
727
+ )
728
+
729
+ await Promise.all(
730
+ this.#localPeers.peers
731
+ .filter(({ status }) => status === 'connected')
732
+ .map((peer) =>
733
+ this.#localPeers.sendDeviceInfo(peer.deviceId, deviceInfo)
734
+ )
735
+ )
736
+
737
+ this.#l.log('set device info %o', deviceInfo)
738
+ }
739
+
740
+ /**
741
+ * @returns {(
742
+ * {
743
+ * deviceId: string;
744
+ * deviceType: DeviceInfoParam['deviceType']
745
+ * } & Partial<DeviceInfoParam>
746
+ * )}
747
+ */
748
+ getDeviceInfo() {
749
+ const row = this.#db
750
+ .select()
751
+ .from(localDeviceInfoTable)
752
+ .where(eq(localDeviceInfoTable.deviceId, this.#deviceId))
753
+ .get()
754
+ return {
755
+ deviceId: this.#deviceId,
756
+ deviceType: 'device_type_unspecified',
757
+ ...row?.deviceInfo,
758
+ }
759
+ }
760
+
761
+ /**
762
+ * @returns {InviteApi}
763
+ */
764
+ get invite() {
765
+ return this.#invite
766
+ }
767
+
768
+ /** @returns {Promise<{ name: string, port: number }>} */
769
+ startLocalPeerDiscoveryServer() {
770
+ return this.#localDiscovery.start()
771
+ }
772
+
773
+ /** @type {LocalDiscovery['stop']} */
774
+ stopLocalPeerDiscoveryServer(opts) {
775
+ return this.#localDiscovery.stop(opts)
776
+ }
777
+
778
+ /** @type {LocalDiscovery['connectPeer']} */
779
+ connectLocalPeer(peer) {
780
+ this.#localDiscovery.connectPeer(peer)
781
+ }
782
+
783
+ /**
784
+ * @returns {Promise<PublicPeerInfo[]>}
785
+ */
786
+ async listLocalPeers() {
787
+ return omitPeerProtomux(this.#localPeers.peers)
788
+ }
789
+
790
+ /**
791
+ * Call this when the app goes into the background.
792
+ *
793
+ * Will gracefully shut down sync.
794
+ *
795
+ * @see {@link onForegrounded}
796
+ * @returns {void}
797
+ */
798
+ onBackgrounded() {
799
+ const projects = this.#activeProjects.values()
800
+ for (const project of projects) project.$sync[kRequestFullStop]()
801
+ }
802
+
803
+ /**
804
+ * Call this when the app goes into the foreground.
805
+ *
806
+ * Will undo the effects of `onBackgrounded`.
807
+ *
808
+ * @see {@link onBackgrounded}
809
+ * @returns {void}
810
+ */
811
+ onForegrounded() {
812
+ const projects = this.#activeProjects.values()
813
+ for (const project of projects) project.$sync[kRescindFullStopRequest]()
814
+ }
815
+
816
+ /**
817
+ * @param {string} projectPublicId
818
+ */
819
+ async leaveProject(projectPublicId) {
820
+ const project = await this.getProject(projectPublicId)
821
+
822
+ await project[kProjectLeave]()
823
+
824
+ const row = this.#db
825
+ .select({
826
+ keysCipher: projectKeysTable.keysCipher,
827
+ projectId: projectKeysTable.projectId,
828
+ projectInfo: projectKeysTable.projectInfo,
829
+ })
830
+ .from(projectKeysTable)
831
+ .where(eq(projectKeysTable.projectPublicId, projectPublicId))
832
+ .get()
833
+
834
+ if (!row) {
835
+ throw new Error(`NotFound: project ID ${projectPublicId} not found`)
836
+ }
837
+
838
+ const { keysCipher, projectId, projectInfo } = row
839
+
840
+ const projectKeys = this.#decodeProjectKeysCipher(keysCipher, projectId)
841
+ const projectInviteId = projectKeyToProjectInviteId(projectKeys.projectKey)
842
+
843
+ const updatedEncryptionKeys = projectKeys.encryptionKeys
844
+ ? // Delete all encryption keys except for auth
845
+ { auth: projectKeys.encryptionKeys.auth }
846
+ : undefined
847
+
848
+ this.#saveToProjectKeysTable({
849
+ projectId,
850
+ projectPublicId,
851
+ projectInviteId,
852
+ projectInfo,
853
+ projectKeys: {
854
+ ...projectKeys,
855
+ encryptionKeys: updatedEncryptionKeys,
856
+ },
857
+ })
858
+
859
+ this.#db
860
+ .delete(projectSettingsTable)
861
+ .where(eq(projectSettingsTable.docId, projectId))
862
+ .run()
863
+
864
+ this.#activeProjects.delete(projectPublicId)
865
+ }
866
+
867
+ async getMapStyleJsonUrl() {
868
+ await pTimeout(this.#fastify.ready(), { milliseconds: 1000 })
869
+ return this.#fastify.mapeoMaps.getStyleJsonUrl()
870
+ }
871
+ }
872
+
873
+ // We use the `protomux` property of connected peers internally, but we don't
874
+ // expose it to the API. I have avoided using a private symbol for this for fear
875
+ // that we could accidentally keep references around of protomux instances,
876
+ // which could cause a memory leak (it shouldn't, but just to eliminate the
877
+ // possibility)
878
+
879
+ /**
880
+ * Remove the protomux property of connected peers
881
+ *
882
+ * @param {import('./local-peers.js').PeerInfo[]} peers
883
+ * @returns {PublicPeerInfo[]}
884
+ */
885
+ function omitPeerProtomux(peers) {
886
+ return peers.map(
887
+ ({
888
+ // @ts-ignore
889
+ // eslint-disable-next-line no-unused-vars
890
+ protomux,
891
+ ...publicPeerInfo
892
+ }) => {
893
+ return publicPeerInfo
894
+ }
895
+ )
896
+ }
897
+
898
+ /**
899
+ * @param {ProjectKeys} projectKeys
900
+ * @returns {asserts projectKeys is ValidatedProjectKeys}
901
+ */
902
+ function validateProjectKeys(projectKeys) {
903
+ if (!projectKeys.encryptionKeys) {
904
+ throw new Error('encryptionKeys should not be undefined')
905
+ }
906
+ }
907
+
908
+ /**
909
+ * @param {Awaited<ReturnType<typeof MapeoManager.prototype.getDeviceInfo>>} partialDeviceInfo
910
+ * @returns {partialDeviceInfo is import('./generated/rpc.js').DeviceInfo}
911
+ */
912
+ function hasSavedDeviceInfo(partialDeviceInfo) {
913
+ return Boolean(partialDeviceInfo.name)
914
+ }