@comapeo/core 1.0.1 → 2.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. package/dist/blob-store/index.d.ts +18 -41
  2. package/dist/blob-store/index.d.ts.map +1 -1
  3. package/dist/config-import.d.ts.map +1 -1
  4. package/dist/core-manager/core-index.d.ts +1 -1
  5. package/dist/core-manager/core-index.d.ts.map +1 -1
  6. package/dist/core-manager/index.d.ts +1 -0
  7. package/dist/core-manager/index.d.ts.map +1 -1
  8. package/dist/core-ownership.d.ts.map +1 -1
  9. package/dist/fastify-controller.d.ts.map +1 -1
  10. package/dist/fastify-plugins/{maps/index.d.ts → maps.d.ts} +8 -8
  11. package/dist/fastify-plugins/maps.d.ts.map +1 -0
  12. package/dist/fastify-plugins/utils.d.ts +4 -0
  13. package/dist/fastify-plugins/utils.d.ts.map +1 -1
  14. package/dist/index.d.ts +1 -3
  15. package/dist/index.d.ts.map +1 -1
  16. package/dist/lib/hashmap.d.ts +2 -2
  17. package/dist/lib/hashmap.d.ts.map +1 -1
  18. package/dist/lib/key-by.d.ts +15 -0
  19. package/dist/lib/key-by.d.ts.map +1 -0
  20. package/dist/lib/noise-secret-stream-helpers.d.ts.map +1 -1
  21. package/dist/lib/omit.d.ts +17 -0
  22. package/dist/lib/omit.d.ts.map +1 -0
  23. package/dist/local-peers.d.ts +3 -2
  24. package/dist/local-peers.d.ts.map +1 -1
  25. package/dist/logger.d.ts +12 -9
  26. package/dist/logger.d.ts.map +1 -1
  27. package/dist/mapeo-manager.d.ts +9 -1
  28. package/dist/mapeo-manager.d.ts.map +1 -1
  29. package/dist/mapeo-project.d.ts +6 -22
  30. package/dist/mapeo-project.d.ts.map +1 -1
  31. package/dist/member-api.d.ts +3 -1
  32. package/dist/member-api.d.ts.map +1 -1
  33. package/dist/roles.d.ts.map +1 -1
  34. package/dist/schema/project.d.ts +1 -1
  35. package/dist/schema/utils.d.ts.map +1 -1
  36. package/dist/sync/core-sync-state.d.ts +10 -3
  37. package/dist/sync/core-sync-state.d.ts.map +1 -1
  38. package/dist/sync/namespace-sync-state.d.ts +8 -12
  39. package/dist/sync/namespace-sync-state.d.ts.map +1 -1
  40. package/dist/sync/peer-sync-controller.d.ts.map +1 -1
  41. package/dist/sync/sync-api.d.ts.map +1 -1
  42. package/dist/sync/sync-state.d.ts +7 -1
  43. package/dist/sync/sync-state.d.ts.map +1 -1
  44. package/dist/translation-api.d.ts +1 -3
  45. package/dist/translation-api.d.ts.map +1 -1
  46. package/dist/types.d.ts +1 -1
  47. package/dist/types.d.ts.map +1 -1
  48. package/dist/utils.d.ts +0 -13
  49. package/dist/utils.d.ts.map +1 -1
  50. package/package.json +12 -11
  51. package/src/blob-store/index.js +17 -2
  52. package/src/config-import.js +0 -1
  53. package/src/core-manager/index.js +13 -10
  54. package/src/core-ownership.js +5 -2
  55. package/src/datastore/README.md +2 -2
  56. package/src/datatype/README.md +1 -1
  57. package/src/fastify-controller.js +7 -1
  58. package/src/fastify-plugins/maps.js +130 -0
  59. package/src/fastify-plugins/utils.js +6 -0
  60. package/src/index-writer/index.js +1 -1
  61. package/src/index.js +1 -3
  62. package/src/lib/hashmap.js +1 -1
  63. package/src/lib/key-by.js +24 -0
  64. package/src/lib/omit.js +28 -0
  65. package/src/local-peers.js +2 -1
  66. package/src/logger.js +52 -16
  67. package/src/mapeo-manager.js +41 -12
  68. package/src/mapeo-project.js +2 -5
  69. package/src/member-api.js +12 -5
  70. package/src/sync/core-sync-state.js +35 -7
  71. package/src/sync/namespace-sync-state.js +26 -24
  72. package/src/sync/peer-sync-controller.js +44 -37
  73. package/src/sync/sync-api.js +9 -6
  74. package/src/sync/sync-state.js +12 -1
  75. package/src/translation-api.js +3 -6
  76. package/src/types.ts +0 -1
  77. package/src/utils.js +11 -39
  78. package/dist/fastify-plugins/maps/index.d.ts.map +0 -1
  79. package/dist/fastify-plugins/maps/offline-fallback-map.d.ts +0 -12
  80. package/dist/fastify-plugins/maps/offline-fallback-map.d.ts.map +0 -1
  81. package/dist/fastify-plugins/maps/static-maps.d.ts +0 -11
  82. package/dist/fastify-plugins/maps/static-maps.d.ts.map +0 -1
  83. package/src/fastify-plugins/maps/index.js +0 -173
  84. package/src/fastify-plugins/maps/offline-fallback-map.js +0 -114
  85. package/src/fastify-plugins/maps/static-maps.js +0 -271
@@ -0,0 +1,130 @@
1
+ import fs from 'node:fs/promises'
2
+ import path from 'node:path'
3
+ import { fetch } from 'undici'
4
+ import { ReaderWatch, Server as SMPServerPlugin } from 'styled-map-package'
5
+
6
+ import { noop } from '../utils.js'
7
+ import { NotFoundError, ENOENTError } from './utils.js'
8
+
9
+ /** @import { FastifyPluginAsync } from 'fastify' */
10
+ /** @import { Stats } from 'node:fs' */
11
+
12
+ export const CUSTOM_MAP_PREFIX = 'custom'
13
+ export const FALLBACK_MAP_PREFIX = 'fallback'
14
+
15
+ /**
16
+ * @typedef {Object} MapsPluginOpts
17
+ *
18
+ * @property {string | URL} defaultOnlineStyleUrl
19
+ * @property {string} [customMapPath]
20
+ * @property {string} fallbackMapPath
21
+ */
22
+
23
+ /** @type {FastifyPluginAsync<MapsPluginOpts>} */
24
+ export async function plugin(fastify, opts) {
25
+ if (opts.customMapPath) {
26
+ const { customMapPath } = opts
27
+
28
+ const customMapReader = new ReaderWatch(customMapPath)
29
+
30
+ fastify.addHook('onClose', () => customMapReader.close().catch(noop))
31
+
32
+ fastify.get(`/${CUSTOM_MAP_PREFIX}/info`, async () => {
33
+ const baseUrl = new URL(fastify.prefix, fastify.listeningOrigin)
34
+
35
+ if (!baseUrl.href.endsWith('/')) {
36
+ baseUrl.href += '/'
37
+ }
38
+
39
+ const customStyleJsonUrl = new URL(
40
+ `${CUSTOM_MAP_PREFIX}/style.json`,
41
+ baseUrl
42
+ )
43
+ const response = await fetch(customStyleJsonUrl)
44
+
45
+ if (response.status === 404) {
46
+ throw new NotFoundError(customStyleJsonUrl.href)
47
+ }
48
+
49
+ if (!response.ok) {
50
+ throw new Error(`Failed to get style from ${customStyleJsonUrl.href}`)
51
+ }
52
+
53
+ /** @type {Stats | undefined} */
54
+ let stats
55
+
56
+ try {
57
+ stats = await fs.stat(customMapPath)
58
+ } catch (err) {
59
+ if (err instanceof Error && 'code' in err && err.code === 'ENOENT') {
60
+ throw new ENOENTError(customMapPath)
61
+ }
62
+
63
+ throw err
64
+ }
65
+
66
+ const style = await response.json()
67
+
68
+ const styleJsonName =
69
+ typeof style === 'object' &&
70
+ style &&
71
+ 'name' in style &&
72
+ typeof style.name === 'string'
73
+ ? style.name
74
+ : undefined
75
+
76
+ return {
77
+ created: stats.ctime,
78
+ size: stats.size,
79
+ name: styleJsonName || path.parse(customMapPath).name,
80
+ }
81
+ })
82
+
83
+ fastify.register(SMPServerPlugin, {
84
+ prefix: CUSTOM_MAP_PREFIX,
85
+ reader: customMapReader,
86
+ })
87
+ }
88
+
89
+ const fallbackMapReader = new ReaderWatch(opts.fallbackMapPath)
90
+
91
+ fastify.addHook('onClose', () => fallbackMapReader.close().catch(noop))
92
+
93
+ fastify.register(SMPServerPlugin, {
94
+ prefix: FALLBACK_MAP_PREFIX,
95
+ reader: fallbackMapReader,
96
+ })
97
+
98
+ fastify.get('/style.json', async (_request, reply) => {
99
+ const baseUrl = new URL(fastify.prefix, fastify.listeningOrigin)
100
+
101
+ // Important for using as a base for creating new URL objects
102
+ if (!baseUrl.href.endsWith('/')) {
103
+ baseUrl.href += '/'
104
+ }
105
+
106
+ /** @type {Array<string | URL>}*/
107
+ const styleUrls = [
108
+ opts.defaultOnlineStyleUrl,
109
+ new URL(`${FALLBACK_MAP_PREFIX}/style.json`, baseUrl),
110
+ ]
111
+
112
+ if (opts.customMapPath) {
113
+ styleUrls.unshift(new URL(`${CUSTOM_MAP_PREFIX}/style.json`, baseUrl))
114
+ }
115
+
116
+ for (const url of styleUrls) {
117
+ const resp = await fetch(url, { method: 'HEAD' }).catch(noop)
118
+
119
+ if (resp && resp.status === 200) {
120
+ return reply
121
+ .headers({
122
+ 'cache-control': 'no-cache',
123
+ })
124
+ .redirect(url.toString())
125
+ }
126
+ }
127
+
128
+ return reply.status(404).send()
129
+ })
130
+ }
@@ -7,6 +7,12 @@ export const NotFoundError = createError(
7
7
  404
8
8
  )
9
9
 
10
+ export const ENOENTError = createError(
11
+ 'FST_ENOENT',
12
+ "ENOENT: no such file or directory '%s'",
13
+ 404
14
+ )
15
+
10
16
  /**
11
17
  * @param {import('node:http').Server} server
12
18
  * @param {{ timeout?: number }} [options]
@@ -97,7 +97,7 @@ export class IndexWriter {
97
97
  const indexer = this.#indexers.get(schemaName)
98
98
  if (!indexer) continue // Won't happen, but TS doesn't know that
99
99
  indexer.batch(docs)
100
- if (this.#l.enabled) {
100
+ if (this.#l.log.enabled) {
101
101
  for (const doc of docs) {
102
102
  this.#l.log(
103
103
  'Indexed %s %S @ %S',
package/src/index.js CHANGED
@@ -3,9 +3,7 @@ import {
3
3
  COORDINATOR_ROLE_ID,
4
4
  MEMBER_ROLE_ID,
5
5
  } from './roles.js'
6
- export { plugin as MapeoStaticMapsFastifyPlugin } from './fastify-plugins/maps/static-maps.js'
7
- export { plugin as MapeoOfflineFallbackMapFastifyPlugin } from './fastify-plugins/maps/offline-fallback-map.js'
8
- export { plugin as MapeoMapsFastifyPlugin } from './fastify-plugins/maps/index.js'
6
+ export { plugin as CoMapeoMapsFastifyPlugin } from './fastify-plugins/maps.js'
9
7
  export { FastifyController } from './fastify-controller.js'
10
8
  export { MapeoManager } from './mapeo-manager.js'
11
9
 
@@ -1,4 +1,4 @@
1
- /** @typedef {string | number | bigint | boolean | undefined | symbol | null} Primitive */
1
+ /** @import { Primitive } from 'type-fest' */
2
2
 
3
3
  /**
4
4
  * `Map` uses same-value-zero equality for keys, which makes it more difficult
@@ -0,0 +1,24 @@
1
+ /**
2
+ * Like [`Map.groupBy`][0], but the result's values aren't arrays.
3
+ *
4
+ * If multiple values resolve to the same key, an error is thrown.
5
+ *
6
+ * [0]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Map/groupBy
7
+ *
8
+ * @template T
9
+ * @template K
10
+ * @param {Iterable<T>} items
11
+ * @param {(item: T) => K} callbackFn
12
+ * @returns {Map<K, T>}
13
+ */
14
+ export function keyBy(items, callbackFn) {
15
+ /** @type {Map<K, T>} */ const result = new Map()
16
+ for (const item of items) {
17
+ const key = callbackFn(item)
18
+ if (result.has(key)) {
19
+ throw new Error(`keyBy found duplicate key ${JSON.stringify(key)}`)
20
+ }
21
+ result.set(key, item)
22
+ }
23
+ return result
24
+ }
@@ -0,0 +1,28 @@
1
+ /**
2
+ * Returns a new object with the own enumerable keys of `obj` that are not in `keys`.
3
+ *
4
+ * In other words, remove some keys from an object.
5
+ *
6
+ * @template {object} T
7
+ * @template {keyof T} K
8
+ * @param {T} obj
9
+ * @param {ReadonlyArray<K>} keys
10
+ * @returns {Omit<T, K>}
11
+ * @example
12
+ * const obj = { foo: 1, bar: 2, baz: 3 }
13
+ * omit(obj, ['foo', 'bar'])
14
+ * // => { baz: 3 }
15
+ */
16
+ export function omit(obj, keys) {
17
+ /** @type {Partial<T>} */ const result = {}
18
+
19
+ /** @type {Set<unknown>} */ const toOmit = new Set(keys)
20
+
21
+ for (const key in obj) {
22
+ if (!Object.hasOwn(obj, key)) continue
23
+ if (toOmit.has(key)) continue
24
+ result[key] = obj[key]
25
+ }
26
+
27
+ return /** @type {Omit<T, K>} */ (result)
28
+ }
@@ -13,6 +13,7 @@ import {
13
13
  import pDefer from 'p-defer'
14
14
  import { Logger } from './logger.js'
15
15
  import pTimeout, { TimeoutError } from 'p-timeout'
16
+ /** @import NoiseStream from '@hyperswarm/secret-stream' */
16
17
  /** @import { OpenedNoiseStream } from './lib/noise-secret-stream-helpers.js' */
17
18
 
18
19
  // Unique identifier for the mapeo rpc protocol
@@ -329,7 +330,7 @@ export class LocalPeers extends TypedEmitter {
329
330
  /**
330
331
  * Connect to a peer over an existing NoiseSecretStream
331
332
  *
332
- * @param {import('./types.js').NoiseStream<any>} stream a NoiseSecretStream from @hyperswarm/secret-stream
333
+ * @param {NoiseStream<any>} stream
333
334
  * @returns {import('./types.js').ReplicationStream}
334
335
  */
335
336
  connect(stream) {
package/src/logger.js CHANGED
@@ -5,6 +5,32 @@ import util from 'util'
5
5
 
6
6
  const TRIM = 7
7
7
 
8
+ const selectColorOriginal = createDebug.selectColor
9
+
10
+ /**
11
+ * Selects a color for a debug namespace (warning: overrides private api).
12
+ * Rather than the default behaviour of creating a unique color for each
13
+ * namespace, we only hash the last 7 characters of the namespace, which is the
14
+ * deviceId. This results in debug output where each deviceId has a different
15
+ * colour, which is more useful for debugging.
16
+ * @param {string} namespace The namespace string for the debug instance to be colored
17
+ * @return {number|string} An ANSI color code for the given namespace
18
+ */
19
+ createDebug.selectColor = function (namespace) {
20
+ if (!namespace.startsWith('mapeo:')) {
21
+ return selectColorOriginal(namespace)
22
+ }
23
+ let hash = 0
24
+
25
+ for (let i = namespace.length - TRIM - 1; i < namespace.length; i++) {
26
+ hash = (hash << 5) - hash + namespace.charCodeAt(i)
27
+ hash |= 0 // Convert to 32bit integer
28
+ }
29
+
30
+ // @ts-expect-error - private debug api
31
+ return createDebug.colors[Math.abs(hash) % createDebug.colors.length]
32
+ }
33
+
8
34
  createDebug.formatters.h = function (v) {
9
35
  if (!Buffer.isBuffer(v)) return '[undefined]'
10
36
  return v.toString('hex').slice(0, TRIM)
@@ -56,13 +82,14 @@ export class Logger {
56
82
  /**
57
83
  * @param {string} ns
58
84
  * @param {Logger} [logger]
85
+ * @param {{ prefix?: string }} [opts]
59
86
  */
60
- static create(ns, logger) {
61
- if (logger) return logger.extend(ns)
87
+ static create(ns, logger, opts) {
88
+ if (logger) return logger.extend(ns, opts)
62
89
  const i = (counts.get(ns) || 0) + 1
63
90
  counts.set(ns, i)
64
91
  const deviceId = String(i).padStart(TRIM, '0')
65
- return new Logger({ deviceId, ns })
92
+ return new Logger({ deviceId, ns, prefix: opts?.prefix })
66
93
  }
67
94
 
68
95
  /**
@@ -70,30 +97,39 @@ export class Logger {
70
97
  * @param {string} opts.deviceId
71
98
  * @param {createDebug.Debugger} [opts.baseLogger]
72
99
  * @param {string} [opts.ns]
100
+ * @param {string} [opts.prefix] optional prefix to add to the start of each log message. Used to add context e.g. the core ID that is syncing. Use this as an alternative to the debug namespace.
73
101
  */
74
- constructor({ deviceId, baseLogger, ns }) {
102
+ constructor({ deviceId, baseLogger, ns, prefix }) {
75
103
  this.deviceId = deviceId
76
104
  this.#baseLogger = baseLogger || createDebug('mapeo' + (ns ? `:${ns}` : ''))
77
- this.#log = this.#baseLogger.extend(this.deviceId.slice(0, TRIM))
78
- }
79
- get enabled() {
80
- return this.#log.enabled
105
+ const log = this.#baseLogger.extend(this.deviceId.slice(0, TRIM))
106
+ if (prefix) {
107
+ this.#log = Object.assign(
108
+ /**
109
+ * @param {any} formatter
110
+ * @param {...any} args
111
+ */
112
+ (formatter, ...args) => {
113
+ return log.apply(null, [`${prefix}${formatter}`, ...args])
114
+ },
115
+ log
116
+ )
117
+ } else {
118
+ this.#log = log
119
+ }
81
120
  }
82
-
83
- /**
84
- * @param {Parameters<createDebug.Debugger>} args
85
- */
86
- log = (...args) => {
87
- this.#log.apply(this, args)
121
+ get log() {
122
+ return this.#log
88
123
  }
89
124
  /**
90
- *
91
125
  * @param {string} ns
126
+ * @param {{ prefix?: string }} [opts]
92
127
  */
93
- extend(ns) {
128
+ extend(ns, { prefix } = {}) {
94
129
  return new Logger({
95
130
  deviceId: this.deviceId,
96
131
  baseLogger: this.#baseLogger.extend(ns),
132
+ prefix,
97
133
  })
98
134
  }
99
135
  }
@@ -8,6 +8,7 @@ import { migrate } from 'drizzle-orm/better-sqlite3/migrator'
8
8
  import Hypercore from 'hypercore'
9
9
  import { TypedEmitter } from 'tiny-typed-emitter'
10
10
  import pTimeout from 'p-timeout'
11
+ import { createRequire } from 'module'
11
12
 
12
13
  import { IndexWriter } from './index-writer/index.js'
13
14
  import {
@@ -33,9 +34,11 @@ import {
33
34
  projectKeyToPublicId,
34
35
  } from './utils.js'
35
36
  import { openedNoiseSecretStream } from './lib/noise-secret-stream-helpers.js'
37
+ import { omit } from './lib/omit.js'
36
38
  import { RandomAccessFilePool } from './core-manager/random-access-file-pool.js'
37
39
  import BlobServerPlugin from './fastify-plugins/blobs.js'
38
40
  import IconServerPlugin from './fastify-plugins/icons.js'
41
+ import { plugin as MapServerPlugin } from './fastify-plugins/maps.js'
39
42
  import { getFastifyServerAddress } from './fastify-plugins/utils.js'
40
43
  import { LocalPeers } from './local-peers.js'
41
44
  import { InviteApi } from './invite-api.js'
@@ -52,7 +55,6 @@ import {
52
55
  /** @import { SetNonNullable } from 'type-fest' */
53
56
  /** @import { CoreStorage, Namespace } from './types.js' */
54
57
  /** @import { DeviceInfoParam } from './schema/client.js' */
55
- /** @import { OpenedNoiseStream } from './lib/noise-secret-stream-helpers.js' */
56
58
 
57
59
  /** @typedef {SetNonNullable<ProjectKeys, 'encryptionKeys'>} ValidatedProjectKeys */
58
60
 
@@ -67,6 +69,15 @@ const MAX_FILE_DESCRIPTORS = 768
67
69
  // Prefix names for routes registered with http server
68
70
  const BLOBS_PREFIX = 'blobs'
69
71
  const ICONS_PREFIX = 'icons'
72
+ const MAPS_PREFIX = 'maps'
73
+
74
+ const require = createRequire(import.meta.url)
75
+ export const DEFAULT_FALLBACK_MAP_FILE_PATH = require.resolve(
76
+ '@comapeo/fallback-smp'
77
+ )
78
+
79
+ export const DEFAULT_ONLINE_STYLE_URL =
80
+ 'https://demotiles.maplibre.org/style.json'
70
81
 
71
82
  export const kRPC = Symbol('rpc')
72
83
  export const kManagerReplicate = Symbol('replicate manager')
@@ -113,6 +124,9 @@ export class MapeoManager extends TypedEmitter {
113
124
  * @param {string | CoreStorage} opts.coreStorage Folder for hypercore storage or a function that returns a RandomAccessStorage instance
114
125
  * @param {import('fastify').FastifyInstance} opts.fastify Fastify server instance
115
126
  * @param {String} [opts.defaultConfigPath]
127
+ * @param {string} [opts.customMapPath] File path to a locally stored Styled Map Package (SMP).
128
+ * @param {string} [opts.fallbackMapPath] File path to a locally stored Styled Map Package (SMP)
129
+ * @param {string} [opts.defaultOnlineStyleUrl] URL for an online-hosted StyleJSON asset.
116
130
  */
117
131
  constructor({
118
132
  rootKey,
@@ -122,6 +136,9 @@ export class MapeoManager extends TypedEmitter {
122
136
  coreStorage,
123
137
  fastify,
124
138
  defaultConfigPath,
139
+ customMapPath,
140
+ fallbackMapPath = DEFAULT_FALLBACK_MAP_FILE_PATH,
141
+ defaultOnlineStyleUrl = DEFAULT_ONLINE_STYLE_URL,
125
142
  }) {
126
143
  super()
127
144
  this.#keyManager = new KeyManager(rootKey)
@@ -190,6 +207,12 @@ export class MapeoManager extends TypedEmitter {
190
207
  prefix: ICONS_PREFIX,
191
208
  getProject: this.getProject.bind(this),
192
209
  })
210
+ this.#fastify.register(MapServerPlugin, {
211
+ prefix: MAPS_PREFIX,
212
+ customMapPath,
213
+ defaultOnlineStyleUrl,
214
+ fallbackMapPath,
215
+ })
193
216
 
194
217
  this.#localDiscovery = new LocalDiscovery({
195
218
  identityKeypair: this.#keyManager.getIdentityKeypair(),
@@ -242,6 +265,10 @@ export class MapeoManager extends TypedEmitter {
242
265
  prefix = ICONS_PREFIX
243
266
  break
244
267
  }
268
+ case 'maps': {
269
+ prefix = MAPS_PREFIX
270
+ break
271
+ }
245
272
  default: {
246
273
  throw new Error(`Unsupported media type ${mediaType}`)
247
274
  }
@@ -416,9 +443,10 @@ export class MapeoManager extends TypedEmitter {
416
443
 
417
444
  // 7. Load config, if relevant
418
445
  // TODO: see how to expose warnings to frontend
419
- /* eslint-disable no-unused-vars */
446
+ // eslint-disable-next-line no-unused-vars
420
447
  let warnings
421
448
  if (configPath) {
449
+ // eslint-disable-next-line no-unused-vars
422
450
  warnings = await project.importConfig({ configPath })
423
451
  }
424
452
 
@@ -675,6 +703,14 @@ export class MapeoManager extends TypedEmitter {
675
703
  isConfigSynced
676
704
  ) {
677
705
  return true
706
+ } else {
707
+ this.#l.log(
708
+ 'Pending initial sync: role %s, projectSettings %o, auth %o, config %o',
709
+ isRoleSynced,
710
+ isProjectSettingsSynced,
711
+ isAuthSynced,
712
+ isConfigSynced
713
+ )
678
714
  }
679
715
  return new Promise((resolve, reject) => {
680
716
  /** @param {import('./sync/sync-state.js').State} syncState */
@@ -866,7 +902,7 @@ export class MapeoManager extends TypedEmitter {
866
902
 
867
903
  async getMapStyleJsonUrl() {
868
904
  await pTimeout(this.#fastify.ready(), { milliseconds: 1000 })
869
- return this.#fastify.mapeoMaps.getStyleJsonUrl()
905
+ return (await this.#getMediaBaseUrl('maps')) + '/style.json'
870
906
  }
871
907
  }
872
908
 
@@ -883,15 +919,8 @@ export class MapeoManager extends TypedEmitter {
883
919
  * @returns {PublicPeerInfo[]}
884
920
  */
885
921
  function omitPeerProtomux(peers) {
886
- return peers.map(
887
- ({
888
- // @ts-ignore
889
- // eslint-disable-next-line no-unused-vars
890
- protomux,
891
- ...publicPeerInfo
892
- }) => {
893
- return publicPeerInfo
894
- }
922
+ return peers.map((peer) =>
923
+ 'protomux' in peer ? omit(peer, ['protomux']) : peer
895
924
  )
896
925
  }
897
926
 
@@ -43,6 +43,7 @@ import {
43
43
  projectKeyToPublicId,
44
44
  valueOf,
45
45
  } from './utils.js'
46
+ import { omit } from './lib/omit.js'
46
47
  import { MemberApi } from './member-api.js'
47
48
  import { SyncApi, kHandleDiscoveryKey } from './sync/sync-api.js'
48
49
  import { Logger } from './logger.js'
@@ -345,7 +346,6 @@ export class MapeoProject extends TypedEmitter {
345
346
 
346
347
  this.#translationApi = new TranslationApi({
347
348
  dataType: this.#dataTypes.translation,
348
- table: translationTable,
349
349
  })
350
350
 
351
351
  ///////// 4. Replicate local peers automatically
@@ -553,7 +553,6 @@ export class MapeoProject extends TypedEmitter {
553
553
  await this.#dataTypes.projectSettings.getByDocId(this.#projectId)
554
554
  )
555
555
  } catch (e) {
556
- this.#l.log('No project settings')
557
556
  return /** @type {EditableProjectSettings} */ (EMPTY_PROJECT_SETTINGS)
558
557
  }
559
558
  }
@@ -889,9 +888,7 @@ export class MapeoProject extends TypedEmitter {
889
888
  * @returns {EditableProjectSettings}
890
889
  */
891
890
  function extractEditableProjectSettings(projectDoc) {
892
- // eslint-disable-next-line no-unused-vars
893
- const { schemaName, ...result } = valueOf(projectDoc)
894
- return result
891
+ return omit(valueOf(projectDoc), ['schemaName'])
895
892
  }
896
893
 
897
894
  /**
package/src/member-api.js CHANGED
@@ -9,6 +9,7 @@ import {
9
9
  projectKeyToId,
10
10
  projectKeyToProjectInviteId,
11
11
  } from './utils.js'
12
+ import { keyBy } from './lib/key-by.js'
12
13
  import { abortSignalAny } from './lib/ponyfills.js'
13
14
  import timingSafeEqual from './lib/timing-safe-equal.js'
14
15
  import { ROLES, isRoleIdForNewInvite } from './roles.js'
@@ -89,6 +90,7 @@ export class MemberApi extends TypedEmitter {
89
90
  * @param {import('./roles.js').RoleIdForNewInvite} opts.roleId
90
91
  * @param {string} [opts.roleName]
91
92
  * @param {string} [opts.roleDescription]
93
+ * @param {Buffer} [opts.__testOnlyInviteId] Hard-code the invite ID. Only for tests.
92
94
  * @returns {Promise<(
93
95
  * typeof InviteResponse_Decision.ACCEPT |
94
96
  * typeof InviteResponse_Decision.REJECT |
@@ -97,7 +99,12 @@ export class MemberApi extends TypedEmitter {
97
99
  */
98
100
  async invite(
99
101
  deviceId,
100
- { roleId, roleName = ROLES[roleId]?.name, roleDescription }
102
+ {
103
+ roleId,
104
+ roleName = ROLES[roleId]?.name,
105
+ roleDescription,
106
+ __testOnlyInviteId,
107
+ }
101
108
  ) {
102
109
  assert(isRoleIdForNewInvite(roleId), 'Invalid role ID for new invite')
103
110
  assert(
@@ -120,7 +127,7 @@ export class MemberApi extends TypedEmitter {
120
127
 
121
128
  abortSignal.throwIfAborted()
122
129
 
123
- const inviteId = crypto.randomBytes(32)
130
+ const inviteId = __testOnlyInviteId || crypto.randomBytes(32)
124
131
  const projectId = projectKeyToId(this.#projectKey)
125
132
  const projectInviteId = projectKeyToProjectInviteId(this.#projectKey)
126
133
  const project = await this.#dataTypes.project.getByDocId(projectId)
@@ -279,6 +286,8 @@ export class MemberApi extends TypedEmitter {
279
286
  this.#dataTypes.deviceInfo.getMany(),
280
287
  ])
281
288
 
289
+ const deviceInfoByConfigCoreId = keyBy(allDeviceInfo, ({ docId }) => docId)
290
+
282
291
  return Promise.all(
283
292
  [...allRoles.entries()].map(async ([deviceId, role]) => {
284
293
  /** @type {MemberInfo} */
@@ -290,9 +299,7 @@ export class MemberApi extends TypedEmitter {
290
299
  'config'
291
300
  )
292
301
 
293
- const deviceInfo = allDeviceInfo.find(
294
- ({ docId }) => docId === configCoreId
295
- )
302
+ const deviceInfo = deviceInfoByConfigCoreId.get(configCoreId)
296
303
 
297
304
  memberInfo.name = deviceInfo?.name
298
305
  memberInfo.deviceType = deviceInfo?.deviceType
@@ -2,6 +2,7 @@ import { keyToId } from '../utils.js'
2
2
  import RemoteBitfield, {
3
3
  BITS_PER_PAGE,
4
4
  } from '../core-manager/remote-bitfield.js'
5
+ import { Logger } from '../logger.js'
5
6
  /** @import { HypercorePeer, HypercoreRemoteBitfield, Namespace } from '../types.js' */
6
7
 
7
8
  /**
@@ -22,7 +23,7 @@ import RemoteBitfield, {
22
23
  * @typedef {object} LocalCoreState
23
24
  * @property {number} have blocks we have
24
25
  * @property {number} want unique blocks we want from any other peer
25
- * @property {number} wanted blocks we want from this peer
26
+ * @property {number} wanted unique blocks any other peer wants from us
26
27
  */
27
28
  /**
28
29
  * @typedef {object} PeerNamespaceState
@@ -71,14 +72,18 @@ export class CoreSyncState {
71
72
  #update
72
73
  #peerSyncControllers
73
74
  #namespace
75
+ #l
74
76
 
75
77
  /**
76
78
  * @param {object} opts
77
79
  * @param {() => void} opts.onUpdate Called when a state update is available (via getState())
78
80
  * @param {Map<string, import('./peer-sync-controller.js').PeerSyncController>} opts.peerSyncControllers
79
81
  * @param {Namespace} opts.namespace
82
+ * @param {Logger} [opts.logger]
80
83
  */
81
- constructor({ onUpdate, peerSyncControllers, namespace }) {
84
+ constructor({ onUpdate, peerSyncControllers, namespace, logger }) {
85
+ // The logger parameter is already namespaced by NamespaceSyncState
86
+ this.#l = logger || Logger.create('css')
82
87
  this.#peerSyncControllers = peerSyncControllers
83
88
  this.#namespace = namespace
84
89
  // Called whenever the state changes, so we clear the cache because next
@@ -150,12 +155,24 @@ export class CoreSyncState {
150
155
  * @param {Uint32Array} bitfield
151
156
  */
152
157
  insertPreHaves(peerId, start, bitfield) {
153
- const peerState = this.#getPeerState(peerId)
158
+ const peerState = this.#getOrCreatePeerState(peerId)
154
159
  peerState.insertPreHaves(start, bitfield)
160
+ const previousLength = Math.max(
161
+ this.#preHavesLength,
162
+ this.#core?.length || 0
163
+ )
155
164
  this.#preHavesLength = Math.max(
156
165
  this.#preHavesLength,
157
166
  peerState.preHavesBitfield.lastSet(start + bitfield.length * 32) + 1
158
167
  )
168
+ if (this.#preHavesLength > previousLength) {
169
+ this.#l.log(
170
+ 'Updated peer %S pre-haves length from %d to %d',
171
+ peerId,
172
+ previousLength,
173
+ this.#preHavesLength
174
+ )
175
+ }
159
176
  this.#update()
160
177
  }
161
178
 
@@ -168,7 +185,7 @@ export class CoreSyncState {
168
185
  * @param {Array<{ start: number, length: number }>} ranges
169
186
  */
170
187
  setPeerWants(peerId, ranges) {
171
- const peerState = this.#getPeerState(peerId)
188
+ const peerState = this.#getOrCreatePeerState(peerId)
172
189
  for (const { start, length } of ranges) {
173
190
  peerState.setWantRange({ start, length })
174
191
  }
@@ -187,7 +204,17 @@ export class CoreSyncState {
187
204
  /**
188
205
  * @param {PeerId} peerId
189
206
  */
190
- #getPeerState(peerId) {
207
+ disconnectPeer(peerId) {
208
+ const wasRemoved = this.#remoteStates.delete(peerId)
209
+ if (wasRemoved) {
210
+ this.#update()
211
+ }
212
+ }
213
+
214
+ /**
215
+ * @param {PeerId} peerId
216
+ */
217
+ #getOrCreatePeerState(peerId) {
191
218
  let peerState = this.#remoteStates.get(peerId)
192
219
  if (!peerState) {
193
220
  peerState = new PeerState()
@@ -207,7 +234,7 @@ export class CoreSyncState {
207
234
  const peerId = keyToId(peer.remotePublicKey)
208
235
 
209
236
  // Update state to ensure this peer is in the state correctly
210
- const peerState = this.#getPeerState(peerId)
237
+ const peerState = this.#getOrCreatePeerState(peerId)
211
238
  peerState.status = 'starting'
212
239
 
213
240
  this.#core?.update({ wait: true }).then(() => {
@@ -243,7 +270,8 @@ export class CoreSyncState {
243
270
  */
244
271
  #onPeerRemove = (peer) => {
245
272
  const peerId = keyToId(peer.remotePublicKey)
246
- const peerState = this.#getPeerState(peerId)
273
+ const peerState = this.#remoteStates.get(peerId)
274
+ if (!peerState) return
247
275
  peerState.status = 'stopped'
248
276
  this.#update()
249
277
  }