@comapeo/core 2.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/blob-store/index.d.ts +23 -49
- package/dist/blob-store/index.d.ts.map +1 -1
- package/dist/constants.d.ts +2 -1
- package/dist/constants.d.ts.map +1 -1
- package/dist/core-manager/index.d.ts +10 -0
- package/dist/core-manager/index.d.ts.map +1 -1
- package/dist/core-ownership.d.ts.map +1 -1
- package/dist/datastore/index.d.ts +5 -4
- package/dist/datastore/index.d.ts.map +1 -1
- package/dist/generated/extensions.d.ts +31 -0
- package/dist/generated/extensions.d.ts.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/lib/drizzle-helpers.d.ts +6 -0
- package/dist/lib/drizzle-helpers.d.ts.map +1 -0
- package/dist/lib/error.d.ts +37 -0
- package/dist/lib/error.d.ts.map +1 -0
- package/dist/lib/get-own.d.ts +9 -0
- package/dist/lib/get-own.d.ts.map +1 -0
- package/dist/lib/is-hostname-ip-address.d.ts +17 -0
- package/dist/lib/is-hostname-ip-address.d.ts.map +1 -0
- package/dist/lib/omit.d.ts +17 -0
- package/dist/lib/omit.d.ts.map +1 -0
- package/dist/lib/ws-core-replicator.d.ts +11 -0
- package/dist/lib/ws-core-replicator.d.ts.map +1 -0
- package/dist/mapeo-manager.d.ts +18 -22
- package/dist/mapeo-manager.d.ts.map +1 -1
- package/dist/mapeo-project.d.ts +454 -37
- package/dist/mapeo-project.d.ts.map +1 -1
- package/dist/member-api.d.ts +40 -1
- package/dist/member-api.d.ts.map +1 -1
- package/dist/schema/client.d.ts +17 -5
- package/dist/schema/client.d.ts.map +1 -1
- package/dist/schema/project.d.ts +211 -1
- package/dist/schema/project.d.ts.map +1 -1
- package/dist/sync/peer-sync-controller.d.ts.map +1 -1
- package/dist/sync/sync-api.d.ts +28 -2
- package/dist/sync/sync-api.d.ts.map +1 -1
- package/dist/translation-api.d.ts.map +1 -1
- package/dist/types.d.ts +3 -2
- package/dist/types.d.ts.map +1 -1
- package/dist/utils.d.ts.map +1 -1
- package/drizzle/client/0001_chubby_cargill.sql +12 -0
- package/drizzle/client/meta/0001_snapshot.json +208 -0
- package/drizzle/client/meta/_journal.json +7 -0
- package/drizzle/project/0001_medical_wendell_rand.sql +22 -0
- package/drizzle/project/meta/0001_snapshot.json +1267 -0
- package/drizzle/project/meta/_journal.json +7 -0
- package/package.json +10 -6
- package/src/blob-store/index.js +20 -4
- package/src/config-import.js +0 -1
- package/src/constants.js +4 -1
- package/src/core-manager/index.js +58 -2
- package/src/core-ownership.js +5 -2
- package/src/datastore/README.md +1 -2
- package/src/datastore/index.js +4 -5
- package/src/fastify-plugins/blobs.js +1 -0
- package/src/fastify-plugins/maps.js +11 -3
- package/src/generated/extensions.d.ts +31 -0
- package/src/generated/extensions.js +150 -0
- package/src/generated/extensions.ts +181 -0
- package/src/index.js +10 -0
- package/src/invite-api.js +1 -1
- package/src/lib/drizzle-helpers.js +79 -0
- package/src/lib/error.js +47 -0
- package/src/lib/get-own.js +10 -0
- package/src/lib/is-hostname-ip-address.js +26 -0
- package/src/lib/omit.js +28 -0
- package/src/lib/ws-core-replicator.js +47 -0
- package/src/mapeo-manager.js +76 -53
- package/src/mapeo-project.js +155 -46
- package/src/member-api.js +253 -2
- package/src/schema/client.js +4 -3
- package/src/schema/project.js +7 -0
- package/src/sync/peer-sync-controller.js +1 -0
- package/src/sync/sync-api.js +171 -3
- package/src/translation-api.js +2 -2
- package/src/types.ts +4 -3
- package/src/utils.js +11 -14
- package/dist/lib/timing-safe-equal.d.ts +0 -15
- package/dist/lib/timing-safe-equal.d.ts.map +0 -1
- package/src/lib/timing-safe-equal.js +0 -34
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@comapeo/core",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.1.0",
|
|
4
4
|
"description": "Offline p2p mapping library",
|
|
5
5
|
"main": "src/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -108,8 +108,9 @@
|
|
|
108
108
|
"homepage": "https://github.com/digidem/comapeo-core#readme",
|
|
109
109
|
"devDependencies": {
|
|
110
110
|
"@bufbuild/buf": "^1.26.1",
|
|
111
|
+
"@comapeo/core2.0.1": "npm:@comapeo/core@2.0.1",
|
|
111
112
|
"@mapeo/default-config": "5.0.0",
|
|
112
|
-
"@mapeo/mock-data": "2.
|
|
113
|
+
"@mapeo/mock-data": "^2.1.1",
|
|
113
114
|
"@sinonjs/fake-timers": "^10.0.2",
|
|
114
115
|
"@types/b4a": "^1.6.0",
|
|
115
116
|
"@types/bogon": "^1.0.2",
|
|
@@ -124,6 +125,7 @@
|
|
|
124
125
|
"@types/sub-encoder": "^2.1.0",
|
|
125
126
|
"@types/throttle-debounce": "^5.0.0",
|
|
126
127
|
"@types/varint": "^6.0.1",
|
|
128
|
+
"@types/ws": "^8.5.12",
|
|
127
129
|
"@types/yauzl-promise": "^4.0.0",
|
|
128
130
|
"@types/yazl": "^2.4.5",
|
|
129
131
|
"bitfield": "^4.2.0",
|
|
@@ -153,7 +155,7 @@
|
|
|
153
155
|
},
|
|
154
156
|
"dependencies": {
|
|
155
157
|
"@comapeo/fallback-smp": "^1.0.0",
|
|
156
|
-
"@comapeo/schema": "1.
|
|
158
|
+
"@comapeo/schema": "1.2.0",
|
|
157
159
|
"@digidem/types": "^2.3.0",
|
|
158
160
|
"@fastify/error": "^3.4.1",
|
|
159
161
|
"@fastify/type-provider-typebox": "^4.1.0",
|
|
@@ -171,7 +173,7 @@
|
|
|
171
173
|
"debug": "^4.3.4",
|
|
172
174
|
"dot-prop": "^9.0.0",
|
|
173
175
|
"drizzle-orm": "^0.30.8",
|
|
174
|
-
"fastify": "
|
|
176
|
+
"fastify": "^4.0.0",
|
|
175
177
|
"fastify-plugin": "^4.5.1",
|
|
176
178
|
"hyperblobs": "2.3.0",
|
|
177
179
|
"hypercore": "10.17.0",
|
|
@@ -181,7 +183,7 @@
|
|
|
181
183
|
"magic-bytes.js": "^1.10.0",
|
|
182
184
|
"map-obj": "^5.0.2",
|
|
183
185
|
"mime": "^4.0.3",
|
|
184
|
-
"multi-core-indexer": "^1.0.0
|
|
186
|
+
"multi-core-indexer": "^1.0.0",
|
|
185
187
|
"p-defer": "^4.0.0",
|
|
186
188
|
"p-event": "^6.0.1",
|
|
187
189
|
"p-timeout": "^6.1.2",
|
|
@@ -191,13 +193,15 @@
|
|
|
191
193
|
"sodium-universal": "^4.0.0",
|
|
192
194
|
"start-stop-state-machine": "^1.2.0",
|
|
193
195
|
"streamx": "^2.19.0",
|
|
194
|
-
"
|
|
196
|
+
"string-timing-safe-equal": "^0.1.0",
|
|
197
|
+
"styled-map-package": "^2.0.0",
|
|
195
198
|
"sub-encoder": "^2.1.1",
|
|
196
199
|
"throttle-debounce": "^5.0.0",
|
|
197
200
|
"tiny-typed-emitter": "^2.1.0",
|
|
198
201
|
"type-fest": "^4.5.0",
|
|
199
202
|
"undici": "^6.13.0",
|
|
200
203
|
"varint": "^6.0.0",
|
|
204
|
+
"ws": "^8.18.0",
|
|
201
205
|
"yauzl-promise": "^4.0.0"
|
|
202
206
|
}
|
|
203
207
|
}
|
package/src/blob-store/index.js
CHANGED
|
@@ -4,7 +4,16 @@ import util from 'node:util'
|
|
|
4
4
|
import { discoveryKey } from 'hypercore-crypto'
|
|
5
5
|
import { TypedEmitter } from 'tiny-typed-emitter'
|
|
6
6
|
import { LiveDownload } from './live-download.js'
|
|
7
|
+
/** @import { JsonObject } from 'type-fest' */
|
|
8
|
+
/** @import { Readable as NodeReadable } from 'node:stream' */
|
|
9
|
+
/** @import { Readable as StreamxReadable, Writable } from 'streamx' */
|
|
7
10
|
/** @import { BlobId } from '../types.js' */
|
|
11
|
+
/** @import { BlobDownloadEvents } from './live-download.js' */
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* @internal
|
|
15
|
+
* @typedef {NodeReadable | StreamxReadable} Readable
|
|
16
|
+
*/
|
|
8
17
|
|
|
9
18
|
/** @typedef {TypedEmitter<{ 'add-drive': (drive: import('hyperdrive')) => void }>} InternalDriveEmitter */
|
|
10
19
|
|
|
@@ -74,12 +83,16 @@ export class BlobStore {
|
|
|
74
83
|
})
|
|
75
84
|
}
|
|
76
85
|
|
|
86
|
+
/**
|
|
87
|
+
* @returns {string}
|
|
88
|
+
*/
|
|
77
89
|
get writerDriveId() {
|
|
78
90
|
return getDiscoveryId(this.#writer.key)
|
|
79
91
|
}
|
|
80
92
|
|
|
81
93
|
/**
|
|
82
94
|
* @param {string} driveId hex-encoded discovery key
|
|
95
|
+
* @returns {Hyperdrive}
|
|
83
96
|
*/
|
|
84
97
|
#getDrive(driveId) {
|
|
85
98
|
const drive = this.#hyperdrives.get(driveId)
|
|
@@ -92,6 +105,7 @@ export class BlobStore {
|
|
|
92
105
|
* @param {object} opts
|
|
93
106
|
* @param {false} [opts.wait=false] Set to `true` to wait for a blob to download, otherwise will throw if blob is not available locally
|
|
94
107
|
* @param {never} [opts.timeout] Optional timeout to wait for a blob to download
|
|
108
|
+
* @returns {Promise<Uint8Array>}
|
|
95
109
|
*/
|
|
96
110
|
async get({ type, variant, name, driveId }, { wait = false, timeout } = {}) {
|
|
97
111
|
const drive = this.#getDrive(driveId)
|
|
@@ -112,7 +126,7 @@ export class BlobStore {
|
|
|
112
126
|
* @param {import('../types.js').BlobFilter} [filter] Filter blob types and/or variants to download. Filter is { [BlobType]: BlobVariants[] }. At least one blob variant must be specified for each blob type.
|
|
113
127
|
* @param {object} options
|
|
114
128
|
* @param {AbortSignal} [options.signal] Optional AbortSignal to cancel in-progress download
|
|
115
|
-
* @returns
|
|
129
|
+
* @returns {TypedEmitter<BlobDownloadEvents>}
|
|
116
130
|
*/
|
|
117
131
|
download(filter, { signal } = {}) {
|
|
118
132
|
return new LiveDownload(this.#hyperdrives.values(), this.#driveEmitter, {
|
|
@@ -126,6 +140,7 @@ export class BlobStore {
|
|
|
126
140
|
* @param {object} [options]
|
|
127
141
|
* @param {boolean} [options.wait=false] Set to `true` to wait for a blob to download, otherwise will throw if blob is not available locally
|
|
128
142
|
* @param {number} [options.timeout] Optional timeout to wait for a blob to download
|
|
143
|
+
* @returns {Readable}
|
|
129
144
|
*/
|
|
130
145
|
createReadStream(
|
|
131
146
|
{ type, variant, name, driveId },
|
|
@@ -146,6 +161,7 @@ export class BlobStore {
|
|
|
146
161
|
* @param {import('hyperdrive').HyperdriveEntry} entry Hyperdrive entry
|
|
147
162
|
* @param {object} [options]
|
|
148
163
|
* @param {boolean} [options.wait=false] Set to `true` to wait for a blob to download, otherwise will throw if blob is not available locally
|
|
164
|
+
* @returns {Promise<Readable>}
|
|
149
165
|
*/
|
|
150
166
|
async createEntryReadStream(driveId, entry, options = { wait: false }) {
|
|
151
167
|
const drive = this.#getDrive(driveId)
|
|
@@ -165,7 +181,6 @@ export class BlobStore {
|
|
|
165
181
|
* @param {import('hyperdrive').HyperdriveEntry} entry Hyperdrive entry
|
|
166
182
|
* @param {object} [opts]
|
|
167
183
|
* @param {number} [opts.length]
|
|
168
|
-
*
|
|
169
184
|
* @returns {Promise<Buffer | null>}
|
|
170
185
|
*/
|
|
171
186
|
async getEntryBlob(driveId, entry, { length } = {}) {
|
|
@@ -186,7 +201,7 @@ export class BlobStore {
|
|
|
186
201
|
* @param {Omit<BlobId, 'driveId'>} blobId
|
|
187
202
|
* @param {Buffer} blob
|
|
188
203
|
* @param {object} [options]
|
|
189
|
-
* @param {
|
|
204
|
+
* @param {JsonObject} [options.metadata] Metadata to store with the blob
|
|
190
205
|
* @returns {Promise<string>} discovery key as hex string of hyperdrive where blob is stored
|
|
191
206
|
*/
|
|
192
207
|
async put({ type, variant, name }, blob, options) {
|
|
@@ -198,7 +213,8 @@ export class BlobStore {
|
|
|
198
213
|
/**
|
|
199
214
|
* @param {Omit<BlobId, 'driveId'>} blobId
|
|
200
215
|
* @param {object} [options]
|
|
201
|
-
* @param {
|
|
216
|
+
* @param {JsonObject} [options.metadata] Metadata to store with the blob
|
|
217
|
+
* @returns {Writable & { driveId: string }}
|
|
202
218
|
*/
|
|
203
219
|
createWriteStream({ type, variant, name }, options) {
|
|
204
220
|
const path = makePath({ type, variant, name })
|
package/src/config-import.js
CHANGED
|
@@ -516,7 +516,6 @@ function parseIcon(filename, buf) {
|
|
|
516
516
|
if (!matches) {
|
|
517
517
|
throw new Error(`Unexpected icon filename ${filename}`)
|
|
518
518
|
}
|
|
519
|
-
/* eslint-disable no-unused-vars */
|
|
520
519
|
const [_, name, size, pixelDensityStr] = matches
|
|
521
520
|
const pixelDensity = Number(pixelDensityStr)
|
|
522
521
|
if (!(pixelDensity === 1 || pixelDensity === 2 || pixelDensity === 3)) {
|
package/src/constants.js
CHANGED
|
@@ -19,7 +19,7 @@ export const DATA_NAMESPACES = NAMESPACES.filter(
|
|
|
19
19
|
)
|
|
20
20
|
|
|
21
21
|
export const NAMESPACE_SCHEMAS = /** @type {const} */ ({
|
|
22
|
-
data: ['observation', 'track'],
|
|
22
|
+
data: ['observation', 'track', 'remoteDetectionAlert'],
|
|
23
23
|
config: [
|
|
24
24
|
'translation',
|
|
25
25
|
'preset',
|
|
@@ -32,3 +32,6 @@ export const NAMESPACE_SCHEMAS = /** @type {const} */ ({
|
|
|
32
32
|
})
|
|
33
33
|
|
|
34
34
|
export const SUPPORTED_CONFIG_VERSION = 1
|
|
35
|
+
|
|
36
|
+
// WARNING: This value is persisted. Be careful when changing it.
|
|
37
|
+
export const DRIZZLE_MIGRATIONS_TABLE = '__drizzle_migrations'
|
|
@@ -4,16 +4,21 @@ import { debounce } from 'throttle-debounce'
|
|
|
4
4
|
import assert from 'node:assert/strict'
|
|
5
5
|
import { sql, eq } from 'drizzle-orm'
|
|
6
6
|
|
|
7
|
-
import {
|
|
7
|
+
import {
|
|
8
|
+
HaveExtension,
|
|
9
|
+
ProjectExtension,
|
|
10
|
+
DownloadIntentExtension,
|
|
11
|
+
} from '../generated/extensions.js'
|
|
8
12
|
import { Logger } from '../logger.js'
|
|
9
13
|
import { NAMESPACES } from '../constants.js'
|
|
10
14
|
import { noop } from '../utils.js'
|
|
11
15
|
import { coresTable } from '../schema/project.js'
|
|
12
16
|
import * as rle from './bitfield-rle.js'
|
|
13
17
|
import { CoreIndex } from './core-index.js'
|
|
18
|
+
import mapObject from 'map-obj'
|
|
14
19
|
|
|
15
20
|
/** @import Hypercore from 'hypercore' */
|
|
16
|
-
/** @import { HypercorePeer, Namespace } from '../types.js' */
|
|
21
|
+
/** @import { BlobFilter, GenericBlobFilter, HypercorePeer, Namespace } from '../types.js' */
|
|
17
22
|
|
|
18
23
|
const WRITER_CORE_PREHAVES_DEBOUNCE_DELAY = 1000
|
|
19
24
|
|
|
@@ -25,6 +30,7 @@ export const kCoreManagerReplicate = Symbol('replicate core manager')
|
|
|
25
30
|
* @typedef {Object} Events
|
|
26
31
|
* @property {(coreRecord: CoreRecord) => void} add-core
|
|
27
32
|
* @property {(namespace: Namespace, msg: { coreDiscoveryId: string, peerId: string, start: number, bitfield: Uint32Array }) => void} peer-have
|
|
33
|
+
* @property {(blobFilter: GenericBlobFilter, peerId: string) => void} peer-download-intent
|
|
28
34
|
*/
|
|
29
35
|
|
|
30
36
|
/**
|
|
@@ -46,6 +52,7 @@ export class CoreManager extends TypedEmitter {
|
|
|
46
52
|
#deviceId
|
|
47
53
|
#l
|
|
48
54
|
#autoDownload
|
|
55
|
+
#downloadIntentExtension
|
|
49
56
|
|
|
50
57
|
static get namespaces() {
|
|
51
58
|
return NAMESPACES
|
|
@@ -158,6 +165,16 @@ export class CoreManager extends TypedEmitter {
|
|
|
158
165
|
},
|
|
159
166
|
})
|
|
160
167
|
|
|
168
|
+
this.#downloadIntentExtension = this.creatorCore.registerExtension(
|
|
169
|
+
'mapeo/download-intent',
|
|
170
|
+
{
|
|
171
|
+
encoding: DownloadIntentCodec,
|
|
172
|
+
onmessage: (msg, peer) => {
|
|
173
|
+
this.#handleDownloadIntentMessage(msg, peer)
|
|
174
|
+
},
|
|
175
|
+
}
|
|
176
|
+
)
|
|
177
|
+
|
|
161
178
|
this.creatorCore.on('peer-add', (peer) => {
|
|
162
179
|
this.#sendHaves(peer, this.#coreIndex).catch(() => {
|
|
163
180
|
this.#l.log('Failed to send pre-haves to newly-connected peer')
|
|
@@ -395,6 +412,23 @@ export class CoreManager extends TypedEmitter {
|
|
|
395
412
|
})
|
|
396
413
|
}
|
|
397
414
|
|
|
415
|
+
/**
|
|
416
|
+
* @param {GenericBlobFilter} blobFilter
|
|
417
|
+
* @param {HypercorePeer} peer
|
|
418
|
+
*/
|
|
419
|
+
#handleDownloadIntentMessage(blobFilter, peer) {
|
|
420
|
+
const peerId = peer.remotePublicKey.toString('hex')
|
|
421
|
+
this.emit('peer-download-intent', blobFilter, peerId)
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
/**
|
|
425
|
+
* @param {BlobFilter} blobFilter
|
|
426
|
+
* @param {HypercorePeer} peer
|
|
427
|
+
*/
|
|
428
|
+
sendDownloadIntents(blobFilter, peer) {
|
|
429
|
+
this.#downloadIntentExtension.send(blobFilter, peer)
|
|
430
|
+
}
|
|
431
|
+
|
|
398
432
|
/**
|
|
399
433
|
*
|
|
400
434
|
* @param {HypercorePeer} peer
|
|
@@ -505,3 +539,25 @@ const HaveExtensionCodec = {
|
|
|
505
539
|
}
|
|
506
540
|
},
|
|
507
541
|
}
|
|
542
|
+
|
|
543
|
+
const DownloadIntentCodec = {
|
|
544
|
+
/** @param {BlobFilter} filter */
|
|
545
|
+
encode(filter) {
|
|
546
|
+
const downloadIntents = mapObject(filter, (key, value) => [
|
|
547
|
+
key,
|
|
548
|
+
{ variants: value || [] },
|
|
549
|
+
])
|
|
550
|
+
return DownloadIntentExtension.encode({ downloadIntents }).finish()
|
|
551
|
+
},
|
|
552
|
+
/**
|
|
553
|
+
* @param {Buffer | Uint8Array} buf
|
|
554
|
+
* @returns {GenericBlobFilter}
|
|
555
|
+
*/
|
|
556
|
+
decode(buf) {
|
|
557
|
+
const msg = DownloadIntentExtension.decode(buf)
|
|
558
|
+
return mapObject(msg.downloadIntents, (key, value) => [
|
|
559
|
+
key + '', // keep TS happy
|
|
560
|
+
value.variants,
|
|
561
|
+
])
|
|
562
|
+
},
|
|
563
|
+
}
|
package/src/core-ownership.js
CHANGED
|
@@ -15,6 +15,7 @@ import { discoveryKey } from 'hypercore-crypto'
|
|
|
15
15
|
import pDefer from 'p-defer'
|
|
16
16
|
import { NAMESPACES } from './constants.js'
|
|
17
17
|
import { TypedEmitter } from 'tiny-typed-emitter'
|
|
18
|
+
import { omit } from './lib/omit.js'
|
|
18
19
|
/**
|
|
19
20
|
* @import {
|
|
20
21
|
* CoreOwnershipWithSignatures,
|
|
@@ -167,8 +168,10 @@ export function mapAndValidateCoreOwnership(doc, { coreDiscoveryKey }) {
|
|
|
167
168
|
if (!verifyCoreOwnership(doc)) {
|
|
168
169
|
throw new Error('Invalid coreOwnership record: signatures are invalid')
|
|
169
170
|
}
|
|
170
|
-
|
|
171
|
-
|
|
171
|
+
const docWithoutSignatures = omit(doc, [
|
|
172
|
+
'identitySignature',
|
|
173
|
+
'coreSignatures',
|
|
174
|
+
])
|
|
172
175
|
docWithoutSignatures.links = []
|
|
173
176
|
return docWithoutSignatures
|
|
174
177
|
}
|
package/src/datastore/README.md
CHANGED
|
@@ -19,6 +19,7 @@ const datastore = new DataStore({
|
|
|
19
19
|
// Process entries here using an indexer...
|
|
20
20
|
},
|
|
21
21
|
namespace: 'data',
|
|
22
|
+
reindex: false,
|
|
22
23
|
})
|
|
23
24
|
|
|
24
25
|
/** @type {MapeoDoc} */
|
|
@@ -33,8 +34,6 @@ datastore.on('index-state', ({ current, remaining, entriesPerSecond }) => {
|
|
|
33
34
|
// show state to user that indexing is happening
|
|
34
35
|
}
|
|
35
36
|
})
|
|
36
|
-
|
|
37
|
-
const { current, remaining, entriesPerSecond } = datastore.getIndexState()
|
|
38
37
|
```
|
|
39
38
|
|
|
40
39
|
## API docs
|
package/src/datastore/index.js
CHANGED
|
@@ -51,8 +51,9 @@ export class DataStore extends TypedEmitter {
|
|
|
51
51
|
* @param {TNamespace} opts.namespace
|
|
52
52
|
* @param {(entries: MultiCoreIndexer.Entry<'binary'>[]) => Promise<import('../index-writer/index.js').IndexedDocIds>} opts.batch
|
|
53
53
|
* @param {MultiCoreIndexer.StorageParam} opts.storage
|
|
54
|
+
* @param {boolean} opts.reindex
|
|
54
55
|
*/
|
|
55
|
-
constructor({ coreManager, namespace, batch, storage }) {
|
|
56
|
+
constructor({ coreManager, namespace, batch, storage, reindex }) {
|
|
56
57
|
super()
|
|
57
58
|
this.#coreManager = coreManager
|
|
58
59
|
this.#namespace = namespace
|
|
@@ -66,6 +67,7 @@ export class DataStore extends TypedEmitter {
|
|
|
66
67
|
this.#coreIndexer = new MultiCoreIndexer(cores, {
|
|
67
68
|
storage,
|
|
68
69
|
batch: (entries) => this.#handleEntries(entries),
|
|
70
|
+
reindex,
|
|
69
71
|
})
|
|
70
72
|
coreManager.on('add-core', (coreRecord) => {
|
|
71
73
|
if (coreRecord.namespace !== namespace) return
|
|
@@ -91,10 +93,6 @@ export class DataStore extends TypedEmitter {
|
|
|
91
93
|
return this.#writerCore
|
|
92
94
|
}
|
|
93
95
|
|
|
94
|
-
getIndexState() {
|
|
95
|
-
return this.#coreIndexer.state
|
|
96
|
-
}
|
|
97
|
-
|
|
98
96
|
/**
|
|
99
97
|
*
|
|
100
98
|
* @param {MultiCoreIndexer.Entry<'binary'>[]} entries
|
|
@@ -167,6 +165,7 @@ export class DataStore extends TypedEmitter {
|
|
|
167
165
|
const deferred = pDefer()
|
|
168
166
|
this.#pendingIndex.set(versionId, deferred)
|
|
169
167
|
await deferred.promise
|
|
168
|
+
this.#pendingIndex.delete(versionId)
|
|
170
169
|
|
|
171
170
|
return /** @type {Extract<MapeoDoc, TDoc>} */ (
|
|
172
171
|
decode(block, { coreDiscoveryKey, index })
|
|
@@ -102,6 +102,7 @@ async function routes(fastify, options) {
|
|
|
102
102
|
// Extract the 'mimeType' property of the metadata and use it for the response header if found
|
|
103
103
|
if (
|
|
104
104
|
metadata &&
|
|
105
|
+
typeof metadata === 'object' &&
|
|
105
106
|
'mimeType' in metadata &&
|
|
106
107
|
typeof metadata.mimeType === 'string'
|
|
107
108
|
) {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import fs from 'node:fs/promises'
|
|
2
2
|
import path from 'node:path'
|
|
3
3
|
import { fetch } from 'undici'
|
|
4
|
-
import { Server as SMPServerPlugin } from 'styled-map-package'
|
|
4
|
+
import { ReaderWatch, Server as SMPServerPlugin } from 'styled-map-package'
|
|
5
5
|
|
|
6
6
|
import { noop } from '../utils.js'
|
|
7
7
|
import { NotFoundError, ENOENTError } from './utils.js'
|
|
@@ -25,6 +25,10 @@ export async function plugin(fastify, opts) {
|
|
|
25
25
|
if (opts.customMapPath) {
|
|
26
26
|
const { customMapPath } = opts
|
|
27
27
|
|
|
28
|
+
const customMapReader = new ReaderWatch(customMapPath)
|
|
29
|
+
|
|
30
|
+
fastify.addHook('onClose', () => customMapReader.close().catch(noop))
|
|
31
|
+
|
|
28
32
|
fastify.get(`/${CUSTOM_MAP_PREFIX}/info`, async () => {
|
|
29
33
|
const baseUrl = new URL(fastify.prefix, fastify.listeningOrigin)
|
|
30
34
|
|
|
@@ -78,13 +82,17 @@ export async function plugin(fastify, opts) {
|
|
|
78
82
|
|
|
79
83
|
fastify.register(SMPServerPlugin, {
|
|
80
84
|
prefix: CUSTOM_MAP_PREFIX,
|
|
81
|
-
|
|
85
|
+
reader: customMapReader,
|
|
82
86
|
})
|
|
83
87
|
}
|
|
84
88
|
|
|
89
|
+
const fallbackMapReader = new ReaderWatch(opts.fallbackMapPath)
|
|
90
|
+
|
|
91
|
+
fastify.addHook('onClose', () => fallbackMapReader.close().catch(noop))
|
|
92
|
+
|
|
85
93
|
fastify.register(SMPServerPlugin, {
|
|
86
94
|
prefix: FALLBACK_MAP_PREFIX,
|
|
87
|
-
|
|
95
|
+
reader: fallbackMapReader,
|
|
88
96
|
})
|
|
89
97
|
|
|
90
98
|
fastify.get('/style.json', async (_request, reply) => {
|
|
@@ -19,6 +19,19 @@ export declare const HaveExtension_Namespace: {
|
|
|
19
19
|
export type HaveExtension_Namespace = typeof HaveExtension_Namespace[keyof typeof HaveExtension_Namespace];
|
|
20
20
|
export declare function haveExtension_NamespaceFromJSON(object: any): HaveExtension_Namespace;
|
|
21
21
|
export declare function haveExtension_NamespaceToNumber(object: HaveExtension_Namespace): number;
|
|
22
|
+
/** A map of blob types and variants that a peer intends to download */
|
|
23
|
+
export interface DownloadIntentExtension {
|
|
24
|
+
downloadIntents: {
|
|
25
|
+
[key: string]: DownloadIntentExtension_DownloadIntent;
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
export interface DownloadIntentExtension_DownloadIntent {
|
|
29
|
+
variants: string[];
|
|
30
|
+
}
|
|
31
|
+
export interface DownloadIntentExtension_DownloadIntentsEntry {
|
|
32
|
+
key: string;
|
|
33
|
+
value: DownloadIntentExtension_DownloadIntent | undefined;
|
|
34
|
+
}
|
|
22
35
|
export declare const ProjectExtension: {
|
|
23
36
|
encode(message: ProjectExtension, writer?: _m0.Writer): _m0.Writer;
|
|
24
37
|
decode(input: _m0.Reader | Uint8Array, length?: number): ProjectExtension;
|
|
@@ -31,6 +44,24 @@ export declare const HaveExtension: {
|
|
|
31
44
|
create<I extends Exact<DeepPartial<HaveExtension>, I>>(base?: I): HaveExtension;
|
|
32
45
|
fromPartial<I extends Exact<DeepPartial<HaveExtension>, I>>(object: I): HaveExtension;
|
|
33
46
|
};
|
|
47
|
+
export declare const DownloadIntentExtension: {
|
|
48
|
+
encode(message: DownloadIntentExtension, writer?: _m0.Writer): _m0.Writer;
|
|
49
|
+
decode(input: _m0.Reader | Uint8Array, length?: number): DownloadIntentExtension;
|
|
50
|
+
create<I extends Exact<DeepPartial<DownloadIntentExtension>, I>>(base?: I): DownloadIntentExtension;
|
|
51
|
+
fromPartial<I extends Exact<DeepPartial<DownloadIntentExtension>, I>>(object: I): DownloadIntentExtension;
|
|
52
|
+
};
|
|
53
|
+
export declare const DownloadIntentExtension_DownloadIntent: {
|
|
54
|
+
encode(message: DownloadIntentExtension_DownloadIntent, writer?: _m0.Writer): _m0.Writer;
|
|
55
|
+
decode(input: _m0.Reader | Uint8Array, length?: number): DownloadIntentExtension_DownloadIntent;
|
|
56
|
+
create<I extends Exact<DeepPartial<DownloadIntentExtension_DownloadIntent>, I>>(base?: I): DownloadIntentExtension_DownloadIntent;
|
|
57
|
+
fromPartial<I extends Exact<DeepPartial<DownloadIntentExtension_DownloadIntent>, I>>(object: I): DownloadIntentExtension_DownloadIntent;
|
|
58
|
+
};
|
|
59
|
+
export declare const DownloadIntentExtension_DownloadIntentsEntry: {
|
|
60
|
+
encode(message: DownloadIntentExtension_DownloadIntentsEntry, writer?: _m0.Writer): _m0.Writer;
|
|
61
|
+
decode(input: _m0.Reader | Uint8Array, length?: number): DownloadIntentExtension_DownloadIntentsEntry;
|
|
62
|
+
create<I extends Exact<DeepPartial<DownloadIntentExtension_DownloadIntentsEntry>, I>>(base?: I): DownloadIntentExtension_DownloadIntentsEntry;
|
|
63
|
+
fromPartial<I extends Exact<DeepPartial<DownloadIntentExtension_DownloadIntentsEntry>, I>>(object: I): DownloadIntentExtension_DownloadIntentsEntry;
|
|
64
|
+
};
|
|
34
65
|
type Builtin = Date | Function | Uint8Array | string | number | boolean | undefined;
|
|
35
66
|
type DeepPartial<T> = T extends Builtin ? T : T extends Array<infer U> ? Array<DeepPartial<U>> : T extends ReadonlyArray<infer U> ? ReadonlyArray<DeepPartial<U>> : T extends {} ? {
|
|
36
67
|
[K in keyof T]?: DeepPartial<T[K]>;
|
|
@@ -169,6 +169,156 @@ export var HaveExtension = {
|
|
|
169
169
|
return message;
|
|
170
170
|
},
|
|
171
171
|
};
|
|
172
|
+
function createBaseDownloadIntentExtension() {
|
|
173
|
+
return { downloadIntents: {} };
|
|
174
|
+
}
|
|
175
|
+
export var DownloadIntentExtension = {
|
|
176
|
+
encode: function (message, writer) {
|
|
177
|
+
if (writer === void 0) { writer = _m0.Writer.create(); }
|
|
178
|
+
Object.entries(message.downloadIntents).forEach(function (_a) {
|
|
179
|
+
var key = _a[0], value = _a[1];
|
|
180
|
+
DownloadIntentExtension_DownloadIntentsEntry.encode({ key: key, value: value }, writer.uint32(10).fork())
|
|
181
|
+
.ldelim();
|
|
182
|
+
});
|
|
183
|
+
return writer;
|
|
184
|
+
},
|
|
185
|
+
decode: function (input, length) {
|
|
186
|
+
var reader = input instanceof _m0.Reader ? input : _m0.Reader.create(input);
|
|
187
|
+
var end = length === undefined ? reader.len : reader.pos + length;
|
|
188
|
+
var message = createBaseDownloadIntentExtension();
|
|
189
|
+
while (reader.pos < end) {
|
|
190
|
+
var tag = reader.uint32();
|
|
191
|
+
switch (tag >>> 3) {
|
|
192
|
+
case 1:
|
|
193
|
+
if (tag !== 10) {
|
|
194
|
+
break;
|
|
195
|
+
}
|
|
196
|
+
var entry1 = DownloadIntentExtension_DownloadIntentsEntry.decode(reader, reader.uint32());
|
|
197
|
+
if (entry1.value !== undefined) {
|
|
198
|
+
message.downloadIntents[entry1.key] = entry1.value;
|
|
199
|
+
}
|
|
200
|
+
continue;
|
|
201
|
+
}
|
|
202
|
+
if ((tag & 7) === 4 || tag === 0) {
|
|
203
|
+
break;
|
|
204
|
+
}
|
|
205
|
+
reader.skipType(tag & 7);
|
|
206
|
+
}
|
|
207
|
+
return message;
|
|
208
|
+
},
|
|
209
|
+
create: function (base) {
|
|
210
|
+
return DownloadIntentExtension.fromPartial(base !== null && base !== void 0 ? base : {});
|
|
211
|
+
},
|
|
212
|
+
fromPartial: function (object) {
|
|
213
|
+
var _a;
|
|
214
|
+
var message = createBaseDownloadIntentExtension();
|
|
215
|
+
message.downloadIntents = Object.entries((_a = object.downloadIntents) !== null && _a !== void 0 ? _a : {}).reduce(function (acc, _a) {
|
|
216
|
+
var key = _a[0], value = _a[1];
|
|
217
|
+
if (value !== undefined) {
|
|
218
|
+
acc[key] = DownloadIntentExtension_DownloadIntent.fromPartial(value);
|
|
219
|
+
}
|
|
220
|
+
return acc;
|
|
221
|
+
}, {});
|
|
222
|
+
return message;
|
|
223
|
+
},
|
|
224
|
+
};
|
|
225
|
+
function createBaseDownloadIntentExtension_DownloadIntent() {
|
|
226
|
+
return { variants: [] };
|
|
227
|
+
}
|
|
228
|
+
export var DownloadIntentExtension_DownloadIntent = {
|
|
229
|
+
encode: function (message, writer) {
|
|
230
|
+
if (writer === void 0) { writer = _m0.Writer.create(); }
|
|
231
|
+
for (var _i = 0, _a = message.variants; _i < _a.length; _i++) {
|
|
232
|
+
var v = _a[_i];
|
|
233
|
+
writer.uint32(10).string(v);
|
|
234
|
+
}
|
|
235
|
+
return writer;
|
|
236
|
+
},
|
|
237
|
+
decode: function (input, length) {
|
|
238
|
+
var reader = input instanceof _m0.Reader ? input : _m0.Reader.create(input);
|
|
239
|
+
var end = length === undefined ? reader.len : reader.pos + length;
|
|
240
|
+
var message = createBaseDownloadIntentExtension_DownloadIntent();
|
|
241
|
+
while (reader.pos < end) {
|
|
242
|
+
var tag = reader.uint32();
|
|
243
|
+
switch (tag >>> 3) {
|
|
244
|
+
case 1:
|
|
245
|
+
if (tag !== 10) {
|
|
246
|
+
break;
|
|
247
|
+
}
|
|
248
|
+
message.variants.push(reader.string());
|
|
249
|
+
continue;
|
|
250
|
+
}
|
|
251
|
+
if ((tag & 7) === 4 || tag === 0) {
|
|
252
|
+
break;
|
|
253
|
+
}
|
|
254
|
+
reader.skipType(tag & 7);
|
|
255
|
+
}
|
|
256
|
+
return message;
|
|
257
|
+
},
|
|
258
|
+
create: function (base) {
|
|
259
|
+
return DownloadIntentExtension_DownloadIntent.fromPartial(base !== null && base !== void 0 ? base : {});
|
|
260
|
+
},
|
|
261
|
+
fromPartial: function (object) {
|
|
262
|
+
var _a;
|
|
263
|
+
var message = createBaseDownloadIntentExtension_DownloadIntent();
|
|
264
|
+
message.variants = ((_a = object.variants) === null || _a === void 0 ? void 0 : _a.map(function (e) { return e; })) || [];
|
|
265
|
+
return message;
|
|
266
|
+
},
|
|
267
|
+
};
|
|
268
|
+
function createBaseDownloadIntentExtension_DownloadIntentsEntry() {
|
|
269
|
+
return { key: "", value: undefined };
|
|
270
|
+
}
|
|
271
|
+
export var DownloadIntentExtension_DownloadIntentsEntry = {
|
|
272
|
+
encode: function (message, writer) {
|
|
273
|
+
if (writer === void 0) { writer = _m0.Writer.create(); }
|
|
274
|
+
if (message.key !== "") {
|
|
275
|
+
writer.uint32(10).string(message.key);
|
|
276
|
+
}
|
|
277
|
+
if (message.value !== undefined) {
|
|
278
|
+
DownloadIntentExtension_DownloadIntent.encode(message.value, writer.uint32(18).fork()).ldelim();
|
|
279
|
+
}
|
|
280
|
+
return writer;
|
|
281
|
+
},
|
|
282
|
+
decode: function (input, length) {
|
|
283
|
+
var reader = input instanceof _m0.Reader ? input : _m0.Reader.create(input);
|
|
284
|
+
var end = length === undefined ? reader.len : reader.pos + length;
|
|
285
|
+
var message = createBaseDownloadIntentExtension_DownloadIntentsEntry();
|
|
286
|
+
while (reader.pos < end) {
|
|
287
|
+
var tag = reader.uint32();
|
|
288
|
+
switch (tag >>> 3) {
|
|
289
|
+
case 1:
|
|
290
|
+
if (tag !== 10) {
|
|
291
|
+
break;
|
|
292
|
+
}
|
|
293
|
+
message.key = reader.string();
|
|
294
|
+
continue;
|
|
295
|
+
case 2:
|
|
296
|
+
if (tag !== 18) {
|
|
297
|
+
break;
|
|
298
|
+
}
|
|
299
|
+
message.value = DownloadIntentExtension_DownloadIntent.decode(reader, reader.uint32());
|
|
300
|
+
continue;
|
|
301
|
+
}
|
|
302
|
+
if ((tag & 7) === 4 || tag === 0) {
|
|
303
|
+
break;
|
|
304
|
+
}
|
|
305
|
+
reader.skipType(tag & 7);
|
|
306
|
+
}
|
|
307
|
+
return message;
|
|
308
|
+
},
|
|
309
|
+
create: function (base) {
|
|
310
|
+
return DownloadIntentExtension_DownloadIntentsEntry.fromPartial(base !== null && base !== void 0 ? base : {});
|
|
311
|
+
},
|
|
312
|
+
fromPartial: function (object) {
|
|
313
|
+
var _a;
|
|
314
|
+
var message = createBaseDownloadIntentExtension_DownloadIntentsEntry();
|
|
315
|
+
message.key = (_a = object.key) !== null && _a !== void 0 ? _a : "";
|
|
316
|
+
message.value = (object.value !== undefined && object.value !== null)
|
|
317
|
+
? DownloadIntentExtension_DownloadIntent.fromPartial(object.value)
|
|
318
|
+
: undefined;
|
|
319
|
+
return message;
|
|
320
|
+
},
|
|
321
|
+
};
|
|
172
322
|
var tsProtoGlobalThis = (function () {
|
|
173
323
|
if (typeof globalThis !== "undefined") {
|
|
174
324
|
return globalThis;
|