@comapeo/core 2.0.1 → 2.1.0

package/drizzle/project/meta/_journal.json

@@ -8,6 +8,13 @@
       "when": 1726514275142,
       "tag": "0000_spooky_lady_ursula",
       "breakpoints": true
+    },
+    {
+      "idx": 1,
+      "version": "5",
+      "when": 1729783892753,
+      "tag": "0001_medical_wendell_rand",
+      "breakpoints": true
     }
   ]
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@comapeo/core",
-  "version": "2.0.1",
+  "version": "2.1.0",
   "description": "Offline p2p mapping library",
   "main": "src/index.js",
   "types": "dist/index.d.ts",
@@ -108,8 +108,9 @@
   "homepage": "https://github.com/digidem/comapeo-core#readme",
   "devDependencies": {
     "@bufbuild/buf": "^1.26.1",
+    "@comapeo/core2.0.1": "npm:@comapeo/core@2.0.1",
     "@mapeo/default-config": "5.0.0",
-    "@mapeo/mock-data": "2.0.0",
+    "@mapeo/mock-data": "^2.1.1",
     "@sinonjs/fake-timers": "^10.0.2",
     "@types/b4a": "^1.6.0",
     "@types/bogon": "^1.0.2",
@@ -124,6 +125,7 @@
     "@types/sub-encoder": "^2.1.0",
     "@types/throttle-debounce": "^5.0.0",
     "@types/varint": "^6.0.1",
+    "@types/ws": "^8.5.12",
     "@types/yauzl-promise": "^4.0.0",
     "@types/yazl": "^2.4.5",
     "bitfield": "^4.2.0",
@@ -153,7 +155,7 @@
   },
   "dependencies": {
     "@comapeo/fallback-smp": "^1.0.0",
-    "@comapeo/schema": "1.0.0",
+    "@comapeo/schema": "1.2.0",
     "@digidem/types": "^2.3.0",
     "@fastify/error": "^3.4.1",
     "@fastify/type-provider-typebox": "^4.1.0",
@@ -171,7 +173,7 @@
     "debug": "^4.3.4",
     "dot-prop": "^9.0.0",
     "drizzle-orm": "^0.30.8",
-    "fastify": ">= 4",
+    "fastify": "^4.0.0",
     "fastify-plugin": "^4.5.1",
     "hyperblobs": "2.3.0",
     "hypercore": "10.17.0",
@@ -181,7 +183,7 @@
     "magic-bytes.js": "^1.10.0",
     "map-obj": "^5.0.2",
     "mime": "^4.0.3",
-    "multi-core-indexer": "^1.0.0-alpha.10",
+    "multi-core-indexer": "^1.0.0",
     "p-defer": "^4.0.0",
     "p-event": "^6.0.1",
     "p-timeout": "^6.1.2",
@@ -191,6 +193,7 @@
     "sodium-universal": "^4.0.0",
     "start-stop-state-machine": "^1.2.0",
     "streamx": "^2.19.0",
+    "string-timing-safe-equal": "^0.1.0",
     "styled-map-package": "^2.0.0",
     "sub-encoder": "^2.1.1",
     "throttle-debounce": "^5.0.0",
@@ -198,6 +201,7 @@
     "type-fest": "^4.5.0",
     "undici": "^6.13.0",
     "varint": "^6.0.0",
+    "ws": "^8.18.0",
     "yauzl-promise": "^4.0.0"
   }
 }

package/src/blob-store/index.js

@@ -4,6 +4,7 @@ import util from 'node:util'
 import { discoveryKey } from 'hypercore-crypto'
 import { TypedEmitter } from 'tiny-typed-emitter'
 import { LiveDownload } from './live-download.js'
+/** @import { JsonObject } from 'type-fest' */
 /** @import { Readable as NodeReadable } from 'node:stream' */
 /** @import { Readable as StreamxReadable, Writable } from 'streamx' */
 /** @import { BlobId } from '../types.js' */
@@ -200,7 +201,7 @@ export class BlobStore {
    * @param {Omit<BlobId, 'driveId'>} blobId
    * @param {Buffer} blob
    * @param {object} [options]
-   * @param {{mimeType: string}} [options.metadata] Metadata to store with the blob
+   * @param {JsonObject} [options.metadata] Metadata to store with the blob
    * @returns {Promise<string>} discovery key as hex string of hyperdrive where blob is stored
    */
   async put({ type, variant, name }, blob, options) {
@@ -212,7 +213,7 @@ export class BlobStore {
   /**
    * @param {Omit<BlobId, 'driveId'>} blobId
    * @param {object} [options]
-   * @param {{mimeType: string}} [options.metadata] Metadata to store with the blob
+   * @param {JsonObject} [options.metadata] Metadata to store with the blob
    * @returns {Writable & { driveId: string }}
    */
   createWriteStream({ type, variant, name }, options) {

package/src/constants.js

@@ -19,7 +19,7 @@ export const DATA_NAMESPACES = NAMESPACES.filter(
 )
 
 export const NAMESPACE_SCHEMAS = /** @type {const} */ ({
-  data: ['observation', 'track'],
+  data: ['observation', 'track', 'remoteDetectionAlert'],
   config: [
     'translation',
     'preset',
@@ -32,3 +32,6 @@ export const NAMESPACE_SCHEMAS = /** @type {const} */ ({
 })
 
 export const SUPPORTED_CONFIG_VERSION = 1
+
+// WARNING: This value is persisted. Be careful when changing it.
+export const DRIZZLE_MIGRATIONS_TABLE = '__drizzle_migrations'

package/src/core-manager/index.js

@@ -4,16 +4,21 @@ import { debounce } from 'throttle-debounce'
 import assert from 'node:assert/strict'
 import { sql, eq } from 'drizzle-orm'
 
-import { HaveExtension, ProjectExtension } from '../generated/extensions.js'
+import {
+  HaveExtension,
+  ProjectExtension,
+  DownloadIntentExtension,
+} from '../generated/extensions.js'
 import { Logger } from '../logger.js'
 import { NAMESPACES } from '../constants.js'
 import { noop } from '../utils.js'
 import { coresTable } from '../schema/project.js'
 import * as rle from './bitfield-rle.js'
 import { CoreIndex } from './core-index.js'
+import mapObject from 'map-obj'
 
 /** @import Hypercore from 'hypercore' */
-/** @import { HypercorePeer, Namespace } from '../types.js' */
+/** @import { BlobFilter, GenericBlobFilter, HypercorePeer, Namespace } from '../types.js' */
 
 const WRITER_CORE_PREHAVES_DEBOUNCE_DELAY = 1000
 
@@ -25,6 +30,7 @@ export const kCoreManagerReplicate = Symbol('replicate core manager')
  * @typedef {Object} Events
  * @property {(coreRecord: CoreRecord) => void} add-core
  * @property {(namespace: Namespace, msg: { coreDiscoveryId: string, peerId: string, start: number, bitfield: Uint32Array }) => void} peer-have
+ * @property {(blobFilter: GenericBlobFilter, peerId: string) => void} peer-download-intent
  */
 
 /**
@@ -46,6 +52,7 @@ export class CoreManager extends TypedEmitter {
   #deviceId
   #l
   #autoDownload
+  #downloadIntentExtension
 
   static get namespaces() {
     return NAMESPACES
@@ -158,6 +165,16 @@ export class CoreManager extends TypedEmitter {
       },
     })
 
+    this.#downloadIntentExtension = this.creatorCore.registerExtension(
+      'mapeo/download-intent',
+      {
+        encoding: DownloadIntentCodec,
+        onmessage: (msg, peer) => {
+          this.#handleDownloadIntentMessage(msg, peer)
+        },
+      }
+    )
+
     this.creatorCore.on('peer-add', (peer) => {
       this.#sendHaves(peer, this.#coreIndex).catch(() => {
         this.#l.log('Failed to send pre-haves to newly-connected peer')
@@ -395,6 +412,23 @@ export class CoreManager extends TypedEmitter {
     })
   }
 
+  /**
+   * @param {GenericBlobFilter} blobFilter
+   * @param {HypercorePeer} peer
+   */
+  #handleDownloadIntentMessage(blobFilter, peer) {
+    const peerId = peer.remotePublicKey.toString('hex')
+    this.emit('peer-download-intent', blobFilter, peerId)
+  }
+
+  /**
+   * @param {BlobFilter} blobFilter
+   * @param {HypercorePeer} peer
+   */
+  sendDownloadIntents(blobFilter, peer) {
+    this.#downloadIntentExtension.send(blobFilter, peer)
+  }
+
   /**
    *
    * @param {HypercorePeer} peer
@@ -505,3 +539,25 @@ const HaveExtensionCodec = {
     }
   },
 }
+
+const DownloadIntentCodec = {
+  /** @param {BlobFilter} filter */
+  encode(filter) {
+    const downloadIntents = mapObject(filter, (key, value) => [
+      key,
+      { variants: value || [] },
+    ])
+    return DownloadIntentExtension.encode({ downloadIntents }).finish()
+  },
+  /**
+   * @param {Buffer | Uint8Array} buf
+   * @returns {GenericBlobFilter}
+   */
+  decode(buf) {
+    const msg = DownloadIntentExtension.decode(buf)
+    return mapObject(msg.downloadIntents, (key, value) => [
+      key + '', // keep TS happy
+      value.variants,
+    ])
+  },
+}

package/src/datastore/README.md

@@ -19,6 +19,7 @@ const datastore = new DataStore({
     // Process entries here using an indexer...
   },
   namespace: 'data',
+  reindex: false,
 })
 
 /** @type {MapeoDoc} */
@@ -33,8 +34,6 @@ datastore.on('index-state', ({ current, remaining, entriesPerSecond }) => {
     // show state to user that indexing is happening
   }
 })
-
-const { current, remaining, entriesPerSecond } = datastore.getIndexState()
 ```
 
 ## API docs

package/src/datastore/index.js

@@ -51,8 +51,9 @@ export class DataStore extends TypedEmitter {
    * @param {TNamespace} opts.namespace
    * @param {(entries: MultiCoreIndexer.Entry<'binary'>[]) => Promise<import('../index-writer/index.js').IndexedDocIds>} opts.batch
    * @param {MultiCoreIndexer.StorageParam} opts.storage
+   * @param {boolean} opts.reindex
    */
-  constructor({ coreManager, namespace, batch, storage }) {
+  constructor({ coreManager, namespace, batch, storage, reindex }) {
     super()
     this.#coreManager = coreManager
     this.#namespace = namespace
@@ -66,6 +67,7 @@ export class DataStore extends TypedEmitter {
     this.#coreIndexer = new MultiCoreIndexer(cores, {
       storage,
       batch: (entries) => this.#handleEntries(entries),
+      reindex,
     })
     coreManager.on('add-core', (coreRecord) => {
       if (coreRecord.namespace !== namespace) return
@@ -91,10 +93,6 @@ export class DataStore extends TypedEmitter {
     return this.#writerCore
   }
 
-  getIndexState() {
-    return this.#coreIndexer.state
-  }
-
   /**
    *
    * @param {MultiCoreIndexer.Entry<'binary'>[]} entries
@@ -167,6 +165,7 @@ export class DataStore extends TypedEmitter {
     const deferred = pDefer()
     this.#pendingIndex.set(versionId, deferred)
     await deferred.promise
+    this.#pendingIndex.delete(versionId)
 
     return /** @type {Extract<MapeoDoc, TDoc>} */ (
       decode(block, { coreDiscoveryKey, index })

package/src/fastify-plugins/blobs.js

@@ -102,6 +102,7 @@ async function routes(fastify, options) {
       // Extract the 'mimeType' property of the metadata and use it for the response header if found
       if (
         metadata &&
+        typeof metadata === 'object' &&
         'mimeType' in metadata &&
         typeof metadata.mimeType === 'string'
       ) {

package/src/generated/extensions.d.ts

@@ -19,6 +19,19 @@ export declare const HaveExtension_Namespace: {
 export type HaveExtension_Namespace = typeof HaveExtension_Namespace[keyof typeof HaveExtension_Namespace];
 export declare function haveExtension_NamespaceFromJSON(object: any): HaveExtension_Namespace;
 export declare function haveExtension_NamespaceToNumber(object: HaveExtension_Namespace): number;
+/** A map of blob types and variants that a peer intends to download */
+export interface DownloadIntentExtension {
+    downloadIntents: {
+        [key: string]: DownloadIntentExtension_DownloadIntent;
+    };
+}
+export interface DownloadIntentExtension_DownloadIntent {
+    variants: string[];
+}
+export interface DownloadIntentExtension_DownloadIntentsEntry {
+    key: string;
+    value: DownloadIntentExtension_DownloadIntent | undefined;
+}
 export declare const ProjectExtension: {
     encode(message: ProjectExtension, writer?: _m0.Writer): _m0.Writer;
     decode(input: _m0.Reader | Uint8Array, length?: number): ProjectExtension;
@@ -31,6 +44,24 @@ export declare const HaveExtension: {
     create<I extends Exact<DeepPartial<HaveExtension>, I>>(base?: I): HaveExtension;
     fromPartial<I extends Exact<DeepPartial<HaveExtension>, I>>(object: I): HaveExtension;
 };
+export declare const DownloadIntentExtension: {
+    encode(message: DownloadIntentExtension, writer?: _m0.Writer): _m0.Writer;
+    decode(input: _m0.Reader | Uint8Array, length?: number): DownloadIntentExtension;
+    create<I extends Exact<DeepPartial<DownloadIntentExtension>, I>>(base?: I): DownloadIntentExtension;
+    fromPartial<I extends Exact<DeepPartial<DownloadIntentExtension>, I>>(object: I): DownloadIntentExtension;
+};
+export declare const DownloadIntentExtension_DownloadIntent: {
+    encode(message: DownloadIntentExtension_DownloadIntent, writer?: _m0.Writer): _m0.Writer;
+    decode(input: _m0.Reader | Uint8Array, length?: number): DownloadIntentExtension_DownloadIntent;
+    create<I extends Exact<DeepPartial<DownloadIntentExtension_DownloadIntent>, I>>(base?: I): DownloadIntentExtension_DownloadIntent;
+    fromPartial<I extends Exact<DeepPartial<DownloadIntentExtension_DownloadIntent>, I>>(object: I): DownloadIntentExtension_DownloadIntent;
+};
+export declare const DownloadIntentExtension_DownloadIntentsEntry: {
+    encode(message: DownloadIntentExtension_DownloadIntentsEntry, writer?: _m0.Writer): _m0.Writer;
+    decode(input: _m0.Reader | Uint8Array, length?: number): DownloadIntentExtension_DownloadIntentsEntry;
+    create<I extends Exact<DeepPartial<DownloadIntentExtension_DownloadIntentsEntry>, I>>(base?: I): DownloadIntentExtension_DownloadIntentsEntry;
+    fromPartial<I extends Exact<DeepPartial<DownloadIntentExtension_DownloadIntentsEntry>, I>>(object: I): DownloadIntentExtension_DownloadIntentsEntry;
+};
 type Builtin = Date | Function | Uint8Array | string | number | boolean | undefined;
 type DeepPartial<T> = T extends Builtin ? T : T extends Array<infer U> ? Array<DeepPartial<U>> : T extends ReadonlyArray<infer U> ? ReadonlyArray<DeepPartial<U>> : T extends {} ? {
     [K in keyof T]?: DeepPartial<T[K]>;

package/src/generated/extensions.js

@@ -169,6 +169,156 @@ export var HaveExtension = {
         return message;
     },
 };
+function createBaseDownloadIntentExtension() {
+    return { downloadIntents: {} };
+}
+export var DownloadIntentExtension = {
+    encode: function (message, writer) {
+        if (writer === void 0) { writer = _m0.Writer.create(); }
+        Object.entries(message.downloadIntents).forEach(function (_a) {
+            var key = _a[0], value = _a[1];
+            DownloadIntentExtension_DownloadIntentsEntry.encode({ key: key, value: value }, writer.uint32(10).fork())
+                .ldelim();
+        });
+        return writer;
+    },
+    decode: function (input, length) {
+        var reader = input instanceof _m0.Reader ? input : _m0.Reader.create(input);
+        var end = length === undefined ? reader.len : reader.pos + length;
+        var message = createBaseDownloadIntentExtension();
+        while (reader.pos < end) {
+            var tag = reader.uint32();
+            switch (tag >>> 3) {
+                case 1:
+                    if (tag !== 10) {
+                        break;
+                    }
+                    var entry1 = DownloadIntentExtension_DownloadIntentsEntry.decode(reader, reader.uint32());
+                    if (entry1.value !== undefined) {
+                        message.downloadIntents[entry1.key] = entry1.value;
+                    }
+                    continue;
+            }
+            if ((tag & 7) === 4 || tag === 0) {
+                break;
+            }
+            reader.skipType(tag & 7);
+        }
+        return message;
+    },
+    create: function (base) {
+        return DownloadIntentExtension.fromPartial(base !== null && base !== void 0 ? base : {});
+    },
+    fromPartial: function (object) {
+        var _a;
+        var message = createBaseDownloadIntentExtension();
+        message.downloadIntents = Object.entries((_a = object.downloadIntents) !== null && _a !== void 0 ? _a : {}).reduce(function (acc, _a) {
+            var key = _a[0], value = _a[1];
+            if (value !== undefined) {
+                acc[key] = DownloadIntentExtension_DownloadIntent.fromPartial(value);
+            }
+            return acc;
+        }, {});
+        return message;
+    },
+};
+function createBaseDownloadIntentExtension_DownloadIntent() {
+    return { variants: [] };
+}
+export var DownloadIntentExtension_DownloadIntent = {
+    encode: function (message, writer) {
+        if (writer === void 0) { writer = _m0.Writer.create(); }
+        for (var _i = 0, _a = message.variants; _i < _a.length; _i++) {
+            var v = _a[_i];
+            writer.uint32(10).string(v);
+        }
+        return writer;
+    },
+    decode: function (input, length) {
+        var reader = input instanceof _m0.Reader ? input : _m0.Reader.create(input);
+        var end = length === undefined ? reader.len : reader.pos + length;
+        var message = createBaseDownloadIntentExtension_DownloadIntent();
+        while (reader.pos < end) {
+            var tag = reader.uint32();
+            switch (tag >>> 3) {
+                case 1:
+                    if (tag !== 10) {
+                        break;
+                    }
+                    message.variants.push(reader.string());
+                    continue;
+            }
+            if ((tag & 7) === 4 || tag === 0) {
+                break;
+            }
+            reader.skipType(tag & 7);
+        }
+        return message;
+    },
+    create: function (base) {
+        return DownloadIntentExtension_DownloadIntent.fromPartial(base !== null && base !== void 0 ? base : {});
+    },
+    fromPartial: function (object) {
+        var _a;
+        var message = createBaseDownloadIntentExtension_DownloadIntent();
+        message.variants = ((_a = object.variants) === null || _a === void 0 ? void 0 : _a.map(function (e) { return e; })) || [];
+        return message;
+    },
+};
+function createBaseDownloadIntentExtension_DownloadIntentsEntry() {
+    return { key: "", value: undefined };
+}
+export var DownloadIntentExtension_DownloadIntentsEntry = {
+    encode: function (message, writer) {
+        if (writer === void 0) { writer = _m0.Writer.create(); }
+        if (message.key !== "") {
+            writer.uint32(10).string(message.key);
+        }
+        if (message.value !== undefined) {
+            DownloadIntentExtension_DownloadIntent.encode(message.value, writer.uint32(18).fork()).ldelim();
+        }
+        return writer;
+    },
+    decode: function (input, length) {
+        var reader = input instanceof _m0.Reader ? input : _m0.Reader.create(input);
+        var end = length === undefined ? reader.len : reader.pos + length;
+        var message = createBaseDownloadIntentExtension_DownloadIntentsEntry();
+        while (reader.pos < end) {
+            var tag = reader.uint32();
+            switch (tag >>> 3) {
+                case 1:
+                    if (tag !== 10) {
+                        break;
+                    }
+                    message.key = reader.string();
+                    continue;
+                case 2:
+                    if (tag !== 18) {
+                        break;
+                    }
+                    message.value = DownloadIntentExtension_DownloadIntent.decode(reader, reader.uint32());
+                    continue;
+            }
+            if ((tag & 7) === 4 || tag === 0) {
+                break;
+            }
+            reader.skipType(tag & 7);
+        }
+        return message;
+    },
+    create: function (base) {
+        return DownloadIntentExtension_DownloadIntentsEntry.fromPartial(base !== null && base !== void 0 ? base : {});
+    },
+    fromPartial: function (object) {
+        var _a;
+        var message = createBaseDownloadIntentExtension_DownloadIntentsEntry();
+        message.key = (_a = object.key) !== null && _a !== void 0 ? _a : "";
+        message.value = (object.value !== undefined && object.value !== null)
+            ? DownloadIntentExtension_DownloadIntent.fromPartial(object.value)
+            : undefined;
+        return message;
+    },
+};
 var tsProtoGlobalThis = (function () {
     if (typeof globalThis !== "undefined") {
         return globalThis;