npm - hypercore-storage - Versions diffs - 2.9.0 → 3.0.1 - Mend

hypercore-storage 2.9.0 → 3.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (8) hide show

package/encoding/external.js +14 -0
package/{spec → encoding/spec}/hyperschema/index.js +212 -78
package/encoding/spec/hyperschema/schema.json +481 -0
package/index.js +76 -9
package/lib/keys.js +33 -2
package/lib/streams.js +18 -3
package/lib/tx.js +46 -3
package/package.json +14 -5

package/encoding/spec/hyperschema/schema.json ADDED Viewed

@@ -0,0 +1,481 @@
+{
+  "version": 2,
+  "schema": [
+    {
+      "name": "allocated",
+      "namespace": "corestore",
+      "compact": true,
+      "flagsPosition": -1,
+      "fields": [
+        {
+          "name": "cores",
+          "required": true,
+          "type": "uint",
+          "version": 1
+        },
+        {
+          "name": "datas",
+          "required": true,
+          "type": "uint",
+          "version": 1
+        }
+      ]
+    },
+    {
+      "name": "head-v1",
+      "namespace": "corestore",
+      "compact": false,
+      "flagsPosition": 0,
+      "fields": [
+        {
+          "name": "allocated",
+          "type": "@corestore/allocated",
+          "version": 1
+        },
+        {
+          "name": "seed",
+          "type": "fixed32",
+          "version": 1
+        },
+        {
+          "name": "defaultDiscoveryKey",
+          "type": "fixed32",
+          "version": 1
+        }
+      ]
+    },
+    {
+      "name": "head-v2",
+      "namespace": "corestore",
+      "compact": false,
+      "flagsPosition": 3,
+      "fields": [
+        {
+          "name": "cores",
+          "required": true,
+          "type": "uint",
+          "version": 1
+        },
+        {
+          "name": "datas",
+          "required": true,
+          "type": "uint",
+          "version": 1
+        },
+        {
+          "name": "groups",
+          "required": true,
+          "type": "uint",
+          "version": 1
+        },
+        {
+          "name": "seed",
+          "type": "fixed32",
+          "version": 1
+        },
+        {
+          "name": "defaultDiscoveryKey",
+          "type": "fixed32",
+          "version": 1
+        }
+      ]
+    },
+    {
+      "name": "head",
+      "namespace": "corestore",
+      "versions": [
+        {
+          "type": "@corestore/head-v1",
+          "map": "headLegacyMap",
+          "version": 1
+        },
+        {
+          "type": "@corestore/head-v2",
+          "map": null,
+          "version": 2
+        }
+      ]
+    },
+    {
+      "name": "alias",
+      "namespace": "corestore",
+      "compact": true,
+      "flagsPosition": -1,
+      "fields": [
+        {
+          "name": "name",
+          "required": true,
+          "type": "string",
+          "version": 1
+        },
+        {
+          "name": "namespace",
+          "required": true,
+          "type": "fixed32",
+          "version": 1
+        }
+      ]
+    },
+    {
+      "name": "core",
+      "namespace": "corestore",
+      "compact": false,
+      "flagsPosition": 3,
+      "fields": [
+        {
+          "name": "version",
+          "required": true,
+          "type": "uint",
+          "version": 1
+        },
+        {
+          "name": "corePointer",
+          "required": true,
+          "type": "uint",
+          "version": 1
+        },
+        {
+          "name": "dataPointer",
+          "required": true,
+          "type": "uint",
+          "version": 1
+        },
+        {
+          "name": "alias",
+          "type": "@corestore/alias",
+          "version": 1
+        }
+      ]
+    },
+    {
+      "name": "hashes",
+      "namespace": "core",
+      "offset": 0,
+      "enum": [
+        {
+          "key": "blake2b",
+          "version": 1
+        }
+      ],
+      "strings": true
+    },
+    {
+      "name": "signatures",
+      "namespace": "core",
+      "offset": 0,
+      "enum": [
+        {
+          "key": "ed25519",
+          "version": 1
+        }
+      ],
+      "strings": true
+    },
+    {
+      "name": "tree-node",
+      "namespace": "core",
+      "compact": true,
+      "flagsPosition": -1,
+      "fields": [
+        {
+          "name": "index",
+          "required": true,
+          "type": "uint",
+          "version": 1
+        },
+        {
+          "name": "size",
+          "required": true,
+          "type": "uint",
+          "version": 1
+        },
+        {
+          "name": "hash",
+          "required": true,
+          "type": "fixed32",
+          "version": 1
+        }
+      ]
+    },
+    {
+      "name": "signer",
+      "namespace": "core",
+      "compact": true,
+      "flagsPosition": -1,
+      "fields": [
+        {
+          "name": "signature",
+          "required": true,
+          "type": "@core/signatures",
+          "version": 1
+        },
+        {
+          "name": "namespace",
+          "required": true,
+          "type": "fixed32",
+          "version": 1
+        },
+        {
+          "name": "publicKey",
+          "required": true,
+          "type": "fixed32",
+          "version": 1
+        }
+      ]
+    },
+    {
+      "name": "prologue",
+      "namespace": "core",
+      "compact": true,
+      "flagsPosition": -1,
+      "fields": [
+        {
+          "name": "hash",
+          "required": true,
+          "type": "fixed32",
+          "version": 1
+        },
+        {
+          "name": "length",
+          "required": true,
+          "type": "uint",
+          "version": 1
+        }
+      ]
+    },
+    {
+      "name": "manifest",
+      "namespace": "core",
+      "compact": false,
+      "flagsPosition": 1,
+      "fields": [
+        {
+          "name": "version",
+          "required": true,
+          "type": "uint",
+          "version": 1
+        },
+        {
+          "name": "hash",
+          "required": true,
+          "type": "@core/hashes",
+          "version": 1
+        },
+        {
+          "name": "quorum",
+          "required": true,
+          "type": "uint",
+          "version": 1
+        },
+        {
+          "name": "allowPatch",
+          "type": "bool",
+          "version": 1
+        },
+        {
+          "name": "signers",
+          "required": true,
+          "array": true,
+          "type": "@core/signer",
+          "version": 1
+        },
+        {
+          "name": "prologue",
+          "type": "@core/prologue",
+          "version": 1
+        },
+        {
+          "name": "linked",
+          "array": true,
+          "type": "fixed32",
+          "version": 1
+        },
+        {
+          "name": "userData",
+          "type": "buffer",
+          "version": 1
+        }
+      ]
+    },
+    {
+      "name": "keyPair",
+      "namespace": "core",
+      "compact": true,
+      "flagsPosition": -1,
+      "fields": [
+        {
+          "name": "publicKey",
+          "required": true,
+          "type": "buffer",
+          "version": 1
+        },
+        {
+          "name": "secretKey",
+          "required": true,
+          "type": "optionalBuffer",
+          "version": 1
+        }
+      ]
+    },
+    {
+      "name": "group",
+      "namespace": "core",
+      "compact": false,
+      "flagsPosition": 0,
+      "fields": [
+        {
+          "name": "key",
+          "type": "fixed32",
+          "version": 1
+        },
+        {
+          "name": "pointer",
+          "type": "uint",
+          "version": 1
+        }
+      ]
+    },
+    {
+      "name": "auth",
+      "namespace": "core",
+      "compact": false,
+      "flagsPosition": 2,
+      "fields": [
+        {
+          "name": "key",
+          "required": true,
+          "type": "fixed32",
+          "version": 1
+        },
+        {
+          "name": "discoveryKey",
+          "required": true,
+          "type": "fixed32",
+          "version": 1
+        },
+        {
+          "name": "manifest",
+          "type": "@core/manifest",
+          "version": 1
+        },
+        {
+          "name": "keyPair",
+          "type": "@core/keyPair",
+          "version": 1
+        },
+        {
+          "name": "encryptionKey",
+          "type": "buffer",
+          "version": 1
+        }
+      ]
+    },
+    {
+      "name": "head",
+      "namespace": "core",
+      "compact": false,
+      "flagsPosition": 4,
+      "fields": [
+        {
+          "name": "fork",
+          "required": true,
+          "type": "uint",
+          "version": 1
+        },
+        {
+          "name": "length",
+          "required": true,
+          "type": "uint",
+          "version": 1
+        },
+        {
+          "name": "rootHash",
+          "required": true,
+          "type": "fixed32",
+          "version": 1
+        },
+        {
+          "name": "signature",
+          "required": true,
+          "type": "optionalBuffer",
+          "version": 1
+        },
+        {
+          "name": "timestamp",
+          "required": false,
+          "type": "uint64",
+          "version": 2
+        }
+      ]
+    },
+    {
+      "name": "hints",
+      "namespace": "core",
+      "compact": false,
+      "flagsPosition": 0,
+      "fields": [
+        {
+          "name": "contiguousLength",
+          "type": "uint",
+          "version": 1
+        },
+        {
+          "name": "remoteContiguousLength",
+          "type": "uint",
+          "version": 1
+        },
+        {
+          "name": "recovering",
+          "type": "uint",
+          "version": 1
+        }
+      ]
+    },
+    {
+      "name": "session",
+      "namespace": "core",
+      "compact": true,
+      "flagsPosition": -1,
+      "fields": [
+        {
+          "name": "name",
+          "required": true,
+          "type": "string",
+          "version": 1
+        },
+        {
+          "name": "dataPointer",
+          "required": true,
+          "type": "uint",
+          "version": 1
+        }
+      ]
+    },
+    {
+      "name": "sessions",
+      "namespace": "core",
+      "array": true,
+      "type": "@core/session"
+    },
+    {
+      "name": "dependency",
+      "namespace": "core",
+      "compact": true,
+      "flagsPosition": -1,
+      "fields": [
+        {
+          "name": "dataPointer",
+          "required": true,
+          "type": "uint",
+          "version": 1
+        },
+        {
+          "name": "length",
+          "required": true,
+          "type": "uint",
+          "version": 1
+        }
+      ]
+    }
+  ]
+}

package/index.js CHANGED Viewed

@@ -4,9 +4,10 @@ const ScopeLock = require('scope-lock')
 const DeviceFile = require('device-file')
 const path = require('path')
 const fs = require('fs')
 const View = require('./lib/view.js')
-const VERSION = 1
+const VERSION = 2
 const COLUMN_FAMILY = 'corestore'
 const { store, core } = require('./lib/keys.js')
@@ -22,7 +23,8 @@ const {
   createUserDataStream,
   createTreeNodeStream,
   createMarkStream,
-  createLocalStream
+  createLocalStream,
+  createGroupUpdateStream
 } = require('./lib/streams.js')
 const EMPTY = new View()
@@ -647,6 +649,10 @@ class CorestoreStorage {
           await require('./migrations/0').store(this, target)
           break
         }
+        case 1: {
+          // implicit migration on write
+          break
+        }
         default: {
           throw new Error(
             'Unsupported version: ' + version + ' - you should probably upgrade your dependencies'
@@ -673,6 +679,10 @@ class CorestoreStorage {
           await require('./migrations/0').core(core, target)
           break
         }
+        case 1: {
+          await require('./migrations/1').core(core, target)
+          break
+        }
         default: {
           throw new Error(
             'Unsupported version: ' + version + ' - you should probably upgrade your dependencies'
@@ -738,7 +748,7 @@ class CorestoreStorage {
     try {
       const head = await this._getHead(view)
-      dataPointer = head.allocated.datas++
+      dataPointer = head.datas++
       tx.setHead(head)
       tx.apply()
@@ -801,6 +811,10 @@ class CorestoreStorage {
     return createDiscoveryKeyStream(this.db, EMPTY, namespace)
   }
+  createGroupUpdateStream(group, { since, reverse } = {}) {
+    return createGroupUpdateStream(this.db, EMPTY, group, { gte: since, reverse })
+  }
   async getAlias(alias) {
     if (this.version === 0) await this._migrateStore()
@@ -822,6 +836,15 @@ class CorestoreStorage {
     return head === null ? null : head.seed
   }
+  async getGroup(topic) {
+    if (this.version === 0) await this._migrateStore()
+    const rx = new CorestoreRX(this.db, EMPTY)
+    const groupPromise = rx.getGroup(topic)
+    rx.tryFlush()
+    return groupPromise
+  }
   async setSeed(seed, { overwrite = true } = {}) {
     if (this.version === 0) await this._migrateStore()
@@ -1079,8 +1102,8 @@ class CorestoreStorage {
     if (head === null) head = initStoreHead()
     if (head.defaultDiscoveryKey === null) head.defaultDiscoveryKey = discoveryKey
-    const corePointer = ptrs ? ptrs.corePointer : head.allocated.cores++
-    const dataPointer = ptrs ? ptrs.dataPointer : head.allocated.datas++
+    const corePointer = ptrs ? ptrs.corePointer : head.cores++
+    const dataPointer = ptrs ? ptrs.dataPointer : head.datas++
     core = { version: VERSION, corePointer, dataPointer, alias }
@@ -1122,6 +1145,51 @@ class CorestoreStorage {
     }
   }
+  // not allowed to throw validation errors as its a shared tx!
+  async _createGroup(view, topic) {
+    const rx = new CorestoreRX(this.db, view)
+    const tx = new CorestoreTX(view)
+    const groupPromise = rx.getGroup(topic)
+    const headPromise = rx.getHead()
+    rx.tryFlush()
+    let group = await groupPromise
+    if (group !== null) {
+      return {
+        key: topic,
+        pointer: group
+      }
+    }
+    let head = await headPromise
+    if (head === null) head = initStoreHead()
+    group = head.groups++
+    tx.putGroup(topic, group)
+    tx.setHead(head)
+    tx.apply()
+    return {
+      key: topic,
+      pointer: group
+    }
+  }
+  async createGroup(topic) {
+    if (this.version === 0) await this._migrateStore()
+    const view = await this._enter()
+    try {
+      return await this._createGroup(view, topic)
+    } finally {
+      await this._exit()
+    }
+  }
   static isParentStorage(storage, parent) {
     return HypercoreStorage.isParentStorage(storage, parent)
   }
@@ -1132,10 +1200,9 @@ module.exports = CorestoreStorage
 function initStoreHead() {
   return {
     version: 0, // cause we wanna run the migration
-    allocated: {
-      datas: 0,
-      cores: 0
-    },
+    datas: 0,
+    cores: 0,
+    groups: 0,
     seed: null,
     defaultDiscoveryKey: null
   }

package/lib/keys.js CHANGED Viewed

@@ -7,11 +7,14 @@ const TL_CORE_BY_DKEY = 1
 const TL_CORE_BY_ALIAS = 2
 const TL_CORE = 3
 const TL_DATA = 4
+const TL_GROUP_BY_TOPIC = 5
+const TL_GROUP = 6
-const TL_END = TL_DATA + 1
+const TL_END = TL_GROUP + 1
 const CORE_AUTH = 0
 const CORE_SESSIONS = 1
+const CORE_GROUP = 2
 const DATA_HEAD = 0
 const DATA_DEPENDENCY = 1
@@ -27,6 +30,7 @@ const slab = { buffer: b4a.allocUnsafe(65536), start: 0, end: 0 }
 const store = {}
 const core = {}
+const group = {}
 store.clear = function () {
   const state = alloc()
@@ -139,6 +143,15 @@ core.auth = function (ptr) {
   return state.buffer.subarray(start, state.start)
 }
+core.group = function (ptr) {
+  const state = alloc()
+  const start = state.start
+  UINT.encode(state, TL_CORE)
+  UINT.encode(state, ptr)
+  UINT.encode(state, CORE_GROUP)
+  return state.buffer.subarray(start, state.start)
+}
 core.sessions = function (ptr) {
   const state = alloc()
   const start = state.start
@@ -300,7 +313,25 @@ core.localKey = function (buffer) {
   return state.buffer.subarray(state.start, state.end)
 }
-module.exports = { store, core }
+group.topic = function (topic) {
+  const state = alloc()
+  const start = state.start
+  UINT.encode(state, TL_GROUP_BY_TOPIC)
+  c.fixed32.encode(state, topic)
+  return state.buffer.subarray(start, state.start)
+}
+group.entry = function (group, timestamp, ptr) {
+  const state = alloc()
+  const start = state.start
+  UINT.encode(state, TL_GROUP)
+  UINT.encode(state, group)
+  c.uint64be.encode(state, timestamp)
+  UINT.encode(state, ptr)
+  return state.buffer.subarray(start, state.start)
+}
+module.exports = { store, core, group }
 function alloc() {
   if (slab.buffer.byteLength - slab.start < 4096) {