hypercore-storage 2.8.0 → 3.0.0

package/encoding/external.js

@@ -0,0 +1,14 @@
+function headLegacyMap(m) {
+  return {
+    version: 2, // version bumped from 1
+    cores: m.allocated.cores,
+    datas: m.allocated.datas,
+    groups: 0,
+    seed: m.seed,
+    defaultDiscoveryKey: m.defaultDiscoveryKey
+  }
+}
+
+module.exports = {
+  headLegacyMap
+}

package/{spec → encoding/spec}/hyperschema/index.js

@@ -5,6 +5,7 @@
 /* eslint-disable space-before-function-paren */
 
 const { c } = require('hyperschema/runtime')
+const external0 = require('../../external.js')
 
 const VERSION = 1
 
@@ -32,10 +33,9 @@ const encoding0 = {
   }
 }
 
-// @corestore/head
+// @corestore/head-v1
 const encoding1 = {
   preencode(state, m) {
-    c.uint.preencode(state, m.version)
     state.end++ // max flag is 4 so always one byte
 
     if (m.allocated) encoding0.preencode(state, m.allocated)
@@ -45,7 +45,6 @@ const encoding1 = {
   encode(state, m) {
     const flags = (m.allocated ? 1 : 0) | (m.seed ? 2 : 0) | (m.defaultDiscoveryKey ? 4 : 0)
 
-    c.uint.encode(state, m.version)
     c.uint.encode(state, flags)
 
     if (m.allocated) encoding0.encode(state, m.allocated)
@@ -53,11 +52,11 @@ const encoding1 = {
     if (m.defaultDiscoveryKey) c.fixed32.encode(state, m.defaultDiscoveryKey)
   },
   decode(state) {
-    const r0 = c.uint.decode(state)
+    const v = c.uint.decode(state)
     const flags = c.uint.decode(state)
 
     return {
-      version: r0,
+      version: v,
       allocated: (flags & 1) !== 0 ? encoding0.decode(state) : null,
       seed: (flags & 2) !== 0 ? c.fixed32.decode(state) : null,
       defaultDiscoveryKey: (flags & 4) !== 0 ? c.fixed32.decode(state) : null
@@ -127,10 +126,10 @@ const encoding4_enum = {
 
 // @core/hashes enum
 const encoding4 = {
-  preencode (state, m) {
+  preencode(state, m) {
     state.end++ // max enum is 0 so always one byte
   },
-  encode (state, m) {
+  encode(state, m) {
     switch (m) {
       case 'blake2b':
         c.uint.encode(state, 0)
@@ -139,11 +138,12 @@ const encoding4 = {
         throw new Error('Unknown enum')
     }
   },
-  decode (state) {
+  decode(state) {
     switch (c.uint.decode(state)) {
       case 0:
         return 'blake2b'
-      default: return null
+      default:
+        return null
     }
   }
 }
@@ -154,10 +154,10 @@ const encoding5_enum = {
 
 // @core/signatures enum
 const encoding5 = {
-  preencode (state, m) {
+  preencode(state, m) {
     state.end++ // max enum is 0 so always one byte
   },
-  encode (state, m) {
+  encode(state, m) {
     switch (m) {
       case 'ed25519':
         c.uint.encode(state, 0)
@@ -166,11 +166,12 @@ const encoding5 = {
         throw new Error('Unknown enum')
     }
   },
-  decode (state) {
+  decode(state) {
     switch (c.uint.decode(state)) {
       case 0:
         return 'ed25519'
-      default: return null
+      default:
+        return null
     }
   }
 }
@@ -299,15 +300,15 @@ const encoding9 = {
 const encoding10 = {
   preencode(state, m) {
     c.buffer.preencode(state, m.publicKey)
-    c.buffer.preencode(state, m.secretKey)
+    c.optionalBuffer.preencode(state, m.secretKey)
   },
   encode(state, m) {
     c.buffer.encode(state, m.publicKey)
-    c.buffer.encode(state, m.secretKey)
+    c.optionalBuffer.encode(state, m.secretKey)
   },
   decode(state) {
     const r0 = c.buffer.decode(state)
-    const r1 = c.buffer.decode(state)
+    const r1 = c.optionalBuffer.decode(state)
 
     return {
       publicKey: r0,
@@ -316,17 +317,43 @@ const encoding10 = {
   }
 }
 
+// @core/group
+const encoding11 = {
+  preencode(state, m) {
+    state.end++ // max flag is 2 so always one byte
+
+    if (m.key) c.fixed32.preencode(state, m.key)
+    if (m.pointer) c.uint.preencode(state, m.pointer)
+  },
+  encode(state, m) {
+    const flags = (m.key ? 1 : 0) | (m.pointer ? 2 : 0)
+
+    c.uint.encode(state, flags)
+
+    if (m.key) c.fixed32.encode(state, m.key)
+    if (m.pointer) c.uint.encode(state, m.pointer)
+  },
+  decode(state) {
+    const flags = c.uint.decode(state)
+
+    return {
+      key: (flags & 1) !== 0 ? c.fixed32.decode(state) : null,
+      pointer: (flags & 2) !== 0 ? c.uint.decode(state) : 0
+    }
+  }
+}
+
 // @core/auth.manifest
-const encoding11_2 = c.frame(encoding9)
+const encoding12_2 = c.frame(encoding9)
 
 // @core/auth
-const encoding11 = {
+const encoding12 = {
   preencode(state, m) {
     c.fixed32.preencode(state, m.key)
     c.fixed32.preencode(state, m.discoveryKey)
     state.end++ // max flag is 4 so always one byte
 
-    if (m.manifest) encoding11_2.preencode(state, m.manifest)
+    if (m.manifest) encoding12_2.preencode(state, m.manifest)
     if (m.keyPair) encoding10.preencode(state, m.keyPair)
     if (m.encryptionKey) c.buffer.preencode(state, m.encryptionKey)
   },
@@ -337,7 +364,7 @@ const encoding11 = {
     c.fixed32.encode(state, m.discoveryKey)
     c.uint.encode(state, flags)
 
-    if (m.manifest) encoding11_2.encode(state, m.manifest)
+    if (m.manifest) encoding12_2.encode(state, m.manifest)
     if (m.keyPair) encoding10.encode(state, m.keyPair)
     if (m.encryptionKey) c.buffer.encode(state, m.encryptionKey)
   },
@@ -349,7 +376,7 @@ const encoding11 = {
     return {
       key: r0,
       discoveryKey: r1,
-      manifest: (flags & 1) !== 0 ? encoding11_2.decode(state) : null,
+      manifest: (flags & 1) !== 0 ? encoding12_2.decode(state) : null,
       keyPair: (flags & 2) !== 0 ? encoding10.decode(state) : null,
       encryptionKey: (flags & 4) !== 0 ? c.buffer.decode(state) : null
     }
@@ -357,36 +384,46 @@ const encoding11 = {
 }
 
 // @core/head
-const encoding12 = {
+const encoding13 = {
   preencode(state, m) {
     c.uint.preencode(state, m.fork)
     c.uint.preencode(state, m.length)
     c.fixed32.preencode(state, m.rootHash)
-    c.buffer.preencode(state, m.signature)
+    c.optionalBuffer.preencode(state, m.signature)
+    state.end++ // max flag is 1 so always one byte
+
+    if (m.timestamp) c.uint64.preencode(state, m.timestamp)
   },
   encode(state, m) {
+    const flags = m.timestamp ? 1 : 0
+
     c.uint.encode(state, m.fork)
     c.uint.encode(state, m.length)
     c.fixed32.encode(state, m.rootHash)
-    c.buffer.encode(state, m.signature)
+    c.optionalBuffer.encode(state, m.signature)
+    c.uint.encode(state, flags)
+
+    if (m.timestamp) c.uint64.encode(state, m.timestamp)
   },
   decode(state) {
     const r0 = c.uint.decode(state)
     const r1 = c.uint.decode(state)
     const r2 = c.fixed32.decode(state)
-    const r3 = c.buffer.decode(state)
+    const r3 = c.optionalBuffer.decode(state)
+    const flags = c.uint.decode(state)
 
     return {
       fork: r0,
       length: r1,
       rootHash: r2,
-      signature: r3
+      signature: r3,
+      timestamp: (flags & 1) !== 0 ? c.uint64.decode(state) : 0
     }
   }
 }
 
 // @core/hints
-const encoding13 = {
+const encoding14 = {
   preencode(state, m) {
     state.end++ // max flag is 4 so always one byte
 
@@ -416,7 +453,7 @@ const encoding13 = {
 }
 
 // @core/session
-const encoding14 = {
+const encoding15 = {
   preencode(state, m) {
     c.string.preencode(state, m.name)
     c.uint.preencode(state, m.dataPointer)
@@ -437,10 +474,10 @@ const encoding14 = {
 }
 
 // @core/sessions
-const encoding15 = c.array(encoding14)
+const encoding16 = c.array(encoding15)
 
 // @core/dependency
-const encoding16 = {
+const encoding17 = {
   preencode(state, m) {
     c.uint.preencode(state, m.dataPointer)
     c.uint.preencode(state, m.length)
@@ -460,6 +497,97 @@ const encoding16 = {
   }
 }
 
+// @corestore/head-v2
+const encoding18 = {
+  preencode(state, m) {
+    c.uint.preencode(state, m.cores)
+    c.uint.preencode(state, m.datas)
+    c.uint.preencode(state, m.groups)
+    state.end++ // max flag is 2 so always one byte
+
+    if (m.seed) c.fixed32.preencode(state, m.seed)
+    if (m.defaultDiscoveryKey) c.fixed32.preencode(state, m.defaultDiscoveryKey)
+  },
+  encode(state, m) {
+    const flags = (m.seed ? 1 : 0) | (m.defaultDiscoveryKey ? 2 : 0)
+
+    c.uint.encode(state, m.cores)
+    c.uint.encode(state, m.datas)
+    c.uint.encode(state, m.groups)
+    c.uint.encode(state, flags)
+
+    if (m.seed) c.fixed32.encode(state, m.seed)
+    if (m.defaultDiscoveryKey) c.fixed32.encode(state, m.defaultDiscoveryKey)
+  },
+  decode(state) {
+    const v = c.uint.decode(state)
+    const r0 = c.uint.decode(state)
+    const r1 = c.uint.decode(state)
+    const r2 = c.uint.decode(state)
+    const flags = c.uint.decode(state)
+
+    return {
+      version: v,
+      cores: r0,
+      datas: r1,
+      groups: r2,
+      seed: (flags & 1) !== 0 ? c.fixed32.decode(state) : null,
+      defaultDiscoveryKey: (flags & 2) !== 0 ? c.fixed32.decode(state) : null
+    }
+  }
+}
+
+// @corestore/head
+const encoding19 = {
+  preencode(state, m) {
+    c.uint.preencode(state, m.version)
+    switch (m.version) {
+      case 0:
+      case 1:
+        encoding1.preencode(state, m)
+        break
+      case 2:
+        encoding18.preencode(state, m)
+        break
+      default:
+        throw new Error('Unsupported version')
+    }
+  },
+  encode(state, m) {
+    c.uint.encode(state, m.version)
+    switch (m.version) {
+      case 0:
+      case 1:
+        encoding1.encode(state, m)
+        break
+      case 2:
+        encoding18.encode(state, m)
+        break
+      default:
+        throw new Error('Unsupported version')
+    }
+  },
+  decode(state) {
+    const start = state.start
+    const v = c.uint.decode(state)
+    state.start = start
+    switch (v) {
+      case 0:
+      case 1: {
+        const decoded = encoding1.decode(state)
+        const map = external0.headLegacyMap
+        return map(decoded)
+      }
+      case 2: {
+        const decoded = encoding18.decode(state)
+        return decoded
+      }
+      default:
+        throw new Error('Unsupported version')
+    }
+  }
+}
+
 function setVersion(v) {
   version = v
 }
@@ -489,7 +617,7 @@ function getEncoding(name) {
   switch (name) {
     case '@corestore/allocated':
       return encoding0
-    case '@corestore/head':
+    case '@corestore/head-v1':
       return encoding1
     case '@corestore/alias':
       return encoding2
@@ -509,18 +637,24 @@ function getEncoding(name) {
       return encoding9
     case '@core/keyPair':
       return encoding10
-    case '@core/auth':
+    case '@core/group':
       return encoding11
-    case '@core/head':
+    case '@core/auth':
       return encoding12
-    case '@core/hints':
+    case '@core/head':
       return encoding13
-    case '@core/session':
+    case '@core/hints':
       return encoding14
-    case '@core/sessions':
+    case '@core/session':
       return encoding15
-    case '@core/dependency':
+    case '@core/sessions':
       return encoding16
+    case '@core/dependency':
+      return encoding17
+    case '@corestore/head-v2':
+      return encoding18
+    case '@corestore/head':
+      return encoding19
     default:
       throw new Error('Encoder not found ' + name)
   }

package/encoding/spec/hyperschema/schema.json

@@ -0,0 +1,481 @@
+{
+  "version": 1,
+  "schema": [
+    {
+      "name": "allocated",
+      "namespace": "corestore",
+      "compact": true,
+      "flagsPosition": -1,
+      "fields": [
+        {
+          "name": "cores",
+          "required": true,
+          "type": "uint",
+          "version": 1
+        },
+        {
+          "name": "datas",
+          "required": true,
+          "type": "uint",
+          "version": 1
+        }
+      ]
+    },
+    {
+      "name": "head-v1",
+      "namespace": "corestore",
+      "compact": false,
+      "flagsPosition": 0,
+      "fields": [
+        {
+          "name": "allocated",
+          "type": "@corestore/allocated",
+          "version": 1
+        },
+        {
+          "name": "seed",
+          "type": "fixed32",
+          "version": 1
+        },
+        {
+          "name": "defaultDiscoveryKey",
+          "type": "fixed32",
+          "version": 1
+        }
+      ]
+    },
+    {
+      "name": "alias",
+      "namespace": "corestore",
+      "compact": true,
+      "flagsPosition": -1,
+      "fields": [
+        {
+          "name": "name",
+          "required": true,
+          "type": "string",
+          "version": 1
+        },
+        {
+          "name": "namespace",
+          "required": true,
+          "type": "fixed32",
+          "version": 1
+        }
+      ]
+    },
+    {
+      "name": "core",
+      "namespace": "corestore",
+      "compact": false,
+      "flagsPosition": 3,
+      "fields": [
+        {
+          "name": "version",
+          "required": true,
+          "type": "uint",
+          "version": 1
+        },
+        {
+          "name": "corePointer",
+          "required": true,
+          "type": "uint",
+          "version": 1
+        },
+        {
+          "name": "dataPointer",
+          "required": true,
+          "type": "uint",
+          "version": 1
+        },
+        {
+          "name": "alias",
+          "type": "@corestore/alias",
+          "version": 1
+        }
+      ]
+    },
+    {
+      "name": "hashes",
+      "namespace": "core",
+      "offset": 0,
+      "enum": [
+        {
+          "key": "blake2b",
+          "version": 1
+        }
+      ],
+      "strings": true
+    },
+    {
+      "name": "signatures",
+      "namespace": "core",
+      "offset": 0,
+      "enum": [
+        {
+          "key": "ed25519",
+          "version": 1
+        }
+      ],
+      "strings": true
+    },
+    {
+      "name": "tree-node",
+      "namespace": "core",
+      "compact": true,
+      "flagsPosition": -1,
+      "fields": [
+        {
+          "name": "index",
+          "required": true,
+          "type": "uint",
+          "version": 1
+        },
+        {
+          "name": "size",
+          "required": true,
+          "type": "uint",
+          "version": 1
+        },
+        {
+          "name": "hash",
+          "required": true,
+          "type": "fixed32",
+          "version": 1
+        }
+      ]
+    },
+    {
+      "name": "signer",
+      "namespace": "core",
+      "compact": true,
+      "flagsPosition": -1,
+      "fields": [
+        {
+          "name": "signature",
+          "required": true,
+          "type": "@core/signatures",
+          "version": 1
+        },
+        {
+          "name": "namespace",
+          "required": true,
+          "type": "fixed32",
+          "version": 1
+        },
+        {
+          "name": "publicKey",
+          "required": true,
+          "type": "fixed32",
+          "version": 1
+        }
+      ]
+    },
+    {
+      "name": "prologue",
+      "namespace": "core",
+      "compact": true,
+      "flagsPosition": -1,
+      "fields": [
+        {
+          "name": "hash",
+          "required": true,
+          "type": "fixed32",
+          "version": 1
+        },
+        {
+          "name": "length",
+          "required": true,
+          "type": "uint",
+          "version": 1
+        }
+      ]
+    },
+    {
+      "name": "manifest",
+      "namespace": "core",
+      "compact": false,
+      "flagsPosition": 1,
+      "fields": [
+        {
+          "name": "version",
+          "required": true,
+          "type": "uint",
+          "version": 1
+        },
+        {
+          "name": "hash",
+          "required": true,
+          "type": "@core/hashes",
+          "version": 1
+        },
+        {
+          "name": "quorum",
+          "required": true,
+          "type": "uint",
+          "version": 1
+        },
+        {
+          "name": "allowPatch",
+          "type": "bool",
+          "version": 1
+        },
+        {
+          "name": "signers",
+          "required": true,
+          "array": true,
+          "type": "@core/signer",
+          "version": 1
+        },
+        {
+          "name": "prologue",
+          "type": "@core/prologue",
+          "version": 1
+        },
+        {
+          "name": "linked",
+          "array": true,
+          "type": "fixed32",
+          "version": 1
+        },
+        {
+          "name": "userData",
+          "type": "buffer",
+          "version": 1
+        }
+      ]
+    },
+    {
+      "name": "keyPair",
+      "namespace": "core",
+      "compact": true,
+      "flagsPosition": -1,
+      "fields": [
+        {
+          "name": "publicKey",
+          "required": true,
+          "type": "buffer",
+          "version": 1
+        },
+        {
+          "name": "secretKey",
+          "required": true,
+          "type": "optionalBuffer",
+          "version": 1
+        }
+      ]
+    },
+    {
+      "name": "group",
+      "namespace": "core",
+      "compact": false,
+      "flagsPosition": 0,
+      "fields": [
+        {
+          "name": "key",
+          "type": "fixed32",
+          "version": 1
+        },
+        {
+          "name": "pointer",
+          "type": "uint",
+          "version": 1
+        }
+      ]
+    },
+    {
+      "name": "auth",
+      "namespace": "core",
+      "compact": false,
+      "flagsPosition": 2,
+      "fields": [
+        {
+          "name": "key",
+          "required": true,
+          "type": "fixed32",
+          "version": 1
+        },
+        {
+          "name": "discoveryKey",
+          "required": true,
+          "type": "fixed32",
+          "version": 1
+        },
+        {
+          "name": "manifest",
+          "type": "@core/manifest",
+          "version": 1
+        },
+        {
+          "name": "keyPair",
+          "type": "@core/keyPair",
+          "version": 1
+        },
+        {
+          "name": "encryptionKey",
+          "type": "buffer",
+          "version": 1
+        }
+      ]
+    },
+    {
+      "name": "head",
+      "namespace": "core",
+      "compact": false,
+      "flagsPosition": 4,
+      "fields": [
+        {
+          "name": "fork",
+          "required": true,
+          "type": "uint",
+          "version": 1
+        },
+        {
+          "name": "length",
+          "required": true,
+          "type": "uint",
+          "version": 1
+        },
+        {
+          "name": "rootHash",
+          "required": true,
+          "type": "fixed32",
+          "version": 1
+        },
+        {
+          "name": "signature",
+          "required": true,
+          "type": "optionalBuffer",
+          "version": 1
+        },
+        {
+          "name": "timestamp",
+          "required": false,
+          "type": "uint64",
+          "version": 1
+        }
+      ]
+    },
+    {
+      "name": "hints",
+      "namespace": "core",
+      "compact": false,
+      "flagsPosition": 0,
+      "fields": [
+        {
+          "name": "contiguousLength",
+          "type": "uint",
+          "version": 1
+        },
+        {
+          "name": "remoteContiguousLength",
+          "type": "uint",
+          "version": 1
+        },
+        {
+          "name": "recovering",
+          "type": "uint",
+          "version": 1
+        }
+      ]
+    },
+    {
+      "name": "session",
+      "namespace": "core",
+      "compact": true,
+      "flagsPosition": -1,
+      "fields": [
+        {
+          "name": "name",
+          "required": true,
+          "type": "string",
+          "version": 1
+        },
+        {
+          "name": "dataPointer",
+          "required": true,
+          "type": "uint",
+          "version": 1
+        }
+      ]
+    },
+    {
+      "name": "sessions",
+      "namespace": "core",
+      "array": true,
+      "type": "@core/session"
+    },
+    {
+      "name": "dependency",
+      "namespace": "core",
+      "compact": true,
+      "flagsPosition": -1,
+      "fields": [
+        {
+          "name": "dataPointer",
+          "required": true,
+          "type": "uint",
+          "version": 1
+        },
+        {
+          "name": "length",
+          "required": true,
+          "type": "uint",
+          "version": 1
+        }
+      ]
+    },
+    {
+      "name": "head-v2",
+      "namespace": "corestore",
+      "compact": false,
+      "flagsPosition": 3,
+      "fields": [
+        {
+          "name": "cores",
+          "required": true,
+          "type": "uint",
+          "version": 1
+        },
+        {
+          "name": "datas",
+          "required": true,
+          "type": "uint",
+          "version": 1
+        },
+        {
+          "name": "groups",
+          "required": true,
+          "type": "uint",
+          "version": 1
+        },
+        {
+          "name": "seed",
+          "type": "fixed32",
+          "version": 1
+        },
+        {
+          "name": "defaultDiscoveryKey",
+          "type": "fixed32",
+          "version": 1
+        }
+      ]
+    },
+    {
+      "name": "head",
+      "namespace": "corestore",
+      "versions": [
+        {
+          "type": "@corestore/head-v1",
+          "map": "headLegacyMap",
+          "version": 1
+        },
+        {
+          "type": "@corestore/head-v2",
+          "map": null,
+          "version": 2
+        }
+      ]
+    }
+  ]
+}

package/index.js

@@ -4,9 +4,10 @@ const ScopeLock = require('scope-lock')
 const DeviceFile = require('device-file')
 const path = require('path')
 const fs = require('fs')
+
 const View = require('./lib/view.js')
 
-const VERSION = 1
+const VERSION = 2
 const COLUMN_FAMILY = 'corestore'
 
 const { store, core } = require('./lib/keys.js')
@@ -22,7 +23,8 @@ const {
   createUserDataStream,
   createTreeNodeStream,
   createMarkStream,
-  createLocalStream
+  createLocalStream,
+  createGroupUpdateStream
 } = require('./lib/streams.js')
 
 const EMPTY = new View()
@@ -647,6 +649,10 @@ class CorestoreStorage {
           await require('./migrations/0').store(this, target)
           break
         }
+        case 1: {
+          // implicit migration on write
+          break
+        }
         default: {
           throw new Error(
             'Unsupported version: ' + version + ' - you should probably upgrade your dependencies'
@@ -738,7 +744,7 @@ class CorestoreStorage {
     try {
       const head = await this._getHead(view)
 
-      dataPointer = head.allocated.datas++
+      dataPointer = head.datas++
 
       tx.setHead(head)
       tx.apply()
@@ -801,6 +807,10 @@ class CorestoreStorage {
     return createDiscoveryKeyStream(this.db, EMPTY, namespace)
   }
 
+  createGroupUpdateStream(group, { since, reverse } = {}) {
+    return createGroupUpdateStream(this.db, EMPTY, group, { gte: since, reverse })
+  }
+
   async getAlias(alias) {
     if (this.version === 0) await this._migrateStore()
 
@@ -822,6 +832,15 @@ class CorestoreStorage {
     return head === null ? null : head.seed
   }
 
+  async getGroup(topic) {
+    if (this.version === 0) await this._migrateStore()
+
+    const rx = new CorestoreRX(this.db, EMPTY)
+    const groupPromise = rx.getGroup(topic)
+    rx.tryFlush()
+    return groupPromise
+  }
+
   async setSeed(seed, { overwrite = true } = {}) {
     if (this.version === 0) await this._migrateStore()
 
@@ -1079,8 +1098,8 @@ class CorestoreStorage {
     if (head === null) head = initStoreHead()
     if (head.defaultDiscoveryKey === null) head.defaultDiscoveryKey = discoveryKey
 
-    const corePointer = ptrs ? ptrs.corePointer : head.allocated.cores++
-    const dataPointer = ptrs ? ptrs.dataPointer : head.allocated.datas++
+    const corePointer = ptrs ? ptrs.corePointer : head.cores++
+    const dataPointer = ptrs ? ptrs.dataPointer : head.datas++
 
     core = { version: VERSION, corePointer, dataPointer, alias }
 
@@ -1122,6 +1141,51 @@ class CorestoreStorage {
     }
   }
 
+  // not allowed to throw validation errors as its a shared tx!
+  async _createGroup(view, topic) {
+    const rx = new CorestoreRX(this.db, view)
+    const tx = new CorestoreTX(view)
+
+    const groupPromise = rx.getGroup(topic)
+    const headPromise = rx.getHead()
+
+    rx.tryFlush()
+
+    let group = await groupPromise
+    if (group !== null) {
+      return {
+        key: topic,
+        pointer: group
+      }
+    }
+
+    let head = await headPromise
+    if (head === null) head = initStoreHead()
+
+    group = head.groups++
+
+    tx.putGroup(topic, group)
+    tx.setHead(head)
+    tx.apply()
+
+    return {
+      key: topic,
+      pointer: group
+    }
+  }
+
+  async createGroup(topic) {
+    if (this.version === 0) await this._migrateStore()
+
+    const view = await this._enter()
+
+    try {
+      return await this._createGroup(view, topic)
+    } finally {
+      await this._exit()
+    }
+  }
+
   static isParentStorage(storage, parent) {
     return HypercoreStorage.isParentStorage(storage, parent)
   }
@@ -1132,10 +1196,9 @@ module.exports = CorestoreStorage
 function initStoreHead() {
   return {
     version: 0, // cause we wanna run the migration
-    allocated: {
-      datas: 0,
-      cores: 0
-    },
+    datas: 0,
+    cores: 0,
+    groups: 0,
     seed: null,
     defaultDiscoveryKey: null
   }

package/lib/keys.js

@@ -7,11 +7,14 @@ const TL_CORE_BY_DKEY = 1
 const TL_CORE_BY_ALIAS = 2
 const TL_CORE = 3
 const TL_DATA = 4
+const TL_GROUP_BY_TOPIC = 5
+const TL_GROUP = 6
 
-const TL_END = TL_DATA + 1
+const TL_END = TL_GROUP + 1
 
 const CORE_AUTH = 0
 const CORE_SESSIONS = 1
+const CORE_GROUP = 2
 
 const DATA_HEAD = 0
 const DATA_DEPENDENCY = 1
@@ -27,6 +30,7 @@ const slab = { buffer: b4a.allocUnsafe(65536), start: 0, end: 0 }
 
 const store = {}
 const core = {}
+const group = {}
 
 store.clear = function () {
   const state = alloc()
@@ -139,6 +143,15 @@ core.auth = function (ptr) {
   return state.buffer.subarray(start, state.start)
 }
 
+core.group = function (ptr) {
+  const state = alloc()
+  const start = state.start
+  UINT.encode(state, TL_CORE)
+  UINT.encode(state, ptr)
+  UINT.encode(state, CORE_GROUP)
+  return state.buffer.subarray(start, state.start)
+}
+
 core.sessions = function (ptr) {
   const state = alloc()
   const start = state.start
@@ -300,7 +313,25 @@ core.localKey = function (buffer) {
   return state.buffer.subarray(state.start, state.end)
 }
 
-module.exports = { store, core }
+group.topic = function (topic) {
+  const state = alloc()
+  const start = state.start
+  UINT.encode(state, TL_GROUP_BY_TOPIC)
+  c.fixed32.encode(state, topic)
+  return state.buffer.subarray(start, state.start)
+}
+
+group.entry = function (group, timestamp, ptr) {
+  const state = alloc()
+  const start = state.start
+  UINT.encode(state, TL_GROUP)
+  UINT.encode(state, group)
+  c.uint64be.encode(state, timestamp)
+  UINT.encode(state, ptr)
+  return state.buffer.subarray(start, state.start)
+}
+
+module.exports = { store, core, group }
 
 function alloc() {
   if (slab.buffer.byteLength - slab.start < 4096) {

package/lib/streams.js

@@ -1,7 +1,7 @@
 const b4a = require('b4a')
 const BlockDependencyStream = require('./block-dependency-stream.js')
-const { core, store } = require('./keys.js')
-const schema = require('../spec/hyperschema')
+const { core, store, group } = require('./keys.js')
+const schema = require('../encoding/spec/hyperschema')
 
 const CORESTORE_CORE = schema.getEncoding('@corestore/core')
 const CORE_TREE_NODE = schema.getEncoding('@core/tree-node')
@@ -16,7 +16,8 @@ module.exports = {
   createDiscoveryKeyStream,
   createTreeNodeStream,
   createMarkStream,
-  createLocalStream
+  createLocalStream,
+  createGroupUpdateStream
 }
 
 function createCoreStream(db, view) {
@@ -49,6 +50,16 @@ function createAliasStream(db, view, namespace) {
   return ite
 }
 
+function createGroupUpdateStream(db, view, groupID, { gte = 0, reverse = true } = {}) {
+  const start = group.entry(groupID, gte, 0)
+  const end = group.entry(groupID + 1, 0, 0)
+
+  const ite = view.iterator(db, start, end, reverse)
+
+  ite._readableState.map = mapGroupUpdate
+  return ite
+}
+
 function createBlockIterator(ptr, db, view, start, end, reverse) {
   if (ptr.dependencies.length > 0) {
     return new BlockDependencyStream(ptr, db, view, start, end, reverse)
@@ -164,6 +175,10 @@ function mapLocal(data) {
   return { key, value: data.value }
 }
 
+function mapGroupUpdate(data) {
+  return data.value
+}
+
 function mapUserData(data) {
   const key = core.userDataKey(data.key)
   return { key, value: data.value }

package/lib/tx.js

@@ -1,13 +1,17 @@
-const schema = require('../spec/hyperschema')
-const { store, core } = require('./keys.js')
-const View = require('./view.js')
 const b4a = require('b4a')
+const c = require('compact-encoding')
 const flat = require('flat-tree')
 
+const schema = require('../encoding/spec/hyperschema')
+const { store, core, group } = require('./keys.js')
+const View = require('./view.js')
+
 const CORESTORE_HEAD = schema.getEncoding('@corestore/head')
 const CORESTORE_CORE = schema.getEncoding('@corestore/core')
+const CORESTORE_GROUP = c.uint
 
 const CORE_AUTH = schema.getEncoding('@core/auth')
+const CORE_GROUP = schema.getEncoding('@core/group')
 const CORE_SESSIONS = schema.getEncoding('@core/sessions')
 const CORE_HEAD = schema.getEncoding('@core/head')
 const CORE_TREE_NODE = schema.getEncoding('@core/tree-node')
@@ -43,6 +47,14 @@ class CoreTX {
     this.changes.push([core.dependency(this.core.dataPointer), encode(CORE_DEPENDENCY, dep), null])
   }
 
+  setGroup(group) {
+    this.changes.push([core.group(this.core.dataPointer), encode(CORE_GROUP, group), null])
+  }
+
+  deleteGroup() {
+    this.changes.push([core.group(this.core.dataPointer), null, null])
+  }
+
   setHints(hints) {
     this.changes.push([core.hints(this.core.dataPointer), encode(CORE_HINTS, hints), null])
   }
@@ -140,6 +152,14 @@ class CoreTX {
     ])
   }
 
+  putGroupUpdate(groupPtr, timestamp, key) {
+    this.changes.push([group.entry(groupPtr, timestamp, this.core.corePointer), key, null])
+  }
+
+  deleteGroupUpdate(groupPtr, timestamp) {
+    this.changes.push([group.entry(groupPtr, timestamp, this.core.corePointer), null, null])
+  }
+
   flush() {
     const changes = this.changes
     if (changes === null) return Promise.resolve(!this.view)
@@ -194,6 +214,13 @@ class CoreRX {
     )
   }
 
+  async getGroup() {
+    return await decode(
+      CORE_GROUP,
+      await this.view.get(this.read, core.group(this.core.corePointer))
+    )
+  }
+
   static async getHints(db, c) {
     return await decode(CORE_HINTS, await db.get(core.hints(c.dataPointer)))
   }
@@ -276,6 +303,14 @@ class CorestoreTX {
     this.changes.push([store.coreByAlias(alias), discoveryKey, null])
   }
 
+  putGroup(topic, groupPtr) {
+    this.changes.push([group.topic(topic), encode(CORESTORE_GROUP, groupPtr), null])
+  }
+
+  deleteGroup(topic) {
+    this.changes.push([group.topic(topic), null, null])
+  }
+
   clear() {
     const [start, end] = store.clear()
     this.changes.push([start, null, end])
@@ -296,6 +331,10 @@ class CorestoreRX {
     view.readStart()
   }
 
+  async getHeadRaw(db) {
+    return await this.view.get(this.read, store.head())
+  }
+
   async getHead() {
     return decode(CORESTORE_HEAD, await this.view.get(this.read, store.head()))
   }
@@ -308,6 +347,10 @@ class CorestoreRX {
     return this.view.get(this.read, store.coreByAlias(alias))
   }
 
+  async getGroup(topic) {
+    return decode(CORESTORE_GROUP, await this.view.get(this.read, group.topic(topic)))
+  }
+
   tryFlush() {
     this.read.tryFlush()
     this._free()

package/migrations/0/messages.js

@@ -364,14 +364,14 @@ const dataUpgrade = {
     c.uint.preencode(state, u.length)
     nodeArray.preencode(state, u.nodes)
     nodeArray.preencode(state, u.additionalNodes)
-    c.buffer.preencode(state, u.signature)
+    c.optionalBuffer.preencode(state, u.signature)
   },
   encode (state, u) {
     c.uint.encode(state, u.start)
     c.uint.encode(state, u.length)
     nodeArray.encode(state, u.nodes)
     nodeArray.encode(state, u.additionalNodes)
-    c.buffer.encode(state, u.signature)
+    c.optionalBuffer.encode(state, u.signature)
   },
   decode (state) {
     return {
@@ -379,7 +379,7 @@ const dataUpgrade = {
       length: c.uint.decode(state),
       nodes: nodeArray.decode(state),
       additionalNodes: nodeArray.decode(state),
-      signature: c.buffer.decode(state)
+      signature: c.optionalBuffer.decode(state)
     }
   }
 }
@@ -404,18 +404,18 @@ const dataSeek = {
 const dataBlock = {
   preencode (state, b) {
     c.uint.preencode(state, b.index)
-    c.buffer.preencode(state, b.value)
+    c.optionalBuffer.preencode(state, b.value)
     nodeArray.preencode(state, b.nodes)
   },
   encode (state, b) {
     c.uint.encode(state, b.index)
-    c.buffer.encode(state, b.value)
+    c.optionalBuffer.encode(state, b.value)
     nodeArray.encode(state, b.nodes)
   },
   decode (state) {
     return {
       index: c.uint.decode(state),
-      value: c.buffer.decode(state) || EMPTY,
+      value: c.optionalBuffer.decode(state) || EMPTY,
       nodes: nodeArray.decode(state)
     }
   }
@@ -633,16 +633,16 @@ wire.extension = {
 const keyValue = {
   preencode (state, p) {
     c.string.preencode(state, p.key)
-    c.buffer.preencode(state, p.value)
+    c.optionalBuffer.preencode(state, p.value)
   },
   encode (state, p) {
     c.string.encode(state, p.key)
-    c.buffer.encode(state, p.value)
+    c.optionalBuffer.encode(state, p.value)
   },
   decode (state) {
     return {
       key: c.string.decode(state),
-      value: c.buffer.decode(state)
+      value: c.optionalBuffer.decode(state)
     }
   }
 }
@@ -652,20 +652,20 @@ const treeUpgrade = {
     c.uint.preencode(state, u.fork)
     c.uint.preencode(state, u.ancestors)
     c.uint.preencode(state, u.length)
-    c.buffer.preencode(state, u.signature)
+    c.optionalBuffer.preencode(state, u.signature)
   },
   encode (state, u) {
     c.uint.encode(state, u.fork)
     c.uint.encode(state, u.ancestors)
     c.uint.encode(state, u.length)
-    c.buffer.encode(state, u.signature)
+    c.optionalBuffer.encode(state, u.signature)
   },
   decode (state) {
     return {
       fork: c.uint.decode(state),
       ancestors: c.uint.decode(state),
       length: c.uint.decode(state),
-      signature: c.buffer.decode(state)
+      signature: c.optionalBuffer.decode(state)
     }
   }
 }
@@ -737,17 +737,17 @@ oplog.entry = {
 
 const keyPair = {
   preencode (state, kp) {
-    c.buffer.preencode(state, kp.publicKey)
-    c.buffer.preencode(state, kp.secretKey)
+    c.optionalBuffer.preencode(state, kp.publicKey)
+    c.optionalBuffer.preencode(state, kp.secretKey)
   },
   encode (state, kp) {
-    c.buffer.encode(state, kp.publicKey)
-    c.buffer.encode(state, kp.secretKey)
+    c.optionalBuffer.encode(state, kp.publicKey)
+    c.optionalBuffer.encode(state, kp.secretKey)
   },
   decode (state) {
     return {
-      publicKey: c.buffer.decode(state),
-      secretKey: c.buffer.decode(state)
+      publicKey: c.optionalBuffer.decode(state),
+      secretKey: c.optionalBuffer.decode(state)
     }
   }
 }
@@ -795,21 +795,21 @@ const treeHeader = {
   preencode (state, t) {
     c.uint.preencode(state, t.fork)
     c.uint.preencode(state, t.length)
-    c.buffer.preencode(state, t.rootHash)
-    c.buffer.preencode(state, t.signature)
+    c.optionalBuffer.preencode(state, t.rootHash)
+    c.optionalBuffer.preencode(state, t.signature)
   },
   encode (state, t) {
     c.uint.encode(state, t.fork)
     c.uint.encode(state, t.length)
-    c.buffer.encode(state, t.rootHash)
-    c.buffer.encode(state, t.signature)
+    c.optionalBuffer.encode(state, t.rootHash)
+    c.optionalBuffer.encode(state, t.signature)
   },
   decode (state) {
     return {
       fork: c.uint.decode(state),
       length: c.uint.decode(state),
-      rootHash: c.buffer.decode(state),
-      signature: c.buffer.decode(state)
+      rootHash: c.optionalBuffer.decode(state),
+      signature: c.optionalBuffer.decode(state)
     }
   }
 }

package/package.json

@@ -1,11 +1,11 @@
 {
   "name": "hypercore-storage",
-  "version": "2.8.0",
+  "version": "3.0.0",
   "main": "index.js",
   "files": [
     "index.js",
     "lib/*.js",
-    "spec/hyperschema/*.js",
+    "encoding/*",
     "migrations/0/*.js"
   ],
   "scripts": {
@@ -41,11 +41,11 @@
     "b4a": "^1.6.7",
     "bare-fs": "^4.0.1",
     "bare-path": "^3.0.0",
-    "compact-encoding": "^2.16.0",
+    "compact-encoding": "^3.1.0",
     "device-file": "^2.1.2",
     "flat-tree": "^1.12.1",
     "hypercore-crypto": "^3.4.2",
-    "hyperschema": "^1.7.0",
+    "hyperschema": "^1.21.0",
     "index-encoder": "^3.3.2",
     "resolve-reject-promise": "^1.0.0",
     "rocksdb-native": "^3.11.0",