hypercore-storage 2.4.0 → 2.5.0

package/index.js

@@ -343,36 +343,6 @@ class HypercoreStorage {
     return this.atomize(atom)
   }
 
-  async createStaticCore() {
-    const rx = this.read()
-
-    const headPromise = rx.getHead()
-    const authPromise = rx.getAuth()
-
-    rx.tryFlush()
-
-    const [head, auth] = await Promise.all([headPromise, authPromise])
-    if (!head || head.length === 0) throw new Error('Must have data')
-
-    const prologue = {
-      length: head.length,
-      hash: head.rootHash
-    }
-
-    const manifest = {
-      version: 1,
-      hash: auth.manifest.hash,
-      quorum: 0,
-      signers: [],
-      prologue
-    }
-
-    return {
-      manifest,
-      core: { ...this.core, dependencies: this.core.dependencies.slice() }
-    }
-  }
-
   _addDependency(dep) {
     const deps = []
 
@@ -577,6 +547,21 @@ class CorestoreStorage {
     }
   }
 
+  async _maybeRecover() {
+    const r = path.join(this.db.path, 'RECOVERING')
+    if (!(await fileExists(r))) return
+    await this.setRecovering()
+    await fs.promises.unlink(r)
+  }
+
+  _maybeRecoverLater(err) {
+    if (!shouldRecover(err)) return
+    fs.writeFileSync(path.join(this.db.path, 'RECOVERING'), '')
+    for (const file of fs.readdirSync(this.db.path)) {
+      if (file.endsWith('.log')) fs.unlinkSync(path.join(this.db.path, file))
+    }
+  }
+
   // runs pre any other mutation and read
   async _migrateStore() {
     const view = await this._enter()
@@ -585,7 +570,15 @@ class CorestoreStorage {
       if (this.version === VERSION) return
 
       await this._preopen
-      await this.db.ready()
+
+      try {
+        await this.db.ready()
+      } catch (err) {
+        this._maybeRecoverLater(err)
+        throw err
+      }
+
+      await this._maybeRecover()
 
       const rx = new CorestoreRX(this.db, view)
       const headPromise = rx.getHead()
@@ -805,6 +798,35 @@ class CorestoreStorage {
     }
   }
 
+  async setRecovering() {
+    const recovering = Date.now()
+    let keys = []
+
+    for await (const data of this.createDiscoveryKeyStream()) {
+      keys.push(data)
+      if (keys.length > 16 * 1024) {
+        await this._flushRecovering(recovering, keys)
+        keys = []
+      }
+    }
+
+    if (keys.length) await this._flushRecovering(recovering, keys)
+  }
+
+  async _flushRecovering(recovering, keys) {
+    const view = await this._enter()
+    const tx = new CorestoreTX(view)
+
+    try {
+      const infos = await this.getInfos(keys, { auth: false, head: false, hints: true })
+      for (const inf of infos) {
+        tx.setCoreHints(inf.core, { ...inf.hints, recovering })
+      }
+    } finally {
+      await this._exit()
+    }
+  }
+
   async getDefaultDiscoveryKey() {
     if (this.version === 0) await this._migrateStore()
 
@@ -1158,6 +1180,19 @@ async function toArray(stream) {
   return all
 }
 
+async function fileExists(path) {
+  try {
+    await fs.promises.stat(path)
+    return true
+  } catch {
+    return false
+  }
+}
+
+function shouldRecover(err) {
+  return err.message.toLowerCase().indexOf('sequence number is being set backwards') > -1
+}
+
 function noop() {}
 
 async function getInfoFromBatch(db, c, discoveryKey, getAuth, getHead, getHints) {

package/lib/tx.js

@@ -244,6 +244,10 @@ class CorestoreTX {
     this.changes.push([store.head(), encode(CORESTORE_HEAD, head), null])
   }
 
+  setCoreHints(c, hints) {
+    this.changes.push([core.hints(c.dataPointer), encode(CORE_HINTS, hints), null])
+  }
+
   putCore(discoveryKey, ptr) {
     this.changes.push([store.core(discoveryKey), encode(CORESTORE_CORE, ptr), null])
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore-storage",
-  "version": "2.4.0",
+  "version": "2.5.0",
   "main": "index.js",
   "files": [
     "index.js",
@@ -11,9 +11,10 @@
   "scripts": {
     "format": "prettier --write .",
     "lint": "prettier --check . && lunte",
-    "test": "node test/all.js",
-    "test:bare": "bare test/all.js",
-    "test:generate": "brittle -r test/all.js test/*.js"
+    "test": "npm run test:node && npm run test:bare",
+    "test:bare": "brittle-bare test/all.js",
+    "test:generate": "brittle -r test/all.js test/*.js",
+    "test:node": "brittle-node test/all.js"
   },
   "author": "Holepunch Inc.",
   "license": "Apache-2.0",
@@ -45,7 +46,7 @@
   },
   "devDependencies": {
     "brittle": "^3.7.0",
-    "lunte": "^1.2.0",
+    "lunte": "^1.3.0",
     "prettier": "^3.6.2",
     "prettier-config-holepunch": "^2.0.0",
     "test-tmp": "^1.3.1"

package/spec/hyperschema/index.js

@@ -43,10 +43,7 @@ const encoding1 = {
     if (m.defaultDiscoveryKey) c.fixed32.preencode(state, m.defaultDiscoveryKey)
   },
   encode(state, m) {
-    const flags =
-      (m.allocated ? 1 : 0) |
-      (m.seed ? 2 : 0) |
-      (m.defaultDiscoveryKey ? 4 : 0)
+    const flags = (m.allocated ? 1 : 0) | (m.seed ? 2 : 0) | (m.defaultDiscoveryKey ? 4 : 0)
 
     c.uint.encode(state, m.version)
     c.uint.encode(state, flags)
@@ -269,10 +266,7 @@ const encoding9 = {
   },
   encode(state, m) {
     const flags =
-      (m.allowPatch ? 1 : 0) |
-      (m.prologue ? 2 : 0) |
-      (m.linked ? 4 : 0) |
-      (m.userData ? 8 : 0)
+      (m.allowPatch ? 1 : 0) | (m.prologue ? 2 : 0) | (m.linked ? 4 : 0) | (m.userData ? 8 : 0)
 
     c.uint.encode(state, m.version)
     c.uint.encode(state, flags)
@@ -337,10 +331,7 @@ const encoding11 = {
     if (m.encryptionKey) c.buffer.preencode(state, m.encryptionKey)
   },
   encode(state, m) {
-    const flags =
-      (m.manifest ? 1 : 0) |
-      (m.keyPair ? 2 : 0) |
-      (m.encryptionKey ? 4 : 0)
+    const flags = (m.manifest ? 1 : 0) | (m.keyPair ? 2 : 0) | (m.encryptionKey ? 4 : 0)
 
     c.fixed32.encode(state, m.key)
     c.fixed32.encode(state, m.discoveryKey)
@@ -397,27 +388,29 @@ const encoding12 = {
 // @core/hints
 const encoding13 = {
   preencode(state, m) {
-    state.end++ // max flag is 2 so always one byte
+    state.end++ // max flag is 4 so always one byte
 
     if (m.contiguousLength) c.uint.preencode(state, m.contiguousLength)
     if (m.remoteContiguousLength) c.uint.preencode(state, m.remoteContiguousLength)
+    if (m.recovering) c.uint.preencode(state, m.recovering)
   },
   encode(state, m) {
     const flags =
-      (m.contiguousLength ? 1 : 0) |
-      (m.remoteContiguousLength ? 2 : 0)
+      (m.contiguousLength ? 1 : 0) | (m.remoteContiguousLength ? 2 : 0) | (m.recovering ? 4 : 0)
 
     c.uint.encode(state, flags)
 
     if (m.contiguousLength) c.uint.encode(state, m.contiguousLength)
     if (m.remoteContiguousLength) c.uint.encode(state, m.remoteContiguousLength)
+    if (m.recovering) c.uint.encode(state, m.recovering)
   },
   decode(state) {
     const flags = c.uint.decode(state)
 
     return {
       contiguousLength: (flags & 1) !== 0 ? c.uint.decode(state) : 0,
-      remoteContiguousLength: (flags & 2) !== 0 ? c.uint.decode(state) : 0
+      remoteContiguousLength: (flags & 2) !== 0 ? c.uint.decode(state) : 0,
+      recovering: (flags & 4) !== 0 ? c.uint.decode(state) : 0
     }
   }
 }