hypercore-fetch 9.0.8 → 9.2.0

package/README.md

@@ -48,6 +48,8 @@ Each response will contain a header for the canonical URL represented as a `Link
 
 There is also an `ETag` header which will be a JSON string containging the drive's current `version`, or the file's sequence number.
 This will change only when the drive has gotten an update of some sort and is monotonically incrementing.
+The `ETag` representing a file's sequence number represents the version the Hyperdrive was at when the file was added.
+Thus you can get the previous version of a file by using `hyper://NAME/$/version/${ETAG}/example.txt`.
 
 If the resource is a file, it may contain the `Last-Modified` header if the file has had a `metadata.mtime` flag set upon update.
 
@@ -188,3 +190,13 @@ The `body` of the request will be used as the payload.
 Please note that only utf8 encoded text is currently supported due to limitations of the event-stream encoding.
 
 Note that this requires the `extensionMessages: true` flag.
+
+### `fetch('hyper://NAME/$/version/VERSION_NUMBER/example.txt')`
+
+You can get older views of data in an archive by using the special `/$/version` folder with a version number to view older states.
+
+`VERSION_NUMBER` should be a number representing the version to check out based on the `ETag` of the root of the archive.
+
+From there, you can use `GET` and `HEAD` requests with allt he same headers and querystring paramters as non-versioned paths to data.
+
+Note that you cannot `PUT` or `DELETE` data in a versioned folder.

package/index.js

@@ -13,6 +13,7 @@ const SPECIAL_DOMAIN = 'localhost'
 const SPECIAL_FOLDER = '$'
 const EXTENSIONS_FOLDER_NAME = 'extensions'
 const EXTENSION_EVENT = 'extension-message'
+const VERSION_FOLDER_NAME = 'version'
 const PEER_OPEN = 'peer-open'
 const PEER_REMOVE = 'peer-remove'
 
@@ -39,6 +40,11 @@ async function DEFAULT_RENDER_INDEX (url, files, fetch) {
 `
 }
 
+// Support gemini files
+mime.define({
+  'text/gemini': ['gmi', 'gemini']
+}, true)
+
 export default async function makeHyperFetch ({
   sdk,
   writable = false,
@@ -54,6 +60,26 @@ export default async function makeHyperFetch ({
   const extensions = new Map()
   const cores = new Map()
 
+  if (extensionMessages) {
+    router.get(`hyper://*/${SPECIAL_FOLDER}/${EXTENSIONS_FOLDER_NAME}/`, listExtensions)
+    router.get(`hyper://*/${SPECIAL_FOLDER}/${EXTENSIONS_FOLDER_NAME}/*`, listenExtension)
+    router.post(`hyper://*/${SPECIAL_FOLDER}/${EXTENSIONS_FOLDER_NAME}/*`, broadcastExtension)
+    router.post(`hyper://*/${SPECIAL_FOLDER}/${EXTENSIONS_FOLDER_NAME}/*/*`, extensionToPeer)
+  }
+
+  if (writable) {
+    router.get(`hyper://${SPECIAL_DOMAIN}/`, getKey)
+    router.post(`hyper://${SPECIAL_DOMAIN}/`, createKey)
+
+    router.put('hyper://*/**', putFiles)
+    router.delete('hyper://*/**', deleteFiles)
+  }
+
+  router.head(`hyper://*/${SPECIAL_FOLDER}/${VERSION_FOLDER_NAME}/**`, headFilesVersioned)
+  router.get(`hyper://*/${SPECIAL_FOLDER}/${VERSION_FOLDER_NAME}/**`, getFilesVersioned)
+  router.get('hyper://*/**', getFiles)
+  router.head('hyper://*/**', headFiles)
+
   async function getCore (hostname) {
     if (cores.has(hostname)) {
       return cores.get(hostname)
@@ -155,225 +181,246 @@ export default async function makeHyperFetch ({
     return [...core.extensions.keys()]
   }
 
-  if (extensionMessages) {
-    router.get(`hyper://*/${SPECIAL_FOLDER}/${EXTENSIONS_FOLDER_NAME}/`, async function listExtensions (request) {
-      const { hostname } = new URL(request.url)
-      const accept = request.headers.get('Accept') || ''
+  async function listExtensions (request) {
+    const { hostname } = new URL(request.url)
+    const accept = request.headers.get('Accept') || ''
 
-      const core = await getCore(`hyper://${hostname}/`)
+    const core = await getCore(`hyper://${hostname}/`)
 
-      if (accept.includes('text/event-stream')) {
-        const events = new EventIterator(({ push }) => {
-          function onMessage (name, content, peer) {
-            const id = peer.remotePublicKey.toString('hex')
-            // TODO: Fancy verification on the `name`?
-            // Send each line of content separately on a `data` line
-            const data = content.split('\n').map((line) => `data:${line}\n`).join('')
+    if (accept.includes('text/event-stream')) {
+      const events = new EventIterator(({ push }) => {
+        function onMessage (name, content, peer) {
+          const id = peer.remotePublicKey.toString('hex')
+          // TODO: Fancy verification on the `name`?
+          // Send each line of content separately on a `data` line
+          const data = content.split('\n').map((line) => `data:${line}\n`).join('')
 
-            push(`id:${id}\nevent:${name}\n${data}\n`)
-          }
-          function onPeerOpen (peer) {
-            const id = peer.remotePublicKey.toString('hex')
-            push(`id:${id}\nevent:${PEER_OPEN}\n\n`)
-          }
-          function onPeerRemove (peer) {
-            // Whatever, probably an uninitialized peer
-            if (!peer.remotePublicKey) return
-            const id = peer.remotePublicKey.toString('hex')
-            push(`id:${id}\nevent:${PEER_REMOVE}\n\n`)
-          }
-          core.on(EXTENSION_EVENT, onMessage)
-          core.on(PEER_OPEN, onPeerOpen)
-          core.on(PEER_REMOVE, onPeerRemove)
-          return () => {
-            core.removeListener(EXTENSION_EVENT, onMessage)
-            core.removeListener(PEER_OPEN, onPeerOpen)
-            core.removeListener(PEER_REMOVE, onPeerRemove)
-          }
-        })
-
-        return {
-          statusCode: 200,
-          headers: {
-            [HEADER_CONTENT_TYPE]: MIME_EVENT_STREAM
-          },
-          body: events
+          push(`id:${id}\nevent:${name}\n${data}\n`)
         }
-      }
+        function onPeerOpen (peer) {
+          const id = peer.remotePublicKey.toString('hex')
+          push(`id:${id}\nevent:${PEER_OPEN}\n\n`)
+        }
+        function onPeerRemove (peer) {
+          // Whatever, probably an uninitialized peer
+          if (!peer.remotePublicKey) return
+          const id = peer.remotePublicKey.toString('hex')
+          push(`id:${id}\nevent:${PEER_REMOVE}\n\n`)
+        }
+        core.on(EXTENSION_EVENT, onMessage)
+        core.on(PEER_OPEN, onPeerOpen)
+        core.on(PEER_REMOVE, onPeerRemove)
+        return () => {
+          core.removeListener(EXTENSION_EVENT, onMessage)
+          core.removeListener(PEER_OPEN, onPeerOpen)
+          core.removeListener(PEER_REMOVE, onPeerRemove)
+        }
+      })
 
-      const extensions = listExtensionNames(core)
       return {
-        status: 200,
-        headers: { [HEADER_CONTENT_TYPE]: MIME_APPLICATION_JSON },
-        body: JSON.stringify(extensions, null, '\t')
+        statusCode: 200,
+        headers: {
+          [HEADER_CONTENT_TYPE]: MIME_EVENT_STREAM
+        },
+        body: events
       }
-    })
-    router.get(`hyper://*/${SPECIAL_FOLDER}/${EXTENSIONS_FOLDER_NAME}/*`, async function listenExtension (request) {
-      const { hostname, pathname: rawPathname } = new URL(request.url)
-      const pathname = decodeURI(rawPathname)
-      const name = pathname.slice(`/${SPECIAL_FOLDER}/${EXTENSIONS_FOLDER_NAME}/`.length)
+    }
 
-      const core = await getCore(`hyper://${hostname}/`)
+    const extensions = listExtensionNames(core)
+    return {
+      status: 200,
+      headers: { [HEADER_CONTENT_TYPE]: MIME_APPLICATION_JSON },
+      body: JSON.stringify(extensions, null, '\t')
+    }
+  }
 
-      await getExtension(core, name)
+  async function listenExtension (request) {
+    const { hostname, pathname: rawPathname } = new URL(request.url)
+    const pathname = decodeURI(rawPathname)
+    const name = pathname.slice(`/${SPECIAL_FOLDER}/${EXTENSIONS_FOLDER_NAME}/`.length)
 
-      const peers = await getExtensionPeers(core, name)
-      const finalPeers = formatPeers(peers)
-      const body = JSON.stringify(finalPeers, null, '\t')
+    const core = await getCore(`hyper://${hostname}/`)
 
-      return {
-        status: 200,
-        body,
-        headers: {
-          [HEADER_CONTENT_TYPE]: MIME_APPLICATION_JSON
-        }
+    await getExtension(core, name)
+
+    const peers = await getExtensionPeers(core, name)
+    const finalPeers = formatPeers(peers)
+    const body = JSON.stringify(finalPeers, null, '\t')
+
+    return {
+      status: 200,
+      body,
+      headers: {
+        [HEADER_CONTENT_TYPE]: MIME_APPLICATION_JSON
       }
-    })
-    router.post(`hyper://*/${SPECIAL_FOLDER}/${EXTENSIONS_FOLDER_NAME}/*`, async function broadcastExtension (request) {
-      const { hostname, pathname: rawPathname } = new URL(request.url)
-      const pathname = decodeURI(rawPathname)
+    }
+  }
 
-      const name = pathname.slice(`/${SPECIAL_FOLDER}/${EXTENSIONS_FOLDER_NAME}/`.length)
+  async function broadcastExtension (request) {
+    const { hostname, pathname: rawPathname } = new URL(request.url)
+    const pathname = decodeURI(rawPathname)
 
-      const core = await getCore(`hyper://${hostname}/`)
+    const name = pathname.slice(`/${SPECIAL_FOLDER}/${EXTENSIONS_FOLDER_NAME}/`.length)
 
-      const extension = await getExtension(core, name)
-      const data = await request.text()
-      extension.broadcast(data)
+    const core = await getCore(`hyper://${hostname}/`)
 
-      return { status: 200 }
-    })
-    router.post(`hyper://*/${SPECIAL_FOLDER}/${EXTENSIONS_FOLDER_NAME}/*/*`, async function extensionToPeer (request) {
-      const { hostname, pathname: rawPathname } = new URL(request.url)
-      const pathname = decodeURI(rawPathname)
+    const extension = await getExtension(core, name)
+    const data = await request.text()
+    extension.broadcast(data)
 
-      const subFolder = pathname.slice(`/${SPECIAL_FOLDER}/${EXTENSIONS_FOLDER_NAME}/`.length)
-      const [name, extensionPeer] = subFolder.split('/')
+    return { status: 200 }
+  }
+
+  async function extensionToPeer (request) {
+    const { hostname, pathname: rawPathname } = new URL(request.url)
+    const pathname = decodeURI(rawPathname)
 
-      const core = await getCore(`hyper://${hostname}/`)
+    const subFolder = pathname.slice(`/${SPECIAL_FOLDER}/${EXTENSIONS_FOLDER_NAME}/`.length)
+    const [name, extensionPeer] = subFolder.split('/')
 
-      const extension = await getExtension(core, name)
-      const peers = await getExtensionPeers(core, name)
-      const peer = peers.find(({ remotePublicKey }) => remotePublicKey.toString('hex') === extensionPeer)
-      if (!peer) {
+    const core = await getCore(`hyper://${hostname}/`)
+
+    const extension = await getExtension(core, name)
+    const peers = await getExtensionPeers(core, name)
+    const peer = peers.find(({ remotePublicKey }) => remotePublicKey.toString('hex') === extensionPeer)
+    if (!peer) {
+      return {
+        status: 404,
+        headers: {
+          [HEADER_CONTENT_TYPE]: MIME_TEXT_PLAIN
+        },
+        body: 'Peer Not Found'
+      }
+    }
+    const data = await request.arrayBuffer()
+    extension.send(data, peer)
+    return { status: 200 }
+  }
+
+  async function getKey (request) {
+    const key = new URL(request.url).searchParams.get('key')
+    if (!key) {
+      return { status: 400, body: 'Must specify key parameter to resolve' }
+    }
+
+    try {
+      const drive = await getDriveFromKey(key, true)
+
+      return { body: drive.url }
+    } catch (e) {
+      if (e.message === ERROR_KEY_NOT_CREATED) {
         return {
-          status: 404,
+          status: 400,
+          body: e.message,
           headers: {
             [HEADER_CONTENT_TYPE]: MIME_TEXT_PLAIN
-          },
-          body: 'Peer Not Found'
+          }
         }
-      }
-      const data = await request.arrayBuffer()
-      extension.send(data, peer)
-      return { status: 200 }
-    })
+      } else throw e
+    }
   }
 
-  if (writable) {
-    router.get(`hyper://${SPECIAL_DOMAIN}/`, async function getKey (request) {
-      const key = new URL(request.url).searchParams.get('key')
-      if (!key) {
-        return { status: 400, body: 'Must specify key parameter to resolve' }
-      }
+  async function createKey (request) {
+    // TODO: Allow importing secret keys here
+    // Maybe specify a seed to use for generating the blobs?
+    // Else we'd need to specify the blobs keys and metadata keys
 
-      try {
-        const drive = await getDriveFromKey(key, true)
+    const key = new URL(request.url).searchParams.get('key')
+    if (!key) {
+      return { status: 400, body: 'Must specify key parameter to resolve' }
+    }
 
-        return { body: drive.url }
-      } catch (e) {
-        if (e.message === ERROR_KEY_NOT_CREATED) {
-          return {
-            status: 400,
-            body: e.message,
-            headers: {
-              [HEADER_CONTENT_TYPE]: MIME_TEXT_PLAIN
-            }
-          }
-        } else throw e
-      }
-    })
-    router.post(`hyper://${SPECIAL_DOMAIN}/`, async function createKey (request) {
-      // TODO: Allow importing secret keys here
-      // Maybe specify a seed to use for generating the blobs?
-      // Else we'd need to specify the blobs keys and metadata keys
-
-      const key = new URL(request.url).searchParams.get('key')
-      if (!key) {
-        return { status: 400, body: 'Must specify key parameter to resolve' }
-      }
+    const drive = await getDriveFromKey(key, false)
 
-      const drive = await getDriveFromKey(key, false)
+    return { body: drive.url }
+  }
 
-      return { body: drive.url }
-    })
+  async function putFiles (request) {
+    const { hostname, pathname: rawPathname } = new URL(request.url)
+    const pathname = decodeURI(rawPathname)
+    const contentType = request.headers.get('Content-Type') || ''
+    const isFormData = contentType.includes('multipart/form-data')
 
-    router.put('hyper://*/**', async function putFiles (request) {
-      const { hostname, pathname: rawPathname } = new URL(request.url)
-      const pathname = decodeURI(rawPathname)
-      const contentType = request.headers.get('Content-Type') || ''
-      const isFormData = contentType.includes('multipart/form-data')
-
-      const drive = await getDrive(`hyper://${hostname}`)
-
-      if (isFormData) {
-        // It's a form! Get the files out and process them
-        const formData = await request.formData()
-        for (const [name, data] of formData) {
-          if (name !== 'file') continue
-          const filePath = posix.join(pathname, data.name)
-          await pipelinePromise(
-            Readable.from(data.stream()),
-            drive.createWriteStream(filePath, {
-              metadata: {
-                mtime: Date.now()
-              }
-            })
-          )
-        }
-      } else {
+    const drive = await getDrive(`hyper://${hostname}`)
+
+    if (isFormData) {
+      // It's a form! Get the files out and process them
+      const formData = await request.formData()
+      for (const [name, data] of formData) {
+        if (name !== 'file') continue
+        const filePath = posix.join(pathname, data.name)
         await pipelinePromise(
-          Readable.from(request.body),
-          drive.createWriteStream(pathname, {
+          Readable.from(data.stream()),
+          drive.createWriteStream(filePath, {
             metadata: {
               mtime: Date.now()
             }
           })
         )
       }
+    } else {
+      await pipelinePromise(
+        Readable.from(request.body),
+        drive.createWriteStream(pathname, {
+          metadata: {
+            mtime: Date.now()
+          }
+        })
+      )
+    }
 
-      return { status: 201, headers: { Location: request.url } }
-    })
-    router.delete('hyper://*/**', async function putFiles (request) {
-      const { hostname, pathname: rawPathname } = new URL(request.url)
-      const pathname = decodeURI(rawPathname)
-
-      const drive = await getDrive(`hyper://${hostname}`)
+    return { status: 201, headers: { Location: request.url } }
+  }
 
-      if (pathname.endsWith('/')) {
-        let didDelete = false
-        for await (const entry of drive.list(pathname)) {
-          await drive.del(entry.key)
-          didDelete = true
-        }
-        if (!didDelete) {
-          return { status: 404, body: 'Not Found', headers: { [HEADER_CONTENT_TYPE]: MIME_TEXT_PLAIN } }
-        }
-        return { status: 200 }
-      }
+  async function deleteFiles (request) {
+    const { hostname, pathname: rawPathname } = new URL(request.url)
+    const pathname = decodeURI(rawPathname)
 
-      const entry = await drive.entry(pathname)
+    const drive = await getDrive(`hyper://${hostname}`)
 
-      if (!entry) {
+    if (pathname.endsWith('/')) {
+      let didDelete = false
+      for await (const entry of drive.list(pathname)) {
+        await drive.del(entry.key)
+        didDelete = true
+      }
+      if (!didDelete) {
         return { status: 404, body: 'Not Found', headers: { [HEADER_CONTENT_TYPE]: MIME_TEXT_PLAIN } }
       }
-      await drive.del(pathname)
-
       return { status: 200 }
-    })
+    }
+
+    const entry = await drive.entry(pathname)
+
+    if (!entry) {
+      return { status: 404, body: 'Not Found', headers: { [HEADER_CONTENT_TYPE]: MIME_TEXT_PLAIN } }
+    }
+    await drive.del(pathname)
+
+    return { status: 200 }
   }
 
-  router.head('hyper://*/**', async function headFiles (request) {
+  async function headFilesVersioned (request) {
+    const url = new URL(request.url)
+    const { hostname, pathname: rawPathname, searchParams } = url
+    const pathname = decodeURI(rawPathname)
+
+    const accept = request.headers.get('Accept') || ''
+    const isRanged = request.headers.get('Range') || ''
+    const noResolve = searchParams.has('noResolve')
+
+    const parts = pathname.split('/')
+    const version = parts[3]
+    const realPath = parts.slice(4).join('/')
+
+    const drive = await getDrive(`hyper://${hostname}`)
+
+    const snapshot = await drive.checkout(version)
+
+    return serveHead(snapshot, realPath, { accept, isRanged, noResolve })
+  }
+
+  async function headFiles (request) {
     const url = new URL(request.url)
     const { hostname, pathname: rawPathname, searchParams } = url
     const pathname = decodeURI(rawPathname)
@@ -381,9 +428,14 @@ export default async function makeHyperFetch ({
     const accept = request.headers.get('Accept') || ''
     const isRanged = request.headers.get('Range') || ''
     const noResolve = searchParams.has('noResolve')
-    const isDirectory = pathname.endsWith('/')
 
     const drive = await getDrive(`hyper://${hostname}`)
+
+    return serveHead(drive, pathname, { accept, isRanged, noResolve })
+  }
+
+  async function serveHead (drive, pathname, { accept, isRanged, noResolve }) {
+    const isDirectory = pathname.endsWith('/')
     const fullURL = new URL(pathname, drive.url).href
 
     const resHeaders = {
@@ -478,19 +530,45 @@ export default async function makeHyperFetch ({
       status: 200,
       headers: resHeaders
     }
-  })
+  }
+
+  async function getFilesVersioned (request) {
+    const url = new URL(request.url)
+    const { hostname, pathname: rawPathname, searchParams } = url
+    const pathname = decodeURI(rawPathname)
+
+    const accept = request.headers.get('Accept') || ''
+    const isRanged = request.headers.get('Range') || ''
+    const noResolve = searchParams.has('noResolve')
+
+    const parts = pathname.split('/')
+    const version = parts[3]
+    const realPath = parts.slice(4).join('/')
+
+    const drive = await getDrive(`hyper://${hostname}`)
+
+    const snapshot = await drive.checkout(version)
+
+    return serveGet(snapshot, realPath, { accept, isRanged, noResolve })
+  }
 
   // TODO: Redirect on directories without trailing slash
-  router.get('hyper://*/**', async function getFiles (request) {
+  async function getFiles (request) {
     const url = new URL(request.url)
     const { hostname, pathname: rawPathname, searchParams } = url
     const pathname = decodeURI(rawPathname)
 
     const accept = request.headers.get('Accept') || ''
+    const isRanged = request.headers.get('Range') || ''
     const noResolve = searchParams.has('noResolve')
-    const isDirectory = pathname.endsWith('/')
 
     const drive = await getDrive(`hyper://${hostname}`)
+
+    return serveGet(drive, pathname, { accept, isRanged, noResolve })
+  }
+
+  async function serveGet (drive, pathname, { accept, isRanged, noResolve }) {
+    const isDirectory = pathname.endsWith('/')
     const fullURL = new URL(pathname, drive.url).href
 
     if (isDirectory) {
@@ -514,12 +592,12 @@ export default async function makeHyperFetch ({
 
       if (!noResolve) {
         if (entries.includes('index.html')) {
-          return serveFile(request.headers, drive, posix.join(pathname, 'index.html'))
+          return serveFile(drive, posix.join(pathname, 'index.html'), isRanged)
         }
       }
 
       if (accept.includes('text/html')) {
-        const body = await renderIndex(url, entries, fetch)
+        const body = await renderIndex(new URL(fullURL), entries, fetch)
         return {
           status: 200,
           body,
@@ -546,14 +624,13 @@ export default async function makeHyperFetch ({
       return { status: 404, body: 'Not Found' }
     }
 
-    return serveFile(request.headers, drive, path)
-  })
+    return serveFile(drive, path, isRanged)
+  }
 
   return fetch
 }
 
-async function serveFile (headers, drive, pathname) {
-  const isRanged = headers.get('Range') || ''
+async function serveFile (drive, pathname, isRanged) {
   const contentType = getMimeType(pathname)
 
   const fullURL = new URL(pathname, drive.url).href

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore-fetch",
-  "version": "9.0.8",
+  "version": "9.2.0",
   "description": "Implementation of Fetch that uses the Dat SDK for loading p2p content",
   "type": "module",
   "main": "index.js",

package/test.js

@@ -447,30 +447,57 @@ test('EventSource extension messages', async (t) => {
 })
 
 test('Resolve DNS', async (t) => {
-  const loadResponse = await fetch('hyper://example2.mauve.moe/')
+  const loadResponse = await fetch('hyper://example2.mauve.moe/?noResolve')
 
   const entries = await loadResponse.json()
 
-  t.pass('Loaded contents and ')
+  t.ok(entries.length, 'Loaded contents with some files present')
 })
 
 test('Error on invalid hostname', async (t) => {
   const loadResponse = await fetch('hyper://example/')
 
-  console.log(loadResponse.status)
-
-  if(loadResponse.ok) {
+  if (loadResponse.ok) {
     throw new Error('Loading without DNS or a public key should have failed')
   } else {
-    t.pass("Invalid names led to an error")
+    t.pass('Invalid names led to an error')
   }
 })
 
+test('GET older version of file from VERSION folder', async (t) => {
+  const created = await nextURL(t)
+
+  const fileName = 'example.txt'
+
+  const data1 = 'Hello World'
+  const data2 = 'Goodbye World'
+
+  const fileURL = new URL(`/${fileName}`, created)
+  const versionFileURL = new URL(`/$/version/2/${fileName}`, created)
+
+  await checkResponse(
+    await fetch(fileURL, { method: 'PUT', body: data1 }), t
+  )
+  await checkResponse(
+    await fetch(fileURL, { method: 'PUT', body: data2 }), t
+  )
+
+  const versionedResponse = await fetch(versionFileURL)
+
+  await checkResponse(versionedResponse, t, 'Able to GET versioned file')
+
+  const versionedData = await versionedResponse.text()
+
+  t.equal(versionedData, data1, 'Old data got loaded')
+})
+
 async function checkResponse (response, t, successMessage = 'Response OK') {
   if (!response.ok) {
     const message = await response.text()
     t.fail(new Error(`HTTP Error ${response.status}:\n${message}`))
+    return false
   } else {
     t.pass(successMessage)
+    return true
   }
 }