hypercore-fetch 9.5.0 → 9.7.0

package/README.md

@@ -142,6 +142,16 @@ Note that you must use the name `file` for uploaded files.
 
 `NAME` can either be the 52 character [z32 encoded](https://github.com/mafintosh/z32) key for a Hyperdrive or Hypercore , or a domain to parse with the [DNSLink](https://www.dnslink.io/) standard.
 
+### `fetch('hyper://NAME/', {method: 'DELETE'})`
+
+You can purge all the stored data for a hyperdrive by sending a `DELETE` to it's root.
+
+If this is a writable drive, your data will get fully clearned and trying to write to it again will lead to data corruption.
+
+If you try to load this drive again data will be loaded from scratch.
+
+`NAME` can either be the 52 character [z32 encoded](https://github.com/mafintosh/z32) key for a Hyperdrive or Hypercore , or a domain to parse with the [DNSLink](https://www.dnslink.io/) standard.
+
 ### `fetch('hyper://NAME/example.txt', {method: 'DELETE'})`
 
 You can delete a file or directory tree in a Hyperdrive by using the `DELETE` method.

package/index.js

@@ -36,13 +36,16 @@ const BASIC_METHODS = [
 ]
 
 export const ERROR_KEY_NOT_CREATED = 'Must create key with POST before reading'
+export const ERROR_DRIVE_EMPTY = 'Could not find data in drive, make sure your key is correct and that there are peers online to load data from'
 
 const INDEX_FILES = [
   'index.html',
   'index.md',
   'index.gmi',
   'index.gemini',
-  'README.md'
+  'index.org',
+  'README.md',
+  'README.org'
 ]
 
 async function DEFAULT_RENDER_INDEX (url, files, fetch) {
@@ -70,7 +73,9 @@ export default async function makeHyperFetch ({
   timeout = DEFAULT_TIMEOUT,
   renderIndex = DEFAULT_RENDER_INDEX
 }) {
-  const { fetch, router } = makeRoutedFetch()
+  const { fetch, router } = makeRoutedFetch({
+    onError
+  })
 
   // Map loaded drive hostnames to their keys
   // TODO: Track LRU + cache clearing
@@ -91,6 +96,7 @@ export default async function makeHyperFetch ({
 
     router.put(`hyper://*/${SPECIAL_FOLDER}/${VERSION_FOLDER_NAME}/**`, putFilesVersioned)
     router.put('hyper://*/**', putFiles)
+    router.delete('hyper://*/', deleteDrive)
     router.delete(`hyper://*/${SPECIAL_FOLDER}/${VERSION_FOLDER_NAME}/**`, deleteFilesVersioned)
     router.delete('hyper://*/**', deleteFiles)
   }
@@ -100,6 +106,16 @@ export default async function makeHyperFetch ({
   router.get('hyper://*/**', getFiles)
   router.head('hyper://*/**', headFiles)
 
+  async function onError (e, request) {
+    return {
+      status: e.statusCode || 500,
+      headers: {
+        'Content-Type': 'text/plain; charset=utf-8'
+      },
+      body: e.stack
+    }
+  }
+
   async function getCore (hostname) {
     if (cores.has(hostname)) {
       return cores.get(hostname)
@@ -128,18 +144,30 @@ export default async function makeHyperFetch ({
     return dbCore
   }
 
-  async function getDrive (hostname) {
+  async function getDrive (hostname, errorOnNew = false) {
     if (drives.has(hostname)) {
       return drives.get(hostname)
     }
 
     const core = await getCore(hostname)
 
+    if (!core.length && errorOnNew) {
+      await core.close()
+      const e = new Error(ERROR_DRIVE_EMPTY)
+      e.statusCode = 404
+      throw e
+    }
+
     const corestore = sdk.namespace(core.id)
     const drive = new Hyperdrive(corestore, core.key)
 
     await drive.ready()
 
+    drive.once('close', () => {
+      drives.delete(drive.core.id)
+      drives.delete(hostname)
+    })
+
     drives.set(drive.core.id, drive)
     drives.set(hostname, drive)
 
@@ -151,8 +179,11 @@ export default async function makeHyperFetch ({
       return drives.get(key)
     }
     const core = await getDBCoreForName(key)
+
     if (!core.length && errorOnNew) {
-      throw new Error(ERROR_KEY_NOT_CREATED)
+      const e = new Error(ERROR_KEY_NOT_CREATED)
+      e.statusCode = 404
+      throw e
     }
 
     const corestore = sdk.namespace(key)
@@ -160,7 +191,14 @@ export default async function makeHyperFetch ({
 
     await drive.ready()
 
+    drive.once('close', () => {
+      drives.delete(key)
+      drives.delete(drive.url)
+      drives.delete(drive.core.id)
+    })
+
     drives.set(key, drive)
+    drives.set(drive.url, drive)
     drives.set(drive.core.id, drive)
 
     return drive
@@ -361,12 +399,14 @@ export default async function makeHyperFetch ({
     const contentType = request.headers.get('Content-Type') || ''
     const isFormData = contentType.includes('multipart/form-data')
 
-    const drive = await getDrive(`hyper://${hostname}`)
+    const drive = await getDrive(`hyper://${hostname}/`, true)
 
     if (!drive.db.feed.writable) {
       return { status: 403, body: `Cannot PUT file to read-only drive: ${drive.url}`, headers: { Location: request.url } }
     }
 
+    const mtime = Date.now()
+
     if (isFormData) {
       // It's a form! Get the files out and process them
       const formData = await request.formData()
@@ -377,7 +417,7 @@ export default async function makeHyperFetch ({
           Readable.from(data.stream()),
           drive.createWriteStream(filePath, {
             metadata: {
-              mtime: Date.now()
+              mtime
             }
           })
         )
@@ -390,25 +430,47 @@ export default async function makeHyperFetch ({
           Readable.from(request.body),
           drive.createWriteStream(pathname, {
             metadata: {
-              mtime: Date.now()
+              mtime
             }
           })
         )
       }
     }
 
-    return { status: 201, headers: { Location: request.url } }
+    const fullURL = new URL(pathname, drive.url).href
+
+    const headers = {
+      Location: request.url,
+      ETag: `${drive.version}`,
+      Link: `<${fullURL}>; rel="canonical"`,
+      [HEADER_LAST_MODIFIED]: isFormData ? undefined : new Date(mtime).toUTCString()
+    }
+
+    return { status: 201, headers }
   }
 
   function putFilesVersioned (request) {
-    return { status: 405, body: 'Cannot PUT file to old version', headers: { Location: request.url } }
+    return {
+      status: 405,
+      body: 'Cannot PUT file to old version',
+      headers: { Location: request.url }
+    }
+  }
+
+  async function deleteDrive (request) {
+    const { hostname } = new URL(request.url)
+    const drive = await getDrive(`hyper://${hostname}/`, true)
+
+    await drive.purge()
+
+    return { status: 200 }
   }
 
   async function deleteFiles (request) {
     const { hostname, pathname: rawPathname } = new URL(request.url)
     const pathname = decodeURI(ensureLeadingSlash(rawPathname))
 
-    const drive = await getDrive(`hyper://${hostname}`)
+    const drive = await getDrive(`hyper://${hostname}/`, true)
 
     if (!drive.db.feed.writable) {
       return { status: 403, body: `Cannot DELETE file in read-only drive: ${drive.url}`, headers: { Location: request.url } }
@@ -433,7 +495,14 @@ export default async function makeHyperFetch ({
     }
     await drive.del(pathname)
 
-    return { status: 200 }
+    const fullURL = new URL(pathname, drive.url).href
+
+    const headers = {
+      ETag: `${drive.version}`,
+      Link: `<${fullURL}>; rel="canonical"`
+    }
+
+    return { status: 200, headers }
   }
 
   function deleteFilesVersioned (request) {
@@ -453,7 +522,7 @@ export default async function makeHyperFetch ({
     const version = parts[3]
     const realPath = ensureLeadingSlash(parts.slice(4).join('/'))
 
-    const drive = await getDrive(`hyper://${hostname}`)
+    const drive = await getDrive(`hyper://${hostname}/`, true)
 
     const snapshot = await drive.checkout(version)
 
@@ -469,7 +538,7 @@ export default async function makeHyperFetch ({
     const isRanged = request.headers.get('Range') || ''
     const noResolve = searchParams.has('noResolve')
 
-    const drive = await getDrive(`hyper://${hostname}`)
+    const drive = await getDrive(`hyper://${hostname}/`, true)
 
     return serveHead(drive, pathname, { accept, isRanged, noResolve })
   }
@@ -555,7 +624,7 @@ export default async function makeHyperFetch ({
       resHeaders[HEADER_LAST_MODIFIED] = date.toUTCString()
     }
 
-    const size = entry.value.byteLength
+    const size = entry.value.blob.byteLength
     if (isRanged) {
       const ranges = parseRange(size, isRanged)
 
@@ -564,7 +633,7 @@ export default async function makeHyperFetch ({
         const length = (end - start + 1)
 
         return {
-          status: 200,
+          status: 204,
           headers: {
             ...resHeaders,
             'Content-Length': `${length}`,
@@ -575,7 +644,7 @@ export default async function makeHyperFetch ({
     }
 
     return {
-      status: 200,
+      status: 204,
       headers: resHeaders
     }
   }
@@ -593,7 +662,7 @@ export default async function makeHyperFetch ({
     const version = parts[3]
     const realPath = ensureLeadingSlash(parts.slice(4).join('/'))
 
-    const drive = await getDrive(`hyper://${hostname}`)
+    const drive = await getDrive(`hyper://${hostname}/`, true)
 
     const snapshot = await drive.checkout(version)
 
@@ -610,7 +679,7 @@ export default async function makeHyperFetch ({
     const isRanged = request.headers.get('Range') || ''
     const noResolve = searchParams.has('noResolve')
 
-    const drive = await getDrive(`hyper://${hostname}`)
+    const drive = await getDrive(`hyper://${hostname}/`, true)
 
     return serveGet(drive, pathname, { accept, isRanged, noResolve })
   }
@@ -741,7 +810,7 @@ function makeToTry (pathname) {
 
 async function resolvePath (drive, pathname, noResolve) {
   if (noResolve) {
-    const entry = drive.entry(pathname)
+    const entry = await drive.entry(pathname)
 
     return { entry, path: pathname }
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore-fetch",
-  "version": "9.5.0",
+  "version": "9.7.0",
   "description": "Implementation of Fetch that uses the Dat SDK for loading p2p content",
   "type": "module",
   "main": "index.js",
@@ -32,7 +32,7 @@
   },
   "devDependencies": {
     "@rangermauve/fetch-event-source": "^1.0.3",
-    "hyper-sdk": "^4.2.3",
+    "hyper-sdk": "^4.3.0",
     "standard": "^17.0.0",
     "tape": "^5.2.2"
   }

package/test.js

@@ -127,6 +127,7 @@ test('HEAD request', async (t) => {
   const headersLastModified = headResponse.headers.get('Last-Modified')
   const headersLink = headResponse.headers.get('Link')
 
+  t.equal(headResponse.status, 204, 'Response had expected status')
   // Version at which the file was added
   t.equal(headersEtag, '2', 'Headers got expected etag')
   t.equal(headersContentType, 'text/plain; charset=utf-8', 'Headers got expected mime type')
@@ -296,6 +297,24 @@ test('DELETE a directory', async (t) => {
   const entries = await listDirRequest.json()
   t.deepEqual(entries, [], 'subfolder got deleted')
 })
+test.only('DELETE a drive from storage', async (t) => {
+  const created = await nextURL(t)
+
+  const uploadLocation = new URL('./subfolder/example.txt', created)
+  const uploadResponse = await fetch(uploadLocation, {
+    method: 'put',
+    body: SAMPLE_CONTENT
+  })
+  await checkResponse(uploadResponse, t)
+
+  const purgeResponse = await fetch(created, { method: 'delete' })
+
+  await checkResponse(purgeResponse, t, 'Able to purge')
+
+  const listDirRequest = await fetch(created)
+
+  t.notOk(listDirRequest.ok, 'Error when trying to read after purge')
+})
 test('Read index.html', async (t) => {
   const created = await nextURL(t)
   const uploadLocation = new URL('./index.html', created)
@@ -333,6 +352,22 @@ test('Ignore index.html with noResolve', async (t) => {
   const entries = await listDirRequest.json()
   t.deepEqual(entries, ['index.html'], 'able to list index.html')
 })
+test('Ensure that noResolve works with file paths', async (t) => {
+  const created = await nextURL(t)
+  const uploadLocation = new URL('./example.txt', created)
+  const uploadResponse = await fetch(uploadLocation, {
+    method: 'put',
+    body: SAMPLE_CONTENT
+  })
+  await checkResponse(uploadResponse, t)
+
+  const noResolve = uploadLocation.href + '?noResolve'
+  const getRequest = await fetch(noResolve)
+  await checkResponse(getRequest, t)
+
+  const headRequest = await fetch(noResolve, { method: 'HEAD' })
+  await checkResponse(headRequest, t)
+})
 test('Render index.gmi', async (t) => {
   const created = await nextURL(t)
   const uploadLocation = new URL('./index.gmi', created)
@@ -403,14 +438,6 @@ test('Resolve pretty markdown URLs', async (t) => {
   t.equal(contentType, 'text/markdown; charset=utf-8', 'Got markdown mime type')
 })
 
-test('Doing a `GET` on an invalid domain should cause an error', async (t) => {
-  const url = 'hyper://example/'
-
-  const failedResponse = await fetch(url)
-
-  t.notOk(failedResponse.ok, 'Response errored out due to invalid domain')
-})
-
 test('EventSource extension messages', async (t) => {
   const domain = await nextURL(t)
 
@@ -474,14 +501,13 @@ test('Resolve DNS', async (t) => {
   t.ok(entries.length, 'Loaded contents with some files present')
 })
 
-test('Error on invalid hostname', async (t) => {
-  const loadResponse = await fetch('hyper://example/')
+test('Doing a `GET` on an invalid domain/public key should cause an error', async (t) => {
+  const invalidDomainResponse = await fetch('hyper://example/')
+  t.notOk(invalidDomainResponse.ok, 'Response errored out due to invalid domain')
 
-  if (loadResponse.ok) {
-    throw new Error('Loading without DNS or a public key should have failed')
-  } else {
-    t.pass('Invalid names led to an error')
-  }
+  const invalidPublicKeyResponse = await fetch('hyper://aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/')
+  t.notOk(invalidPublicKeyResponse.ok, 'Response errored out due to invalid public key')
+  t.equal(invalidPublicKeyResponse.status, 404, 'Invalid public key should 404')
 })
 
 test('Old versions in VERSION folder', async (t) => {