hypercore-fetch 9.6.0 → 9.8.0

package/README.md

@@ -120,13 +120,18 @@ Note that this is only available with the `writable: true` flag.
 
 ### `fetch('hyper://NAME/example.txt', {method: 'PUT', body: 'Hello World'})`
 
-You can add files to archives using a `PUT` method along with a `body`.
+You can add files to archives using a `PUT` method along with a
+`body`. Note that this is only available with the `writable: true`
+flag.
 
 The `body` can be any of the options supported by the Fetch API such as a `String`, `Blob`, `FormData`, or `ReadableStream`.
 
 `NAME` can either be the 52 character [z32 encoded](https://github.com/mafintosh/z32) key for a Hyperdrive or Hypercore , or a domain to parse with the [DNSLink](https://www.dnslink.io/) standard.
 
-Note that this is only available with the `writable: true` flag.
+The mtime metadata is automatically set to the current time when
+uploading. To override this value, pass a `Last-Modified` header with a value
+set to a date string according to [RFC
+7231](https://datatracker.ietf.org/doc/html/rfc7231#section-7.1.1.1).
 
 An attempt to `PUT` a file to a hyperdrive which is not writable will
 fail with status `403`.
@@ -142,6 +147,16 @@ Note that you must use the name `file` for uploaded files.
 
 `NAME` can either be the 52 character [z32 encoded](https://github.com/mafintosh/z32) key for a Hyperdrive or Hypercore , or a domain to parse with the [DNSLink](https://www.dnslink.io/) standard.
 
+### `fetch('hyper://NAME/', {method: 'DELETE'})`
+
+You can purge all the stored data for a hyperdrive by sending a `DELETE` to it's root.
+
+If this is a writable drive, your data will get fully clearned and trying to write to it again will lead to data corruption.
+
+If you try to load this drive again data will be loaded from scratch.
+
+`NAME` can either be the 52 character [z32 encoded](https://github.com/mafintosh/z32) key for a Hyperdrive or Hypercore , or a domain to parse with the [DNSLink](https://www.dnslink.io/) standard.
+
 ### `fetch('hyper://NAME/example.txt', {method: 'DELETE'})`
 
 You can delete a file or directory tree in a Hyperdrive by using the `DELETE` method.

package/index.js

@@ -3,7 +3,7 @@ import { posix } from 'path'
 import { Readable, pipelinePromise } from 'streamx'
 import Hyperdrive from 'hyperdrive'
 import { makeRoutedFetch } from 'make-fetch'
-import mime from 'mime/lite.js'
+import mime from 'mime/index.js'
 import parseRange from 'range-parser'
 import { EventIterator } from 'event-iterator'
 
@@ -36,13 +36,16 @@ const BASIC_METHODS = [
 ]
 
 export const ERROR_KEY_NOT_CREATED = 'Must create key with POST before reading'
+export const ERROR_DRIVE_EMPTY = 'Could not find data in drive, make sure your key is correct and that there are peers online to load data from'
 
 const INDEX_FILES = [
   'index.html',
   'index.md',
   'index.gmi',
   'index.gemini',
-  'README.md'
+  'index.org',
+  'README.md',
+  'README.org'
 ]
 
 async function DEFAULT_RENDER_INDEX (url, files, fetch) {
@@ -70,7 +73,9 @@ export default async function makeHyperFetch ({
   timeout = DEFAULT_TIMEOUT,
   renderIndex = DEFAULT_RENDER_INDEX
 }) {
-  const { fetch, router } = makeRoutedFetch()
+  const { fetch, router } = makeRoutedFetch({
+    onError
+  })
 
   // Map loaded drive hostnames to their keys
   // TODO: Track LRU + cache clearing
@@ -91,6 +96,7 @@ export default async function makeHyperFetch ({
 
     router.put(`hyper://*/${SPECIAL_FOLDER}/${VERSION_FOLDER_NAME}/**`, putFilesVersioned)
     router.put('hyper://*/**', putFiles)
+    router.delete('hyper://*/', deleteDrive)
     router.delete(`hyper://*/${SPECIAL_FOLDER}/${VERSION_FOLDER_NAME}/**`, deleteFilesVersioned)
     router.delete('hyper://*/**', deleteFiles)
   }
@@ -100,6 +106,16 @@ export default async function makeHyperFetch ({
   router.get('hyper://*/**', getFiles)
   router.head('hyper://*/**', headFiles)
 
+  async function onError (e, request) {
+    return {
+      status: e.statusCode || 500,
+      headers: {
+        'Content-Type': 'text/plain; charset=utf-8'
+      },
+      body: e.stack
+    }
+  }
+
   async function getCore (hostname) {
     if (cores.has(hostname)) {
       return cores.get(hostname)
@@ -128,18 +144,30 @@ export default async function makeHyperFetch ({
     return dbCore
   }
 
-  async function getDrive (hostname) {
+  async function getDrive (hostname, errorOnNew = false) {
     if (drives.has(hostname)) {
       return drives.get(hostname)
     }
 
     const core = await getCore(hostname)
 
+    if (!core.length && errorOnNew) {
+      await core.close()
+      const e = new Error(ERROR_DRIVE_EMPTY)
+      e.statusCode = 404
+      throw e
+    }
+
     const corestore = sdk.namespace(core.id)
     const drive = new Hyperdrive(corestore, core.key)
 
     await drive.ready()
 
+    drive.once('close', () => {
+      drives.delete(drive.core.id)
+      drives.delete(hostname)
+    })
+
     drives.set(drive.core.id, drive)
     drives.set(hostname, drive)
 
@@ -151,8 +179,11 @@ export default async function makeHyperFetch ({
       return drives.get(key)
     }
     const core = await getDBCoreForName(key)
+
     if (!core.length && errorOnNew) {
-      throw new Error(ERROR_KEY_NOT_CREATED)
+      const e = new Error(ERROR_KEY_NOT_CREATED)
+      e.statusCode = 404
+      throw e
     }
 
     const corestore = sdk.namespace(key)
@@ -160,7 +191,14 @@ export default async function makeHyperFetch ({
 
     await drive.ready()
 
+    drive.once('close', () => {
+      drives.delete(key)
+      drives.delete(drive.url)
+      drives.delete(drive.core.id)
+    })
+
     drives.set(key, drive)
+    drives.set(drive.url, drive)
     drives.set(drive.core.id, drive)
 
     return drive
@@ -359,16 +397,15 @@ export default async function makeHyperFetch ({
     const { hostname, pathname: rawPathname } = new URL(request.url)
     const pathname = decodeURI(ensureLeadingSlash(rawPathname))
     const contentType = request.headers.get('Content-Type') || ''
+    const mtime = Date.parse(request.headers.get('Last-Modified')) || Date.now()
     const isFormData = contentType.includes('multipart/form-data')
 
-    const drive = await getDrive(`hyper://${hostname}`)
+    const drive = await getDrive(`hyper://${hostname}/`, true)
 
     if (!drive.db.feed.writable) {
       return { status: 403, body: `Cannot PUT file to read-only drive: ${drive.url}`, headers: { Location: request.url } }
     }
 
-    const mtime = Date.now()
-
     if (isFormData) {
       // It's a form! Get the files out and process them
       const formData = await request.formData()
@@ -419,11 +456,20 @@ export default async function makeHyperFetch ({
     }
   }
 
+  async function deleteDrive (request) {
+    const { hostname } = new URL(request.url)
+    const drive = await getDrive(`hyper://${hostname}/`, true)
+
+    await drive.purge()
+
+    return { status: 200 }
+  }
+
   async function deleteFiles (request) {
     const { hostname, pathname: rawPathname } = new URL(request.url)
     const pathname = decodeURI(ensureLeadingSlash(rawPathname))
 
-    const drive = await getDrive(`hyper://${hostname}`)
+    const drive = await getDrive(`hyper://${hostname}/`, true)
 
     if (!drive.db.feed.writable) {
       return { status: 403, body: `Cannot DELETE file in read-only drive: ${drive.url}`, headers: { Location: request.url } }
@@ -475,7 +521,7 @@ export default async function makeHyperFetch ({
     const version = parts[3]
     const realPath = ensureLeadingSlash(parts.slice(4).join('/'))
 
-    const drive = await getDrive(`hyper://${hostname}`)
+    const drive = await getDrive(`hyper://${hostname}/`, true)
 
     const snapshot = await drive.checkout(version)
 
@@ -491,7 +537,7 @@ export default async function makeHyperFetch ({
     const isRanged = request.headers.get('Range') || ''
     const noResolve = searchParams.has('noResolve')
 
-    const drive = await getDrive(`hyper://${hostname}`)
+    const drive = await getDrive(`hyper://${hostname}/`, true)
 
     return serveHead(drive, pathname, { accept, isRanged, noResolve })
   }
@@ -577,7 +623,7 @@ export default async function makeHyperFetch ({
       resHeaders[HEADER_LAST_MODIFIED] = date.toUTCString()
     }
 
-    const size = entry.value.byteLength
+    const size = entry.value.blob.byteLength
     if (isRanged) {
       const ranges = parseRange(size, isRanged)
 
@@ -586,7 +632,7 @@ export default async function makeHyperFetch ({
         const length = (end - start + 1)
 
         return {
-          status: 200,
+          status: 204,
           headers: {
             ...resHeaders,
             'Content-Length': `${length}`,
@@ -597,7 +643,7 @@ export default async function makeHyperFetch ({
     }
 
     return {
-      status: 200,
+      status: 204,
       headers: resHeaders
     }
   }
@@ -615,7 +661,7 @@ export default async function makeHyperFetch ({
     const version = parts[3]
     const realPath = ensureLeadingSlash(parts.slice(4).join('/'))
 
-    const drive = await getDrive(`hyper://${hostname}`)
+    const drive = await getDrive(`hyper://${hostname}/`, true)
 
     const snapshot = await drive.checkout(version)
 
@@ -632,7 +678,7 @@ export default async function makeHyperFetch ({
     const isRanged = request.headers.get('Range') || ''
     const noResolve = searchParams.has('noResolve')
 
-    const drive = await getDrive(`hyper://${hostname}`)
+    const drive = await getDrive(`hyper://${hostname}/`, true)
 
     return serveGet(drive, pathname, { accept, isRanged, noResolve })
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore-fetch",
-  "version": "9.6.0",
+  "version": "9.8.0",
   "description": "Implementation of Fetch that uses the Dat SDK for loading p2p content",
   "type": "module",
   "main": "index.js",
@@ -32,7 +32,7 @@
   },
   "devDependencies": {
     "@rangermauve/fetch-event-source": "^1.0.3",
-    "hyper-sdk": "^4.2.3",
+    "hyper-sdk": "^4.3.0",
     "standard": "^17.0.0",
     "tape": "^5.2.2"
   }

package/test.js

@@ -127,6 +127,7 @@ test('HEAD request', async (t) => {
   const headersLastModified = headResponse.headers.get('Last-Modified')
   const headersLink = headResponse.headers.get('Link')
 
+  t.equal(headResponse.status, 204, 'Response had expected status')
   // Version at which the file was added
   t.equal(headersEtag, '2', 'Headers got expected etag')
   t.equal(headersContentType, 'text/plain; charset=utf-8', 'Headers got expected mime type')
@@ -141,9 +142,13 @@ test('PUT file', async (t) => {
 
   const uploadLocation = new URL('./example.txt', created)
 
+  const fakeDate = new Date(Date.parse(0)).toUTCString()
   const uploadResponse = await fetch(uploadLocation, {
     method: 'put',
-    body: SAMPLE_CONTENT
+    body: SAMPLE_CONTENT,
+    headers: {
+      'Last-Modified': fakeDate
+    }
   })
 
   await checkResponse(uploadResponse, t, 'upload successful')
@@ -158,7 +163,7 @@ test('PUT file', async (t) => {
 
   t.equal(contentType, 'text/plain; charset=utf-8', 'Content got expected mime type')
   t.equal(content, SAMPLE_CONTENT, 'Got uploaded content back out')
-  t.ok(lastModified, 'Last-Modified header got set')
+  t.equal(lastModified, fakeDate, 'Last-Modified header was set to value of Date header')
 })
 test('PUT FormData', async (t) => {
   const created = await nextURL(t)
@@ -286,7 +291,7 @@ test('DELETE a directory', async (t) => {
   })
   await checkResponse(uploadResponse, t)
 
-  const deleteResponse = await fetch(created, {
+  const deleteResponse = await fetch(uploadLocation, {
     method: 'delete'
   })
   await checkResponse(deleteResponse, t, 'Able to DELETE')
@@ -296,6 +301,24 @@ test('DELETE a directory', async (t) => {
   const entries = await listDirRequest.json()
   t.deepEqual(entries, [], 'subfolder got deleted')
 })
+test('DELETE a drive from storage', async (t) => {
+  const created = await nextURL(t)
+
+  const uploadLocation = new URL('./subfolder/example.txt', created)
+  const uploadResponse = await fetch(uploadLocation, {
+    method: 'put',
+    body: SAMPLE_CONTENT
+  })
+  await checkResponse(uploadResponse, t)
+
+  const purgeResponse = await fetch(created, { method: 'delete' })
+
+  await checkResponse(purgeResponse, t, 'Able to purge')
+
+  const listDirRequest = await fetch(created)
+
+  t.notOk(listDirRequest.ok, 'Error when trying to read after purge')
+})
 test('Read index.html', async (t) => {
   const created = await nextURL(t)
   const uploadLocation = new URL('./index.html', created)
@@ -419,14 +442,6 @@ test('Resolve pretty markdown URLs', async (t) => {
   t.equal(contentType, 'text/markdown; charset=utf-8', 'Got markdown mime type')
 })
 
-test('Doing a `GET` on an invalid domain should cause an error', async (t) => {
-  const url = 'hyper://example/'
-
-  const failedResponse = await fetch(url)
-
-  t.notOk(failedResponse.ok, 'Response errored out due to invalid domain')
-})
-
 test('EventSource extension messages', async (t) => {
   const domain = await nextURL(t)
 
@@ -490,14 +505,13 @@ test('Resolve DNS', async (t) => {
   t.ok(entries.length, 'Loaded contents with some files present')
 })
 
-test('Error on invalid hostname', async (t) => {
-  const loadResponse = await fetch('hyper://example/')
+test('Doing a `GET` on an invalid domain/public key should cause an error', async (t) => {
+  const invalidDomainResponse = await fetch('hyper://example/')
+  t.notOk(invalidDomainResponse.ok, 'Response errored out due to invalid domain')
 
-  if (loadResponse.ok) {
-    throw new Error('Loading without DNS or a public key should have failed')
-  } else {
-    t.pass('Invalid names led to an error')
-  }
+  const invalidPublicKeyResponse = await fetch('hyper://aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/')
+  t.notOk(invalidPublicKeyResponse.ok, 'Response errored out due to invalid public key')
+  t.equal(invalidPublicKeyResponse.status, 404, 'Invalid public key should 404')
 })
 
 test('Old versions in VERSION folder', async (t) => {
@@ -575,17 +589,10 @@ test('Handle empty string pathname', async (t) => {
     await fetch(urlNoTrailingSlash, {
       method: 'put',
       body: formData
-    }), t
+    }),
+    t
   )
 
-  // DELETE
-  await checkResponse(await fetch(urlNoTrailingSlash, { method: 'DELETE' }), t)
-
-  // HEAD
-  const headResponse = await fetch(urlNoTrailingSlash, { method: 'HEAD' })
-  await checkResponse(headResponse, t)
-  t.deepEqual(headResponse.headers.get('Etag'), '5', 'HEAD request returns correct Etag')
-
   // HEAD (versioned)
   const versionedHeadResponse = await fetch(versionedURLNoTrailingSlash, { method: 'HEAD' })
   await checkResponse(versionedHeadResponse, t)
@@ -594,12 +601,16 @@ test('Handle empty string pathname', async (t) => {
   // GET
   const getResponse = await fetch(urlNoTrailingSlash)
   await checkResponse(getResponse, t)
-  t.deepEqual(await getResponse.json(), [], 'Returns empty root directory')
+  t.deepEqual(await getResponse.json(), ['example.txt', 'example2.txt'], 'Returns directory listing')
 
   // GET (versioned)
   const versionedGetResponse = await fetch(versionedURLNoTrailingSlash)
   await checkResponse(versionedGetResponse, t)
   t.deepEqual(await versionedGetResponse.json(), ['example.txt', 'example2.txt'], 'Returns root directory prior to DELETE')
+
+  
+  // DELETE
+  await checkResponse(await fetch(urlNoTrailingSlash, { method: 'DELETE' }), t, 'Able to delete root')
 })
 
 test('Return status 403 Forbidden on attempt to modify read-only hyperdrive', async (t) => {