hypercore-fetch 8.6.1 → 9.0.0

package/package.json

@@ -1,10 +1,8 @@
 {
   "name": "hypercore-fetch",
-  "version": "8.6.1",
+  "version": "9.0.0",
   "description": "Implementation of Fetch that uses the Dat SDK for loading p2p content",
-  "bin": {
-    "hypercore-fetch": "bin.js"
-  },
+  "type": "module",
   "main": "index.js",
   "scripts": {
     "test": "node test",
@@ -25,22 +23,16 @@
   },
   "homepage": "https://github.com/RangerMauve/hypercore-fetch#readme",
   "dependencies": {
-    "busboy": "^0.3.1",
     "event-iterator": "^2.0.0",
-    "fetch-headers": "^2.0.0",
-    "hyper-dns": "^0.12.0",
-    "hyper-sdk": "^3.0.9",
-    "make-dir": "^3.1.0",
-    "make-fetch": "^2.3.3",
-    "mime": "^2.4.4",
+    "hyperdrive": "^11.0.0-alpha.10",
+    "make-fetch": "^3.1.0",
+    "mime": "^3.0.0",
     "range-parser": "^1.2.1",
-    "resolve-dat-path": "^2.0.0",
-    "sodium-universal": "^3.0.2",
-    "streamx": "^2.10.0"
+    "streamx": "^2.13.0"
   },
   "devDependencies": {
-    "form-data": "^4.0.0",
-    "random-access-memory": "^3.1.1",
+    "@rangermauve/fetch-event-source": "^1.0.3",
+    "hyper-sdk": "^4.1.1",
     "standard": "^17.0.0",
     "tape": "^5.2.2"
   }

package/test.js

@@ -1,282 +1,387 @@
-const SDK = require('hyper-sdk')
-const test = require('tape')
-const FormData = require('form-data')
+/* global FormData, Blob */
+import * as SDK from 'hyper-sdk'
+import test from 'tape'
+import createEventSource from '@rangermauve/fetch-event-source'
+import { once } from 'events'
 
-runTests()
+import makeHyperFetch from './index.js'
 
 const SAMPLE_CONTENT = 'Hello World'
+let count = 0
+function next () {
+  return count++
+}
 
-async function runTests () {
-  const { Hyperdrive, close } = await SDK({
-    persist: false
+async function nextURL (t) {
+  const createResponse = await fetch(`hyper://localhost/?key=example${next()}`, {
+    method: 'post'
   })
+  await checkResponse(createResponse, t, 'Created new drive')
 
-  const { Hyperdrive: Hyperdrive2, close: close2 } = await SDK({
-    persist: false
-  })
+  const created = await createResponse.text()
+  return created
+}
 
-  const fetch = require('./')({
-    Hyperdrive,
-    writable: true
-  })
+const sdk1 = await SDK.create({ storage: false })
+const sdk2 = await SDK.create({ storage: false })
+
+const fetch = await makeHyperFetch({
+  sdk: sdk1,
+  writable: true
+})
 
-  const fetch2 = require('./')({
-    Hyperdrive: Hyperdrive2,
-    writable: true
+const fetch2 = await makeHyperFetch({
+  sdk: sdk2,
+  writable: true
+})
+
+test.onFinish(() => {
+  sdk1.close()
+  sdk2.close()
+})
+
+test('Quick check', async (t) => {
+  const createResponse = await fetch(`hyper://localhost/?key=example${next()}`, {
+    method: 'post'
   })
 
-  test.onFinish(() => {
-    close()
-    close2()
+  await checkResponse(createResponse, t, 'Created new drive')
+
+  const created = await createResponse.text()
+
+  t.ok(created.startsWith('hyper://'), 'Created drive has hyper URL')
+
+  const existsResponse = await fetch(created)
+
+  await checkResponse(existsResponse, t)
+
+  t.deepEqual(await existsResponse.json(), [], 'Empty dir on create')
+
+  const uploadLocation = new URL('./example.txt', created)
+
+  const uploadResponse = await fetch(uploadLocation, {
+    method: 'put',
+    body: SAMPLE_CONTENT
   })
 
-  test('Read index.html', async (t) => {
-    const archive = Hyperdrive('example1')
+  await checkResponse(uploadResponse, t)
 
-    const FILE_LOCATION = '/index.html'
-    const FILE_DATA = '<h1>Hello World!</h1>'
+  const uploadedContentResponse = await fetch(uploadLocation)
 
-    await archive.writeFile(FILE_LOCATION, FILE_DATA)
+  await checkResponse(uploadedContentResponse, t)
 
-    const url = `hyper://${archive.key.toString('hex')}${FILE_LOCATION}`
+  const content = await uploadedContentResponse.text()
+  const contentType = uploadedContentResponse.headers.get('Content-Type')
 
-    t.pass('Prepped archive ' + url)
+  t.equal(contentType, 'text/plain; charset=utf-8', 'Content got expected mime type')
+  t.equal(content, SAMPLE_CONTENT, 'Got uploaded content back out')
 
-    const response = await fetch(url)
+  const dirResponse = await fetch2(created)
 
-    t.pass('got response')
+  await checkResponse(dirResponse, t)
 
-    const text = await response.text()
+  t.deepEqual(await dirResponse.json(), ['example.txt'], 'File got added')
+})
 
-    t.pass('got response text')
+test('PUT file', async (t) => {
+  const created = await nextURL(t)
 
-    const contentType = response.headers.get('content-type')
+  const uploadLocation = new URL('./example.txt', created)
 
-    t.equal(contentType, 'text/html; charset=utf-8')
-    t.equal(text, FILE_DATA)
-    t.pass('Headers ' + [...response.headers.entries()])
+  const uploadResponse = await fetch(uploadLocation, {
+    method: 'put',
+    body: SAMPLE_CONTENT
   })
 
-  test('GET .well-known/dat', async (t) => {
-    const response = await fetch('hyper://example/.well-known/dat')
-    t.ok(response, 'Got response')
-    t.equal(response.status, 200, 'Got OK response code')
-    const text = await response.text()
-    t.ok(text.startsWith('dat://'), 'Returned dat URL')
-  })
+  await checkResponse(uploadResponse, t, 'upload successful')
 
-  test('GET .well-known/hyper', async (t) => {
-    const response = await fetch('hyper://example/.well-known/hyper')
-    t.ok(response, 'Got response')
-    t.equal(response.status, 200, 'Got OK response code')
-    const text = await response.text()
-    t.ok(text.startsWith('hyper://'), 'Returned dat URL')
-  })
+  const uploadedContentResponse = await fetch(uploadLocation)
 
-  test('PUT file', async (t) => {
-    const response1 = await fetch('hyper://example/checkthis.txt', { method: 'PUT', body: SAMPLE_CONTENT })
+  await checkResponse(uploadedContentResponse, t, 'able to load content')
 
-    t.equal(response1.status, 200, 'Got OK response on write')
+  const content = await uploadedContentResponse.text()
+  const contentType = uploadedContentResponse.headers.get('Content-Type')
 
-    const response2 = await fetch('hyper://example/checkthis.txt')
+  t.equal(contentType, 'text/plain; charset=utf-8', 'Content got expected mime type')
+  t.equal(content, SAMPLE_CONTENT, 'Got uploaded content back out')
+})
+test('PUT FormData', async (t) => {
+  const created = await nextURL(t)
 
-    t.equal(response2.status, 200, 'Got OK response on read')
+  const formData = new FormData()
+  formData.append('file', new Blob([SAMPLE_CONTENT]), 'example.txt')
+  formData.append('file', new Blob([SAMPLE_CONTENT]), 'example2.txt')
 
-    t.equal(await response2.text(), SAMPLE_CONTENT, 'Read back written data')
+  const uploadedResponse = await fetch(created, {
+    method: 'put',
+    body: formData
   })
 
-  test('PUT FormData to directory', async (t) => {
-    const form = new FormData()
-
-    form.append('file', SAMPLE_CONTENT, {
-      filename: 'example.txt'
-    })
-    const body = form.getBuffer()
-    const headers = form.getHeaders()
+  await checkResponse(uploadedResponse, t)
 
-    const response1 = await fetch('hyper://example/foo/bar/', {
-      method: 'PUT',
-      headers,
-      body
-    })
+  const file2URL = new URL('/example2.txt', created)
+  const file2Response = await fetch(file2URL)
 
-    t.equal(response1.status, 200, 'Got OK response on directory upload')
+  await checkResponse(file2Response, t)
+  const file2Content = await file2Response.text()
 
-    console.log(await response1.text())
+  t.equal(file2Content, SAMPLE_CONTENT, 'file contents got uploaded')
 
-    const response2 = await fetch('hyper://example/foo/bar/example.txt')
+  const listDirRequest = await fetch(created)
+  await checkResponse(listDirRequest, t)
+  const entries = await listDirRequest.json()
+  t.deepEqual(entries, ['example.txt', 'example2.txt'], 'new files are listed')
+})
+test('PUT into new directory', async (t) => {
+  const created = await nextURL(t)
 
-    t.equal(response2.status, 200, 'Got OK response on read')
+  const uploadLocation = new URL('./subfolder/example.txt', created)
 
-    t.equal(await response2.text(), SAMPLE_CONTENT, 'Read back written data')
+  const uploadResponse = await fetch(uploadLocation, {
+    method: 'put',
+    body: SAMPLE_CONTENT
   })
 
-  test('PUT file in new directory', async (t) => {
-    const response1 = await fetch('hyper://example/fizz/buzz/example.txt', { method: 'PUT', body: SAMPLE_CONTENT })
+  await checkResponse(uploadResponse, t)
 
-    t.equal(response1.status, 200, 'Got OK response on directory/file creation')
-  })
+  const uploadedContentResponse = await fetch(uploadLocation)
 
-  test('PUT to overwrite a file', async (t) => {
-    const response1 = await fetch('hyper://example/baz/index.html', { method: 'PUT', body: SAMPLE_CONTENT })
-    t.ok(response1.ok)
-    const response2 = await fetch('hyper://example/baz/index.html', { method: 'PUT', body: SAMPLE_CONTENT })
+  await checkResponse(uploadedContentResponse, t)
 
-    t.equal(response2.status, 200, 'Got OK response on file overwrite')
-  })
+  const content = await uploadedContentResponse.text()
+  const contentType = uploadedContentResponse.headers.get('Content-Type')
 
-  test('DELETE file', async (t) => {
-    const response1 = await fetch('hyper://example/test.txt', { method: 'PUT', body: SAMPLE_CONTENT })
-    t.ok(response1.ok)
+  t.equal(contentType, 'text/plain; charset=utf-8', 'Content got expected mime type')
+  t.equal(content, SAMPLE_CONTENT, 'Got uploaded content back out')
 
-    const response2 = await fetch('hyper://example/test.txt', { method: 'DELETE' })
+  const listDirRequest = await fetch(created)
+  await checkResponse(listDirRequest, t)
+  const entries = await listDirRequest.json()
+  t.deepEqual(entries, ['subfolder/'], 'new files are listed')
+})
+test('PUT to overwrite a file', async (t) => {
+  const created = await nextURL(t)
 
-    t.equal(response2.status, 200, 'Got OK response on file delete')
+  const uploadLocation = new URL('./example.txt', created)
 
-    const response3 = await fetch('hyper://example/test.txt', { method: 'GET' })
+  const uploadResponse = await fetch(uploadLocation, {
+    method: 'put',
+    body: SAMPLE_CONTENT
+  })
+  await checkResponse(uploadResponse, t)
 
-    t.equal(response3.status, 404, 'Got not found on deleted file')
+  const SHORTER_CONTENT = 'Hello'
+
+  const overWriteResponse = await fetch(uploadLocation, {
+    method: 'put',
+    body: SHORTER_CONTENT
   })
+  await checkResponse(overWriteResponse, t)
+
+  const uploadedContentResponse = await fetch(uploadLocation)
 
-  test('GET index.html', async (t) => {
-    const response1 = await fetch('hyper://example/baz')
+  await checkResponse(uploadedContentResponse, t)
 
-    t.equal(await response1.text(), SAMPLE_CONTENT, 'Got index.html content')
+  const content = await uploadedContentResponse.text()
+  const contentType = uploadedContentResponse.headers.get('Content-Type')
 
-    const response2 = await fetch('hyper://example/baz?noResolve')
+  t.equal(contentType, 'text/plain; charset=utf-8', 'Content got expected mime type')
+  t.equal(content, SHORTER_CONTENT, 'Got uploaded content back out')
+})
+test('DELETE a file', async (t) => {
+  const created = await nextURL(t)
 
-    t.equal(response2.headers.get('content-type'), 'application/json; charset=utf-8', 'noResolve flag yields JSON by default')
-    t.deepEqual(await response2.json(), ['index.html'], 'Listed directory')
+  const formData = new FormData()
+  formData.append('file', new Blob([SAMPLE_CONTENT]), 'example.txt')
+  formData.append('file', new Blob([SAMPLE_CONTENT]), 'example2.txt')
 
-    const response3 = await fetch('hyper://example/baz?noResolve')
-    t.equal(response3.headers.get('content-type'), 'application/json; charset=utf-8', 'noResolve flag yields JSON by default')
-    t.deepEqual(await response3.json(), ['index.html'], 'Listed directory')
+  const uploadedResponse = await fetch(created, {
+    method: 'put',
+    body: formData
   })
+  await checkResponse(uploadedResponse, t)
 
-  test('Create and read tags', async (t) => {
-    await fetch('hyper://example/test.txt', { method: 'PUT', body: SAMPLE_CONTENT })
+  const file2URL = new URL('/example2.txt', created)
+  const deleteResponse = await fetch(file2URL, {
+    method: 'delete'
+  })
 
-    const response2 = await fetch('hyper://example/$/tags/tag1', { method: 'PUT' })
-    t.ok(response2.ok, 'Able to create tag')
+  await checkResponse(deleteResponse, t, 'Able to DELETE')
 
-    const version = await response2.json()
+  const dirResponse = await fetch(created)
 
-    const response3 = await fetch('hyper://example/$/tags/')
+  await checkResponse(dirResponse, t)
 
-    t.ok(response3.ok, 'Able to ask for tags')
-    t.deepEqual(await response3.json(), { tag1: version }, 'Tag got created')
+  t.deepEqual(await dirResponse.json(), ['example.txt'], 'Only one file remains')
+})
+test('DELETE a directory', async (t) => {
+  const created = await nextURL(t)
 
-    // Insert a file which won't be available with the old tag
-    await fetch('hyper://example/notaccessible.txt', { method: 'PUT', body: 'test' })
+  const uploadLocation = new URL('./subfolder/example.txt', created)
 
-    const response4 = await fetch('hyper://example+tag1/notaccessible.txt')
+  const uploadResponse = await fetch(uploadLocation, {
+    method: 'put',
+    body: SAMPLE_CONTENT
+  })
+  await checkResponse(uploadResponse, t)
 
-    t.equal(response4.status, 404, 'Newer file not found in older tag')
+  const deleteResponse = await fetch(created, {
+    method: 'delete'
+  })
+  await checkResponse(deleteResponse, t, 'Able to DELETE')
+
+  const listDirRequest = await fetch(created)
+  await checkResponse(listDirRequest, t)
+  const entries = await listDirRequest.json()
+  t.deepEqual(entries, [], 'subfolder got deleted')
+})
+test('Read index.html', async (t) => {
+  const created = await nextURL(t)
+  const uploadLocation = new URL('./index.html', created)
+
+  const uploadResponse = await fetch(uploadLocation, {
+    method: 'put',
+    body: SAMPLE_CONTENT
+  })
+  await checkResponse(uploadResponse, t)
 
-    const response5 = await fetch('hyper://example/$/tags/tag1', { method: 'DELETE' })
+  const uploadedContentResponse = await fetch(uploadLocation)
 
-    t.ok(response5.ok, 'Able to delete tag')
+  await checkResponse(uploadedContentResponse, t)
 
-    const response6 = await fetch('hyper://example/$/tags/')
+  const content = await uploadedContentResponse.text()
+  const contentType = uploadedContentResponse.headers.get('Content-Type')
 
-    t.deepEqual(await response6.json(), {}, 'No tags left after delete')
+  t.equal(contentType, 'text/html; charset=utf-8', 'got HTML mime type')
+  t.equal(content, SAMPLE_CONTENT, 'loaded index.html content')
+})
+test('Ignore index.html with noResolve', async (t) => {
+  const created = await nextURL(t)
+  const uploadLocation = new URL('./index.html', created)
+
+  const uploadResponse = await fetch(uploadLocation, {
+    method: 'put',
+    body: SAMPLE_CONTENT
   })
+  await checkResponse(uploadResponse, t)
 
-  test('Load Mauve\'s blog', async (t) => {
-    const response = await fetch('hyper://blog.mauve.moe/')
+  const noResolve = created + '?noResolve'
 
-    t.ok(response.ok, 'Succesfully loaded homepage')
-  })
+  const listDirRequest = await fetch(noResolve)
+  await checkResponse(listDirRequest, t)
+  const entries = await listDirRequest.json()
+  t.deepEqual(entries, ['index.html'], 'able to list index.html')
+})
+test('Read directory as HTML', async (t) => {
+  const created = await nextURL(t)
 
-  test('Watch for changes', async (t) => {
-    const response = await fetch('hyper://example/', {
-      headers: {
-        Accept: 'text/event-stream'
-      }
-    })
+  const formData = new FormData()
+  formData.append('file', new Blob([SAMPLE_CONTENT]), 'example.txt')
+  formData.append('file', new Blob([SAMPLE_CONTENT]), 'example2.txt')
+
+  const uploadedResponse = await fetch(created, {
+    method: 'put',
+    body: formData
+  })
+  await checkResponse(uploadedResponse, t)
 
-    t.ok(response.ok, 'Able to open request')
-    t.equal(response.headers.get('Content-Type'), 'text/event-stream', 'Response is event stream')
+  const listDirRequest = await fetch(created, {
+    headers: {
+      Accept: 'text/html'
+    }
+  })
+  await checkResponse(listDirRequest, t, 'Able to list HTML')
 
-    const reader = await response.body.getReader()
+  const html = await listDirRequest.text()
 
-    const [data] = await Promise.all([
-      reader.read(),
-      fetch('hyper://example/example4.txt', { method: 'PUT', body: 'Hello World' })
-    ])
+  t.equal(listDirRequest.headers.get('Content-Type'), 'text/html; charset=utf-8', 'Returned HTML in mime type')
+  t.ok(html.includes('<title'), 'Listing has title')
+  t.ok(html.includes('./example.txt'), 'Listing has link to file')
+})
+test('Resolve pretty markdown URLs', async (t) => {
+  const created = await nextURL(t)
 
-    t.ok(data.value, 'Got eventsource data after writing')
-    t.ok(data.value.includes('event:change'), 'Eventsource data represents a change event')
-    t.ok(data.value.endsWith('\n\n'), 'Ends with two newlines')
+  const uploadLocation = new URL('./example.md', created)
 
-    await reader.cancel()
+  const uploadResponse = await fetch(uploadLocation, {
+    method: 'put',
+    body: SAMPLE_CONTENT
   })
+  await checkResponse(uploadResponse, t)
 
-  test('Send extension from one peer to another', async (t) => {
-    const domainResponse = await fetch('hyper://example/.well-known/hyper')
-    const domain = (await domainResponse.text()).split('\n')[0]
+  const resolvedLocation = new URL('/example', created)
 
-    const extensionURL = `${domain}/$/extensions/example`
-    const extensionListURL = `${domain}/$/extensions/`
+  const uploadedContentResponse = await fetch(resolvedLocation)
 
-    // Load up extension message on peer 1
-    await fetch(extensionURL)
-    // Load up extension message on peer 2
-    await fetch2(extensionURL)
+  await checkResponse(uploadedContentResponse, t, 'able to load content')
 
-    t.pass('Able to initialize extensions')
+  const content = await uploadedContentResponse.text()
+  const contentType = uploadedContentResponse.headers.get('Content-Type')
 
-    const extensionListRequest = await fetch(extensionListURL)
-    const extensionList = await extensionListRequest.json()
+  t.equal(content, SAMPLE_CONTENT, 'Got original content out')
+  t.equal(contentType, 'text/markdown; charset=utf-8', 'Got markdown mime type')
+})
 
-    // Extension list will always be alphabetically sorted
-    t.deepEqual(extensionList, ['example', 'hypertrie'], 'Got expected list of extensions')
+test('EventSource extension messages', async (t) => {
+  const domain = await nextURL(t)
 
-    // Wait a bit for them to connect
-    // TODO: Peers API
-    await delay(2000)
+  const extensionURL = `${domain}$/extensions/example`
+  const extensionListURL = `${domain}$/extensions/`
 
-    const peerResponse1 = await fetch(extensionURL)
-    const peerList1 = await peerResponse1.json()
+  // Load up extension message on peer 1
+  const extensionLoadResponse1 = await fetch(extensionURL)
+  await checkResponse(extensionLoadResponse1, t)
+  // Load up extension message on peer 2
+  const extensionLoadResponse2 = await fetch2(extensionURL)
+  await checkResponse(extensionLoadResponse2, t)
 
-    t.equal(peerList1.length, 1, 'Got one peer for extension message on peer1')
+  const extensionListRequest = await fetch(extensionListURL)
+  const extensionList = await extensionListRequest.json()
 
-    const peerResponse2 = await fetch2(extensionURL)
-    const peerList2 = await peerResponse2.json()
+  // Extension list will always be alphabetically sorted
+  t.deepEqual(extensionList, ['example'], 'Got expected list of extensions')
 
-    t.equal(peerList2.length, 1, 'Got one peer for extension message on peer2')
+  const peerResponse1 = await fetch(extensionURL)
+  const peerList1 = await peerResponse1.json()
 
-    const eventRequest = await fetch(extensionListURL, {
-      headers: {
-        Accept: 'text/event-stream'
-      }
-    })
+  t.equal(peerList1.length, 1, 'Got one peer for extension message on peer1')
 
-    t.ok(eventRequest.ok, 'Able to open request')
-    t.equal(eventRequest.headers.get('Content-Type'), 'text/event-stream', 'Response is event stream')
+  const peerResponse2 = await fetch2(extensionURL)
+  const peerList2 = await peerResponse2.json()
 
-    const reader = await eventRequest.body.getReader()
+  t.equal(peerList2.length, 1, 'Got one peer for extension message on peer2')
 
-    const toRead = reader.read()
+  const { EventSource } = createEventSource(fetch)
+  const source = new EventSource(extensionListURL)
 
-    await delay(500)
+  await Promise.race([
+    once(source, 'open'),
+    once(source, 'error').then(([e]) => { throw e })
+  ])
 
-    const broadcastRequest = await fetch2(extensionURL, { method: 'POST', body: 'Hello World' })
+  const toRead = Promise.race([
+    once(source, 'example'),
+    once(source, 'error').then(([e]) => { throw e })
+  ])
 
-    t.ok(broadcastRequest.ok, 'Able to broadcast to peers')
+  const broadcastRequest = await fetch2(extensionURL, { method: 'POST', body: SAMPLE_CONTENT })
 
-    const data = await toRead
+  t.ok(broadcastRequest.ok, 'Able to broadcast to peers')
 
-    t.ok(data.value, 'Got eventsource data after writing')
-    t.ok(data.value.includes('event:example\n'), 'EventSource data represents an example event')
-    t.ok(data.value.includes('data:Hello World\n'), 'EventSource data contains expected body')
-    t.ok(data.value.includes('id:'), 'EventSource data contains an ID')
-    t.ok(data.value.endsWith('\n\n'), 'Ends with two newlines')
+  const [event] = await toRead
 
-    await reader.cancel()
-  })
-}
+  const { type, data, lastEventId } = event
+
+  t.equal(data, SAMPLE_CONTENT, 'Got data from event')
+  t.equal(type, 'example', 'Event got set to extension message name')
+  t.ok(lastEventId, 'Event contained peer ID')
+})
 
-function delay (time) {
-  return new Promise((resolve) => setTimeout(resolve, time))
+async function checkResponse (response, t, successMessage = 'Response OK') {
+  if (!response.ok) {
+    const message = await response.text()
+    t.fail(new Error(`HTTP Error ${response.status}:\n${message}`))
+  } else {
+    t.pass(successMessage)
+  }
 }

package/bin.js

@@ -1,27 +0,0 @@
-#!/usr/bin/env node
-
-const eosp = require('end-of-stream-promise')
-const { Readable } = require('streamx')
-
-run()
-  .catch((e) => process.nextTick(() => {
-    throw e
-  }))
-
-async function run () {
-  const fetch = require('./')()
-
-  try {
-    const url = process.argv[2]
-
-    const response = await fetch(url)
-
-    const stream = Readable.from(response.body)
-
-    stream.pipe(process.stdout)
-
-    await eosp(stream)
-  } finally {
-    fetch.close()
-  }
-}