hypercore-fetch 8.3.3 → 8.6.0

package/README.md

@@ -130,6 +130,15 @@ The `body` can be either a `String`, an `ArrayBuffer`, a `Blob`, a WHATWG `Reada
 
 Your `NAME` will likely be a `name` in most cases to ensure you have a writeable archive.
 
+### `fetch('hyper://NAME/folder/', {method: 'PUT', body: new FormData()})`
+
+You can add multiple files to a folder using the `PUT` method with a [FormData](https://developer.mozilla.org/en-US/docs/Web/API/FormData) body.
+
+You can [append](https://developer.mozilla.org/en-US/docs/Web/API/FormData) to a FormData with `formData.append(fieldname, content, 'filename.txt')` where `fieldname` gets ignored (use something like `file`?) the `content` can either be a String, Blob, or some sort of stream.
+The `filename` will be the filename within the directory that gets created.
+
+`NAME` can either be the 64 character hex key for an archive, a domain to parse with [dat-dns](https://www.npmjs.com/package/dat-dns), or a name for an archive which allows you to write to it.
+
 ### `fetch('hyper://NAME/example.txt', {method: 'DELETE'})`
 
 You can delete a file in an archive by using the `DELETE` method.
@@ -212,6 +221,10 @@ Using the `text/event-stream` content type in the `Accept` header will get back
 
 The `event` will be the name of the extension you got the data for, the `id` (accessible by `e.lastEventId` in EventSource) will be set to the ID of the peer that sent it.
 
+Only extension messages that have been queried before via a `GET` to the EXTENSION_NAME will be visible in this stream.
+
+There are also two special events: `peer-open` which gets emitted whena new peer has connected, and `peer-remove` which gets emitted when an existing peer disconnects.
+
 ### `fetch('hyper://NAME/$/extensions/EXTENSION_NAME', {method: 'POST', body: 'Example'})`
 
 You can broadcast an extension message to all peers that are replicating that extension type with a `POST` to the extension's URL.

package/index.js

@@ -7,6 +7,8 @@ const makeDir = require('make-dir')
 const { Readable } = require('streamx')
 const makeFetch = require('make-fetch')
 const { EventIterator } = require('event-iterator')
+const Busboy = require('busboy')
+const posixPath = require('path').posix
 
 const DEFAULT_TIMEOUT = 5000
 
@@ -24,6 +26,8 @@ const TAGS_FOLDER = SPECIAL_FOLDER + TAGS_FOLDER_NAME
 const EXTENSIONS_FOLDER_NAME = 'extensions/'
 const EXTENSIONS_FOLDER = SPECIAL_FOLDER + EXTENSIONS_FOLDER_NAME
 const EXTENSION_EVENT = 'extension-message'
+const PEER_OPEN = 'peer-open'
+const PEER_REMOVE = 'peer-remove'
 
 // TODO: Add caching support
 const { resolveURL: DEFAULT_RESOLVE_URL } = require('hyper-dns')
@@ -279,9 +283,23 @@ module.exports = function makeHyperFetch (opts = {}) {
                   const data = content.split('\n').map((line) => `data:${line}\n`).join('')
                   push(`id:${id}\nevent:${name}\n${data}\n`)
                 }
+                function onPeerOpen (peer) {
+                  const id = peer.remotePublicKey.toString('hex')
+                  push(`id:${id}\nevent:${PEER_OPEN}\n\n`)
+                }
+                function onPeerRemove (peer) {
+                  // Whatever, probably an uninitialized peer
+                  if (!peer.remotePublicKey) return
+                  const id = peer.remotePublicKey.toString('hex')
+                  push(`id:${id}\nevent:${PEER_REMOVE}\n\n`)
+                }
                 archive.on(EXTENSION_EVENT, onMessage)
+                archive.on(PEER_OPEN, onPeerOpen)
+                archive.on(PEER_REMOVE, onPeerRemove)
                 return () => {
-                  archive.removeListener('extension-message', onMessage)
+                  archive.removeListener(EXTENSION_EVENT, onMessage)
+                  archive.removeListener(PEER_OPEN, onPeerOpen)
+                  archive.removeListener(PEER_REMOVE, onPeerRemove)
                 }
               })
 
@@ -364,9 +382,56 @@ module.exports = function makeHyperFetch (opts = {}) {
 
       if (method === 'PUT') {
         checkWritable(archive)
+        const contentType = headers.get('Content-Type') || headers.get('content-type')
+        const isFormData = contentType && contentType.includes('multipart/form-data')
+
         if (path.endsWith('/')) {
           await makeDir(path, { fs: archive })
+          const busboy = new Busboy({ headers: rawHeaders })
+
+          const toUpload = new EventIterator(({ push, stop, fail }) => {
+            busboy.once('error', fail)
+            busboy.once('finish', stop)
+
+            busboy.on('file', async (fieldName, fileData, fileName) => {
+              const finalPath = posixPath.join(path, fileName)
+
+              const source = Readable.from(fileData)
+              const destination = archive.createWriteStream(finalPath)
+
+              source.pipe(destination)
+              try {
+                Promise.race([
+                  once(source, 'error').then((e) => { throw e }),
+                  once(destination, 'error').then((e) => { throw e }),
+                  once(source, 'end')
+                ])
+              } catch (e) {
+                fail(e)
+              }
+            })
+
+            // TODO: Does busboy need to be GC'd?
+            return () => {}
+          })
+
+          Readable.from(body).pipe(busboy)
+
+          await Promise.all(await collect(toUpload))
+
+          return {
+            statusCode: 200,
+            headers: responseHeaders,
+            data: intoAsyncIterable(canonical)
+          }
         } else {
+          if (isFormData) {
+            return {
+              statusCode: 400,
+              headers: responseHeaders,
+              data: intoAsyncIterable('FormData only supported for folders (ending with a /)')
+            }
+          }
           const parentDir = path.split('/').slice(0, -1).join('/')
           if (parentDir) {
             await makeDir(parentDir, { fs: archive })
@@ -389,7 +454,7 @@ module.exports = function makeHyperFetch (opts = {}) {
         return {
           statusCode: 200,
           headers: responseHeaders,
-          data: intoAsyncIterable('')
+          data: intoAsyncIterable(canonical)
         }
       } else if (method === 'DELETE') {
         if (headers.get('x-clear') === 'cache') {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hypercore-fetch",
-  "version": "8.3.3",
+  "version": "8.6.0",
   "description": "Implementation of Fetch that uses the Dat SDK for loading p2p content",
   "bin": {
     "hypercore-fetch": "bin.js"
@@ -24,6 +24,7 @@
   },
   "homepage": "https://github.com/RangerMauve/hypercore-fetch#readme",
   "dependencies": {
+    "busboy": "^0.3.1",
     "event-iterator": "^2.0.0",
     "fetch-headers": "^2.0.0",
     "hyper-dns": "^0.12.0",
@@ -37,6 +38,7 @@
     "streamx": "^2.10.0"
   },
   "devDependencies": {
+    "form-data": "^4.0.0",
     "random-access-memory": "^3.1.1",
     "tape": "^5.2.2"
   }

package/test.js

@@ -1,5 +1,6 @@
 const SDK = require('hyper-sdk')
 const test = require('tape')
+const FormData = require('form-data')
 
 runTests()
 
@@ -84,10 +85,30 @@ async function runTests () {
     t.equal(await response2.text(), SAMPLE_CONTENT, 'Read back written data')
   })
 
-  test('PUT directory', async (t) => {
-    const response1 = await fetch('hyper://example/foo/bar/', { method: 'PUT' })
+  test.only('PUT FormData to directory', async (t) => {
+    const form = new FormData()
 
-    t.equal(response1.status, 200, 'Got OK response on directory creation')
+    form.append('file', SAMPLE_CONTENT, {
+      filename: 'example.txt'
+    })
+    const body = form.getBuffer()
+    const headers = form.getHeaders()
+
+    const response1 = await fetch('hyper://example/foo/bar/', {
+      method: 'PUT',
+      headers,
+      body
+    })
+
+    t.equal(response1.status, 200, 'Got OK response on directory upload')
+
+    console.log(await response1.text())
+
+    const response2 = await fetch('hyper://example/foo/bar/example.txt')
+
+    t.equal(response2.status, 200, 'Got OK response on read')
+
+    t.equal(await response2.text(), SAMPLE_CONTENT, 'Read back written data')
   })
 
   test('PUT file in new directory', async (t) => {