@rpcbase/client 0.196.0 → 0.197.0-fileupload.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rpcbase/client",
-  "version": "0.196.0",
+  "version": "0.197.0-fileupload.0",
   "scripts": {
     "build": "../../node_modules/.bin/wireit",
     "test": "../../node_modules/.bin/wireit"

package/ui/FileUpload/FileUploadContext.tsx

@@ -0,0 +1,160 @@
+import Promise from "bluebird"
+import { ReactNode } from "react"
+import debug from "debug"
+import { createContext, useContext, useState, useRef, useEffect } from "react"
+
+// import complete_upload from "rpc!server/items/drives/files/complete_upload";
+import {EVENTS, UPLOAD_MAX_CONCURRENCY} from "./constants"
+
+
+const log = debug("drive:upload_context")
+
+export const FileUploadContext = createContext()
+
+export const FileUploadContextProvider = ({
+  onUploadComplete,
+  children,
+}: {
+  onUploadComplete?: (files: any) => Promise<void>,
+  children: ReactNode,
+}) => {
+  const workerRef = useRef(null)
+
+  const [canUpload, setCanUpload] = useState(false)
+  const [isUploading, setIsUploading] = useState(false)
+  const [selectedFiles, setSelectedFiles] = useState([])
+  const [processedFiles, setProcessedFiles] = useState(Object.create(null))
+
+  const pendingUploadCallbacksRef = useRef({})
+
+
+  const updateProcessedFile = (filename, payload) => {
+    setProcessedFiles((previousProcessedFiles) => {
+      const nextProcessedFiles = Object.create(null)
+      Object.assign(nextProcessedFiles, previousProcessedFiles)
+
+      if (!nextProcessedFiles[filename]) nextProcessedFiles[filename] = {}
+
+      Object.assign(nextProcessedFiles[filename], payload)
+      return nextProcessedFiles
+    })
+  }
+
+  // when all files are ready to upload update parent context
+  useEffect(() => {
+    if (!(Object.keys(processedFiles).length > 0)) return
+
+    let isComplete = true
+    Object.keys(processedFiles).forEach((k) => {
+      if (processedFiles[k]?.status !== EVENTS.HASH_COMPLETE) {
+        isComplete = false
+      }
+    })
+
+    if (isComplete && !canUpload) {
+      // wait a bit for transitions to finish
+      setTimeout(() => {
+        setCanUpload(true)
+      }, 300)
+    }
+  }, [processedFiles, canUpload, setCanUpload])
+
+  useEffect(() => {
+    workerRef.current = new Worker(
+      new URL("./upload-worker/index.js", import.meta.url),
+    )
+
+    const onMessage = async({ data }) => {
+      log("worker message:", data)
+      const { type, payload } = data
+
+      if (type === EVENTS.UPLOAD_COMPLETE) {
+        const {filename, hash} = payload
+        updateProcessedFile(filename, {
+          status: EVENTS.UPLOAD_COMPLETE,
+        })
+        const cb = pendingUploadCallbacksRef.current[hash]
+        cb.resolve()
+      } else if (type === EVENTS.UPLOAD_ERROR) {
+        const cb = pendingUploadCallbacksRef.current[hash]
+        cb.reject()
+      } else if ([EVENTS.HASH_PROGRESS, EVENTS.HASH_COMPLETE].includes(type)) {
+        //
+        console.log("got hash event", type, payload)
+      } else {
+        throw new Error("unknown message event type")
+      }
+    }
+
+    workerRef.current.addEventListener("message", onMessage)
+
+    return () => {
+      workerRef.current.removeEventListener("message", onMessage)
+    }
+  }, [])
+
+
+  const startUploadAndWait = ({file, hash}) => new Promise((resolve, reject) => {
+    console.log("startUploadAndWait", {file, hash})
+
+    workerRef.current.postMessage({type: EVENTS.UPLOAD_FILE, file, hash})
+
+    pendingUploadCallbacksRef.current[hash] = {resolve, reject}
+  })
+
+
+  const uploadAllFiles = async() => {
+    setCanUpload(false)
+    setIsUploading(true)
+
+    const getUploadFiles = () => selectedFiles.map((f) => {
+      const processedFile = processedFiles[f.name]
+      if (!processedFile) {
+        throw new Error(`unable to retrieve processed file: ${f.name}`)
+      }
+
+      return {
+        file: f,
+        ...processedFile,
+      }
+    })
+
+    await Promise.map(getUploadFiles(), startUploadAndWait, {concurrency: UPLOAD_MAX_CONCURRENCY})
+
+    if (typeof onUploadComplete === "function") {
+      await onUploadComplete(getUploadFiles())
+    }
+
+    setCanUpload(true)
+    setIsUploading(false)
+    setSelectedFiles([])
+    setProcessedFiles(Object.create(null))
+
+  }
+
+  return (
+    <FileUploadContext.Provider
+      value={{
+        workerRef,
+        //
+        canUpload,
+        setCanUpload,
+        //
+        isUploading,
+        setIsUploading,
+        //
+        selectedFiles,
+        setSelectedFiles,
+        //
+        processedFiles,
+        updateProcessedFile,
+        //
+        uploadAllFiles,
+      }}
+    >
+      {children}
+    </FileUploadContext.Provider>
+  )
+}
+
+export const useFileUploadContext = () => useContext(FileUploadContext)

package/ui/FileUpload/FileUploadForm/index.tsx

@@ -0,0 +1,137 @@
+import assert from "assert"
+import {useEffect} from "react"
+import _set from "lodash/set"
+
+import {formatFileSize} from "@rpcbase/std"
+
+import {useFileUploadContext} from "../FileUploadContext"
+
+import usePreventUnload from "./usePreventUnload"
+
+import {EVENTS} from "../constants"
+
+
+const STATUS_MESSAGES = {
+  HASH_COMPLETE: "Ready to upload!",
+  HASHING: "Checking file...",
+}
+
+const FileUploadForm = () => {
+  const {workerRef, selectedFiles, setSelectedFiles, processedFiles, updateProcessedFile} = useFileUploadContext()
+
+  usePreventUnload()
+
+  useEffect(() => {
+    if (!workerRef.current) return
+
+    const onMessage = ({data: {type, payload}}) => {
+      // done hashing
+      if (type === EVENTS.HASH_COMPLETE) {
+        const {filename, hash} = payload
+        updateProcessedFile(filename, {
+          status: EVENTS.HASH_COMPLETE,
+          hash,
+          progress: 100,
+        })
+      }
+      // hash_progress
+      else if (type === EVENTS.HASH_PROGRESS) {
+        updateProcessedFile(payload.filename, {
+          status: EVENTS.HASHING,
+          progress: payload.progress,
+        })
+      }
+    }
+
+    workerRef.current.addEventListener("message", onMessage)
+
+    return () => {
+      workerRef.current.removeEventListener("message", onMessage)
+    }
+  }, [workerRef.current])
+
+  const onClickPicker = async() => {
+    const dirHandle = await window.showDirectoryPicker({
+      startIn: "desktop",
+    })
+    console.log("got dir hanlde", dirHandle)
+  }
+
+  useEffect(() => {}, [selectedFiles])
+
+  const onFilesChange = (e) => {
+    const {files} = e.target
+
+    // TODO: we should have a way to limit to a maximum number of hashes at the same time
+    Array.from(files).forEach((file) => {
+      assert(file.webkitRelativePath === "", "expected webkitRelativePath to be ''")
+
+      workerRef.current.postMessage({type: EVENTS.BEGIN_HASH, file})
+    })
+    setSelectedFiles(Array.from(files))
+  }
+
+  return (
+    <div className="px-3">
+      <div className="mb-3">
+        <label htmlFor="file-upload-input-files" className="form-label fw-bold">
+          Add Files
+        </label>
+        <input
+          className="form-control"
+          type="file"
+          id="file-upload-input-files"
+          accept="*"
+          multiple
+          onChange={onFilesChange}
+        />
+      </div>
+
+      <div className="text-secondary">OR</div>
+
+      <button
+        className="btn btn-light mt-2 d-flex flex-row align-items-center"
+        onClick={onClickPicker}
+      >
+        <img src="/static/icons/drive/folder-xs.svg" />
+        <span className="ms-2">Upload a Folder</span>
+      </button>
+
+      {selectedFiles.length > 0 && <hr className="mt-4" />}
+
+      <div className="files-list">
+        {selectedFiles.map((f, i) => {
+          const progress = processedFiles[f.name]?.progress || 0
+          const status = STATUS_MESSAGES[processedFiles[f.name]?.status] || "Preparing..."
+
+          return (
+            <div key={`file-${i}`}>
+              <div className="d-flex flex-row align-items-center mx-2 mt-3 mb-1">
+                <img src="/static/icons/drive/file-xs.svg" />
+                <div className="ms-2 text-truncate">{f.name}</div>
+                <div className="ms-2 text-secondary text-monospace">{formatFileSize(f.size)}</div>
+              </div>
+
+              <div className="d-flex flex-row align-items-center">
+                <div className="progress mt-0" style={{width: 120}}>
+                  <div
+                    className="progress-bar"
+                    role="progressbar"
+                    style={{width: `${progress}%`}}
+                    aria-valuenow={progress}
+                    aria-valuemin="0"
+                    aria-valuemax="100"
+                  />
+                </div>
+                <div className="ms-2 text-secondary">{progress}&nbsp;%</div>
+                <div className="ms-2 text-secondary">Status: {status}</div>
+              </div>
+            </div>
+          )
+        })}
+      </div>
+    </div>
+  )
+}
+
+export default FileUploadForm

package/ui/FileUpload/FileUploadForm/usePreventUnload.js

@@ -0,0 +1,21 @@
+/* eslint-disable react-hooks/rules-of-hooks */
+/* @flow */
+import {useEffect} from "react"
+
+
+const usePreventUnload = () => {
+  if (__DEV__) return
+
+  useEffect(() => {
+    const onBeforeUnload = (e) => {
+      e.preventDefault()
+      e.returnValue = ""
+    }
+
+    window.addEventListener("beforeunload", onBeforeUnload)
+
+    return () => window.removeEventListener("beforeunload", onBeforeUnload)
+  }, [])
+}
+
+export default usePreventUnload

package/ui/FileUpload/FileUploadModal.tsx

@@ -0,0 +1,53 @@
+import ActivityIndicator from "../ActivityIndicator"
+import Modal from "../Modal"
+
+import {FileUploadContextProvider} from "./FileUploadContext"
+import FileUploadForm from "./FileUploadForm"
+import UploadButton from "./UploadButton"
+
+import "./file-upload.scss"
+
+
+// TODO: option to disable compression
+
+export const FileUploadModal = ({onHide}) => {
+  const isLoading = false
+
+  const onHideFn = () => null
+
+  return (
+    <FileUploadContextProvider>
+      <Modal className="file-upload-modal" show scrollable={true} onHide={onHideFn}>
+        <Modal.Header className="close-top" closeButton>
+          <img
+            width={22}
+            height={22}
+            className="me-2 align-self-start mt-1"
+            src={`/static/icons/drive/file-upload-s.svg`}
+          />
+          <div>
+            <div>File Upload</div>
+            <small className="text-secondary fw-normal">Upload any type of file, any size.</small>
+          </div>
+        </Modal.Header>
+        <Modal.Body className="py-2">
+          {isLoading && (
+            <div className="d-flex flex-row align-items-center">
+              <ActivityIndicator size={24} />
+              <div className="ms-2">Loading text...</div>
+            </div>
+          )}
+
+          <FileUploadForm />
+        </Modal.Body>
+        <Modal.Footer className="d-flex justify-content-between">
+          <div>
+            <a href="/docs/drives">/docs/drives</a>
+          </div>
+
+          <UploadButton />
+        </Modal.Footer>
+      </Modal>
+    </FileUploadContextProvider>
+  )
+}

package/ui/FileUpload/UploadButton.tsx

@@ -0,0 +1,23 @@
+import {SubmitButton} from "../SubmitButton"
+import {useFileUploadContext} from "./FileUploadContext"
+
+
+export const UploadButton = () => {
+  const {canUpload, isUploading, uploadAllFiles} =
+    useFileUploadContext()
+
+  const onStartUpload = async() => {
+    await uploadAllFiles()
+  }
+
+  return (
+    <SubmitButton
+      id="file-start-upload-button"
+      disabled={!canUpload}
+      isLoading={isUploading}
+      onClick={onStartUpload}
+      title="Start Upload"
+      submittingTitle="Uploading Files..."
+    />
+  )
+}

package/ui/FileUpload/constants.ts

@@ -0,0 +1,16 @@
+export const UPLOAD_MAX_CONCURRENCY = 1
+
+export const HASH_CHUNK_SIZE = 2 * 1024 * 1024 // 1 MB
+
+// must match server chunk size to avoid mongodb chunking again
+export const UPLOAD_CHUNK_SIZE = 1 * 1024 * 1024 // 1MB
+
+export const EVENTS = {
+  BEGIN_HASH: "BEGIN_HASH",
+  HASH_PROGRESS: "HASH_PROGRESS",
+  HASHING: "HASHING",
+  HASH_COMPLETE: "HASH_COMPLETE",
+  UPLOAD_FILE: "UPLOAD_FILE",
+  UPLOAD_COMPLETE: "UPLOAD_COMPLETE",
+  UPLOAD_ERROR: "UPLOAD_ERROR",
+}

package/ui/FileUpload/file-upload.scss

@@ -0,0 +1 @@
+@import "helpers";

package/ui/FileUpload/index.tsx

@@ -0,0 +1,21 @@
+import { FileUploadContextProvider } from "./FileUploadContext"
+import FileUploadForm from "./FileUploadForm"
+import {UploadButton} from "./UploadButton"
+
+import "./file-upload.scss"
+
+
+export const FileUpload = ({
+  onUploadComplete,
+}: {
+  onUploadComplete?: (files: any) => void | Promise<void>
+}) => {
+
+  return (
+    <FileUploadContextProvider onUploadComplete={onUploadComplete}>
+      <FileUploadForm />
+
+      <UploadButton />
+    </FileUploadContextProvider>
+  )
+}

package/ui/FileUpload/upload-worker/get_file_hash.js

@@ -0,0 +1,63 @@
+/* @flow */
+import {createXXHash128} from "hash-wasm"
+
+import {EVENTS, HASH_CHUNK_SIZE} from "../constants"
+
+
+// read and hash the file
+const get_file_hash = async(file) => {
+  if (!file) return
+
+  const hasher = await createXXHash128()
+
+  const reader = new FileReader()
+  const size = file.size
+  let chunks_count = 0
+
+  let offset = 0
+  let bytes_read = 0
+
+  reader.onloadend = async(e) => {
+    if (e.target.readyState === FileReader.DONE) {
+      const chunk = new Uint8Array(e.target.result)
+      bytes_read += chunk.length
+
+      chunks_count++
+
+      console.log(`${chunks_count} hash chunks // ${bytes_read} bytes_read...`)
+
+      await hasher.update(chunk)
+
+      self.postMessage({
+        type: EVENTS.HASH_PROGRESS,
+        payload: {
+          // TODO: use relative path here as well in case we have two same filenames from two dirs
+          filename: file.name,
+          progress: ((bytes_read / size) * 100).toFixed(1),
+        },
+      })
+
+      if (offset < size) {
+        offset += HASH_CHUNK_SIZE
+        const blob = file.slice(offset, offset + HASH_CHUNK_SIZE)
+        reader.readAsArrayBuffer(blob)
+      } else {
+        const hash = hasher.digest()
+
+        self.postMessage({
+          type: EVENTS.HASH_COMPLETE,
+          payload: {
+            filename: file.name,
+            hash,
+          },
+        })
+      }
+    }
+  }
+
+  const blob = file.slice(offset, offset + HASH_CHUNK_SIZE)
+
+  reader.readAsArrayBuffer(blob)
+}
+
+export default get_file_hash

package/ui/FileUpload/upload-worker/index.js

@@ -0,0 +1,15 @@
+/* @flow */
+import upload_file from "./upload_file"
+import get_file_hash from "./get_file_hash"
+
+import {EVENTS} from "../constants"
+
+self.onmessage = ({data: {file, type, hash}}) => {
+  if (type === EVENTS.BEGIN_HASH) {
+    get_file_hash(file)
+  } else if (type === EVENTS.UPLOAD_FILE) {
+    upload_file(file, hash)
+  } else {
+    console.log("upload worker: unknown message", type)
+  }
+}

package/ui/FileUpload/upload-worker/no_compress_exts.ts

@@ -0,0 +1,33 @@
+// we consider these file types to be already compressed
+export const NO_COMPRESS_EXTS = [
+  ".jpg",
+  ".jpeg",
+  ".png",
+  ".gif",
+  ".tif",
+  ".tiff",
+  ".webp",
+  ".heif",
+  ".heic",
+  ".mp3",
+  ".aac",
+  ".ogg",
+  ".flac",
+  ".m4a",
+  ".mp4",
+  ".m4v",
+  ".avi",
+  ".mov",
+  ".mkv",
+  ".webm",
+  ".zip",
+  ".gzip",
+  ".gz",
+  ".rar",
+  ".7z",
+  ".bz2",
+  ".tar.gz",
+  ".tgz",
+  ".jar",
+  ".apk",
+]

package/ui/FileUpload/upload-worker/upload_file.js

@@ -0,0 +1,127 @@
+/* @flow */
+import assert from "assert"
+// import debug from "debug"
+import brotliPromise from "brotli-wasm"
+
+import getBaseUrl from "../../../getBaseUrl"
+
+import {NO_COMPRESS_EXTS} from "./no_compress_exts"
+import {UPLOAD_CHUNK_SIZE, EVENTS} from "../constants"
+
+// const log = debug("rb:upload_worker")
+const log = (...args) => console.log(...args)
+
+let _brotli
+
+
+const check_should_compress = (file) => {
+  for (const ext of NO_COMPRESS_EXTS) {
+    if (file.name.endsWith(ext)) return false
+  }
+  return true
+}
+
+const upload_file = async(file, hash) => {
+  assert(file, "upload_file has no file")
+
+  const should_compress = check_should_compress(file)
+  log("filename:", file.name, "should_compress:", should_compress)
+
+  if (should_compress && !_brotli) {
+    _brotli = await brotliPromise
+  }
+
+  const reader = new FileReader()
+  const size = file.size
+
+  log("total file size", file.size)
+
+  const total_chunks = Math.ceil(size / UPLOAD_CHUNK_SIZE)
+
+  let current_chunks_count = 0
+
+  let offset = 0
+  let total_bytes_read = 0
+
+  reader.onloadend = async(e) => {
+    if (e.target.readyState === FileReader.DONE) {
+      const chunk_index = current_chunks_count
+
+      // how many bytes did we just read
+      const read_chunk_length = e.target.result.byteLength
+
+      const chunk = should_compress
+        ? _brotli.compress(new Uint8Array(e.target.result))
+        : e.target.result
+
+      log("will upload chunk of size:", chunk.size)
+
+      const data = new FormData()
+
+      data.append("file_chunk", new Blob([chunk]))
+      data.append("is_compressed", should_compress ? "yes" : "no")
+      data.append("chunk_index", chunk_index)
+      data.append("total_chunks", total_chunks)
+      data.append("chunk_size", UPLOAD_CHUNK_SIZE)
+      data.append("original_filename", file.name)
+      data.append("mime_type", file.type)
+      data.append("hash", hash)
+      let res_json
+      try {
+        const res = await fetch(`${getBaseUrl()}/rb-api/v1/files/upload_chunk`, {
+          method: "POST",
+          credentials: "include",
+          body: data,
+        })
+        res_json = await res.json()
+        assert(res_json.status === "ok", "failed to fetch")
+      } catch (error) {
+        log("Error uploading chunk:", error)
+        console.log("error json", res_json)
+        // self.postMessage({
+        //   type: EVENTS.UPLOAD_ERROR,
+        //   payload: {
+        //     filename: file.name,
+        //     hash,
+        //     error: error.message,
+        //   },
+        // })
+        //
+        return
+      }
+
+      total_bytes_read += read_chunk_length
+
+      current_chunks_count++
+
+      log(`upload chunks ${current_chunks_count} chunk // ${total_bytes_read} total_bytes_read...`)
+
+      // there are more bytes to be read
+      if (total_bytes_read < size) {
+        offset += UPLOAD_CHUNK_SIZE
+        const blob = file.slice(offset, offset + UPLOAD_CHUNK_SIZE)
+        reader.readAsArrayBuffer(blob)
+      }
+      // reading complete, finalize upload
+      else {
+        console.log("upload complete!")
+        console.log("FINALLL RESS", res_json)
+        self.postMessage({
+          type: EVENTS.UPLOAD_COMPLETE,
+          payload: {
+            filename: file.name,
+            hash,
+          },
+        })
+      }
+    } else {
+      log("unknown filereader state", e.target.readyState)
+    }
+  }
+
+  // start reading the first chunk
+  const blob = file.slice(offset, offset + UPLOAD_CHUNK_SIZE)
+  reader.readAsArrayBuffer(blob)
+}
+
+export default upload_file

package/ui/SubmitButton/{index.js → index.tsx}

@@ -1,31 +1,40 @@
+import {ReactNode} from "react"
 import {useFormContext} from "react-hook-form"
+
 import ActivityIndicator from "../ActivityIndicator"
 
 
-type Props = {
-  id?: string,
-  className: string,
-  disabled: boolean,
-  isLoading: boolean,
-  onClick: Function,
+interface Props {
+  id?: string;
+  className?: string;
+  disabled: boolean;
+  isLoading: boolean;
+  title: string;
+  submittingTitle?: string;
+  onClick: () => void | Promise<void>;
+  children?: ReactNode;
 }
 
 export const SubmitButton = ({
-  className = "",
   id = "btn-submit",
+  className = "",
   title,
   submittingTitle,
   children,
   ...props
 }: Props) => {
 
-  const {formState} = useFormContext()
+  const formContext = useFormContext()
 
-  const {isSubmitting} = formState
+  let isSubmitting = formContext?.formState?.isSubmitting
 
   const isDisabled = props.disabled || props.isLoading || isSubmitting
   const isLoading = props.isLoading || isSubmitting
 
+  if (!formContext && isLoading) {
+    isSubmitting = true
+  }
+
   if (title && children) {
     throw new Error("button cannot have both title and children props")
   }