@juzi/file-box 1.7.0

package/src/file-box.type.ts

@@ -0,0 +1,145 @@
+import type http    from 'http'
+import type {
+  Readable,
+  Writable,
+}                   from 'stream'
+
+import type {
+  FileBox,
+}                 from './file-box.js'
+
+interface Pipeable {
+  // pipe: typeof Readable.prototype.pipe,
+  pipe<T extends Writable>(destination: T, options?: { end?: boolean; }): T;
+}
+
+/**
+ * Huan(202002):
+ *  We need to keep this enum number to be consistent
+ *  because of toJSON & fromJSON need the same type number across versoins.
+ *  and gRPC maybe will use those numbers in the future as well.
+ */
+enum FileBoxType {
+  Unknown = 0,
+
+  /**
+   * 1. toJSON() Serializable
+   *  - Base64
+   *  - Url
+   *  - QRCode
+   *  - UUID
+   *
+   * 2. toJSON() NOT Serializable: need to convert to FileBoxType.Base64 before call toJSON()
+   *  - Buffer
+   *  - Stream
+   *  - File
+   */
+  Base64  = 1,
+  Url     = 2,
+  QRCode  = 3,
+
+  Buffer  = 4,
+  File    = 5,
+  Stream  = 6,
+  Uuid    = 7,
+}
+
+interface Metadata {
+  [key: string]: any,
+}
+
+/**
+ * URI to the file
+ * See:
+ *  https://nodejs.org/api/fs.html#fs_url_object_support
+ *  https://danielmiessler.com/study/url-uri/
+ *
+ * FileType: LOCAL, REMOTE, BUFFER, STREAM
+ *
+ */
+interface FileBoxOptionsCommon {
+  /**
+   * File base name: name + ext
+   *  like: "file.txt"
+   */
+  name: string
+
+  /**
+   * Can be only set once
+   */
+  metadata?: Metadata,
+
+  /**
+   * Size
+   */
+  size?: number
+  md5?: string
+}
+
+interface FileBoxOptionsFile {
+  type : FileBoxType.File
+  path : string
+}
+interface FileBoxOptionsUrl {
+  type     : FileBoxType.Url
+  url      : string
+  headers? : http.OutgoingHttpHeaders
+}
+interface FileBoxOptionsBuffer {
+  type   : FileBoxType.Buffer
+  buffer : Buffer
+}
+interface FileBoxOptionsStream {
+  type   : FileBoxType.Stream
+  stream : Readable
+}
+interface FileBoxOptionsQRCode {
+  type   : FileBoxType.QRCode,
+  qrCode : string,
+}
+interface FileBoxOptionsBase64 {
+  type   : FileBoxType.Base64,
+  base64 : string,
+}
+interface FileBoxOptionsUuid {
+  type : FileBoxType.Uuid
+  uuid : string
+}
+
+type FileBoxOptions = FileBoxOptionsCommon & (
+    never
+  | FileBoxOptionsBase64
+  | FileBoxOptionsBuffer
+  | FileBoxOptionsFile
+  | FileBoxOptionsQRCode
+  | FileBoxOptionsStream
+  | FileBoxOptionsUrl
+  | FileBoxOptionsUuid
+)
+
+type FileBoxJsonObject =  FileBoxOptionsCommon & (
+  | FileBoxOptionsBase64
+  | FileBoxOptionsUrl
+  | FileBoxOptionsQRCode
+  | FileBoxOptionsUuid
+)
+
+type UuidLoader = (this: FileBox, uuid: string)      => Promise<Readable>
+type UuidSaver  = (this: FileBox, stream: Readable)  => Promise<string>
+
+export type {
+  FileBoxJsonObject,
+  FileBoxOptions,
+  FileBoxOptionsBase64,
+  FileBoxOptionsCommon,
+  FileBoxOptionsQRCode,
+  FileBoxOptionsUrl,
+  FileBoxOptionsUuid,
+  Metadata,
+  Pipeable,
+  UuidSaver,
+  UuidLoader,
+}
+export {
+  FileBoxType,
+}

package/src/interface.spec.ts

@@ -0,0 +1,17 @@
+#!/usr/bin/env -S node --no-warnings --loader ts-node/esm
+
+// tslint:disable:no-shadowed-variable
+import { test }  from 'tstest'
+
+import type {
+  FileBoxInterface,
+}                         from './interface.js'
+
+import {
+  FileBox,
+}           from './file-box.js'
+
+test('FileBoxInterface', async t => {
+  const fileBox: FileBoxInterface = FileBox.fromQRCode('test')
+  t.ok(fileBox, 'should be ok with interface')
+})

package/src/interface.ts

@@ -0,0 +1,47 @@
+import type {
+  Constructor,
+}                       from 'clone-class'
+import type {
+  FileBox,
+}                       from './file-box.js'
+
+interface FileBoxInterface {
+  type      : FileBox['type']
+
+  name      : FileBox['name']
+  size      : FileBox['size']
+  md5       : FileBox['md5']
+  mediaType : FileBox['mediaType']
+  metadata  : FileBox['metadata']
+
+  // version: any
+  // ready: any
+  // syncRemote: any
+  // transformBufferToStream: any
+  // transformBase64ToStream: any
+  // transformFileToStream: any
+  // ransformUrlToStream: any
+  // transformQRCodeToStream: any
+  // transformUrlToStream: any
+
+  toBase64  : FileBox['toBase64']
+  toBuffer  : FileBox['toBuffer']
+  toDataURL : FileBox['toDataURL']
+  toFile    : FileBox['toFile']
+  toJSON    : FileBox['toJSON']
+  toQRCode  : FileBox['toQRCode']
+  toStream  : FileBox['toStream']
+  toUuid    : FileBox['toUuid']
+
+  pipe: FileBox['pipe']
+}
+
+/**
+ * Huan(202110): TODO support static methods after TypeScript 4.5: fromXXX()
+ */
+type FileBoxConstructor = Constructor<FileBoxInterface>
+
+export type {
+  FileBoxInterface,
+  FileBoxConstructor,
+}

package/src/misc.spec.ts

@@ -0,0 +1,70 @@
+#!/usr/bin/env -S node --no-warnings --loader ts-node/esm
+
+// tslint:disable:no-shadowed-variable
+import { test }  from 'tstest'
+
+import {
+  dataUrlToBase64,
+  httpHeaderToFileName,
+  httpHeadHeader,
+  httpStream,
+  streamToBuffer,
+}                         from './misc.js'
+
+test('dataUrl to base64', async t => {
+  const base64 = [
+    'R0lGODlhEAAQAMQAAORHHOVSKudfOulrSOp3WOyDZu6QdvCchPGolfO0o/XBs/fNwfjZ0frl',
+    '3/zy7////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA',
+    'ACH5BAkAABAALAAAAAAQABAAAAVVICSOZGlCQAosJ6mu7fiyZeKqNKToQGDsM8hBADgUXoGA',
+    'iqhSvp5QAnQKGIgUhwFUYLCVDFCrKUE1lBavAViFIDlTImbKC5Gm2hB0SlBCBMQiB0UjIQA7',
+  ].join('')
+  const dataUrl = [
+    'data:image/png;base64,',
+    base64,
+  ].join('')
+
+  t.equal(base64, dataUrlToBase64(dataUrl), 'should get base64 from dataUrl')
+})
+
+test('httpHeadHeader', async t => {
+  const URL = 'https://github.com/huan/file-box/archive/v0.6.tar.gz'
+
+  const EXPECTED_HEADERS_KEY   = 'content-disposition'
+  const EXPECTED_HEADERS_VALUE = 'attachment; filename=file-box-0.6.tar.gz'
+
+  const headers = await httpHeadHeader(URL)
+
+  t.equal(headers[EXPECTED_HEADERS_KEY], EXPECTED_HEADERS_VALUE, 'should get the headers right')
+})
+
+test('httpHeaderToFileName', async t => {
+  const HEADERS_QUOTATION_MARK: any = {
+    'content-disposition': 'attachment; filename="db-0.0.19.zip"',
+  }
+  const HEADERS_NO_QUOTATION_MARK: any = {
+    'content-disposition': 'attachment; filename=db-0.0.19.zip',
+  }
+  const EXPECTED_FILE_NAME = 'db-0.0.19.zip'
+
+  let filename = httpHeaderToFileName(HEADERS_QUOTATION_MARK)
+  t.equal(filename, EXPECTED_FILE_NAME, 'should get filename with quotation mark')
+
+  filename = httpHeaderToFileName(HEADERS_NO_QUOTATION_MARK)
+  t.equal(filename, EXPECTED_FILE_NAME, 'should get filename with no quotation mark')
+})
+
+test('httpStream', async t => {
+  const URL = 'https://httpbin.org/headers'
+
+  const MOL_KEY = 'Mol'
+  const MOL_VAL = '42'
+
+  const headers = {} as { [idx: string]: string }
+  headers[MOL_KEY] = MOL_VAL
+
+  const res = await httpStream(URL, headers)
+
+  const buffer = await streamToBuffer(res)
+  const obj = JSON.parse(buffer.toString())
+  t.equal(obj.headers[MOL_KEY], MOL_VAL, 'should send the header right')
+})

package/src/misc.ts

@@ -0,0 +1,157 @@
+import http    from 'http'
+import https   from 'https'
+import { URL } from 'url'
+import type stream   from 'stream'
+
+import { HTTP_TIMEOUT } from './config.js'
+
+export function dataUrlToBase64 (dataUrl: string): string {
+  const dataList = dataUrl.split(',')
+  return dataList[dataList.length - 1]!
+}
+
+/**
+ * Get http headers for specific `url`
+ * follow 302 redirection for max `REDIRECT_TTL` times.
+ *
+ * @credit https://stackoverflow.com/a/43632171/1123955
+ */
+export async function httpHeadHeader (url: string): Promise<http.IncomingHttpHeaders> {
+
+  let REDIRECT_TTL = 7
+
+  while (true) {
+    if (REDIRECT_TTL-- <= 0) {
+      throw new Error(`ttl expired! too many(>${REDIRECT_TTL}) 302 redirection.`)
+    }
+
+    const res = await _headHeader(url)
+
+    if (!/^3/.test(String(res.statusCode))) {
+      return res.headers
+    }
+
+    // console.log('302 found for ' + url)
+
+    if (!res.headers.location) {
+      throw new Error('302 found but no location!')
+    }
+
+    url = res.headers.location
+  }
+
+  async function _headHeader (destUrl: string): Promise<http.IncomingMessage> {
+    const parsedUrl = new URL(destUrl)
+    const options = {
+      method   : 'HEAD',
+      // method   : 'GET',
+    }
+
+    let request: typeof http.request
+
+    if (parsedUrl.protocol === 'https:') {
+      request = https.request
+    } else if (parsedUrl.protocol === 'http:') {
+      request = http.request
+    } else {
+      throw new Error('unknown protocol: ' + parsedUrl.protocol)
+    }
+
+    return new Promise<http.IncomingMessage>((resolve, reject) => {
+      let res: undefined | http.IncomingMessage
+      const req = request(parsedUrl, options, (response) => {
+        res = response
+        resolve(res)
+      })
+        .on('error', reject)
+        .setTimeout(HTTP_TIMEOUT, () => {
+          const e = new Error(`FileBox: Http request timeout (${HTTP_TIMEOUT})!`)
+          req.emit('error', e)
+          req.destroy()
+        })
+        .end()
+    })
+  }
+}
+
+export function httpHeaderToFileName (
+  headers: http.IncomingHttpHeaders,
+): null | string {
+  const contentDisposition = headers['content-disposition']
+
+  if (!contentDisposition) {
+    return null
+  }
+
+  // 'content-disposition': 'attachment; filename=db-0.0.19.zip'
+  const matches = contentDisposition.match(/attachment; filename="?(.+[^"])"?$/i)
+
+  if (matches && matches[1]) {
+    return matches[1]
+  }
+
+  return null
+}
+
+export async function httpStream (
+  url     : string,
+  headers : http.OutgoingHttpHeaders = {},
+): Promise<http.IncomingMessage> {
+  const parsedUrl = new URL(url)
+
+  const protocol  = parsedUrl.protocol
+
+  const options: http.RequestOptions = {}
+
+  let get: typeof https.get
+
+  if (!protocol) {
+    throw new Error('protocol is empty')
+  }
+
+  if (protocol.match(/^https:/i)) {
+    get           = https.get
+    options.agent = https.globalAgent
+  } else if (protocol.match(/^http:/i)) {
+    get           = http.get
+    options.agent = http.globalAgent
+  } else {
+    throw new Error('protocol unknown: ' + protocol)
+  }
+
+  options.headers = {
+    ...headers,
+  }
+
+  return new Promise<http.IncomingMessage>((resolve, reject) => {
+    let res: http.IncomingMessage | null = null
+    const req = get(parsedUrl, options, (response) => {
+      res = response
+      resolve(res)
+    })
+      .on('error', reject)
+      .setTimeout(HTTP_TIMEOUT, () => {
+        const e = new Error(`FileBox: Http request timeout (${HTTP_TIMEOUT})!`)
+        if (res) {
+          res.emit('error', e)
+        }
+        req.emit('error', e)
+        req.destroy()
+      })
+      .end()
+  })
+}
+
+export async function streamToBuffer (
+  stream: stream.Readable,
+): Promise<Buffer> {
+  return new Promise<Buffer>((resolve, reject) => {
+    const bufferList: Buffer[] = []
+    stream.once('error', reject)
+    stream.once('end', () => {
+      const fullBuffer = Buffer.concat(bufferList)
+      resolve(fullBuffer)
+    })
+    stream.on('data', buffer => bufferList.push(buffer))
+  })
+}

package/src/mod.ts

@@ -0,0 +1,33 @@
+import {
+  FileBoxType,
+}                       from './file-box.type.js'
+import {
+  VERSION,
+}                       from './config.js'
+import {
+  FileBox,
+}                       from './file-box.js'
+import type {
+  FileBoxInterface,
+  FileBoxConstructor,
+}                       from './interface.js'
+import type {
+  UuidSaver,
+  UuidLoader,
+}                       from './file-box.type.js'
+import {
+  UniformResourceNameRegistry,
+}                             from './urn-registry/uniform-resource-name-registry.js'
+
+export type {
+  UuidSaver,
+  UuidLoader,
+  FileBoxInterface,
+  FileBoxConstructor,
+}
+export {
+  FileBox,
+  FileBoxType,
+  UniformResourceNameRegistry,
+  VERSION,
+}

package/src/pure-functions/sized-chunk-transformer.spec.ts

@@ -0,0 +1,52 @@
+#!/usr/bin/env -S node --no-warnings --loader ts-node/esm
+
+/* eslint @typescript-eslint/no-unused-vars:off */
+
+import { test } from 'tstest'
+
+import {
+  PassThrough,
+  Readable,
+}                     from 'stream'
+
+import { sizedChunkTransformer } from './sized-chunk-transformer.js'
+
+test('chunkerTransformStream()', async t => {
+  const DATA_LIST = [
+    'a',
+    'b',
+  ]
+  const DATA = DATA_LIST.join('')
+
+  const createStream = () => {
+    const stream = new PassThrough()
+    stream.end(DATA)
+    return stream
+  }
+
+  const getDataList = (stream: Readable) => {
+    return new Promise<any[]>(resolve => {
+      const list = [] as any[]
+      stream.on('end', () => resolve(list))
+      stream.on('data', chunk => list.push(chunk))
+    })
+  }
+
+  const newStream0 = createStream()
+  const dataList0 = await getDataList(newStream0)
+
+  t.equal(dataList0.length, 1, 'should get 1 chunks')
+  t.equal(dataList0[0].toString(), DATA, 'should get data')
+
+  const newStream1 = createStream().pipe(sizedChunkTransformer(2))
+  const dataList1 = await getDataList(newStream1)
+
+  t.equal(dataList1.length, 1, 'should get 1 chunks')
+  t.equal(dataList1[0].toString(), DATA, 'should get data')
+
+  const newStream2 = createStream().pipe(sizedChunkTransformer(1))
+  const dataList2 = await getDataList(newStream2)
+
+  t.equal(dataList2.length, 2, 'should get 2 chunks')
+  t.equal(dataList2.join(''), DATA, 'should get data')
+})

package/src/pure-functions/sized-chunk-transformer.ts

@@ -0,0 +1,49 @@
+/**
+ * ChunkerTransformStream, a transform stream to take arbitrary chunk sizes and make them consistent
+ * https://codereview.stackexchange.com/q/57492/185709
+ */
+import stream from 'stream'
+
+/**
+ * TCP streaming workload tuning
+ *  https://www.ibm.com/docs/en/aix/7.2?topic=tuning-tcp-streaming-workload
+ */
+const DEFAULT_CHUNK_BYTE = 256 * 1024 // 256 KB
+
+/**
+ * @param chunkByte The size of the chunks to be created
+ * @returns
+ */
+function sizedChunkTransformer (chunkByte = DEFAULT_CHUNK_BYTE) {
+  let buffer = Buffer.from([])
+
+  const transform: stream.TransformOptions['transform'] = function (chunk, _, done) {
+    buffer = Buffer.concat([ buffer, chunk ])
+
+    while (buffer.length >= chunkByte) {
+      this.push(buffer.slice(0, chunkByte))
+      buffer = buffer.slice(chunkByte)
+    }
+
+    done()
+  }
+
+  const flush: stream.TransformOptions['flush'] = function (done) {
+    if (buffer.length) {
+      this.push(buffer)
+    }
+    done()
+  }
+
+  const chunker = new stream.Transform({
+    flush,
+    objectMode: true,
+    transform,
+  })
+
+  return chunker
+}
+
+export {
+  sizedChunkTransformer,
+}

package/src/qrcode.spec.ts

@@ -0,0 +1,55 @@
+#!/usr/bin/env -S node --no-warnings --loader ts-node/esm
+
+import { test } from 'tstest'
+
+import {
+  bufferToQrValue,
+  qrValueToStream,
+}                     from './qrcode.js'
+
+test('imageBase64ToQrCode()', async t => {
+  const QRCODE_IMAGE_BASE64 = [
+    'iVBORw0KGgoAAAANSUhEUgAAAMgAAADIAQMAAACXljzdAAAABlBMVEX///8AAABVwtN+AAAA',
+    'CXBIWXMAAA7EAAAOxAGVKw4bAAAA7klEQVRYw+2WsQ3EIAxFjShSMgKjZLRktIzCCJQpIv7Z',
+    'hCiXO/qzT/wCWXo0X3wbEw0NWVaEKM187KHW2QLZ+AhpXovfQ+J6skEWHELqBa5NEeCwR7iS',
+    'V7BDzuzAiZ9eqn5IWjfWXHf7VCO5tPAM6U9AjSRideyHFn4FiuvDqV5CM9rZXuF2pZmIAjZy',
+    'x4S0MDdBxEmu3TrliPf7iglPvuLlRydfU3P70UweCSK+ZYK0mUg1O4AVcv0/8itGkC7SdiTH',
+    '0+Mz19oJZ4NkhhSPbIhQkQGI8u1HJzmzs7p7pzNAru2pJb6z8ykkQ0P/pheK6vjurjf7+wAA',
+    'AABJRU5ErkJggg==',
+  ].join('')
+  const EXPECTED_QRCODE_TEXT = 'hello, world!'
+
+  const buf = Buffer.from(QRCODE_IMAGE_BASE64, 'base64')
+  const text = await bufferToQrValue(buf)
+  t.equal(text, EXPECTED_QRCODE_TEXT, 'should decode qrcode image base64')
+})
+
+test('qrValueToStream()', async t => {
+  const QRCODE_VALUE = 'hello, world!'
+  const EXPECTED_QRCODE_IMAGE_BASE64 = [
+    'iVBORw0KGgoAAAANSUhEUgAAAHQAAAB0CAYAAABUmhYnAAAAAklEQVR4AewaftIAAAKcSURBVO3BQY7',
+    'cQAwEwSxC//9yeo88NSBIsx7TjIg/WGMUa5RijVKsUYo1SrFGKdYoxRqlWKMUa5RijVKsUYo1SrFGKd',
+    'YoxRrl4qEk/CaVkyTcoXKShN+k8kSxRinWKMUa5eJlKm9Kwh0qn6TypiS8qVijFGuUYo1y8WFJuEPlj',
+    'iR0Kl0SOpUuCZ3KHUm4Q+WTijVKsUYp1igXw6n8T4o1SrFGKdYoF8MloVOZrFijFGuUYo1y8WEq3yQJ',
+    'ncoTKt+kWKMUa5RijXLxsiR8M5UuCZ3KSRK+WbFGKdYoxRrl4iGVf5nKicq/pFijFGuUYo0Sf/BAEjq',
+    'VLglvUnkiCZ3KSRLepPJJxRqlWKMUa5SLlyXhROU3JaFT6ZJwonKShCeS0Kk8UaxRijVKsUaJP3hREj',
+    'qVO5JwotIloVO5Iwl3qJwk4Q6VNxVrlGKNUqxRLv6yJJyodEl4Igl3qHRJ6FTuUPmkYo1SrFGKNcrFQ',
+    '0k4ScITSehUuiScJKFT6ZLQqXRJuEPljiR0Kk8Ua5RijVKsUS4eUvkmSbhDpUvCHUk4UflNxRqlWKMU',
+    'a5SLh5Lwm1Q6lS4JdyThCZWTJJyovKlYoxRrlGKNcvEylTcl4SQJnUqXhCdUuiScJOFvKtYoxRqlWKN',
+    'cfFgS7lB5k0qXhDuS0Kl0SehUTpLQJaFTeaJYoxRrlGKNcjFcEjqVJ5JwkoROpVP5pGKNUqxRijXKxX',
+    '8mCScqXRJOVE6S0Kl8UrFGKdYoxRrl4sNUPknlDpUuCV0SOpWTJHyTYo1SrFGKNcrFy5Lwm5LQqXQqX',
+    'RI6lZMkdConKidJ6FTeVKxRijVKsUaJP1hjFGuUYo1SrFGKNUqxRinWKMUapVijFGuUYo1SrFGKNUqx',
+    'RinWKMUa5Q8Ztu740xD9iQAAAABJRU5ErkJggg==',
+  ].join('')
+
+  const stream = await qrValueToStream(QRCODE_VALUE)
+
+  const chunks = [] as Buffer[]
+  for await (const chunk of stream) {
+    chunks.push(chunk as Buffer)
+  }
+  const buf = Buffer.concat(chunks)
+  const base64Text = buf.toString('base64')
+
+  t.equal(base64Text, EXPECTED_QRCODE_IMAGE_BASE64, 'should encode QR Code value to expected image')
+})

package/src/qrcode.ts

@@ -0,0 +1,38 @@
+import {
+  PassThrough,
+  Readable,
+}               from 'stream'
+
+// The npm package of my best choice for QR code decoding on Angular SPA
+// https://dev.to/j_sakamoto/the-npm-package-of-my-best-choice-for-qr-code-decoding-on-angular-spa-4747?returning-user=true
+import Jimp             from 'jimp'
+import jsQR             from 'jsqr'
+
+/**
+ * https://www.npmjs.com/package/qrcode
+ *  Huan(202002): This module is encode only.
+ */
+import { toFileStream } from 'qrcode'
+
+export async function bufferToQrValue (buf: Buffer): Promise<string> {
+  const image = await Jimp.read(buf)
+  const qrCodeImageArray = new Uint8ClampedArray(image.bitmap.data.buffer)
+
+  const qrCodeResult = jsQR(
+    qrCodeImageArray,
+    image.bitmap.width,
+    image.bitmap.height,
+  )
+
+  if (qrCodeResult) {
+    return qrCodeResult.data
+  } else {
+    throw new Error('bufferToQrcode(buf) fail!')
+  }
+}
+
+export async function qrValueToStream (value: string): Promise<Readable> {
+  const stream = new PassThrough()
+  await toFileStream(stream, value) // only support .png for now
+  return stream
+}

package/src/typings.d.ts

@@ -0,0 +1 @@
+declare module 'jsqr'

package/src/urn-registry/mod.ts

@@ -0,0 +1,7 @@
+import {
+  UniformResourceNameRegistry,
+}                                 from './uniform-resource-name-registry.js'
+
+export {
+  UniformResourceNameRegistry,
+}

package/src/urn-registry/random-uuid.ts

@@ -0,0 +1,7 @@
+import * as uuid from 'uuid'
+
+const randomUuid = uuid.v4
+
+export {
+  randomUuid,
+}