@juzi/file-box 1.7.0

package/src/urn-registry/uniform-resource-name-registry.spec.ts

@@ -0,0 +1,109 @@
+#!/usr/bin/env -S node --no-warnings --loader ts-node/esm
+
+import {
+  test,
+  sinon,
+}           from 'tstest'
+
+import { FileBox } from '../file-box.js'
+
+import { UniformResourceNameRegistry } from './uniform-resource-name-registry.js'
+
+test('UniformResourceNameRegistry class', async t => {
+  const QRCODE = 'test qrcode'
+
+  const urnRegistry = new UniformResourceNameRegistry()
+  await urnRegistry.init()
+
+  const fileBox = FileBox.fromQRCode(QRCODE)
+  const stream = await fileBox.toStream()
+
+  const uuid = await urnRegistry.save(stream)
+
+  const stream2 = await urnRegistry.load(uuid)
+  t.ok(stream2, 'should load stream')
+
+  const fileBox2 = FileBox.fromStream(stream2!, 'test')
+  const qrcode = await fileBox2.toQRCode()
+
+  t.equal(qrcode, QRCODE, 'should get back the qrcode data')
+
+  await t.resolves(() => urnRegistry.load(uuid), 'should not reject when load a UUID again')
+
+  await urnRegistry.destroy()
+})
+
+test('expireMilliseconds: in time', async t => {
+  const expireMilliseconds = 3
+  const urnRegistry = new UniformResourceNameRegistry({
+    expireMilliseconds,
+  })
+  await urnRegistry.init()
+
+  const uuid = await urnRegistry.save(await FileBox.fromQRCode('qr').toStream())
+  await new Promise(resolve => setTimeout(resolve, 1))
+  await t.resolves(() => urnRegistry.load(uuid), `should not expire after 1ms (with ${expireMilliseconds}ms expire)`)
+
+  await urnRegistry.destroy()
+})
+
+test('expireMilliseconds: time out', async t => {
+  const sandbox = sinon.createSandbox({
+    useFakeTimers: true,
+  })
+
+  const expireMilliseconds = 10 * 60 * 1000  // 10 minute
+  const urnRegistry = new UniformResourceNameRegistry({
+    expireMilliseconds,
+  })
+  await urnRegistry.init()
+
+  /**
+   * Time: 0
+   */
+  const uuid1 = await urnRegistry.save(await FileBox.fromQRCode('qr').toStream())
+  await t.resolves(() => urnRegistry.load(uuid1), 'should load uuid1 successfully')
+
+  /**
+   * Time: 5
+   */
+  await sandbox.clock.tickAsync(5 * 60 * 1000)
+  const uuid2 = await urnRegistry.save(await FileBox.fromQRCode('qr2').toStream())
+  await t.resolves(() => urnRegistry.load(uuid1), 'should load uuid1 successfully after 5 minutes')
+  await t.resolves(() => urnRegistry.load(uuid2), 'should load uuid2 successfully')
+
+  /**
+   * Time: 11
+   */
+  await sandbox.clock.tickAsync(6 * 60 * 1000)
+  await t.rejects(() => urnRegistry.load(uuid1), 'should load uuid1 fail because it is expired')
+  await t.resolves(() => urnRegistry.load(uuid2), 'should load uuid2 successfully 6 minutes after it has been saved')
+
+  /**
+   * Time 20
+   */
+  await sandbox.clock.tickAsync(9 * 60 * 1000)
+  await t.rejects(() => urnRegistry.load(uuid2), 'should load uuid2 fail because it is expired')
+
+  await urnRegistry.destroy()
+  sandbox.restore()
+})
+
+test('URN FileBox helper smoke testing', async t => {
+  const QRCODE = 'test qrcode'
+
+  const urnRegistry = new UniformResourceNameRegistry()
+
+  const UUIDFileBox = urnRegistry.getFileBox()
+
+  const uuid = await UUIDFileBox
+    .fromQRCode(QRCODE)
+    .toUuid()
+
+  const qrcode = await UUIDFileBox
+    .fromUuid(uuid)
+    .toQRCode()
+
+  t.equal(qrcode, QRCODE, 'should get back the qrcode data')
+  urnRegistry.destroy()
+})

package/src/urn-registry/uniform-resource-name-registry.ts

@@ -0,0 +1,342 @@
+/**
+ * Huan(202110): Assignment and Resolution of Uniform Resource Names
+ *  https://datatracker.ietf.org/wg/urn/about/
+ */
+
+/**
+ * RFC 4122: A Universally Unique IDentifier (UUID) URN Namespace
+ *  ------------------------------------------------------------
+ *  This specification defines a Uniform Resource Name namespace for
+ *  UUIDs (Universally Unique IDentifier), also known as GUIDs (Globally
+ *  Unique IDentifier).  A UUID is 128 bits long, and can guarantee
+ *  uniqueness across space and time.  UUIDs were originally used in the
+ *  Apollo Network Computing System and later in the Open Software
+ *  Foundation's (OSF) Distributed Computing Environment (DCE), and then
+ *  in Microsoft Windows platforms.
+ *
+ *  The information here is meant to be a concise guide for those wishing
+ *  to implement services using UUIDs as URNs.  Nothing in this document
+ *  should be construed to override the DCE standards that defined UUIDs.
+ */
+
+/**
+ * RFC 2141: Uniform Resource Names (URNs) Syntax
+ * ----------------------------------------------
+ *  Uniform Resource Names (URNs) are intended to serve as persistent,
+ *  location-independent, resource identifiers. This document sets
+ *  forward the canonical syntax for URNs.  A discussion of both existing
+ *  legacy and new namespaces and requirements for URN presentation and
+ *  transmission are presented.  Finally, there is a discussion of URN
+ *  equivalence and how to determine it.
+ */
+import fs     from 'fs'
+import os     from 'os'
+import path   from 'path'
+
+import type { Readable } from 'stream'
+
+import { instanceToClass }  from 'clone-class'
+import { log }              from 'brolog'
+
+import {
+  FileBox,
+}                         from '../file-box.js'
+
+import {
+  randomUuid,
+}               from './random-uuid.js'
+
+/**
+ * A UUID will be only keep for a certain time.
+ */
+const DEFAULT_UUID_EXPIRE_MINUTES         = 30
+const DEFAULT_UUID_PURGE_INTERVAL_MINUTES = 1
+
+interface UniformResourceNameRegistryOptions {
+  expireMilliseconds? : number,
+  storeDir?           : string,
+}
+
+class UniformResourceNameRegistry {
+
+  protected static processExitMap = new WeakMap<
+    UniformResourceNameRegistry,
+    Function
+  >()
+
+  /**
+   * The directory that store all UUID files
+   */
+  protected storeDir: string
+
+  protected expireMilliseconds: number
+
+  /**
+   * Key: expiretime
+   * Value: the array of UUID that will expires after the `expiretime` (Key)
+   */
+  protected uuidExpiringTable: Map<
+    number,
+    string[]
+  >
+
+  protected purgerTimer?: ReturnType<typeof setInterval>
+
+  constructor (
+    options: UniformResourceNameRegistryOptions = {},
+  ) {
+    log.verbose('UniformResourceNameRegistry', 'constructor("%s")', JSON.stringify(options))
+
+    this.uuidExpiringTable = new Map()
+
+    this.expireMilliseconds = options.expireMilliseconds ?? (DEFAULT_UUID_EXPIRE_MINUTES * 60 * 1000 * 1000)
+    this.storeDir = options.storeDir || path.join(
+      os.tmpdir(),
+      'file-box-urn-registry.' + String(process.pid),
+    )
+  }
+
+  /**
+   * Return a FileBox Interface with the current URN Registry for conience
+   */
+  getFileBox (): typeof FileBox {
+    this.init()
+
+    class UUIDFileBox extends FileBox {}
+    UUIDFileBox.setUuidLoader(this.load.bind(this))
+    UUIDFileBox.setUuidSaver(this.save.bind(this))
+    return UUIDFileBox
+  }
+
+  /**
+   * init the UUID registry
+   *
+   * must be called before use.
+   */
+  init () {
+    log.verbose('UniformResourceNameRegistry', 'init()')
+
+    try {
+      const stat = fs.statSync(this.storeDir)
+      if (!stat.isDirectory()) {
+        throw new Error(this.storeDir + ' is Not a directory')
+      }
+    } catch (e) {
+      if ((e as NodeJS.ErrnoException).code === 'ENOENT') {
+        fs.mkdirSync(this.storeDir, { recursive: true })
+      } else {
+        throw e
+      }
+    }
+
+    if (!this.purgerTimer) {
+      this.addProcessExitListener()
+
+      this.purgerTimer = setInterval(
+        () => this.purgeExpiredUuid(),
+        DEFAULT_UUID_PURGE_INTERVAL_MINUTES * 60 * 1000,
+      )
+    }
+
+  }
+
+  protected purgeExpiredUuid () {
+    log.verbose('UniformResourceNameRegistry', 'purgeExpiredUuid()')
+    const expireTimeList = [ ...this.uuidExpiringTable.keys() ]
+      .sort((a, b) => Number(a) - Number(b))
+
+    for (const expireTime of expireTimeList) {
+      if (Date.now() < expireTime) {
+        // The earliest expire time is in the future
+        break
+      }
+
+      const uuidList = this.uuidExpiringTable.get(expireTime) || []
+      this.uuidExpiringTable.delete(expireTime)
+
+      for (const uuid of uuidList) {
+        this.purge(uuid).catch(console.error)
+      }
+    }
+  }
+
+  /**
+   * Clean up by calling this.destroy() before process exit
+   */
+  protected addProcessExitListener () {
+    log.verbose('UniformResourceNameRegistry', 'addProcessExitListener()')
+
+    const Klass = instanceToClass(this, UniformResourceNameRegistry)
+
+    /**
+     * If we have already registered the listener, do nothing.
+     */
+    if (Klass.processExitMap.has(this)) {
+      return
+    }
+
+    const destroyCallback = () => this.destroy()
+
+    process.addListener('exit', destroyCallback)
+    Klass.processExitMap.set(
+      this,
+      () => process.removeListener('exit', destroyCallback),
+    )
+  }
+
+  protected uuidFileName (uuid: string): string {
+    return path.join(
+      this.storeDir,
+      uuid + '.dat',
+    )
+  }
+
+  /**
+   * @deprecated use `load()` instead
+   */
+  async resolve (uuid: string): Promise<Readable> {
+    log.warn('UniformResourceNameRegistry', 'resolve() is deprecated: use `load()` instead.\n%s', new Error().stack)
+    return this.load(uuid)
+  }
+
+  /**
+   * `resolve()` can only be used once.
+   *  after resolve(), the UUID will be not exist any more
+   */
+  async load (uuid: string): Promise<Readable> {
+    log.verbose('UniformResourceNameRegistry', 'load(%s)', uuid)
+
+    const filename  = this.uuidFileName(uuid)
+    const stream    = fs.createReadStream(filename)
+
+    await new Promise<void>((resolve, reject) => {
+      stream.on('ready', resolve)
+      stream.on('error', reject)
+    })
+
+    return stream
+  }
+
+  /**
+   * @deprecated use `save()` instead
+   */
+  async register (stream: Readable): Promise<string> {
+    log.verbose('UniformResourceNameRegistry', 'register() deprecated: use save() instead.\n%s', new Error().stack)
+    return this.save(stream)
+  }
+
+  /**
+   * Save the `Readable` stream and return a random UUID
+   *  The UUID will be expired after MAX_KEEP_MINUTES
+   */
+  async save (stream: Readable): Promise<string> {
+    log.verbose('UniformResourceNameRegistry', 'save(stream)')
+
+    const uuid = randomUuid()
+
+    const fileStream = fs.createWriteStream(this.uuidFileName(uuid))
+    const future = new Promise<void>((resolve, reject) => {
+      stream.on('end',        resolve)
+      stream.on('error',      reject)
+      fileStream.on('error',  reject)
+    })
+    stream.pipe(fileStream)
+    await future
+
+    this.addToExpiringTable(uuid)
+
+    return uuid
+  }
+
+  /**
+   * Set a timer to execute delete callback after `expireMilliseconds`
+   */
+  protected addToExpiringTable (uuid: string): void {
+    log.verbose('UniformResourceNameRegistry', 'addToExpiringTable(%s)', uuid)
+
+    const expireTime         = Date.now() + this.expireMilliseconds
+    const expireTimeInterval = DEFAULT_UUID_PURGE_INTERVAL_MINUTES * 60 * 1000
+
+    // https://stackoverflow.com/a/22687090
+    const expireTimeNearestMinute = Math.ceil(expireTime / expireTimeInterval) * expireTimeInterval
+
+    const uuidList = this.uuidExpiringTable.get(expireTime) || []
+    uuidList.push(uuid)
+    this.uuidExpiringTable.set(expireTimeNearestMinute, uuidList)
+
+    log.silly('UniformResourceNameRegistry', 'addToExpiringTable() uuidList.length = %s, expireTime = %s',
+      uuidList.length,
+      expireTimeNearestMinute,
+    )
+  }
+
+  protected async purge (uuid: string): Promise<void> {
+    log.verbose('UniformResourceNameRegistry', 'purge(%s)', uuid)
+
+    const file = this.uuidFileName(uuid)
+    try {
+      await fs.promises.unlink(file)
+      log.silly('UniformResourceNameRegistry', 'purge() %s', file)
+    } catch (e) {
+      log.warn('UniformResourceNameRegistry', 'purge() rejection:', (e as Error).message)
+    }
+  }
+
+  /**
+   * destroy the urn registry.
+   *
+   * This function will be called automatically at `process.on(exit)`
+   * however, it till need to be called before the program ends
+   *  because there have some timers in eventloop task list
+   */
+  destroy () {
+    log.verbose('UniformResourceNameRegistry', 'destroy() %s UUIDs left',
+      [ ...this.uuidExpiringTable.values() ].flat().length,
+    )
+
+    if (this.purgerTimer) {
+      log.verbose('UniformResourceNameRegistry', 'destroy() clearing purger timer ...')
+      clearInterval(this.purgerTimer)
+      this.purgerTimer = undefined
+    }
+
+    const Klass = instanceToClass(this, UniformResourceNameRegistry)
+
+    /**
+     * Remove process exit listener
+     */
+    if (Klass.processExitMap.has(this)) {
+      log.verbose('UniformResourceNameRegistry', 'destroy() remove process `exit` listener ...')
+      const fn = Klass.processExitMap.get(this)
+      Klass.processExitMap.delete(this)
+      fn && fn()
+    }
+
+    /**
+     * Clean up all the files
+     */
+    log.verbose('UniformResourceNameRegistry', 'destroy() fs.rmSync(%s) ...', this.storeDir)
+    try {
+      /**
+       * Huan(202110):
+       *  Check for the `this.uuidDir` exist or not
+       *    when we are running unit tests, we might instanciate multiple UniformResourceNameRegistry
+       *    which will cause the `this.destroy()` to be registered multiple times
+       */
+      fs.statSync(this.storeDir)
+
+      fs.rmSync(this.storeDir, { recursive: true })
+      log.verbose('UniformResourceNameRegistry', 'destroy() fs.rmSync(%s) done', this.storeDir)
+
+    } catch (e) {
+      if ((e as NodeJS.ErrnoException).code === 'ENOENT') {
+        log.verbose('UniformResourceNameRegistry', 'destroy() %s not exist', this.storeDir)
+        return
+      }
+      log.warn('UniformResourceNameRegistry', 'destroy() fs.rmSync(%s) exception: %s', (e as Error).message)
+    }
+  }
+
+}
+
+export { UniformResourceNameRegistry }

package/src/urn-registry/uuid-to-big-int.spec.ts

@@ -0,0 +1,19 @@
+#!/usr/bin/env -S node --no-warnings --loader ts-node/esm
+
+import { test }  from 'tstest'
+
+import { uuidToBigInt } from './uuid-to-big-int.js'
+
+test('uuidToBigInt() Nil UUID', async t => {
+  const uuid = '00000000-0000-0000-0000-000000000000'
+  const bigInt = uuidToBigInt(uuid)
+  t.equal(bigInt.toString(), '0', 'should get zero for a Nil UUID')
+})
+
+test('uuidToBigInt() Nil UUID', async t => {
+  const UUID = '00bd7c03-a690-48d3-a1fe-1314574e4fc1'
+  const EXPECTED = BigInt('0x00bd7c03a69048d3a1fe1314574e4fc1')
+
+  const bigInt = uuidToBigInt(UUID)
+  t.equal(bigInt, EXPECTED, 'should convert UUID right')
+})

package/src/urn-registry/uuid-to-big-int.ts

@@ -0,0 +1,16 @@
+/**
+ * Convert UUID to BigInt
+ * @param uuid
+ * @returns BigInt
+ */
+
+function uuidToBigInt (uuid: string): BigInt {
+  // credit: https://stackoverflow.com/a/58014300/1123955
+  const hexBytes    = `0x${uuid.replace(/-/g, '')}`
+  const bigInteger  = BigInt(hexBytes)
+  return bigInteger
+}
+
+export {
+  uuidToBigInt,
+}

package/src/version.spec.ts

@@ -0,0 +1,9 @@
+#!/usr/bin/env -S node --no-warnings --loader ts-node/esm
+
+import { test }  from 'tstest'
+
+import { VERSION } from './version.js'
+
+test('Make sure the VERSION is fresh in source code', async t => {
+  t.equal(VERSION, '0.0.0', 'version should be 0.0.0 in source code, only updated before publish to NPM')
+})

package/src/version.ts

@@ -0,0 +1,4 @@
+/**
+ * This file was auto generated from scripts/generate-version.sh
+ */
+export const VERSION: string = '1.7.0'