@zuplo/cli 7.0.2 → 7.1.1

package/node_modules/thread-stream/index.js

@@ -16,9 +16,11 @@ const assert = require('assert')
 
 const kImpl = Symbol('kImpl')
 
-// V8 limit for string size
+// Maximum pending buffered data before forcing a synchronous drain
 const MAX_STRING = buffer.constants.MAX_STRING_LENGTH
 
+function noop () {}
+
 function updateState (stream, fn) {
   Atomics.add(stream[kImpl].state, SEQ_INDEX, 1)
   fn()
@@ -26,6 +28,13 @@ function updateState (stream, fn) {
   Atomics.notify(stream[kImpl].state, SEQ_INDEX)
 }
 
+function resetIndexes (stream) {
+  updateState(stream, () => {
+    Atomics.store(stream[kImpl].state, READ_INDEX, 0)
+    Atomics.store(stream[kImpl].state, WRITE_INDEX, 0)
+  })
+}
+
 class FakeWeakRef {
   constructor (value) {
     this._value = value
@@ -62,6 +71,7 @@ function createWorker (stream, opts) {
 
   const worker = new Worker(toExecute, {
     ...opts.workerOpts,
+    name: opts.workerOpts?.name || 'thread-stream',
     trackUnmanagedFds: false,
     workerData: {
       filename: filename.indexOf('file://') === 0
@@ -98,70 +108,46 @@ function drain (stream) {
 }
 
 function nextFlush (stream) {
-  const writeIndex = Atomics.load(stream[kImpl].state, WRITE_INDEX)
-  let leftover = stream[kImpl].data.length - writeIndex
+  while (true) {
+    const writeIndex = Atomics.load(stream[kImpl].state, WRITE_INDEX)
+    const leftover = stream[kImpl].data.length - writeIndex
+
+    if (leftover > 0) {
+      if (stream[kImpl].bufLen === 0) {
+        stream[kImpl].flushing = false
 
-  if (leftover > 0) {
-    if (stream[kImpl].buf.length === 0) {
-      stream[kImpl].flushing = false
+        if (stream[kImpl].ending) {
+          end(stream)
+        } else if (stream[kImpl].needDrain) {
+          process.nextTick(drain, stream)
+        }
 
-      if (stream[kImpl].ending) {
-        end(stream)
-      } else if (stream[kImpl].needDrain) {
-        process.nextTick(drain, stream)
+        return
       }
 
-      return
+      write(stream, leftover, noop)
+      continue
     }
 
-    let toWrite = stream[kImpl].buf.slice(0, leftover)
-    let toWriteBytes = Buffer.byteLength(toWrite)
-    if (toWriteBytes <= leftover) {
-      stream[kImpl].buf = stream[kImpl].buf.slice(leftover)
-      // process._rawDebug('writing ' + toWrite.length)
-      write(stream, toWrite, nextFlush.bind(null, stream))
-    } else {
-      // multi-byte utf-8
-      stream.flush(() => {
-        // err is already handled in flush()
+    if (leftover === 0) {
+      if (writeIndex === 0 && stream[kImpl].bufLen === 0) {
+        // we had a flushSync in the meanwhile
+        return
+      }
+      waitForRead(stream, () => {
         if (stream.destroyed) {
           return
         }
 
-        updateState(stream, () => {
-          Atomics.store(stream[kImpl].state, READ_INDEX, 0)
-          Atomics.store(stream[kImpl].state, WRITE_INDEX, 0)
-        })
-        Atomics.notify(stream[kImpl].state, READ_INDEX)
-
-        // Find a toWrite length that fits the buffer
-        // it must exists as the buffer is at least 4 bytes length
-        // and the max utf-8 length for a char is 4 bytes.
-        while (toWriteBytes > stream[kImpl].data.length) {
-          leftover = leftover / 2
-          toWrite = stream[kImpl].buf.slice(0, leftover)
-          toWriteBytes = Buffer.byteLength(toWrite)
-        }
-        stream[kImpl].buf = stream[kImpl].buf.slice(leftover)
-        write(stream, toWrite, nextFlush.bind(null, stream))
+        resetIndexes(stream)
+        nextFlush(stream)
       })
-    }
-  } else if (leftover === 0) {
-    if (writeIndex === 0 && stream[kImpl].buf.length === 0) {
-      // we had a flushSync in the meanwhile
       return
     }
-    stream.flush(() => {
-      updateState(stream, () => {
-        Atomics.store(stream[kImpl].state, READ_INDEX, 0)
-        Atomics.store(stream[kImpl].state, WRITE_INDEX, 0)
-      })
-      Atomics.notify(stream[kImpl].state, READ_INDEX)
-      nextFlush(stream)
-    })
-  } else {
+
     // This should never happen
     destroy(stream, new Error('overwritten'))
+    return
   }
 }
 
@@ -186,7 +172,7 @@ function onWorkerMessage (msg) {
       // proper one.
       this.stream = new WeakRef(stream)
 
-      stream.flush(() => {
+      waitForRead(stream, () => {
         stream[kImpl].ready = true
         stream.emit('ready')
       })
@@ -201,6 +187,19 @@ function onWorkerMessage (msg) {
         stream.emit(msg.name, msg.args)
       }
       break
+    case 'FLUSHED': {
+      if (msg.context !== 'thread-stream') {
+        destroy(stream, new Error('this should not happen: ' + msg.code))
+        break
+      }
+
+      const cb = stream[kImpl].flushCallbacks.get(msg.id)
+      if (cb) {
+        stream[kImpl].flushCallbacks.delete(msg.id)
+        process.nextTick(cb)
+      }
+      break
+    }
     case 'WARNING':
       process.emitWarning(msg.err)
       break
@@ -244,7 +243,11 @@ class ThreadStream extends EventEmitter {
     this[kImpl].finished = false
     this[kImpl].errored = null
     this[kImpl].closed = false
-    this[kImpl].buf = ''
+    this[kImpl].buf = []
+    this[kImpl].bufHead = 0
+    this[kImpl].bufLen = 0
+    this[kImpl].flushCallbacks = new Map()
+    this[kImpl].nextFlushId = 0
 
     // TODO (fix): Make private?
     this.worker = createWorker(this, opts) // TODO (fix): make private
@@ -254,6 +257,7 @@ class ThreadStream extends EventEmitter {
   }
 
   write (data) {
+    const dataBuf = Buffer.isBuffer(data) ? data : Buffer.from(data)
     if (this[kImpl].destroyed) {
       error(this, new Error('the worker has exited'))
       return false
@@ -264,7 +268,7 @@ class ThreadStream extends EventEmitter {
       return false
     }
 
-    if (this[kImpl].flushing && this[kImpl].buf.length + data.length >= MAX_STRING) {
+    if (this[kImpl].flushing && this[kImpl].bufLen + dataBuf.length >= MAX_STRING) {
       try {
         writeSync(this)
         this[kImpl].flushing = true
@@ -274,7 +278,8 @@ class ThreadStream extends EventEmitter {
       }
     }
 
-    this[kImpl].buf += data
+    this[kImpl].buf.push(dataBuf)
+    this[kImpl].bufLen += dataBuf.length
 
     if (this[kImpl].sync) {
       try {
@@ -291,7 +296,7 @@ class ThreadStream extends EventEmitter {
       setImmediate(nextFlush, this)
     }
 
-    this[kImpl].needDrain = this[kImpl].data.length - this[kImpl].buf.length - Atomics.load(this[kImpl].state, WRITE_INDEX) <= 0
+    this[kImpl].needDrain = this[kImpl].data.length - this[kImpl].bufLen - Atomics.load(this[kImpl].state, WRITE_INDEX) <= 0
     return !this[kImpl].needDrain
   }
 
@@ -305,28 +310,15 @@ class ThreadStream extends EventEmitter {
   }
 
   flush (cb) {
-    if (this[kImpl].destroyed) {
-      if (typeof cb === 'function') {
-        process.nextTick(cb, new Error('the worker has exited'))
-      }
-      return
-    }
+    cb = typeof cb === 'function' ? cb : noop
 
-    // TODO write all .buf
-    const writeIndex = Atomics.load(this[kImpl].state, WRITE_INDEX)
-    // process._rawDebug(`(flush) readIndex (${Atomics.load(this.state, READ_INDEX)}) writeIndex (${Atomics.load(this.state, WRITE_INDEX)})`)
-    wait(this[kImpl].state, READ_INDEX, writeIndex, Infinity, (err, res) => {
+    flushBuffer(this, (err) => {
       if (err) {
-        destroy(this, err)
         process.nextTick(cb, err)
         return
       }
-      if (res === 'not-equal') {
-        // TODO handle deadlock
-        this.flush(cb)
-        return
-      }
-      process.nextTick(cb)
+
+      requestWorkerFlush(this, cb)
     })
   }
 
@@ -384,6 +376,93 @@ class ThreadStream extends EventEmitter {
   }
 }
 
+function flushBuffer (stream, cb) {
+  if (stream[kImpl].destroyed) {
+    process.nextTick(cb, new Error('the worker has exited'))
+    return
+  }
+
+  if (!stream[kImpl].sync && (stream[kImpl].flushing || stream[kImpl].bufLen > 0)) {
+    setImmediate(flushBuffer, stream, cb)
+    return
+  }
+
+  waitForRead(stream, cb)
+}
+
+function waitForRead (stream, cb) {
+  const writeIndex = Atomics.load(stream[kImpl].state, WRITE_INDEX)
+  wait(stream[kImpl].state, READ_INDEX, writeIndex, Infinity, (err, res) => {
+    if (err) {
+      destroy(stream, err)
+      cb(err)
+      return
+    }
+
+    if (res !== 'ok') {
+      waitForRead(stream, cb)
+      return
+    }
+
+    cb()
+  })
+}
+
+function requestWorkerFlush (stream, cb) {
+  if (stream[kImpl].destroyed) {
+    process.nextTick(cb, new Error('the worker has exited'))
+    return
+  }
+
+  if (!stream[kImpl].ready) {
+    const onReady = () => {
+      cleanup()
+      requestWorkerFlush(stream, cb)
+    }
+    const onClose = () => {
+      cleanup()
+      process.nextTick(cb, new Error('the worker has exited'))
+    }
+    const cleanup = () => {
+      stream.off('ready', onReady)
+      stream.off('close', onClose)
+    }
+
+    stream.once('ready', onReady)
+    stream.once('close', onClose)
+    return
+  }
+
+  const id = ++stream[kImpl].nextFlushId
+  stream[kImpl].flushCallbacks.set(id, cb)
+
+  try {
+    stream.worker.postMessage({
+      code: 'FLUSH',
+      context: 'thread-stream',
+      id
+    })
+  } catch (err) {
+    stream[kImpl].flushCallbacks.delete(id)
+    destroy(stream, err)
+    process.nextTick(cb, err)
+  }
+}
+
+function failPendingFlushCallbacks (stream, err) {
+  const callbacks = stream[kImpl].flushCallbacks
+  if (callbacks.size === 0) {
+    return
+  }
+
+  const flushErr = err || new Error('the worker has exited')
+
+  for (const cb of callbacks.values()) {
+    process.nextTick(cb, flushErr)
+  }
+  callbacks.clear()
+}
+
 function error (stream, err) {
   setImmediate(() => {
     stream.emit('error', err)
@@ -395,6 +474,7 @@ function destroy (stream, err) {
     return
   }
   stream[kImpl].destroyed = true
+  failPendingFlushCallbacks(stream, err)
 
   if (err) {
     stream[kImpl].errored = err
@@ -416,13 +496,42 @@ function destroy (stream, err) {
   }
 }
 
-function write (stream, data, cb) {
+function write (stream, maxBytes, cb) {
   // data is smaller than the shared buffer length
   const current = Atomics.load(stream[kImpl].state, WRITE_INDEX)
-  const length = Buffer.byteLength(data)
-  stream[kImpl].data.write(data, current)
+  let offset = current
+  let remaining = maxBytes
+
+  while (remaining > 0 && stream[kImpl].bufLen !== 0) {
+    const head = stream[kImpl].bufHead
+    const buf = stream[kImpl].buf[head]
+
+    if (buf.length <= remaining) {
+      buf.copy(stream[kImpl].data, offset)
+      offset += buf.length
+      remaining -= buf.length
+      stream[kImpl].bufLen -= buf.length
+      stream[kImpl].bufHead = head + 1
+
+      if (stream[kImpl].bufHead === stream[kImpl].buf.length) {
+        stream[kImpl].buf.length = 0
+        stream[kImpl].bufHead = 0
+      } else if (stream[kImpl].bufHead >= 1024 && stream[kImpl].bufHead * 2 >= stream[kImpl].buf.length) {
+        stream[kImpl].buf.splice(0, stream[kImpl].bufHead)
+        stream[kImpl].bufHead = 0
+      }
+      continue
+    }
+
+    buf.copy(stream[kImpl].data, offset, 0, remaining)
+    stream[kImpl].buf[head] = buf.subarray(remaining)
+    stream[kImpl].bufLen -= remaining
+    offset += remaining
+    remaining = 0
+  }
+
   updateState(stream, () => {
-    Atomics.store(stream[kImpl].state, WRITE_INDEX, current + length)
+    Atomics.store(stream[kImpl].state, WRITE_INDEX, offset)
   })
   cb()
   return true
@@ -483,48 +592,19 @@ function writeSync (stream) {
   }
   stream[kImpl].flushing = false
 
-  while (stream[kImpl].buf.length !== 0) {
+  while (stream[kImpl].bufLen !== 0) {
     const writeIndex = Atomics.load(stream[kImpl].state, WRITE_INDEX)
-    let leftover = stream[kImpl].data.length - writeIndex
+    const leftover = stream[kImpl].data.length - writeIndex
     if (leftover === 0) {
       flushSync(stream)
-      updateState(stream, () => {
-        Atomics.store(stream[kImpl].state, READ_INDEX, 0)
-        Atomics.store(stream[kImpl].state, WRITE_INDEX, 0)
-      })
-      Atomics.notify(stream[kImpl].state, READ_INDEX)
+      resetIndexes(stream)
       continue
     } else if (leftover < 0) {
       // stream should never happen
       throw new Error('overwritten')
     }
 
-    let toWrite = stream[kImpl].buf.slice(0, leftover)
-    let toWriteBytes = Buffer.byteLength(toWrite)
-    if (toWriteBytes <= leftover) {
-      stream[kImpl].buf = stream[kImpl].buf.slice(leftover)
-      // process._rawDebug('writing ' + toWrite.length)
-      write(stream, toWrite, cb)
-    } else {
-      // multi-byte utf-8
-      flushSync(stream)
-      updateState(stream, () => {
-        Atomics.store(stream[kImpl].state, READ_INDEX, 0)
-        Atomics.store(stream[kImpl].state, WRITE_INDEX, 0)
-      })
-      Atomics.notify(stream[kImpl].state, READ_INDEX)
-
-      // Find a toWrite length that fits the buffer
-      // it must exists as the buffer is at least 4 bytes length
-      // and the max utf-8 length for a char is 4 bytes.
-      while (toWriteBytes > stream[kImpl].buf.length) {
-        leftover = leftover / 2
-        toWrite = stream[kImpl].buf.slice(0, leftover)
-        toWriteBytes = Buffer.byteLength(toWrite)
-      }
-      stream[kImpl].buf = stream[kImpl].buf.slice(leftover)
-      write(stream, toWrite, cb)
-    }
+    write(stream, leftover, cb)
   }
 }
 

package/node_modules/thread-stream/lib/wait.js

@@ -1,61 +1,77 @@
 'use strict'
 
-const MAX_TIMEOUT = 1000
+// Maximum wait time for a single waitAsync call
+// Used as a fallback poll interval in case notifications are missed
+// Keep this low enough for good throughput but high enough to not busy-loop
+const WAIT_MS = 10000
 
 function wait (state, index, expected, timeout, done) {
-  const max = Date.now() + timeout
-  let current = Atomics.load(state, index)
-  if (current === expected) {
-    done(null, 'ok')
-    return
-  }
-  let prior = current
-  const check = (backoff) => {
-    if (Date.now() > max) {
+  const max = timeout === Infinity ? Infinity : Date.now() + timeout
+
+  const check = () => {
+    const current = Atomics.load(state, index)
+    if (current === expected) {
+      done(null, 'ok')
+      return
+    }
+
+    if (max !== Infinity && Date.now() > max) {
       done(null, 'timed-out')
+      return
+    }
+
+    // Wait for any change from current value
+    const remaining = max === Infinity ? WAIT_MS : Math.min(WAIT_MS, Math.max(1, max - Date.now()))
+    const result = Atomics.waitAsync(state, index, current, remaining)
+
+    if (result.async) {
+      result.value.then(check)
     } else {
-      setTimeout(() => {
-        prior = current
-        current = Atomics.load(state, index)
-        if (current === prior) {
-          check(backoff >= MAX_TIMEOUT ? MAX_TIMEOUT : backoff * 2)
-        } else {
-          if (current === expected) done(null, 'ok')
-          else done(null, 'not-equal')
-        }
-      }, backoff)
+      // Value already changed (not-equal) - recheck on next tick
+      setImmediate(check)
     }
   }
-  check(1)
+
+  check()
 }
 
-// let waitDiffCount = 0
 function waitDiff (state, index, expected, timeout, done) {
-  // const id = waitDiffCount++
-  // process._rawDebug(`>>> waitDiff ${id}`)
-  const max = Date.now() + timeout
-  let current = Atomics.load(state, index)
-  if (current !== expected) {
-    done(null, 'ok')
-    return
-  }
-  const check = (backoff) => {
-    // process._rawDebug(`${id} ${index} current ${current} expected ${expected}`)
-    // process._rawDebug('' + backoff)
-    if (Date.now() > max) {
+  const max = timeout === Infinity ? Infinity : Date.now() + timeout
+
+  const check = () => {
+    const current = Atomics.load(state, index)
+    if (current !== expected) {
+      done(null, 'ok')
+      return
+    }
+
+    if (max !== Infinity && Date.now() > max) {
       done(null, 'timed-out')
-    } else {
-      setTimeout(() => {
-        current = Atomics.load(state, index)
-        if (current !== expected) {
+      return
+    }
+
+    // Wait for value to change from expected.
+    // If we are notified, resume immediately even if the value cycled back
+    // to the same number before we could re-read it.
+    const remaining = max === Infinity ? WAIT_MS : Math.min(WAIT_MS, Math.max(1, max - Date.now()))
+    const result = Atomics.waitAsync(state, index, expected, remaining)
+
+    if (result.async) {
+      result.value.then((res) => {
+        if (res === 'ok') {
           done(null, 'ok')
-        } else {
-          check(backoff >= MAX_TIMEOUT ? MAX_TIMEOUT : backoff * 2)
+          return
         }
-      }, backoff)
+
+        check()
+      })
+    } else {
+      // Value already changed (not-equal) - recheck on next tick
+      setImmediate(check)
     }
   }
-  check(1)
+
+  check()
 }
 
 module.exports = { wait, waitDiff }