@contrail/cli 3.4.5 → 3.4.6-alpha.cffb28e

package/node_modules/thread-stream/lib/worker.js

@@ -0,0 +1,179 @@
+'use strict'
+
+const { realImport, realRequire } = require('real-require')
+const { workerData, parentPort } = require('worker_threads')
+const { WRITE_INDEX, READ_INDEX } = require('./indexes')
+const { waitDiff } = require('./wait')
+
+const {
+  dataBuf,
+  filename,
+  stateBuf
+} = workerData
+
+let destination
+
+const state = new Int32Array(stateBuf)
+const data = Buffer.from(dataBuf)
+
+// Keep the event loop alive - Atomics.waitAsync promises don't prevent worker exit
+const keepAlive = setInterval(() => {}, 60 * 60 * 1000)
+
+async function start () {
+  let worker
+  try {
+    worker = (await realImport(filename))
+  } catch (error) {
+    // A yarn user that tries to start a ThreadStream for an external module
+    // provides a filename pointing to a zip file.
+    // eg. require.resolve('pino-elasticsearch') // returns /foo/pino-elasticsearch-npm-6.1.0-0c03079478-6915435172.zip/bar.js
+    // The `import` will fail to try to load it.
+    // This catch block executes the `require` fallback to load the module correctly.
+    // In fact, yarn modifies the `require` function to manage the zipped path.
+    // More details at https://github.com/pinojs/pino/pull/1113
+    // The error codes may change based on the node.js version (ENOTDIR > 12, ERR_MODULE_NOT_FOUND <= 12 )
+    if ((error.code === 'ENOTDIR' || error.code === 'ERR_MODULE_NOT_FOUND') &&
+     filename.startsWith('file://')) {
+      worker = realRequire(decodeURIComponent(filename.replace('file://', '')))
+    } else if (error.code === undefined || error.code === 'ERR_VM_DYNAMIC_IMPORT_CALLBACK_MISSING') {
+      // When bundled with pkg, an undefined error is thrown when called with realImport
+      // When bundled with pkg and using node v20, an ERR_VM_DYNAMIC_IMPORT_CALLBACK_MISSING error is thrown when called with realImport
+      // More info at: https://github.com/pinojs/thread-stream/issues/143
+      try {
+        worker = realRequire(decodeURIComponent(filename.replace(process.platform === 'win32' ? 'file:///' : 'file://', '')))
+      } catch {
+        throw error
+      }
+    } else if (filename.endsWith('.ts') || filename.endsWith('.cts')) {
+      // Native TypeScript import failed (type stripping not enabled).
+      // Fall back to ts-node for TypeScript files.
+      try {
+        if (!process[Symbol.for('ts-node.register.instance')]) {
+          realRequire('ts-node/register')
+        } else if (process.env.TS_NODE_DEV) {
+          realRequire('ts-node-dev')
+        }
+        worker = realRequire(decodeURIComponent(filename.replace(process.platform === 'win32' ? 'file:///' : 'file://', '')))
+      } catch {
+        throw error
+      }
+    } else {
+      throw error
+    }
+  }
+
+  // Depending on how the default export is performed, and on how the code is
+  // transpiled, we may find cases of two nested "default" objects.
+  // See https://github.com/pinojs/pino/issues/1243#issuecomment-982774762
+  if (typeof worker === 'object') worker = worker.default
+  if (typeof worker === 'object') worker = worker.default
+
+  destination = await worker(workerData.workerData)
+
+  destination.on('error', function (err) {
+    Atomics.store(state, WRITE_INDEX, -2)
+    Atomics.notify(state, WRITE_INDEX)
+
+    Atomics.store(state, READ_INDEX, -2)
+    Atomics.notify(state, READ_INDEX)
+
+    parentPort.postMessage({
+      code: 'ERROR',
+      err
+    })
+  })
+
+  destination.on('close', function () {
+    // process._rawDebug('worker close emitted')
+    const end = Atomics.load(state, WRITE_INDEX)
+    Atomics.store(state, READ_INDEX, end)
+    Atomics.notify(state, READ_INDEX)
+    clearInterval(keepAlive)
+    setImmediate(() => {
+      process.exit(0)
+    })
+  })
+}
+
+// No .catch() handler,
+// in case there is an error it goes
+// to unhandledRejection
+start().then(function () {
+  parentPort.postMessage({
+    code: 'READY'
+  })
+
+  process.nextTick(run)
+})
+
+function run () {
+  const current = Atomics.load(state, READ_INDEX)
+  const end = Atomics.load(state, WRITE_INDEX)
+
+  // process._rawDebug(`pre state ${current} ${end}`)
+
+  if (end === current) {
+    if (end === data.length) {
+      waitDiff(state, READ_INDEX, end, Infinity, run)
+    } else {
+      waitDiff(state, WRITE_INDEX, end, Infinity, run)
+    }
+    return
+  }
+
+  // process._rawDebug(`post state ${current} ${end}`)
+
+  if (end === -1) {
+    // process._rawDebug('end')
+    destination.end()
+    return
+  }
+
+  const toWrite = data.toString('utf8', current, end)
+  // process._rawDebug('worker writing: ' + toWrite)
+
+  const res = destination.write(toWrite)
+
+  if (res) {
+    Atomics.store(state, READ_INDEX, end)
+    Atomics.notify(state, READ_INDEX)
+    setImmediate(run)
+  } else {
+    destination.once('drain', function () {
+      Atomics.store(state, READ_INDEX, end)
+      Atomics.notify(state, READ_INDEX)
+      run()
+    })
+  }
+}
+
+process.on('unhandledRejection', function (err) {
+  parentPort.postMessage({
+    code: 'ERROR',
+    err
+  })
+  process.exit(1)
+})
+
+process.on('uncaughtException', function (err) {
+  parentPort.postMessage({
+    code: 'ERROR',
+    err
+  })
+  process.exit(1)
+})
+
+process.once('exit', exitCode => {
+  if (exitCode !== 0) {
+    process.exit(exitCode)
+    return
+  }
+  if (destination?.writableNeedDrain && !destination?.writableEnded) {
+    parentPort.postMessage({
+      code: 'WARNING',
+      err: new Error('ThreadStream: process exited before destination stream was drained. this may indicate that the destination stream try to write to a another missing stream')
+    })
+  }
+
+  process.exit(0)
+})

package/node_modules/thread-stream/package.json

@@ -0,0 +1,52 @@
+{
+  "name": "thread-stream",
+  "version": "4.0.0",
+  "description": "A streaming way to send data to a Node.js Worker Thread",
+  "main": "index.js",
+  "types": "index.d.ts",
+  "engines": {
+    "node": ">=20"
+  },
+  "dependencies": {
+    "real-require": "^0.2.0"
+  },
+  "devDependencies": {
+    "@types/node": "^22.0.0",
+    "@yao-pkg/pkg": "^6.0.0",
+    "borp": "^0.21.0",
+    "desm": "^1.3.0",
+    "eslint": "^9.39.1",
+    "fastbench": "^1.0.1",
+    "husky": "^9.0.6",
+    "neostandard": "^0.12.2",
+    "pino-elasticsearch": "^8.0.0",
+    "sonic-boom": "^4.0.1",
+    "ts-node": "^10.8.0",
+    "typescript": "~5.7.3"
+  },
+  "scripts": {
+    "build": "tsc --noEmit",
+    "lint": "eslint",
+    "test": "npm run lint && npm run build && npm run transpile && borp --pattern 'test/*.test.{js,mjs}'",
+    "test:ci": "npm run lint && npm run transpile && borp --pattern 'test/*.test.{js,mjs}'",
+    "test:yarn": "npm run transpile && borp --pattern 'test/*.test.js'",
+    "transpile": "sh ./test/ts/transpile.sh",
+    "prepare": "husky install"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/mcollina/thread-stream.git"
+  },
+  "keywords": [
+    "worker",
+    "thread",
+    "threads",
+    "stream"
+  ],
+  "author": "Matteo Collina <hello@matteocollina.com>",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/mcollina/thread-stream/issues"
+  },
+  "homepage": "https://github.com/mcollina/thread-stream#readme"
+}

package/node_modules/thread-stream/test/base.test.js

@@ -0,0 +1,259 @@
+'use strict'
+
+const { test } = require('node:test')
+const assert = require('node:assert')
+const { join } = require('path')
+const { readFile } = require('fs')
+const { file } = require('./helper')
+const ThreadStream = require('..')
+const { MessageChannel } = require('worker_threads')
+const { once } = require('events')
+
+test('base sync=true', function (t, done) {
+  const dest = file()
+  const stream = new ThreadStream({
+    filename: join(__dirname, 'to-file.js'),
+    workerData: { dest },
+    sync: true
+  })
+
+  assert.deepStrictEqual(stream.writableObjectMode, false)
+
+  assert.deepStrictEqual(stream.writableFinished, false)
+  stream.on('finish', () => {
+    assert.deepStrictEqual(stream.writableFinished, true)
+    readFile(dest, 'utf8', (err, data) => {
+      assert.ifError(err)
+      assert.strictEqual(data, 'hello world\nsomething else\n')
+    })
+  })
+
+  assert.deepStrictEqual(stream.closed, false)
+  stream.on('close', () => {
+    assert.deepStrictEqual(stream.closed, true)
+    assert.ok(!stream.writable)
+    done()
+  })
+
+  assert.deepStrictEqual(stream.writableNeedDrain, false)
+  assert.ok(stream.write('hello world\n'))
+  assert.ok(stream.write('something else\n'))
+  assert.ok(stream.writable)
+
+  assert.deepStrictEqual(stream.writableEnded, false)
+  stream.end()
+  assert.deepStrictEqual(stream.writableEnded, true)
+})
+
+test('overflow sync=true', function (t, done) {
+  const dest = file()
+  const stream = new ThreadStream({
+    bufferSize: 128,
+    filename: join(__dirname, 'to-file.js'),
+    workerData: { dest },
+    sync: true
+  })
+
+  let count = 0
+
+  // Write 10 chars, 20 times
+  function write () {
+    if (count++ === 20) {
+      stream.end()
+      return
+    }
+
+    stream.write('aaaaaaaaaa')
+    // do not wait for drain event
+    setImmediate(write)
+  }
+
+  write()
+
+  stream.on('close', () => {
+    readFile(dest, 'utf8', (err, data) => {
+      assert.ifError(err)
+      assert.strictEqual(data.length, 200)
+      done()
+    })
+  })
+})
+
+test('overflow sync=false', function (t, done) {
+  const dest = file()
+  const stream = new ThreadStream({
+    bufferSize: 128,
+    filename: join(__dirname, 'to-file.js'),
+    workerData: { dest },
+    sync: false
+  })
+
+  let count = 0
+
+  assert.deepStrictEqual(stream.writableNeedDrain, false)
+
+  // Write 10 chars, 20 times
+  function write () {
+    if (count++ === 20) {
+      stream.end()
+      return
+    }
+
+    if (!stream.write('aaaaaaaaaa')) {
+      assert.deepStrictEqual(stream.writableNeedDrain, true)
+    }
+    // do not wait for drain event
+    setImmediate(write)
+  }
+
+  write()
+
+  stream.on('drain', () => {
+    assert.deepStrictEqual(stream.writableNeedDrain, false)
+  })
+
+  stream.on('close', () => {
+    readFile(dest, 'utf8', (err, data) => {
+      assert.ifError(err)
+      assert.strictEqual(data.length, 200)
+      done()
+    })
+  })
+})
+
+test('over the bufferSize at startup', function (t, done) {
+  const dest = file()
+  const stream = new ThreadStream({
+    bufferSize: 10,
+    filename: join(__dirname, 'to-file.js'),
+    workerData: { dest },
+    sync: true
+  })
+
+  stream.on('finish', () => {
+    readFile(dest, 'utf8', (err, data) => {
+      assert.ifError(err)
+      assert.strictEqual(data, 'hello world\nsomething else\n')
+    })
+  })
+
+  stream.on('close', () => {
+    done()
+  })
+
+  assert.ok(stream.write('hello'))
+  assert.ok(stream.write(' world\n'))
+  assert.ok(stream.write('something else\n'))
+
+  stream.end()
+})
+
+test('over the bufferSize at startup (async)', function (t, done) {
+  const dest = file()
+  const stream = new ThreadStream({
+    bufferSize: 10,
+    filename: join(__dirname, 'to-file.js'),
+    workerData: { dest },
+    sync: false
+  })
+
+  assert.ok(stream.write('hello'))
+  assert.ok(!stream.write(' world\n'))
+  assert.ok(!stream.write('something else\n'))
+
+  stream.end()
+
+  stream.on('finish', () => {
+    readFile(dest, 'utf8', (err, data) => {
+      assert.ifError(err)
+      assert.strictEqual(data, 'hello world\nsomething else\n')
+    })
+  })
+
+  stream.on('close', () => {
+    done()
+  })
+})
+
+test('flushSync sync=false', function (t, done) {
+  const dest = file()
+  const stream = new ThreadStream({
+    bufferSize: 128,
+    filename: join(__dirname, 'to-file.js'),
+    workerData: { dest },
+    sync: false
+  })
+
+  stream.on('drain', () => {
+    stream.end()
+  })
+
+  stream.on('close', () => {
+    readFile(dest, 'utf8', (err, data) => {
+      assert.ifError(err)
+      assert.strictEqual(data.length, 200)
+      done()
+    })
+  })
+
+  for (let count = 0; count < 20; count++) {
+    stream.write('aaaaaaaaaa')
+  }
+  stream.flushSync()
+})
+
+test('pass down MessagePorts', async function (t) {
+  const { port1, port2 } = new MessageChannel()
+  const stream = new ThreadStream({
+    filename: join(__dirname, 'port.js'),
+    workerData: { port: port1 },
+    workerOpts: {
+      transferList: [port1]
+    },
+    sync: false
+  })
+  t.after(() => {
+    stream.end()
+  })
+
+  assert.ok(stream.write('hello world\n'))
+  assert.ok(stream.write('something else\n'))
+
+  const [strings] = await once(port2, 'message')
+
+  assert.strictEqual(strings, 'hello world\nsomething else\n')
+})
+
+test('destroy does not error', function (t, done) {
+  const dest = file()
+  const stream = new ThreadStream({
+    filename: join(__dirname, 'to-file.js'),
+    workerData: { dest },
+    sync: false
+  })
+
+  stream.on('ready', () => {
+    stream.worker.terminate()
+  })
+
+  stream.on('error', (err) => {
+    assert.strictEqual(err.message, 'the worker thread exited')
+    stream.flush((err) => {
+      assert.strictEqual(err.message, 'the worker has exited')
+    })
+    assert.doesNotThrow(() => stream.flushSync())
+    assert.doesNotThrow(() => stream.end())
+    done()
+  })
+})
+
+test('syntax error', function (t, done) {
+  const stream = new ThreadStream({
+    filename: join(__dirname, 'syntax-error.mjs')
+  })
+
+  stream.on('error', (err) => {
+    assert.strictEqual(err.message, 'Unexpected end of input')
+    done()
+  })
+})

package/node_modules/thread-stream/test/bench.test.js

@@ -0,0 +1,38 @@
+'use strict'
+
+const { test } = require('node:test')
+const { join } = require('path')
+const ThreadStream = require('..')
+const { file } = require('./helper')
+
+const MAX = 1000
+
+let str = ''
+
+for (let i = 0; i < 10; i++) {
+  str += 'hello'
+}
+
+test('base', function (t, done) {
+  const dest = file()
+  const stream = new ThreadStream({
+    filename: join(__dirname, 'to-file.js'),
+    workerData: { dest }
+  })
+  let runs = 0
+  function benchThreadStream () {
+    if (++runs === 1000) {
+      stream.end()
+      return
+    }
+
+    for (let i = 0; i < MAX; i++) {
+      stream.write(str)
+    }
+    setImmediate(benchThreadStream)
+  }
+  benchThreadStream()
+  stream.on('finish', function () {
+    done()
+  })
+})

package/node_modules/thread-stream/test/bundlers.test.js

@@ -0,0 +1,59 @@
+'use strict'
+
+const { test } = require('node:test')
+const assert = require('node:assert')
+const { join } = require('path')
+const { file } = require('./helper')
+const ThreadStream = require('..')
+
+test('bundlers support with .js file', function (t, done) {
+  globalThis.__bundlerPathsOverrides = {
+    'thread-stream-worker': join(__dirname, 'custom-worker.js')
+  }
+
+  const dest = file()
+
+  process.on('uncaughtException', error => {
+    console.log(error)
+  })
+
+  const stream = new ThreadStream({
+    filename: join(__dirname, 'to-file.js'),
+    workerData: { dest },
+    sync: true
+  })
+
+  stream.worker.removeAllListeners('message')
+  stream.worker.once('message', message => {
+    assert.strictEqual(message.code, 'CUSTOM-WORKER-CALLED')
+    done()
+  })
+
+  stream.end()
+})
+
+test('bundlers support with .mjs file', function (t, done) {
+  globalThis.__bundlerPathsOverrides = {
+    'thread-stream-worker': join(__dirname, 'custom-worker.js')
+  }
+
+  const dest = file()
+
+  process.on('uncaughtException', error => {
+    console.log(error)
+  })
+
+  const stream = new ThreadStream({
+    filename: join(__dirname, 'to-file.mjs'),
+    workerData: { dest },
+    sync: true
+  })
+
+  stream.worker.removeAllListeners('message')
+  stream.worker.once('message', message => {
+    assert.strictEqual(message.code, 'CUSTOM-WORKER-CALLED')
+    done()
+  })
+
+  stream.end()
+})

package/node_modules/thread-stream/test/close-on-gc.js

@@ -0,0 +1,37 @@
+'use strict'
+
+const { join } = require('path')
+const ThreadStream = require('..')
+const assert = require('assert')
+
+let worker = null
+
+function setup () {
+  const stream = new ThreadStream({
+    filename: join(__dirname, 'to-file.js'),
+    workerData: { dest: process.argv[2] },
+    sync: true
+  })
+
+  worker = stream.worker
+
+  stream.write('hello')
+  stream.write(' ')
+  stream.write('world\n')
+  stream.flushSync()
+  stream.unref()
+
+  // the stream object goes out of scope here
+  setImmediate(gc) // eslint-disable-line
+}
+
+setup()
+
+let exitEmitted = false
+worker.on('exit', function () {
+  exitEmitted = true
+})
+
+process.on('exit', function () {
+  assert.strictEqual(exitEmitted, true)
+})

package/node_modules/thread-stream/test/commonjs-fallback.test.js

@@ -0,0 +1,75 @@
+'use strict'
+
+const { test } = require('node:test')
+const assert = require('node:assert')
+const { join } = require('path')
+const { MessageChannel } = require('worker_threads')
+const { once } = require('events')
+const ThreadStream = require('..')
+
+const isYarnPnp = process.versions.pnp !== undefined
+
+test('yarn module resolution', { skip: !isYarnPnp }, (t, done) => {
+  const modulePath = require.resolve('pino-elasticsearch')
+  assert.match(modulePath, /.*\.zip.*/)
+
+  const stream = new ThreadStream({
+    filename: modulePath,
+    workerData: { node: null },
+    sync: true
+  })
+
+  assert.deepStrictEqual(stream.writableErrored, null)
+  stream.on('error', (err) => {
+    assert.deepStrictEqual(stream.writableErrored, err)
+  })
+
+  assert.ok(stream.write('hello world\n'))
+  assert.ok(stream.writable)
+  stream.end()
+  done()
+})
+
+test('yarn module resolution for directories with special characters', { skip: !isYarnPnp }, async t => {
+  const { port1, port2 } = new MessageChannel()
+  const stream = new ThreadStream({
+    filename: join(__dirname, 'dir with spaces', 'test-package.zip', 'worker.js'),
+    workerData: { port: port1 },
+    workerOpts: {
+      transferList: [port1]
+    },
+    sync: false
+  })
+  t.after(() => {
+    stream.end()
+  })
+
+  assert.ok(stream.write('hello world\n'))
+  assert.ok(stream.write('something else\n'))
+
+  const [strings] = await once(port2, 'message')
+
+  assert.strictEqual(strings, 'hello world\nsomething else\n')
+})
+
+test('yarn module resolution for typescript commonjs modules', { skip: !isYarnPnp }, async t => {
+  const { port1, port2 } = new MessageChannel()
+  const stream = new ThreadStream({
+    filename: join(__dirname, 'ts-commonjs-default-export.zip', 'worker.js'),
+    workerData: { port: port1 },
+    workerOpts: {
+      transferList: [port1]
+    },
+    sync: false
+  })
+  t.after(() => {
+    stream.end()
+  })
+
+  assert.ok(stream.write('hello world\n'))
+  assert.ok(stream.write('something else\n'))
+
+  const [strings] = await once(port2, 'message')
+
+  assert.strictEqual(strings, 'hello world\nsomething else\n')
+})