@platformatic/basic 2.0.0-alpha.10

package/lib/schema.js

@@ -0,0 +1,80 @@
+import { schemaComponents as utilsSchemaComponents } from '@platformatic/utils'
+import { readFileSync } from 'node:fs'
+
+export const packageJson = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf-8'))
+
+const application = {
+  type: 'object',
+  properties: {
+    basePath: {
+      type: 'string'
+    },
+    outputDirectory: {
+      type: 'string',
+      default: 'dist'
+    },
+    include: {
+      type: 'array',
+      items: {
+        type: 'string'
+      },
+      default: ['dist']
+    },
+    commands: {
+      type: 'object',
+      properties: {
+        install: {
+          type: 'string',
+          default: 'npm ci --omit-dev'
+        },
+        // All the following options purposely don't have a default so
+        // that stackables can detect if the user explicitly set them.
+        build: {
+          type: 'string'
+        },
+        development: {
+          type: 'string'
+        },
+        production: {
+          type: 'string'
+        }
+      },
+      default: {},
+      additionalProperties: false
+    }
+  },
+  additionalProperties: false,
+  default: {}
+}
+
+const watch = {
+  anyOf: [
+    utilsSchemaComponents.watch,
+    {
+      type: 'boolean'
+    },
+    {
+      type: 'string'
+    }
+  ]
+}
+
+export const schemaComponents = { application, watch }
+
+export const schema = {
+  $id: `https://schemas.platformatic.dev/@platformatic/basic/${packageJson.version}.json`,
+  $schema: 'http://json-schema.org/draft-07/schema#',
+  title: 'Platformatic Stackable',
+  type: 'object',
+  properties: {
+    $schema: {
+      type: 'string'
+    }
+  },
+  additionalProperties: true
+}
+
+/* c8 ignore next 3 */
+if (process.argv[1] === import.meta.filename) {
+  console.log(JSON.stringify(schema, null, 2))
+}

package/lib/utils.js

@@ -0,0 +1,73 @@
+import { createRequire } from 'node:module'
+import { pathToFileURL } from 'node:url'
+import { request } from 'undici'
+
+export function getServerUrl (server) {
+  const { family, address, port } = server.address()
+
+  return new URL(family === 'IPv6' ? `http://[${address}]:${port}` : `http://${address}:${port}`).origin
+}
+
+export async function injectViaRequest (baseUrl, injectParams, onInject) {
+  const url = new URL(injectParams.url, baseUrl).href
+  const requestParams = { method: injectParams.method, headers: injectParams.headers }
+
+  if (injectParams.body) {
+    const body = injectParams.body
+    requestParams.body = typeof body === 'object' ? JSON.stringify(body) : body
+  }
+
+  try {
+    const { statusCode, headers, body } = await request(url, requestParams)
+
+    const rawPayload = Buffer.from(await body.arrayBuffer())
+    const payload = rawPayload.toString()
+    const response = { statusCode, headers, body: payload, payload, rawPayload }
+
+    if (onInject) {
+      return onInject(null, response)
+    }
+
+    return response
+  } catch (error) {
+    if (onInject) {
+      onInject(error)
+      return
+    }
+
+    throw error
+  }
+}
+
+export function ensureFileUrl (pathOrUrl) {
+  if (!pathOrUrl) {
+    return pathOrUrl
+  }
+
+  pathOrUrl = pathOrUrl.toString()
+
+  if (pathOrUrl.startsWith('file://')) {
+    return pathOrUrl
+  }
+
+  return pathToFileURL(pathOrUrl)
+}
+
+// This is to avoid common path/URL problems on Windows
+export function importFile (path) {
+  return import(ensureFileUrl(path))
+}
+
+export function resolvePackage (root, pkg) {
+  const require = createRequire(root)
+
+  return require.resolve(pkg, { paths: [root, ...require.main.paths] })
+}
+
+export function cleanBasePath (basePath) {
+  return basePath ? `/${basePath}`.replaceAll(/\/+/g, '/').replace(/\/$/, '') : '/'
+}
+
+export function ensureTrailingSlash (basePath) {
+  return basePath ? `${basePath}${basePath.endsWith('/') ? '' : '/'}` : '/'
+}

package/lib/worker/child-manager.js

@@ -0,0 +1,219 @@
+import { ITC, generateNotification } from '@platformatic/itc'
+import { createDirectory, errors } from '@platformatic/utils'
+import { once } from 'node:events'
+import { rm, writeFile } from 'node:fs/promises'
+import { createServer } from 'node:http'
+import { register } from 'node:module'
+import { platform, tmpdir } from 'node:os'
+import { dirname, resolve } from 'node:path'
+import { workerData } from 'node:worker_threads'
+import { request } from 'undici'
+import { WebSocketServer } from 'ws'
+import { exitCodes } from '../errors.js'
+import { ensureFileUrl } from '../utils.js'
+
+export const isWindows = platform() === 'win32'
+
+// In theory we could use the context.id to namespace even more, but due to
+// UNIX socket length limitation on MacOS, we don't.
+function generateChildrenId (context) {
+  return [process.pid, Date.now()].join('-')
+}
+
+export function getSocketPath (id) {
+  let socketPath = null
+  if (platform() === 'win32') {
+    socketPath = `\\\\.\\pipe\\plt-${id}`
+  } else {
+    // As stated in https://nodejs.org/dist/latest-v20.x/docs/api/net.html#identifying-paths-for-ipc-connections,
+    // Node will take care of deleting the file for us
+    socketPath = resolve(tmpdir(), 'platformatic', 'runtimes', `${id}.socket`)
+  }
+
+  return socketPath
+}
+
+export class ChildManager extends ITC {
+  #id
+  #loader
+  #context
+  #scripts
+  #logger
+  #server
+  #socketPath
+  #clients
+  #requests
+  #currentClient
+  #listener
+  #originalNodeOptions
+  #dataPath
+
+  constructor (opts) {
+    let { loader, context, scripts, handlers, ...itcOpts } = opts
+
+    context ??= {}
+    scripts ??= []
+
+    super({
+      name: 'child-manager',
+      ...itcOpts,
+      handlers: {
+        log: message => {
+          return this.#log(message)
+        },
+        fetch: request => {
+          return this.#fetch(request)
+        },
+        ...itcOpts.handlers
+      }
+    })
+
+    this.#id = generateChildrenId(context)
+    this.#loader = loader
+    this.#context = context
+    this.#scripts = scripts
+    this.#originalNodeOptions = process.env.NODE_OPTIONS
+    this.#logger = globalThis.platformatic.logger
+    this.#server = createServer()
+    this.#socketPath ??= getSocketPath(this.#id)
+    this.#clients = new Set()
+    this.#requests = new Map()
+  }
+
+  async listen () {
+    super.listen()
+
+    if (!isWindows) {
+      await createDirectory(dirname(this.#socketPath))
+    }
+
+    const wssServer = new WebSocketServer({ server: this.#server })
+
+    wssServer.on('connection', ws => {
+      this.#clients.add(ws)
+
+      ws.on('message', raw => {
+        try {
+          const message = JSON.parse(raw)
+          this.#requests.set(message.reqId, ws)
+          this.#listener(message)
+        } catch (error) {
+          this.#handleUnexpectedError(error, 'Handling a message failed.', exitCodes.MANAGER_MESSAGE_HANDLING_FAILED)
+        }
+      })
+
+      ws.on('close', () => {
+        this.#clients.delete(ws)
+      })
+
+      ws.on('error', error => {
+        this.#handleUnexpectedError(
+          error,
+          'Error while communicating with the children process.',
+          exitCodes.MANAGER_SOCKET_ERROR
+        )
+      })
+    })
+
+    return new Promise((resolve, reject) => {
+      this.#server.listen({ path: this.#socketPath }, resolve).on('error', reject)
+    })
+  }
+
+  async close (signal) {
+    await rm(this.#dataPath)
+
+    for (const client of this.#clients) {
+      this.#currentClient = client
+      this._send(generateNotification('close', signal))
+    }
+
+    super.close()
+  }
+
+  async inject () {
+    await this.listen()
+
+    // Serialize data into a JSON file for the stackable to use
+    this.#dataPath = resolve(tmpdir(), 'platformatic', 'runtimes', `${this.#id}.json`)
+    await createDirectory(dirname(this.#dataPath))
+
+    // We write all the data to a JSON file
+    await writeFile(
+      this.#dataPath,
+      JSON.stringify(
+        {
+          data: this.#context,
+          loader: ensureFileUrl(this.#loader),
+          scripts: this.#scripts.map(s => ensureFileUrl(s))
+        },
+        null,
+        2
+      ),
+      'utf-8'
+    )
+
+    process.env.PLT_MANAGER_ID = this.#id
+    process.env.NODE_OPTIONS =
+      `--import="${new URL('./child-process.js', import.meta.url)}" ${process.env.NODE_OPTIONS ?? ''}`.trim()
+  }
+
+  async eject () {
+    process.env.NODE_OPTIONS = this.#originalNodeOptions
+    process.env.PLT_MANAGER_ID = ''
+  }
+
+  register () {
+    register(this.#loader, { data: this.#context })
+  }
+
+  send (client, name, message) {
+    this.#currentClient = client
+    super.send(name, message)
+  }
+
+  _send (message) {
+    if (!this.#currentClient) {
+      this.#currentClient = this.#requests.get(message.reqId)
+      this.#requests.delete(message.reqId)
+
+      if (!this.#currentClient) {
+        return
+      }
+    }
+
+    this.#currentClient.send(JSON.stringify(message))
+    this.#currentClient = null
+  }
+
+  _setupListener (listener) {
+    this.#listener = listener
+  }
+
+  _createClosePromise () {
+    return once(this.#server, 'exit')
+  }
+
+  _close () {
+    this.#server.close()
+  }
+
+  #log (message) {
+    const logs = Array.isArray(message.logs) ? message.logs : [message.logs]
+    workerData.loggingPort.postMessage({ logs: logs.map(m => JSON.stringify(m)) })
+  }
+
+  async #fetch (opts) {
+    const { statusCode, headers, body } = await request(opts)
+
+    const rawPayload = Buffer.from(await body.arrayBuffer())
+    const payload = rawPayload.toString()
+
+    return { statusCode, headers, body: payload, payload, rawPayload }
+  }
+
+  #handleUnexpectedError (error, message, exitCode) {
+    this.#logger.error({ err: errors.ensureLoggableError(error) }, message)
+    process.exit(exitCode)
+  }
+}

package/lib/worker/child-process.js

@@ -0,0 +1,232 @@
+import { ITC } from '@platformatic/itc'
+import { createPinoWritable, errors } from '@platformatic/utils'
+import { tracingChannel } from 'node:diagnostics_channel'
+import { once } from 'node:events'
+import { readFile } from 'node:fs/promises'
+import { register } from 'node:module'
+import { platform, tmpdir } from 'node:os'
+import { basename, resolve } from 'node:path'
+import { isMainThread } from 'node:worker_threads'
+import pino from 'pino'
+import { getGlobalDispatcher, setGlobalDispatcher } from 'undici'
+import { WebSocket } from 'ws'
+import { exitCodes } from '../errors.js'
+import { importFile } from '../utils.js'
+import { getSocketPath, isWindows } from './child-manager.js'
+
+function createInterceptor (itc) {
+  return function (dispatch) {
+    return async (opts, handler) => {
+      let url = opts.origin
+      if (!(url instanceof URL)) {
+        url = new URL(opts.path, url)
+      }
+
+      // Other URLs are handled normally
+      if (!url.hostname.endsWith('.plt.local')) {
+        return dispatch(opts, handler)
+      }
+
+      const headers = {
+        ...opts?.headers
+      }
+
+      delete headers.connection
+      delete headers['transfer-encoding']
+      headers.host = url.host
+
+      const requestOpts = {
+        ...opts,
+        headers
+      }
+      delete requestOpts.dispatcher
+
+      itc
+        .send('fetch', requestOpts)
+        .then(res => {
+          if (res.rawPayload && !Buffer.isBuffer(res.rawPayload)) {
+            res.rawPayload = Buffer.from(res.rawPayload.data)
+          }
+
+          const headers = []
+          for (const [key, value] of Object.entries(res.headers)) {
+            if (Array.isArray(value)) {
+              for (const v of value) {
+                headers.push(key)
+                headers.push(v)
+              }
+            } else {
+              headers.push(key)
+              headers.push(value)
+            }
+          }
+
+          handler.onHeaders(res.statusCode, headers, () => {}, res.statusMessage)
+          handler.onData(res.rawPayload)
+          handler.onComplete([])
+        })
+        .catch(e => {
+          handler.onError(new Error(e.message))
+        })
+
+      return true
+    }
+  }
+}
+
+export class ChildProcess extends ITC {
+  #listener
+  #socket
+  #child
+  #logger
+  #pendingMessages
+
+  constructor () {
+    super({ throwOnMissingHandler: false, name: `${process.env.PLT_MANAGER_ID}-child-process` })
+
+    const protocol = platform() === 'win32' ? 'ws+unix:' : 'ws+unix://'
+    this.#socket = new WebSocket(`${protocol}${getSocketPath(process.env.PLT_MANAGER_ID)}`)
+    this.#pendingMessages = []
+
+    this.listen()
+    this.#setupLogger()
+    this.#setupHandlers()
+    this.#setupServer()
+    this.#setupInterceptors()
+
+    this.on('close', signal => {
+      process.kill(process.pid, signal)
+    })
+  }
+
+  _setupListener (listener) {
+    this.#listener = listener
+
+    this.#socket.on('open', () => {
+      // Never hang the process on this socket.
+      this.#socket._socket.unref()
+
+      for (const message of this.#pendingMessages) {
+        this.#socket.send(message)
+      }
+    })
+
+    this.#socket.on('message', message => {
+      try {
+        this.#listener(JSON.parse(message))
+      } catch (error) {
+        this.#logger.error({ err: errors.ensureLoggableError(error) }, 'Handling a message failed.')
+        process.exit(exitCodes.PROCESS_MESSAGE_HANDLING_FAILED)
+      }
+    })
+
+    this.#socket.on('error', error => {
+      process._rawDebug(error)
+      // There is nothing to log here as the connection with the parent thread is lost. Exit with a special code
+      process.exit(exitCodes.PROCESS_SOCKET_ERROR)
+    })
+  }
+
+  _send (message) {
+    if (this.#socket.readyState === WebSocket.CONNECTING) {
+      this.#pendingMessages.push(JSON.stringify(message))
+      return
+    }
+
+    this.#socket.send(JSON.stringify(message))
+  }
+
+  _createClosePromise () {
+    return once(this.#socket, 'close')
+  }
+
+  _close () {
+    this.#socket.close()
+  }
+
+  #setupLogger () {
+    // Since this is executed by user code, make sure we only override this in the main thread
+    // The rest will be intercepted by the BaseStackable.
+    if (isMainThread) {
+      this.#logger = pino({
+        level: 'info',
+        name: globalThis.platformatic.id,
+        transport: {
+          target: new URL('./child-transport.js', import.meta.url).toString(),
+          options: { id: globalThis.platformatic.id }
+        }
+      })
+
+      Reflect.defineProperty(process, 'stdout', { value: createPinoWritable(this.#logger, 'info') })
+      Reflect.defineProperty(process, 'stderr', { value: createPinoWritable(this.#logger, 'error') })
+    } else {
+      this.#logger = pino({ level: 'info', name: globalThis.platformatic.id })
+    }
+  }
+
+  #setupServer () {
+    const subscribers = {
+      asyncStart ({ options }) {
+        const port = globalThis.platformatic.port
+
+        if (port !== false) {
+          options.port = typeof port === 'number' ? port : 0
+        }
+      },
+      asyncEnd: ({ server }) => {
+        tracingChannel('net.server.listen').unsubscribe(subscribers)
+
+        const { family, address, port } = server.address()
+        const url = new URL(family === 'IPv6' ? `http://[${address}]:${port}` : `http://${address}:${port}`).origin
+
+        this.notify('url', url)
+      },
+      error: error => {
+        tracingChannel('net.server.listen').unsubscribe(subscribers)
+        this.notify('error', error)
+      }
+    }
+
+    tracingChannel('net.server.listen').subscribe(subscribers)
+  }
+
+  #setupInterceptors () {
+    setGlobalDispatcher(getGlobalDispatcher().compose(createInterceptor(this)))
+  }
+
+  #setupHandlers () {
+    function handleUnhandled (type, err) {
+      process._rawDebug(globalThis.platformatic.id, err)
+      this.#logger.error(
+        { err: errors.ensureLoggableError(err) },
+        `Child process for service ${globalThis.platformatic.id} threw an ${type}.`
+      )
+
+      process.exit(exitCodes.PROCESS_UNHANDLED_ERROR)
+    }
+
+    process.on('uncaughtException', handleUnhandled.bind(this, 'uncaught exception'))
+    process.on('unhandledRejection', handleUnhandled.bind(this, 'unhandled rejection'))
+  }
+}
+
+async function main () {
+  const dataPath = resolve(tmpdir(), 'platformatic', 'runtimes', `${process.env.PLT_MANAGER_ID}.json`)
+  const { data, loader, scripts } = JSON.parse(await readFile(dataPath))
+
+  globalThis.platformatic = data
+
+  if (loader) {
+    register(loader, { data })
+  }
+
+  for (const script of scripts) {
+    await importFile(script)
+  }
+
+  globalThis[Symbol.for('plt.children.itc')] = new ChildProcess()
+}
+
+if (!isWindows || basename(process.argv.at(-1)) !== 'npm-prefix.js') {
+  await main()
+}

package/lib/worker/child-transport.js

@@ -0,0 +1,54 @@
+import { generateRequest } from '@platformatic/itc'
+import { errors } from '@platformatic/utils'
+import { once } from 'node:events'
+import { platform } from 'node:os'
+import { workerData } from 'node:worker_threads'
+import build from 'pino-abstract-transport'
+import { WebSocket } from 'ws'
+import { getSocketPath } from './child-manager.js'
+
+function logDirectError (message, error) {
+  process._rawDebug(`Logger thread for child process of service ${workerData.id} ${message}.`, {
+    error: errors.ensureLoggableError(error)
+  })
+}
+
+function handleUnhandled (type, error) {
+  logDirectError(`threw an ${type}`, error)
+  process.exit(6)
+}
+
+process.on('uncaughtException', handleUnhandled.bind(null, 'uncaught exception'))
+process.on('unhandledRejection', handleUnhandled.bind(null, 'unhandled rejection'))
+
+export default async function (opts) {
+  try {
+    const protocol = platform() === 'win32' ? 'ws+unix:' : 'ws+unix://'
+    const socket = new WebSocket(`${protocol}${getSocketPath(process.env.PLT_MANAGER_ID)}`)
+
+    await once(socket, 'open')
+
+    // Do not process responses but empty the socket inbound queue
+    socket.on('message', () => {})
+
+    socket.on('error', error => {
+      logDirectError('threw a socket error', error)
+    })
+
+    return build(
+      async function (source) {
+        for await (const obj of source) {
+          socket.send(JSON.stringify(generateRequest('log', { logs: [obj] })))
+        }
+      },
+      {
+        close (_, cb) {
+          socket.close()
+          cb()
+        }
+      }
+    )
+  } catch (error) {
+    logDirectError('threw a connection error', error)
+  }
+}

package/lib/worker/listeners.js

@@ -0,0 +1,51 @@
+import { withResolvers } from '@platformatic/utils'
+import { subscribe, tracingChannel, unsubscribe } from 'node:diagnostics_channel'
+
+export function createServerListener (overridePort = true) {
+  const { promise, resolve, reject } = withResolvers()
+
+  const subscribers = {
+    asyncStart ({ options }) {
+      if (overridePort !== false) {
+        options.port = typeof overridePort === 'number' ? overridePort : 0
+      }
+    },
+    asyncEnd ({ server }) {
+      resolve(server)
+    },
+    error (error) {
+      cancel()
+      reject(error)
+    }
+  }
+
+  function cancel () {
+    tracingChannel('net.server.listen').unsubscribe(subscribers)
+  }
+
+  tracingChannel('net.server.listen').subscribe(subscribers)
+  promise.finally(cancel)
+  promise.cancel = resolve.bind(null, null)
+
+  return promise
+}
+
+export function createChildProcessListener () {
+  const { promise, resolve } = withResolvers()
+
+  const handler = ({ process: child }) => {
+    unsubscribe('child_process', handler)
+    resolve(child)
+  }
+
+  function cancel () {
+    unsubscribe('child_process', handler)
+  }
+
+  subscribe('child_process', handler)
+
+  promise.finally(cancel)
+  promise.cancel = resolve.bind(null, null)
+
+  return promise
+}