@uknows/setup 0.0.0-stage → 0.1.0

package/lib/stacks/node.mjs

@@ -0,0 +1,235 @@
+/*
+ * Node.js: add @opentelemetry/api and @opentelemetry/auto-instrumentations-node
+ * with the package manager of the repo, then load the register module when
+ * node starts: NODE_OPTIONS in the Dockerfile, or the option right after
+ * `node` in scripts.start.
+ *
+ * Safety rule: node crashes at start when the register module is missing, so
+ * the start changes only when the packages surely reach the running app.
+ * Package managers run with install scripts turned off, so no code of the
+ * project or its dependencies runs.
+ */
+import { Doc, shellWords } from '../doc.mjs'
+import { addOption, dockerStart, parseDockerfile } from '../dockerfile.mjs'
+import { NODE_REGISTER, PINS, STACKS } from '../contract.mjs'
+import { list } from '../ui.mjs'
+import { upFrom } from '../workspace.mjs'
+import { envOverrides } from './k8s.mjs'
+
+const STACK = STACKS.find((s) => s.id === 'node')
+const LOCKS = [['package-lock.json', 'npm'], ['npm-shrinkwrap.json', 'npm'], ['pnpm-lock.yaml', 'pnpm'], ['yarn.lock', 'yarn'],
+  ['bun.lockb', 'bun'], ['bun.lock', 'bun']]
+const DEFAULT_LOCK = { npm: 'package-lock.json', pnpm: 'pnpm-lock.yaml', yarn: 'yarn.lock' }
+const RUNS_NODE = /^(node|nodejs|npm|npx|yarn|pnpm)$|\.[cm]?js$/
+const BUNDLER = /\b(esbuild|webpack|rollup|tsup|ncc)\b/
+const VALUE_FLAGS = new Set(['-r', '--require', '--import', '--loader', '--experimental-loader', '--env-file', '-C', '--conditions'])
+
+/** @type {import('../types.mjs').Applier} */
+export const node = {
+  id: 'node',
+  plan(service, ctx) {
+    const { ws } = ctx
+    const out = { summary: [], edits: [], commands: [], extraSettings: [], manual: [], notes: [] }
+    const pkgFile = join(service.dir, 'package.json')
+    const pkgText = read(ws, pkgFile)
+    let pkg = null
+    try { pkg = JSON.parse(pkgText) } catch { /* handled below */ }
+    if (pkgText === null) {
+      out.manual.push({ title: 'Add OpenTelemetry to your Node.js app', lines: [`This tool did not find ${pkgFile}.`,
+        'Add the packages with your package manager. Then start node with the option below.'],
+      snippets: [{ caption: 'Install', language: 'bash', code: STACK.install }, { caption: 'Start command', language: 'bash', code: STACK.start }] })
+      return out
+    }
+    if (!pkg || typeof pkg !== 'object' || Array.isArray(pkg)) {
+      out.manual.push({ title: `Fix ${pkgFile}`, lines: [`This tool cannot read ${pkgFile} as JSON.`, 'Fix the file. Then run npx @uknows/setup again.'] })
+      return out
+    }
+    // Too old for the packages: with --import or --require of the register module, node would not start.
+    const v = /^(\d+)(?:\.(\d+))?/.exec(service.runtimeVersion ?? '')
+    const [major, minor] = v ? [Number(v[1]), v[2] === undefined ? 99 : Number(v[2])] : [99, 99]
+    if (major < 18 || major === 19 || (major === 18 && minor < 19) || (major === 20 && minor < 6)) {
+      out.manual.push({ title: 'Use Node.js 18.19 or later', lines: [`This app uses Node.js ${service.runtimeVersion}.`,
+        'The OpenTelemetry packages need Node.js 18.19 or later on version 18, or 20.6 or later.',
+        'Update the Node.js version of the app. Then run npx @uknows/setup again.'] })
+      return out
+    }
+    const docker = service.dockerfile ? read(ws, service.dockerfile) : null
+    const info = docker !== null ? parseDockerfile(docker) : null
+    const start = info ? dockerStart(info) : null
+    const script = typeof pkg.scripts?.start === 'string' ? pkg.scripts.start : null
+    const entry = entryOf(info ? start?.argv : service.start?.argv ?? shellWords(script ?? ''), script)
+    const esm = /\.m[jt]s$/.test(entry ?? '') || (pkg.type === 'module' && !/\.c[jt]s$/.test(entry ?? ''))
+    const flag = esm ? PINS.node.esmFlag : PINS.node.requireFlag
+
+    const guided = guidedBlock(service, ws, pkg, info, entry, flag)
+    if (guided) { out.manual.push(guided); return out }
+    const pm = packageManager(service.dir, pkg, ctx.repo ?? ws)
+    if (pm.outside) {
+      out.manual.push({ title: 'Run setup from the repo root', lines: [
+        `${service.dir === '.' ? 'This folder' : service.dir} is part of a workspace. Its ${base(pm.outside)} is in ${dirOf(pm.outside)}, `
+          + 'outside the folder this tool works in, so this tool cannot update it.',
+        'Run npx @uknows/setup in the repo root without --path. Then pick this service from the list, or use --all.'] })
+      out.stop = true
+      return out
+    }
+    const wanted = [...PINS.node.packages, ...(esm ? [PINS.node.esmPackage] : [])]
+    if (pm.cmd === 'bun' || pm.cmd === 'other' || pm.pnp) {
+      out.manual.push({ title: 'Add OpenTelemetry with your package manager',
+        lines: [pm.pnp ? 'This repo uses Yarn Plug\'n\'Play, so node cannot load the packages with --require alone.'
+          : 'This tool adds packages with npm, pnpm or yarn only.', 'Add the packages below as normal dependencies.',
+        pm.pnp ? 'Start the app with yarn node and the option below.' : 'Start node with the option below.'],
+        snippets: [{ caption: 'Packages', language: 'text', code: wanted.join('\n') },
+          { caption: 'Start command', language: 'bash', code: `${pm.pnp ? 'yarn ' : ''}${startLine(flag, entry)}` }] })
+      return out
+    }
+    const added = wanted.filter((name) => !pkg.dependencies?.[name])
+    if (added.length) {
+      const lock = pm.lock ?? (pm.cmd === 'npm' ? null : join(service.dir, DEFAULT_LOCK[pm.cmd]))
+      // No install scripts. npm with a lockfile and pnpm write only package.json and the lockfile.
+      const flags = pm.cmd === 'npm' ? ['--ignore-scripts', pm.lock ? '--package-lock-only' : '--no-package-lock']
+        : pm.cmd === 'pnpm' ? ['--lockfile-only', ...(ws.exists(join(service.dir, 'pnpm-workspace.yaml')) ? ['-w'] : [])]
+          : [pm.berry ? '--mode=skip-build' : '--ignore-scripts']
+      const why = `Add ${list(added)} with ${pm.cmd}.`
+      const lockOnly = pm.cmd === 'pnpm' || (pm.cmd === 'npm' && Boolean(pm.lock))
+      out.commands.push({ cwd: service.dir, cmd: pm.cmd, why, added, lockOnly,
+        args: pm.cmd === 'npm' ? ['install', '--save', ...flags, ...added] : ['add', ...flags, ...added],
+        undo: { cmd: pm.cmd, args: [pm.cmd === 'npm' ? 'uninstall' : 'remove', ...flags, ...added] },
+        touches: [pkgFile, ...(lock ? [lock] : [])] })
+      out.summary.push(why)
+      if (lockOnly) out.notes.push(`To start the app on this computer, run ${pm.cmd} install first. Your build installs the packages from ${base(lock)}.`)
+    }
+
+    if (info) {
+      const runsNode = start && (start.argv ?? [start.raw]).flatMap((w) => w.split(/\s+/)).some((w) => RUNS_NODE.test(base(w.replace(/^["']|["']$/g, ''))))
+      if (!runsNode) out.manual.push(startBlock(flag, entry, `the start command in ${service.dockerfile}`))
+      else {
+        const has = (value) => value.includes(NODE_REGISTER)
+        const after = addOption(docker, 'NODE_OPTIONS', flag, has)
+        if (after !== docker) {
+          out.edits.push({ file: service.dockerfile, before: docker, after, why: 'Load OpenTelemetry when node starts.' })
+          out.summary.push(`Load OpenTelemetry when node starts (NODE_OPTIONS in ${service.dockerfile}).`)
+          if (info.env.has('NODE_OPTIONS') || info.inherited.has('NODE_OPTIONS')) {
+            out.notes.push(`${service.dockerfile} already sets NODE_OPTIONS. A new ENV line before the start keeps that value and adds the option.`)
+          }
+        }
+        out.manual.push(...envOverrides(service, 'NODE_OPTIONS', flag, has))
+      }
+    } else if (script !== null && service.start?.kind !== 'procfile' && shellWords(script)?.[0] === 'node') {
+      if (!script.includes(NODE_REGISTER)) {
+        const after = editStartScript(pkgText, flag)
+        if (after === null) out.manual.push(startBlock(flag, entry, `scripts.start in ${pkgFile}`))
+        else {
+          out.edits.push({ file: pkgFile, before: pkgText, after, why: 'Load OpenTelemetry when node starts.' })
+          out.summary.push(`Load OpenTelemetry when node starts (start script in ${pkgFile}).`)
+        }
+      }
+    } else {
+      out.manual.push(startBlock(flag, entry, script !== null ? `scripts.start in ${pkgFile}` : null))
+    }
+    return out
+  },
+}
+
+/** Cases where loading the register module cannot work: guided only, no packages. */
+function guidedBlock(service, ws, pkg, info, entry, flag) {
+  if (service.frameworks.includes('next')) {
+    return { title: 'Next.js needs its own OpenTelemetry setup', lines: ['Next.js builds its server code, so the --require option cannot load OpenTelemetry.',
+      'Follow the OpenTelemetry guide in the Next.js docs. It adds an instrumentation.ts file.', 'Keep the 5 UKNOWS settings. That setup reads them.'] }
+  }
+  if (info && (service.frameworks.includes('nitro') || service.frameworks.includes('nuxt'))) {
+    const install = info.final.some((x) => x.keyword === 'RUN' && /\b(npm (ci|install|i)|pnpm (install|i)|yarn)\b/.test(x.args))
+    const copies = info.final.filter((x) => x.keyword === 'COPY' || x.keyword === 'ADD')
+    if (!install && !copies.some((x) => /node_modules/.test(x.args) || !/\.output/.test(x.args))) {
+      return { title: 'Put node_modules in the final image', lines: [`The final stage of ${service.dockerfile} copies only the built server, so node cannot load OpenTelemetry.`,
+        'Install the packages in the final stage, or copy node_modules into it.', 'Then run npx @uknows/setup again.'] }
+    }
+  }
+  const bundler = BUNDLER.exec(typeof pkg.scripts?.build === 'string' ? pkg.scripts.build : '')
+  if (bundler && !(entry && ws.exists(join(service.dir, entry.replace(/^\.\//, ''))))) {
+    return { title: 'Load OpenTelemetry in a bundled app', lines: [`The build script bundles the app with ${bundler[1]}. OpenTelemetry cannot change bundled packages.`,
+      'Add the packages below as normal dependencies. In the bundler config, mark them and the libraries to trace as external.',
+      'Start node with the option below.'],
+    snippets: [{ caption: 'Install', language: 'bash', code: STACK.install }, { caption: 'Start command', language: 'bash', code: startLine(flag, entry) }] }
+  }
+  return null
+}
+
+/**
+ * The package manager from the packageManager field, else from the lockfile of the service folder or of
+ * the workspace root above it (a parent folder whose package.json has workspaces, or with pnpm-workspace.yaml).
+ * The search goes on above the --path folder, up to the top of the git repo (view.top). A workspace root or
+ * its lockfile up there is `outside`: this tool cannot update it.
+ */
+function packageManager(dir, pkg, view) {
+  const field = typeof pkg.packageManager === 'string' ? /^(npm|pnpm|yarn|bun)@(\d+)/.exec(pkg.packageManager) : null
+  let cmd = field ? field[1] : typeof pkg.packageManager === 'string' ? 'other' : null
+  let lock = null, outside = null
+  for (const d of upFrom(dir, view.top ?? '.')) {
+    const pnpmRoot = view.exists(join(d, 'pnpm-workspace.yaml'))
+    const marker = pnpmRoot || /"workspaces"\s*:/.test(read(view, join(d, 'package.json')) ?? '')
+    const hit = (d === dir || marker) && LOCKS.find(([file, owner]) => (!cmd || cmd === owner) && read(view, join(d, file)) !== null)
+    if ((d === '..' || d.startsWith('../')) && (marker || hit)) { outside = join(d, hit ? hit[0] : pnpmRoot ? 'pnpm-workspace.yaml' : 'package.json'); break }
+    if (hit) { lock = join(d, hit[0]); cmd = hit[1]; break }
+  }
+  cmd ??= 'npm'
+  const home = lock ? dirOf(lock) : dir
+  const rc = read(view, join(home, '.yarnrc.yml'))
+  const berry = cmd === 'yarn' && (field ? Number(field[2]) >= 2 : rc !== null || /^__metadata:/m.test(read(view, lock ?? '') ?? ''))
+  const pnp = cmd === 'yarn' && (view.exists(join(home, '.pnp.cjs')) || view.exists(join(home, '.pnp.js'))
+    || (berry && !/^\s*nodeLinker\s*:\s*["']?(node-modules|pnpm)/m.test(rc ?? '')))
+  return { cmd, lock, pnp, berry, outside }
+}
+
+/** The JS file that node runs, from a start argv. `npm start` and friends read scripts.start. */
+function entryOf(argv, script) {
+  let words = argv ?? null
+  if (words && /^(npm|yarn|pnpm)$/.test(base(words[0])) && (words[1] === 'start' || (words[1] === 'run' && words[2] === 'start'))) words = shellWords(script ?? '')
+  if (!words?.length) return null
+  if (/\.[cm]?[jt]s$/.test(words[0])) return words[0]
+  if (base(words[0]) !== 'node') return null
+  for (let i = 1; i < words.length; i++) {
+    if (!words[i].startsWith('-')) return words[i]
+    if (VALUE_FLAGS.has(words[i])) i++
+  }
+  return null
+}
+
+/** Insert the option right after `node` in scripts.start. Changes only that JSON string. null when unsure. */
+function editStartScript(text, flag) {
+  const doc = new Doc(text)
+  const from = doc.find(/"scripts"\s*:/)
+  const i = from < 0 ? -1 : doc.find(/"start"\s*:\s*"/, from)
+  if (i < 0) return null
+  const line = doc.get(i)
+  const m = /("start"\s*:\s*)("(?:[^"\\]|\\.)*")/.exec(line)
+  if (!m) return null
+  const value = JSON.parse(m[2]).replace(/^(\s*node)(?=\s|$)/, `$1 ${flag}`)
+  doc.set(i, line.slice(0, m.index + m[1].length) + JSON.stringify(value) + line.slice(m.index + m[0].length))
+  const after = doc.toString()
+  try {
+    const expected = JSON.parse(text)
+    expected.scripts.start = value
+    return JSON.stringify(JSON.parse(after)) === JSON.stringify(expected) ? after : null
+  } catch { return null }
+}
+
+function startLine(flag, entry) {
+  return STACK.start.replace(PINS.node.requireFlag, flag).replace(/server\.js$/, entry ?? 'server.js')
+}
+
+function startBlock(flag, entry, where) {
+  return { title: 'Load OpenTelemetry when node starts',
+    lines: [where ? `This tool cannot change ${where} safely.` : 'This tool did not find the command that starts the app.',
+      'Add the option below to the node command that starts the app. Or set NODE_OPTIONS to it where the app runs.',
+      ...STACK.notes.filter((n) => !entry && /server\.js/.test(n))],
+    snippets: [{ caption: 'Start command', language: 'bash', code: startLine(flag, entry) },
+      { caption: 'Or set NODE_OPTIONS', language: 'bash', code: `NODE_OPTIONS="${flag}"` }] }
+}
+
+const base = (word) => (word ?? '').split('/').pop()
+const join = (dir, file) => (dir === '.' ? file : `${dir}/${file}`)
+const dirOf = (path) => (path.includes('/') ? path.slice(0, path.lastIndexOf('/')) : '.')
+
+function read(ws, path) {
+  try { return path ? ws.read(path) : null } catch { return null }
+}

package/lib/stacks/otel.mjs

@@ -0,0 +1,11 @@
+// The app already uses OpenTelemetry: keep its packages and start command. settings.mjs adds the 5 settings.
+import { STACKS } from '../contract.mjs'
+
+/** @type {import('../types.mjs').Applier} */
+export const otel = {
+  id: 'otel',
+  plan() {
+    return { summary: ['Keep your OpenTelemetry packages. Point them at UKNOWS with the 5 settings.'], edits: [], commands: [],
+      extraSettings: [], manual: [], notes: [...STACKS.find((s) => s.id === 'otel').notes] }
+  },
+}

package/lib/stacks/python-bootstrap.mjs

@@ -0,0 +1,51 @@
+/*
+ * What `opentelemetry-bootstrap -a requirements` prints, without a Python
+ * environment. Source: bootstrap_gen.py of opentelemetry-python-contrib
+ * v0.66b1. Upstream checks the installed packages; this tool reads the
+ * requirement names instead, plus a few packages that the common frameworks
+ * always pull in (IMPLIED).
+ *
+ * Left out on purpose: openai-v2 and vertexai (not pinned to 0.66b1),
+ * psutil -> system-metrics (metrics are off) and the upstream quirk that maps
+ * pydantic to tortoiseorm.
+ */
+
+/** Always printed. */
+export const DEFAULTS = ['asyncio', 'dbapi', 'exceptions', 'logging', 'sqlite3', 'threading', 'urllib', 'wsgi']
+
+/** [normalized library name, instrumentation suffixes], in upstream order. */
+export const LIBRARIES = [
+  ['aio-pika', ['aio-pika']], ['aiohttp', ['aiohttp-client', 'aiohttp-server']], ['aiokafka', ['aiokafka']],
+  ['aiopg', ['aiopg']], ['asgiref', ['asgi']], ['asyncclick', ['asyncclick']], ['asyncpg', ['asyncpg']],
+  ['boto3', ['boto3sqs']], ['botocore', ['botocore']], ['aiobotocore', ['botocore']],
+  ['cassandra-driver', ['cassandra']], ['scylla-driver', ['cassandra']], ['celery', ['celery']], ['click', ['click']],
+  ['confluent-kafka', ['confluent-kafka']], ['django', ['django']], ['falcon', ['falcon']], ['fastapi', ['fastapi']],
+  ['flask', ['flask']], ['grpcio', ['grpc']], ['httpx', ['httpx']], ['httpx2', ['httpx']], ['jinja2', ['jinja2']],
+  ['kafka-python', ['kafka-python']], ['mysql-connector-python', ['mysql']], ['mysqlclient', ['mysqlclient']],
+  ['pika', ['pika']], ['psycopg', ['psycopg']], ['psycopg2', ['psycopg2']], ['psycopg2-binary', ['psycopg2']],
+  ['pymemcache', ['pymemcache']], ['pymongo', ['pymongo']], ['pymssql', ['pymssql']], ['pymysql', ['pymysql']],
+  ['pyramid', ['pyramid']], ['redis', ['redis']], ['remoulade', ['remoulade']], ['requests', ['requests']],
+  ['sqlalchemy', ['sqlalchemy']], ['starlette', ['starlette']], ['structlog', ['structlog']], ['tornado', ['tornado']],
+  ['tortoise-orm', ['tortoiseorm']], ['urllib3', ['urllib3']],
+]
+
+/** Packages that a framework always installs, so upstream would see them too. */
+export const IMPLIED = { fastapi: ['starlette'], django: ['asgiref'], flask: ['jinja2'], requests: ['urllib3'], boto3: ['botocore'] }
+
+/** PEP 503 name: lower case, runs of - _ . become -. */
+export const normalize = (name) => name.toLowerCase().replace(/[-_.]+/g, '-')
+
+/**
+ * The instrumentation packages for these requirement names: the defaults,
+ * then 1 per matching library in table order, without repeats.
+ * @param {Iterable<string>} names  normalized requirement names
+ * @param {string} version          e.g. '0.66b1'
+ * @returns {string[]} e.g. ['opentelemetry-instrumentation-asyncio==0.66b1', ...]
+ */
+export function bootstrap(names, version) {
+  const have = new Set(names)
+  for (const name of [...have]) for (const extra of IMPLIED[name] ?? []) have.add(extra)
+  const suffixes = [...DEFAULTS]
+  for (const [library, list] of LIBRARIES) if (have.has(library)) suffixes.push(...list)
+  return [...new Set(suffixes)].map((x) => `opentelemetry-instrumentation-${x}==${version}`)
+}