@uknows/setup 0.1.0 → 0.2.0

package/lib/settings.mjs

@@ -1,15 +1,37 @@
-// Where the 5 UKNOWS settings go for 1 service, and the edits that put them there: the final stage of
-// the Dockerfile, else the env of the Kubernetes container, else <dir>/otel.env. Never the key.
+// Where the UKNOWS settings go for 1 service, and the edits that put them there: the final stage of
+// the Dockerfile, else the env of the Kubernetes container, else <dir>/otel.env when the repo has it or a repo
+// file loads it (a systemd EnvironmentFile=, a Compose env_file:, --env-file or . ./otel.env). Otherwise no file:
+// a required step prints the settings in the form of the platform. Never the key.
 // Settings that would keep traces away from UKNOWS (another address, a traces-only override, a Kubernetes
 // env that replaces the image value) become manual steps, never silent successes.
-import { INGEST_ENDPOINT, SECRET, SETTINGS, SETTINGS_FILE, STACKS } from './contract.mjs'
+//
+// The settings depend on the mode of the service (ctx.mode):
+//   standard       the 5 settings, with OTEL_METRICS_EXPORTER=none and OTEL_LOGS_EXPORTER=otlp (traces and logs).
+//   channel        the same 5 settings. The agent comes from the UKNOWS agent image (stacks/channel.mjs).
+//   otel           the 3 shared settings. Never the none lines: the app keeps its metrics and logs. The traces-only
+//                  names replace the endpoint and protocol when the code also sends metrics or logs over OTLP, and for
+//                  Node.js, Python and Java, whose SDKs send metrics and logs over OTLP by default (unless the target
+//                  turns both off with none).
+//
+// An endpoint value that an earlier setup wrote (a change in .uknows/setup.json added that line) is this tool's own,
+// not another address to keep. A newer setup replaces it, for example the ingest address of 0.1.0.
+//   azure-monitor  the 3 shared settings, and OTEL_SERVICE_NAME only when the app does not set it: it also sets
+//                  the App Insights Cloud Role Name.
+import { posix } from 'node:path'
+import { AZURE_MONITOR, DOCS_URL, DOTNET_TRACES_ONLY, FUNCTIONS, INGEST_ENDPOINT, SECRET, SETTINGS, SETTINGS_ELSEWHERE, SETTINGS_FILE, SHARED_SETTINGS, STACKS,
+  TRACES_ONLY_PLAN, settingsSnippet } from './contract.mjs'
+import { isDockerfile } from './detect.mjs'
 import { Doc, indentOf } from './doc.mjs'
 import { parseDockerfile, setEnv } from './dockerfile.mjs'
-import { list } from './ui.mjs'
+import { BOM, sha256 } from './manifest.mjs'
+import { list, stackLabel } from './ui.mjs'
 
 const NAMES = SETTINGS.map((s) => s.name)
-const [SERVICE, ENDPOINT] = NAMES
-const COLLECTOR = STACKS.find((s) => s.id === 'otel').notes.find((n) => /Collector/.test(n))
+const [SERVICE, ENDPOINT, PROTOCOL] = NAMES
+const TRACES_ENDPOINT = 'OTEL_EXPORTER_OTLP_TRACES_ENDPOINT', TRACES_PROTOCOL = 'OTEL_EXPORTER_OTLP_TRACES_PROTOCOL'
+const fill = (text, values) => text.replace(/\{(\w+)\}/g, (m, key) => (key in values ? String(values[key]) : m))
+/** The Collector note, with the link to that section: the stack tab of the docs page that has it folded (F-012-09). */
+const COLLECTOR = `${STACKS.find((s) => s.id === 'otel').notes.find((n) => /Collector/.test(n))} See ${DOCS_URL}?method=manual&stack=otel#install-app.`
 const at = (dir, name) => (dir === '.' ? name : `${dir}/${name}`)
 const blank = (line) => !line.trim() || /^\s*#/.test(line)
 const quote = (value) => `"${value.replace(/["\\]/g, '\\$&')}"`
@@ -43,25 +65,75 @@ const overrideRules = (endpoint) => [
  */
 export function settingsPlan(service, ctx, extra = []) {
   const { ws } = ctx
+  const mode = ctx.mode ?? 'standard'
   const ingest = ctx.endpoint ?? INGEST_ENDPOINT
-  const sameEndpoint = sameAs(ingest)
-  const pairs = [...SETTINGS.map((s) => [s.name, s.name === SERVICE ? ctx.name : s.name === ENDPOINT ? ingest : s.value]),
-    ...extra.filter((s) => s.name !== SECRET.name && !NAMES.includes(s.name)).map((s) => [s.name, String(s.value)])]
+  const envFile = at(service.dir, SETTINGS_FILE.path)
   const target = (service.dockerfile && ws.read(service.dockerfile) !== null && dockerTarget(ws, service.dockerfile))
     || (service.k8s ?? []).map((k) => k8sTarget(ws, k)).find(Boolean)
-    || envTarget(ws, at(service.dir, SETTINGS_FILE.path))
-  const plan = { target: { kind: target.kind, file: target.file }, edits: [], summary: [], manual: [], notes: [] }
+    || envFileTarget(ws, ctx.repo ?? ws, service, envFile)
+  const plan = { target: target.kind === 'none' ? null : { kind: target.kind, file: target.file, ...(target.loader ? { loader: target.loader } : {}) },
+    edits: [], summary: [], manual: [], notes: [], pairs: [], tracesOnly: false }
+  if (target.loader) plan.notes.push(fill(SETTINGS_ELSEWHERE.loaded, { ...target.loader, path: envFile }))
+  const anchors = ctx.existing?.anchors ?? []
+  // otel mode: metrics or logs over OTLP in the code (or set in the target) keep the general names. Traces get their own.
+  // The SDKs of Node.js, Python and Java send metrics and logs over OTLP by default, unless the target turns both off.
+  const signal = anchors.find((a) => a.kind === 'otlp-signal')
+  const signals = ['OTEL_METRICS_EXPORTER', 'OTEL_LOGS_EXPORTER']
+  const otlpSet = signals.some((n) => /\botlp\b/.test(target.current.get(n) ?? ''))
+  const bothOff = signals.every((n) => (target.current.get(n) ?? '').trim().toLowerCase() === 'none')
+  const sdkDefault = SDK_SIGNALS.has(service.lang) && !bothOff
+  const tracesOnly = mode === 'otel' && Boolean(signal || otlpSet || sdkDefault)
+  // A value that an earlier setup wrote in this file: replace it, never keep it as another address.
+  const wrote = (name, current) => current !== null && current !== undefined && setupWrote(ctx.manifest, target.file, name, current, target.before)
+  const value = (s) => (s.name === SERVICE ? ctx.name : s.name === ENDPOINT ? (tracesOnly ? `${ingest}/v1/traces` : ingest) : s.value)
+  const rename = (name) => (!tracesOnly ? name : name === ENDPOINT ? TRACES_ENDPOINT : name === PROTOCOL ? TRACES_PROTOCOL : name)
+  let base = (mode === 'standard' || mode === 'channel' ? SETTINGS : SHARED_SETTINGS).map((s) => [rename(s.name), value(s)])
+  // azure-monitor: OTEL_SERVICE_NAME is also the App Insights Cloud Role Name. Set it only when the app does not, and
+  // say how to see the current role name and keep it.
+  if (mode === 'azure-monitor') {
+    const elsewhere = anchors.find((a) => a.kind === 'service-name' && a.file !== target.file)
+    const set = target.current.has(SERVICE) ? { file: target.file } : elsewhere
+    if (set) base = base.filter(([name]) => name !== SERVICE)
+    const R = AZURE_MONITOR.plan.roleName
+    plan.manual.push(set ? { title: R.title, required: false, lines: [fill(R.kept, { where: `${set.file}${set.line ? ` line ${set.line}` : ''}` })] }
+      : { title: R.title, required: false, lines: R.set.map((l) => fill(l, { name: ctx.name })),
+        snippets: [{ caption: 'App Insights Logs', language: 'kusto', code: R.query }] })
+  }
+  const ours = base.map(([name]) => name)
+  const endpointName = tracesOnly ? TRACES_ENDPOINT : ENDPOINT
+  const sameEndpoint = (v) => sameAs(ingest)(tracesOnly ? v.replace(/\/v1\/traces\/?$/, '') : v)
+  const pairs = [...base, ...extra.filter((s) => s.name !== SECRET.name && !NAMES.includes(s.name) && !ours.includes(s.name)).map((s) => [s.name, String(s.value)])]
+  plan.pairs = pairs
+  plan.tracesOnly = tracesOnly
 
   const keyFiles = [...new Set([service.dockerfile, ...(service.k8s ?? []).map((k) => k.file),
     at(service.dir, SETTINGS_FILE.path), at(service.dir, '.env.example')])].filter((f) => f && hasKey(ws.read(f)))
   // A file that holds a key is never edited: the diff and the manifest hunks must not show it.
   const keyInTarget = keyFiles.includes(target.file)
   if (keyFiles.length) {
-    plan.manual.push({ title: 'Remove the key from your repo', lines: [...keyFiles.map((f) => `Remove ${SECRET.name} from ${f}.`),
+    plan.manual.push({ title: 'Remove the key from your repo', verify: true, lines: [...keyFiles.map((f) => `Remove ${SECRET.name} from ${f}.`),
       'Get a new key. The old key stays in your git history.', SECRET.summary,
       ...(keyInTarget ? [`Then run npx @uknows/setup again. It adds the settings to ${target.file}.`] : [])] })
   }
-  if (keyInTarget) return plan
+  // The settings cannot go in: the service is not set up until the key is out of the file.
+  if (keyInTarget) return { ...plan, stop: true }
+  // An Azure Functions app (stacks/functions.mjs): the settings are app settings of the function app, never a file.
+  if (service.functions && (mode === 'standard' || mode === 'azure-monitor')) {
+    const fill = (text) => text.replace(/\{count\}/g, String(pairs.length))
+    return { ...plan, target: null, manual: [...plan.manual, { title: FUNCTIONS.settings.title, required: true, lines: FUNCTIONS.settings.lines.map(fill),
+      snippets: [settingsSnippet('azure-functions', pairs, ctx.name)] }] }
+  }
+  if (target.kind === 'none') {
+    // No file to put the settings in: the settings go where the app runs, in the form of each platform it runs on.
+    const hosts = [...new Set((ctx.detection?.hosts ?? []).map((h) => h.id).filter((id) => id !== 'github-actions'))]
+    const snippets = (hosts.length ? hosts : ['vm-docker']).map((id) => settingsSnippet(id, pairs, ctx.name))
+    // The same words as the start file advice of Node.js: this tool did not pick a Dockerfile, and these are the ones it saw.
+    const seen = unpicked(ctx)
+    const why = seen.length ? fill(SETTINGS_ELSEWHERE.notPicked, { dockerfiles: list(seen.slice(0, 5)) }) : SETTINGS_ELSEWHERE.why
+    plan.manual.push({ title: SETTINGS_ELSEWHERE.title, lines: [why, ...SETTINGS_ELSEWHERE.steps],
+      snippets: snippets.filter((x, i) => snippets.findIndex((y) => y.caption === x.caption) === i) })
+    return plan
+  }
   if (target.flow) {
     plan.manual.push({ title: `Add the UKNOWS settings to ${target.file}`,
       lines: [`The env of ${target.where} is on 1 line. This tool does not change that format.`, 'Add these settings to the env of the container.'],
@@ -70,38 +142,79 @@ export function settingsPlan(service, ctx, extra = []) {
   }
 
   // Another address is kept. Then its protocol and exporters belong to that pipeline: keep them, add none.
-  const endpoint = target.current.get(ENDPOINT)
-  const kept = endpoint !== undefined && !(endpoint !== null && sameEndpoint(endpoint))
+  // Traces only: the traces go to the general address now when the traces address is not set.
+  const general = tracesOnly && !target.current.has(TRACES_ENDPOINT)
+  const keptName = general ? ENDPOINT : endpointName
+  const endpoint = target.current.get(keptName)
+  const kept = endpoint !== undefined && !(endpoint !== null && (general ? sameAs(ingest)(endpoint) : sameEndpoint(endpoint))) && !wrote(keptName, endpoint)
+  if (tracesOnly && !kept) {
+    const why = signal || otlpSet ? fill(TRACES_ONLY_PLAN.code, { where: signal ? `${signal.file} line ${signal.line}` : target.where })
+      : fill(TRACES_ONLY_PLAN.sdk, { lang: stackLabel(service.lang) })
+    plan.notes.push(`${why} ${TRACES_ONLY_PLAN.set} ${TRACES_ONLY_PLAN.key}`)
+    // .NET AddOtlpExporter() does not read the traces-only settings: the code must pass the address on.
+    const add = anchors.find((a) => a.kind === 'add-otlp-exporter')
+    const passed = anchors.some((a) => a.kind === 'traces-endpoint') && anchors.some((a) => a.kind === 'traces-headers')
+    if (service.lang === 'dotnet' && add && !anchors.some((a) => a.kind === 'use-otlp-exporter') && !passed) {
+      plan.manual.push({ title: 'Pass the traces address on to AddOtlpExporter', verify: true, lines: [`${add.file} line ${add.line} adds an OTLP exporter with AddOtlpExporter().`,
+        DOTNET_TRACES_ONLY.note, 'Change the AddOtlpExporter() call of the traces as below. Then traces go to UKNOWS, and metrics and logs keep their address.'],
+      snippets: [{ caption: add.file, language: 'csharp', code: DOTNET_TRACES_ONLY.code }] })
+    }
+    // The key goes in the traces-only variable too (the note above), so the metrics and logs exporters never send it.
+    plan.manual.push({ title: TRACES_ONLY_PLAN.keyTitle, required: false, lines: [TRACES_ONLY_PLAN.key, TRACES_ONLY_PLAN.keyCheck] })
+  }
   if (kept) {
     plan.manual.push({ title: 'Check where this app sends data', lines: [
-      endpoint === null ? `${target.where} sets ${ENDPOINT} from another source.` : `${target.where} sends data to ${hideUser(endpoint)}.`,
+      endpoint === null ? `${target.where} sets ${keptName} from another source.` : `${target.where} sends data to ${hideUser(endpoint)}.`,
       'This tool keeps that address and the other OpenTelemetry settings of the app.', COLLECTOR,
       'To send data to UKNOWS directly, set these settings instead.'],
-    snippets: [{ caption: 'Settings for UKNOWS', language: 'text', code: pairs.filter(([name]) => NAMES.includes(name) && name !== SERVICE)
+    snippets: [{ caption: 'Settings for UKNOWS', language: 'text', code: pairs.filter(([name]) => ours.includes(name) && name !== SERVICE)
       .map(([name, value]) => `${name}=${value}`).join('\n') }] })
   }
   const set = []
   for (const [name, value] of pairs) {
     const current = target.current.get(name)
-    if (current === undefined) { if (!kept || !NAMES.includes(name) || name === SERVICE) set.push([name, value]) }
-    else if (current === value || name === ENDPOINT) continue
+    const stale = ours.includes(name) && name !== SERVICE && wrote(name, current)
+    if (current === undefined) { if (!kept || !ours.includes(name) || name === SERVICE) set.push([name, value]) }
+    else if (current === value || (name === endpointName && !stale)) continue
     else if (current === null) plan.notes.push(`${target.where} sets ${name} from another source. This tool keeps it.`)
-    else if (!NAMES.includes(name) || kept) plan.notes.push(`${target.where} already sets ${name}. This tool keeps that value.`)
+    else if ((!ours.includes(name) || kept) && !stale) plan.notes.push(`${target.where} already sets ${name}. This tool keeps that value.`)
     else {
       if (name !== SERVICE) plan.notes.push(`${target.where} sets ${name} to ${current}. This tool changes it to ${value}.`)
       set.push([name, value])
     }
   }
-  plan.manual.push(...overrides(service, target, pairs, ingest))
+  plan.manual.push(...overrides(service, target, pairs, ingest, { ours, tracesOnly }))
   if (!set.length) return plan
-  const added = set.every(([name]) => target.current.get(name) === undefined) && NAMES.every((n) => set.some(([name]) => name === n))
-  const extras = set.map(([name]) => name).filter((name) => !NAMES.includes(name))
-  plan.summary.push(added ? `Add the 5 UKNOWS settings${extras.length ? ` and ${list(extras)}` : ''} to ${target.where}.`
+  const added = set.every(([name]) => target.current.get(name) === undefined) && ours.every((n) => set.some(([name]) => name === n))
+  const extras = set.map(([name]) => name).filter((name) => !ours.includes(name))
+  plan.summary.push(added ? `Add the ${ours.length} UKNOWS settings${extras.length ? ` and ${list(extras)}` : ''} to ${target.where}.`
     : `Set ${list(set.map(([name]) => name))} in ${target.where}.`)
   plan.edits.push({ file: target.file, before: target.before, after: target.write(set), why: added ? 'Add the UKNOWS settings.' : 'Set the UKNOWS settings.' })
   return plan
 }
 
+/** Languages whose OpenTelemetry SDKs send metrics and logs over OTLP by default. */
+const SDK_SIGNALS = new Set(['node', 'python', 'java'])
+
+/**
+ * True when an earlier setup run wrote name=value in this file: a change in .uknows/setup.json added that line (an
+ * ENV or env-file line, or a Kubernetes name: and value: pair), or created the file (text, as it is now) and it is
+ * unchanged since.
+ * @param {{changes: {file: string, created: boolean, afterSha256?: string, hunks: {after: string[]}[]}[]}|null} manifest
+ */
+export function setupWrote(manifest, file, name, value, text = null) {
+  if (!manifest || !file) return false
+  const esc = (t) => t.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+  const line = new RegExp(`(?:^|\\s)${esc(name)}=["']?${esc(value)}["']?(?:\\s|\\\\|$)`)
+  const yaml = new RegExp(`^\\s*-?\\s*name:\\s*["']?${esc(name)}["']?\\s*$`)
+  const yamlValue = new RegExp(`^\\s*value:\\s*["']?${esc(value)}["']?\\s*$`)
+  const lines = (list) => list.map((l) => l.replace(/\r?\n$/, ''))
+  const has = (list) => list.some((l, i) => line.test(l) || (yaml.test(l) && yamlValue.test(list[i + 1] ?? '')))
+  const same = (sha) => text !== null && [text, BOM + text].some((t) => sha256(Buffer.from(t, 'utf8')) === sha)
+  return manifest.changes.some((c) => c.file === file && (c.created ? same(c.afterSha256) && has(text.split(/\r?\n/))
+    : c.hunks.some((h) => has(lines(h.after)))))
+}
+
 /**
  * True when the text sets OTEL_EXPORTER_OTLP_HEADERS to something that is not a placeholder (in any env format,
  * Kubernetes flow style too), or holds x-uknows-key=<a real value> anywhere. Never returns the value.
@@ -128,7 +241,7 @@ export const redactKey = (line) => line.replace(KEY_HEADER, (m, value) => m.slic
  * Manual steps for settings that keep traces from UKNOWS: an overrideRules() value where the settings go, and a
  * Kubernetes container env that replaces the image values with something else.
  */
-function overrides(service, target, pairs, endpoint) {
+function overrides(service, target, pairs, endpoint, { ours = NAMES, tracesOnly = false } = {}) {
   const sameEndpoint = sameAs(endpoint)
   const sources = [{ where: target.where, file: target.file, env: target.current }]
   if (target.kind !== 'k8s') {
@@ -139,18 +252,20 @@ function overrides(service, target, pairs, endpoint) {
   const blocks = []
   for (const { where, file, env, k8s } of sources) {
     for (const [name, bad, fix] of overrideRules(endpoint)) {
+      // Traces-only settings that this tool sets itself are not overrides (another address is kept above).
+      if (tracesOnly && ours.includes(name) && !k8s) continue
       if (!env.has(name) || (env.get(name) !== null && !bad(env.get(name)))) continue
       const value = env.get(name)
       const shown = value === null ? ' from another source' : /HEADERS/.test(name) ? '' : ` to ${hideUser(value)}`
-      blocks.push({ title: `Check ${name}`, lines: [`${where} sets ${name}${shown}.`,
+      blocks.push({ title: `Check ${name}`, verify: true, lines: [`${where} sets ${name}${shown}.`,
         name === 'OTEL_SDK_DISABLED' ? 'That turns OpenTelemetry off.' : 'It wins over the UKNOWS settings, so traces would not reach UKNOWS.', fix] })
     }
     if (!k8s) continue
     // The container env replaces the ENV of the image for the same names.
     const differs = (name, value, current) => current === null || (current !== value && !(name === ENDPOINT && sameEndpoint(current)))
-    const differ = pairs.filter(([name, value]) => NAMES.includes(name) && env.has(name) && differs(name, value, env.get(name)))
+    const differ = pairs.filter(([name, value]) => ours.includes(name) && env.has(name) && differs(name, value, env.get(name)))
     if (differ.length) {
-      blocks.push({ title: `Check the UKNOWS settings in ${file}`, lines: [
+      blocks.push({ title: `Check the UKNOWS settings in ${file}`, verify: true, lines: [
         `${where} sets ${list(differ.map(([name]) => name))}. The container env replaces the values in ${target.file}.`,
         'Set them to the values below there, or remove them.'],
       snippets: [{ caption: file, language: 'text', code: differ.map(([name, value]) => `${name}=${value}`).join('\n') }] })
@@ -167,6 +282,71 @@ function dockerTarget(ws, file) {
   return { kind: 'dockerfile', file, where: file, before, current, write: (set) => setEnv(before, set) }
 }
 
+/** Repo files that can load otel.env when the app starts. */
+const LOADER_FILE = /\.(?:service|ya?ml|sh|bash|ps1)$|(?:^|\/)(?:Makefile|Procfile|package\.json|justfile)$/
+const COMPOSE_FILE = /(?:^|\/)(?:docker-)?compose(?:\.[^/]+)?\.ya?ml$/
+/** [regex, files]: the group is the path of the env file. systemd EnvironmentFile=, --env-file of docker run or node, and . or source in a shell. */
+const LOADERS = [
+  [/^[ \t]*EnvironmentFile[ \t]*=[ \t]*-?["']?([^\s"']*otel\.env)(?=["'\s]|$)/m, /\.service$/],
+  [/--env-file(?:[ \t]+|=)["']?([^\s"';&|)]*otel\.env)(?=["'\s;&|)]|$)/m, LOADER_FILE],
+  [/(?:^|[\s;&|(`])(?:\.|source)[ \t]+["']?([^\s"';&|)]*otel\.env)(?=["'\s;&|)]|$)/m, LOADER_FILE],
+]
+
+/**
+ * otel.env as the target when the repo has it or a repo file loads it. Else a target of kind none: there is no
+ * file that the app surely reads, so the settings become a required step.
+ */
+function envFileTarget(ws, view, service, file) {
+  if (ws.read(file) !== null) return envTarget(ws, file)
+  const loader = findLoader(view, service, file)
+  return loader ? { ...envTarget(ws, file), loader } : { kind: 'none', file: null, where: null, before: null, current: new Map() }
+}
+
+/**
+ * The first repo file that loads this otel.env: {file, line}. A relative path counts when it is the otel.env of the
+ * service from the folder of that file, the service folder or the repo root. A full path or a path with a variable
+ * names a file on the host, so it counts for every service.
+ */
+function findLoader(view, service, envFile) {
+  let files = []
+  try { files = view.files() } catch { /* unreadable */ }
+  const ours = (from, path) => /^[/~$]/.test(path) || path.includes('$')
+    || [posix.dirname(from), service.dir, '.'].some((dir) => posix.normalize(at(dir, path.replace(/^\.\//, ''))) === envFile)
+  for (const from of files) {
+    if (from.startsWith('../') || !(LOADER_FILE.test(from) || COMPOSE_FILE.test(from))) continue
+    let text = null
+    try { text = (view.stat?.(from)?.size ?? 0) <= 262144 ? view.read(from) : null } catch { text = null }
+    if (!text?.includes(SETTINGS_FILE.path)) continue
+    const code = text.replace(/^[ \t]*#.*$/gm, '')
+    const hits = [...(COMPOSE_FILE.test(from) ? composeEnvFiles(code) : []),
+      ...LOADERS.filter(([, re]) => re.test(from)).flatMap(([re]) => [...code.matchAll(new RegExp(re.source, 'gm'))]
+        .map((m) => ({ path: m[1], line: code.slice(0, m.index + m[0].lastIndexOf(m[1])).split('\n').length })))]
+    const hit = hits.find((h) => ours(from, h.path))
+    if (hit) return { file: from, line: hit.line }
+  }
+  return null
+}
+
+/** The env files of a Compose file: env_file: with 1 path, a [list], or list items below it (- path or - path: path). */
+function composeEnvFiles(text) {
+  const lines = text.split(/\r?\n/), out = []
+  const paths = (value, line) => value.replace(/\s#.*$/, '').replace(/^\[|\]$/g, '').split(',')
+    .map((v) => v.trim().replace(/^-\s*/, '').replace(/^path\s*:\s*/, '').replace(/^["']|["']$/g, '')).filter((v) => v.endsWith(SETTINGS_FILE.path))
+    .forEach((path) => out.push({ path, line }))
+  lines.forEach((line, i) => {
+    const m = /^(\s*)env_file\s*:\s*(.*)$/.exec(line)
+    if (!m) return
+    if (m[2].trim()) return paths(m[2], i + 1)
+    for (let j = i + 1; j < lines.length; j++) {
+      if (!lines[j].trim()) continue
+      const indent = /^\s*/.exec(lines[j])[0].length
+      if (indent < m[1].length || (indent === m[1].length && !/^\s*-/.test(lines[j]))) break
+      paths(lines[j].trim(), j + 1)
+    }
+  })
+  return out
+}
+
 function envTarget(ws, file) {
   const before = ws.read(file)
   const doc = new Doc(before ?? '')
@@ -278,3 +458,11 @@ function underContainers(doc, start) {
   }
   return false
 }
+
+/** The Dockerfiles of the repo that detection did not pick for any service: the same list for every step that names them. */
+export function unpicked(ctx) {
+  let files = []
+  try { files = (ctx.repo ?? ctx.ws).files() } catch { /* unreadable */ }
+  const picked = new Set((ctx.detection?.services ?? []).map((s) => s.dockerfile).filter(Boolean))
+  return files.filter((f) => isDockerfile(f) && !picked.has(f) && !/(?:^|\/)node_modules\//.test(f))
+}

package/lib/signin.mjs

@@ -0,0 +1,259 @@
+/*
+ * Sign in from the terminal. When setup runs in a terminal and has no key, it asks UKNOWS for a sign-in link, opens
+ * the browser, and waits while the person signs in and picks the workspace and the environment. The key then comes
+ * to setup by itself: nobody creates, shows, copies or pastes it.
+ *
+ *   1. POST {app}/api/v1/cli/links {client, version, secretHash, app?}: 201 {linkId, userCode, verificationUrl,
+ *      expiresIn, interval}. secretHash is the base64url SHA-256 of a device secret: 32 random bytes that only this
+ *      process holds, in memory. So only this process can collect the key.
+ *   2. POST {app}/api/v1/cli/links/{linkId}/token {deviceSecret}, every interval seconds: 202 waiting, 200 the key,
+ *      403 cancelled in the browser, 410 expired, 429 slow down (Retry-After).
+ *
+ * A UKNOWS without these routes answers the links request with 404 or 405, or with 401 when its sign-in check runs
+ * before its routes (a test copy of UKNOWS older than this sign-in does). The links route itself needs no sign-in, so
+ * all 3 mean the same: this UKNOWS cannot make a sign-in link, and setup asks for the key instead. 5xx answers and
+ * network errors are failures: setup tries again, then says what went wrong.
+ *
+ * The key and the device secret never print and are never written anywhere. A pipeline, --yes, --json and a run
+ * without a terminal never start a sign-in (engine.mjs decides, before it calls signIn).
+ */
+import { createHash, randomBytes } from 'node:crypto'
+import { request } from './check.mjs'
+import { CLI } from './contract.mjs'
+import { VERSION } from './manifest.mjs'
+import { list } from './ui.mjs'
+
+export const LINKS_PATH = '/api/v1/cli/links'
+export const CLIENT = '@uknows/setup'
+/** The answers of a UKNOWS that has no links route: it cannot make a sign-in link, so the person pastes the key. */
+export const NO_LINKS_STATUS = new Set([401, 404, 405])
+/** Network errors and 5xx answers in a row before setup says that it cannot reach UKNOWS. */
+const MAX_FAILURES = 5
+const LOCAL = new Set(['localhost', '127.0.0.1', '[::1]'])
+
+/** The clock of a real run. Tests pass one whose time moves only when something sleeps. */
+export const REAL_CLOCK = {
+  now: () => Date.now(),
+  sleep(ms) {
+    let timer
+    const done = new Promise((resolve) => { timer = setTimeout(resolve, Math.max(0, ms)) })
+    done.cancel = () => clearTimeout(timer)
+    return done
+  },
+}
+
+/** base64url of the SHA-256 of the device secret: what UKNOWS keeps, so only this process can collect the key. */
+export const secretHash = (secret) => createHash('sha256').update(secret).digest('base64url')
+
+/** The words of the sign-in. Exported for the tests and the README. */
+export const SIGN_IN = {
+  opening: 'Sign in to UKNOWS to connect this app. Opening your browser...',
+  noBrowser: 'Sign in to UKNOWS to connect this app. Open this link in a browser:',
+  code: (code) => `Your code: ${code} (check the browser shows the same code)`,
+  keys: 'Press Enter to open the browser again, p to paste a key instead, or Ctrl+C to stop.',
+  waitingPlain: 'Waiting for you to sign in in the browser. The link works for {minutes}.',
+  waiting: 'Waiting for you to sign in... {left} left',
+  slow: ' (UKNOWS is slow to answer, still trying)',
+  signedIn: 'Signed in.',
+  denied: 'The sign-in was cancelled in the browser.',
+  expired: 'The sign-in link expired.',
+  noLinks: (host) => `${host} cannot make a sign-in link, so paste your key instead.`,
+  /** Sign-in is off: a test address is set (names: the settings that are set), but the app address of that UKNOWS is not. */
+  noApp: (names) => [`Setup cannot sign you in from this terminal: ${list(names)} ${names.length > 1 ? 'are' : 'is'} set, but ${CLI.appEnv} is not.`,
+    `To sign in, set ${CLI.appEnv} to the app address of your test copy of UKNOWS, for example ${CLI.appEnv}=https://app-test.example.com, and run setup again. For now, paste your key.`],
+}
+
+/**
+ * Sign in, and come back with the key. Returns one of:
+ *   {approved: true, key, workspace: {id, name}, environment, environmentLabel, keyId, ingestEndpoint, siteUrl}
+ *   {paste: true}      the person chose to paste a key instead (p, or after a failed sign-in). pressed: p while waiting.
+ *   {stopped: true}    q after a failed sign-in, or the end of the input. interrupt: true for Ctrl+C.
+ * ctx: {base: the app address, app?: {name, repo, platform}, env, openUrl(url, env, say) => boolean, request?, clock?}
+ */
+export async function signIn(ui, ctx) {
+  for (let round = 0; ; round++) {
+    const made = await makeLink(ui, ctx)
+    let result = made
+    if (made.link) result = await waitForSignIn(ui, ctx, made.link, made.deviceSecret)
+    if (result.approved || result.paste || result.stopped) return result
+    ui.print(result.why)
+    const next = await whatNext(ui, result.again)
+    if (next !== 'again') return next === 'paste' ? { paste: true } : { stopped: true }
+    ui.print()
+  }
+}
+
+/**
+ * POST /api/v1/cli/links. {link, deviceSecret}, or {why, again} when UKNOWS cannot be reached, or {paste} when this
+ * UKNOWS has no links route (401, 404 or 405).
+ */
+async function makeLink(ui, { base, app, request: send = request, clock = REAL_CLOCK }) {
+  const deviceSecret = randomBytes(32).toString('base64url')
+  const body = { client: CLIENT, version: VERSION, secretHash: secretHash(deviceSecret), ...(app ? { app: clean(app) } : {}) }
+  const url = new URL(`${base}${LINKS_PATH}`)
+  for (let failures = 0, tries = 0; tries < 20; tries++) {
+    const r = await post(send, url, body)
+    if (r.status === 201 || r.status === 200) {
+      const link = readLink(r.json)
+      return link ? { link, deviceSecret } : { why: `${url.hostname} sent a sign-in link that setup cannot read.`, again: 'try again' }
+    }
+    if (NO_LINKS_STATUS.has(r.status)) {
+      ui.print(SIGN_IN.noLinks(url.hostname))
+      return { paste: true }
+    }
+    if (r.status === 429) { await clock.sleep(retryAfter(r.headers, clock) ?? 5000); continue }
+    if (r.error || r.status >= 500) {
+      if (++failures >= 3) return { why: unreachable(url, r), again: 'try again' }
+      await clock.sleep(1000 * 2 ** (failures - 1))
+      continue
+    }
+    return { why: `${url.hostname} did not make a sign-in link (status ${r.status}).`, again: 'try again' }
+  }
+  return { why: `${url.hostname} is busy. It asked setup to wait too many times.`, again: 'try again' }
+}
+
+/**
+ * Show the link and the code, open the browser, and poll until UKNOWS answers. Enter opens the browser again, p goes to
+ * the paste question, Ctrl+C stops. Returns {approved, ...}, {paste}, {stopped}, or {why, again} (cancelled in the
+ * browser, expired, or no answer from UKNOWS).
+ */
+async function waitForSignIn(ui, ctx, link, deviceSecret) {
+  const { env, openUrl, request: send = request, clock = REAL_CLOCK } = ctx
+  const open = () => openUrl(link.verificationUrl, env, () => {})
+  const opened = open()
+  ui.print(opened ? SIGN_IN.opening : SIGN_IN.noBrowser)
+  ui.print(`  ${ui.cyan(link.verificationUrl)}`)
+  ui.print(`  ${SIGN_IN.code(ui.bold(link.userCode))}`)
+  ui.print()
+  const keys = ui.keys()
+  if (keys) {
+    ui.print(ui.dim(`  ${SIGN_IN.keys}`))
+    ui.print()
+  } else ui.print(SIGN_IN.waitingPlain.replace('{minutes}', minutes(link.expiresIn)))
+  const expires = clock.now() + link.expiresIn * 1000
+  const url = new URL(`${new URL(ctx.base).origin}${LINKS_PATH}/${encodeURIComponent(link.linkId)}/token`)
+  let interval = link.interval * 1000, failures = 0, note = ''
+  let nextPoll = clock.now() + interval
+  const left = () => {
+    const s = Math.max(0, Math.ceil((expires - clock.now()) / 1000))
+    return `${Math.floor(s / 60)}:${String(s % 60).padStart(2, '0')}`
+  }
+  const spin = ui.progress(() => `${SIGN_IN.waiting.replace('{left}', left())}${note}`)
+  let keyWait = null
+  const end = (result, line = null) => { spin.end(line); keys?.close(); return result }
+  try {
+    for (;;) {
+      const now = clock.now()
+      if (now >= expires) return end({ why: SIGN_IN.expired, again: 'start a new sign-in' })
+      if (now >= nextPoll) {
+        const r = await post(send, url, { deviceSecret })
+        if (r.status === 200) {
+          const data = readToken(r.json)
+          if (data) return end({ approved: true, ...data }, `${ui.cyan(ui.marks.done)} ${SIGN_IN.signedIn}`)
+          return end({ why: `${url.hostname} sent an answer that setup cannot read.`, again: 'try again' })
+        }
+        if (r.status === 403) return end({ why: SIGN_IN.denied, again: 'sign in again' })
+        if (r.status === 410 || r.status === 404) return end({ why: SIGN_IN.expired, again: 'start a new sign-in' })
+        if (r.status === 202) { failures = 0; note = ''; nextPoll = clock.now() + interval; continue }
+        if (r.status === 429) {
+          // Slow down: wait as long as UKNOWS asks, and poll less often from now on.
+          const wait = retryAfter(r.headers, clock)
+          interval = Math.min(Math.max(interval, wait ?? interval * 2), 60000)
+          nextPoll = clock.now() + Math.max(interval, wait ?? 0)
+          continue
+        }
+        if (r.error || r.status >= 500) {
+          if (++failures >= MAX_FAILURES) return end({ why: unreachable(url, r), again: 'try again' })
+          note = SIGN_IN.slow
+          nextPoll = clock.now() + Math.min(interval * 2 ** failures, 30000)
+          continue
+        }
+        return end({ why: `${url.hostname} answered the sign-in with status ${r.status}.`, again: 'try again' })
+      }
+      const sleep = clock.sleep(Math.min(nextPoll, expires) - now)
+      if (!keys) { await sleep; continue }
+      keyWait ??= keys.next()
+      const key = await Promise.race([sleep.then(() => undefined), keyWait])
+      sleep.cancel?.()
+      if (key === undefined) continue
+      keyWait = null
+      if (key === 'ctrl-c' || key === null) return end({ stopped: true, interrupt: true }, null)
+      if (key === 'p') {
+        // People often press Enter after p by habit. That Enter belongs to p, not to the paste question.
+        await Promise.race([keys.next(), new Promise((resolve) => setTimeout(resolve, 400))])
+        return end({ paste: true, pressed: true }, null)
+      }
+      if (key === 'enter' && !open()) ui.print(`  ${ui.cyan(link.verificationUrl)}`)
+    }
+  } finally {
+    spin.stop()
+    keys?.close()
+  }
+}
+
+/** After a failed sign-in: again (Enter), paste (p) or stop (q, or the end of the input). */
+async function whatNext(ui, again) {
+  for (let tries = 0; tries < 3; tries++) {
+    const answer = await ui.line(`Press Enter to ${again}, type p to paste a key instead, or type q to stop:`)
+    if (answer === null) return 'stop'
+    const a = answer.toLowerCase()
+    if (a === '') return 'again'
+    if (a === 'p' || a === 'paste') return 'paste'
+    if (a === 'q' || a === 'quit' || a === 'stop') return 'stop'
+    ui.print('Press Enter, or type p or q.')
+  }
+  return 'stop'
+}
+
+/** The link of a 201 answer, checked and tidied, or null. */
+function readLink(json) {
+  if (!json || typeof json !== 'object') return null
+  const { linkId, userCode, verificationUrl } = json
+  if (typeof linkId !== 'string' || !/^[A-Za-z0-9._~-]{1,200}$/.test(linkId)) return null
+  if (typeof userCode !== 'string' || !/^[A-Za-z0-9-]{4,20}$/.test(userCode)) return null
+  let url
+  try { url = new URL(String(verificationUrl)) } catch { return null }
+  const safe = url.protocol === 'https:' || (url.protocol === 'http:' && LOCAL.has(url.hostname))
+  if (!safe || url.username || url.password) return null
+  const number = (value, fallback, min, max) => (Number.isFinite(value) && value > 0 ? Math.min(Math.max(value, min), max) : fallback)
+  return { linkId, userCode, verificationUrl: url.href, expiresIn: number(json.expiresIn, 600, 30, 3600), interval: number(json.interval, 3, 0.05, 60) }
+}
+
+/** The key of a 200 answer, checked, or null. A key fits in a header: printable, no spaces, no commas. */
+function readToken(json) {
+  if (!json || typeof json !== 'object' || typeof json.key !== 'string' || !/^[\x21-\x7e]{8,512}$/.test(json.key) || json.key.includes(',')) return null
+  const text = (value) => (typeof value === 'string' && value.trim() ? value.replace(/[\u0000-\u001f\u007f]+/g, ' ').trim().slice(0, 200) : null)
+  const workspace = json.workspace && typeof json.workspace === 'object' ? { id: text(json.workspace.id), name: text(json.workspace.name) } : { id: null, name: null }
+  return { key: json.key, workspace, environment: text(json.environment), environmentLabel: text(json.environmentLabel),
+    keyId: text(json.keyId), ingestEndpoint: text(json.ingestEndpoint), siteUrl: text(json.siteUrl) }
+}
+
+/** Retry-After in ms: seconds, or an HTTP date. null when missing. */
+function retryAfter(headers = {}, clock = REAL_CLOCK) {
+  const raw = String(headers?.['retry-after'] ?? '').trim()
+  if (!raw) return null
+  if (/^\d+$/.test(raw)) return Math.min(Number(raw), 120) * 1000
+  const at = Date.parse(raw)
+  return Number.isFinite(at) ? Math.min(Math.max(at - clock.now(), 0), 120000) : null
+}
+
+function unreachable(url, r) {
+  return r.error ? `Could not reach ${url.hostname} (${r.error.code || 'network error'}). Check your internet connection.`
+    : `${url.hostname} did not answer the sign-in (status ${r.status}).`
+}
+
+async function post(send, url, body) {
+  const r = await send(url, 'POST', { 'content-type': 'application/json', accept: 'application/json' }, JSON.stringify(body), 10000)
+  let json = null
+  try { json = r.body ? JSON.parse(r.body) : null } catch { json = null }
+  return { ...r, json }
+}
+
+/** The app as UKNOWS shows it in the browser: only text fields that are set. */
+function clean(app) {
+  const out = {}
+  for (const key of ['name', 'repo', 'platform']) if (typeof app[key] === 'string' && app[key].trim()) out[key] = app[key].trim().slice(0, 200)
+  return out
+}
+
+const minutes = (seconds) => (seconds % 60 === 0 ? `${seconds / 60} ${seconds === 60 ? 'minute' : 'minutes'}` : `${Math.round(seconds)} seconds`)