@reclaimprotocol/agent 0.1.23 → 0.1.25

package/src/live/cloudflared.ts

@@ -0,0 +1,348 @@
+/**
+ * Finding — and if necessary fetching — the `cloudflared` binary the live view
+ * tunnels through.
+ *
+ * Resolution order, first hit wins:
+ *  1. `RECLAIM_CLOUDFLARED_PATH`, for an air-gapped box or a corporate mirror.
+ *  2. `cloudflared` on PATH — someone who installed it with a package manager
+ *     keeps their own copy, updated their way.
+ *  3. `~/.reclaim/bin/cloudflared`, ours from a previous run.
+ *  4. Download it from Cloudflare's GitHub releases into `~/.reclaim/bin`.
+ *
+ * Step 4 happens on FIRST USE, never at install time. The npm `cloudflared`
+ * package would have done this for us, but its postinstall pulls a ~38 MB
+ * binary on every `npm install` of this package — a cost paid by everyone who
+ * never shares a live view. `run_proof`'s ZK circuits are deferred for exactly
+ * the same reason.
+ *
+ * Every download is SHA-256 verified. Cloudflare publishes a checksum table in
+ * the release notes (not as a release asset), so we read the release metadata
+ * first, take the digest for our asset, and refuse to install anything that
+ * doesn't match. The release TAG from that same response is used to build the
+ * download URL, so the checksum and the bytes always come from one release —
+ * `/latest/download/` could otherwise hand us a newer build than the table we
+ * just read.
+ *
+ * Verification is not optional: this writes an executable to the developer's
+ * machine. If the checksum can't be established, we fail rather than install.
+ * The escape hatch is to install cloudflared yourself — step 2 then wins and
+ * nothing is downloaded.
+ */
+import { execFile } from 'node:child_process'
+import { createHash } from 'node:crypto'
+import {
+	chmod,
+	mkdir,
+	rename,
+	rm,
+	stat,
+	writeFile
+} from 'node:fs/promises'
+import { arch, platform } from 'node:os'
+import { join } from 'node:path'
+import { promisify } from 'node:util'
+import { gunzipSync } from 'node:zlib'
+import { LOGGER } from '../logger.ts'
+import { reclaimHomeDirPath } from '../paths.ts'
+
+const execFileAsync = promisify(execFile)
+
+/** Release metadata: the tag to pin to, and the checksum table in its notes. */
+const LATEST_RELEASE_API =
+	'https://api.github.com/repos/cloudflare/cloudflared/releases/latest'
+/** Asset downloads, keyed by the tag we read the checksums from. */
+function releaseDownloadUrl(tag: string, asset: string): string {
+	return 'https://github.com/cloudflare/cloudflared/releases/download'
+		+ `/${tag}/${asset}`
+}
+
+/** Give up on a slow or stalled download rather than hanging the tool. */
+const DOWNLOAD_TIMEOUT_MS = 180_000
+
+export interface CloudflaredBinary {
+	/** Path to run. */
+	path: string
+	/** How it was found — surfaced so the caller can tell the developer when a
+	 *  download just happened on their machine. */
+	source: 'env' | 'path' | 'cached' | 'downloaded'
+}
+
+/**
+ * The release asset for this machine, or `undefined` when Cloudflare ships no
+ * build for it. macOS assets are gzipped tarballs; everything else is the bare
+ * executable.
+ */
+function assetFor(
+	os: string,
+	cpu: string,
+): { name: string, tarball: boolean } | undefined {
+	if(os === 'darwin') {
+		const suffix = cpu === 'arm64' ? 'arm64' : 'amd64'
+		return { name: `cloudflared-darwin-${suffix}.tgz`, tarball: true }
+	}
+
+	if(os === 'linux') {
+		const map: Record<string, string> = {
+			x64: 'amd64',
+			arm64: 'arm64',
+			arm: 'arm',
+			ia32: '386',
+		}
+		const suffix = map[cpu]
+		return suffix
+			? { name: `cloudflared-linux-${suffix}`, tarball: false }
+			: undefined
+	}
+
+	if(os === 'win32') {
+		const suffix = cpu === 'ia32' ? '386' : 'amd64'
+		return { name: `cloudflared-windows-${suffix}.exe`, tarball: false }
+	}
+
+	return undefined
+}
+
+/** Our install location. Kept beside the other `~/.reclaim` caches. */
+function cachedBinaryPath(): string {
+	const name = platform() === 'win32' ? 'cloudflared.exe' : 'cloudflared'
+	return join(reclaimHomeDirPath(), 'bin', name)
+}
+
+/** Whether `command` runs. Used both for a PATH probe and to smoke-test a
+ *  freshly downloaded binary before we hand it back. */
+async function runsOk(command: string): Promise<boolean> {
+	try {
+		await execFileAsync(command, ['--version'], { timeout: 15_000 })
+		return true
+	} catch{
+		return false
+	}
+}
+
+async function exists(path: string): Promise<boolean> {
+	try {
+		await stat(path)
+		return true
+	} catch{
+		return false
+	}
+}
+
+/**
+ * Resolve a runnable cloudflared, downloading one only if `autoInstall` allows
+ * it and nothing else turned up. Throws with an actionable message when this
+ * platform has no build, or the download fails.
+ */
+export async function resolveCloudflared(
+	{ autoInstall = true }: { autoInstall?: boolean } = {},
+): Promise<CloudflaredBinary> {
+	const override = process.env.RECLAIM_CLOUDFLARED_PATH?.trim()
+	if(override) {
+		if(!await runsOk(override)) {
+			throw new Error(
+				`RECLAIM_CLOUDFLARED_PATH is set to "${override}", but that did not `
+				+ 'run. Point it at a working cloudflared binary, or unset it to let '
+				+ 'the agent find one.',
+			)
+		}
+
+		return { path: override, source: 'env' }
+	}
+
+	if(await runsOk('cloudflared')) {
+		return { path: 'cloudflared', source: 'path' }
+	}
+
+	const cached = cachedBinaryPath()
+	if(await exists(cached) && await runsOk(cached)) {
+		return { path: cached, source: 'cached' }
+	}
+
+	if(!autoInstall) {
+		throw new Error(
+			'cloudflared is not installed. Install it (macOS: `brew install '
+			+ 'cloudflared`; other platforms: https://developers.cloudflare.com/'
+			+ 'cloudflare-one/connections/connect-networks/downloads/), or allow '
+			+ 'the agent to fetch it.',
+		)
+	}
+
+	await install(cached)
+	return { path: cached, source: 'downloaded' }
+}
+
+/** `cloudflared-linux-amd64: <64 hex>` lines inside the notes' fenced block. */
+const CHECKSUM_LINE = /^\s*(\S+):\s*([a-f0-9]{64})\s*$/gim
+
+/**
+ * The current release's tag and its published SHA-256 table.
+ *
+ * Both come from one response on purpose: pinning the download to this tag is
+ * what makes the digest meaningful.
+ */
+async function fetchRelease(): Promise<{
+	tag: string
+	checksums: Map<string, string>
+}> {
+	const res = await fetch(LATEST_RELEASE_API, {
+		headers: {
+			accept: 'application/vnd.github+json',
+			'user-agent': '@reclaimprotocol/agent',
+		},
+		signal: AbortSignal.timeout(30_000),
+	})
+	if(!res.ok) {
+		throw new Error(
+			`Could not read cloudflared release metadata: ${res.status} `
+			+ `${res.statusText}. Needed to verify the download's checksum. `
+			+ 'Install cloudflared yourself, or set RECLAIM_CLOUDFLARED_PATH.',
+		)
+	}
+
+	const release = await res.json() as { tag_name?: string, body?: string }
+	const tag = release.tag_name
+	if(!tag) {
+		throw new Error('cloudflared release metadata carried no tag.')
+	}
+
+	const checksums = new Map<string, string>()
+	for(const match of (release.body ?? '').matchAll(CHECKSUM_LINE)) {
+		checksums.set(match[1], match[2].toLowerCase())
+	}
+
+	return { tag, checksums }
+}
+
+/** Download the release asset for this machine to `target`, verified. */
+async function install(target: string): Promise<void> {
+	const asset = assetFor(platform(), arch())
+	if(!asset) {
+		throw new Error(
+			`Cloudflare publishes no cloudflared build for ${platform()}/${arch()}. `
+			+ 'Install it another way and set RECLAIM_CLOUDFLARED_PATH, or use '
+			+ '`attach_browser` in builder mode, whose Popcorn browser returns a '
+			+ 'hosted liveViewUrl and needs no tunnel.',
+		)
+	}
+
+	// Read the checksum table BEFORE downloading, and pin the download to the
+	// tag it came from.
+	const { tag, checksums } = await fetchRelease()
+	const expected = checksums.get(asset.name)
+	if(!expected) {
+		throw new Error(
+			`cloudflared release ${tag} publishes no SHA-256 for ${asset.name}, so `
+			+ 'the download cannot be verified. Refusing to install an unverified '
+			+ 'binary. Install cloudflared yourself, or set '
+			+ 'RECLAIM_CLOUDFLARED_PATH.',
+		)
+	}
+
+	const url = releaseDownloadUrl(tag, asset.name)
+	LOGGER.info({ url, target, tag }, 'cloudflared: downloading (first use)')
+	const res = await fetch(url, {
+		redirect: 'follow',
+		signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS),
+	})
+	if(!res.ok) {
+		throw new Error(
+			`Downloading cloudflared failed: ${res.status} ${res.statusText} `
+			+ `from ${url}. Install it manually and retry, or set `
+			+ 'RECLAIM_CLOUDFLARED_PATH.',
+		)
+	}
+
+	const downloaded = Buffer.from(await res.arrayBuffer())
+
+	// The macOS assets are tarballs, so unpack before hashing. Unpacked
+	// in-process rather than by shelling out to `tar`: that would add a
+	// dependency on something being on PATH, which is exactly the assumption
+	// this module exists to stop making.
+	//
+	// This ordering is not incidental. Cloudflare's table lists a digest for
+	// `cloudflared-darwin-arm64.tgz`, but that digest is of the BINARY INSIDE,
+	// not of the tarball (verified against a real release: the tarball hashes
+	// to something else entirely, and the binary self-reports the published
+	// value as its own checksum). The linux and windows entries, which ship a
+	// bare executable, do match the downloaded file. Hashing the payload we are
+	// about to write covers both, and is the stronger invariant anyway: what
+	// gets verified is what gets executed.
+	const payload = asset.tarball
+		? readTarEntry(gunzipSync(downloaded), 'cloudflared')
+		: downloaded
+
+	const actual = createHash('sha256').update(payload).digest('hex')
+	if(actual !== expected) {
+		// Nothing is written to disk on a mismatch — not even a partial file.
+		throw new Error(
+			`cloudflared checksum mismatch for ${asset.name} (release ${tag}). `
+			+ `Expected ${expected}, got ${actual}. The download was discarded. `
+			+ 'Retry; if it persists, install cloudflared yourself and set '
+			+ 'RECLAIM_CLOUDFLARED_PATH.',
+		)
+	}
+
+	LOGGER.info({ asset: asset.name, tag }, 'cloudflared: checksum verified')
+
+	// Write beside the target, then rename: a half-written or unverified
+	// binary is never left at a path a later run would trust.
+	await mkdir(join(target, '..'), { recursive: true })
+	const partial = `${target}.download`
+	await writeFile(partial, payload)
+	await rename(partial, target)
+
+	if(platform() !== 'win32') {
+		await chmod(target, 0o755)
+	}
+
+	if(!await runsOk(target)) {
+		await rm(target, { force: true })
+		throw new Error(
+			'Downloaded cloudflared, but it did not run. Removed it. Install '
+			+ 'cloudflared manually and retry.',
+		)
+	}
+
+	LOGGER.info({ target }, 'cloudflared: installed')
+}
+
+/** A tar header block is 512 bytes, and so is every data block. */
+const TAR_BLOCK = 512
+
+/**
+ * Pull one file out of an uncompressed tar archive.
+ *
+ * Only as much of the format as this needs: walk the 512-byte headers, read
+ * each entry's name and octal size, and return the payload of the first
+ * regular file whose basename matches. Cloudflare's macOS tarball holds a
+ * single binary, so there is nothing more to handle.
+ */
+function readTarEntry(tar: Buffer, wanted: string): Buffer {
+	let offset = 0
+	while(offset + TAR_BLOCK <= tar.length) {
+		const header = tar.subarray(offset, offset + TAR_BLOCK)
+		const name = header.subarray(0, 100).toString('utf8').replace(/\0.*$/, '')
+		if(!name) {
+			break // two zero blocks mark the end of the archive
+		}
+
+		const size = parseInt(
+			header.subarray(124, 136).toString('utf8').replace(/\0.*$/, '').trim(),
+			8,
+		) || 0
+		const type = header.subarray(156, 157).toString('utf8')
+		const dataStart = offset + TAR_BLOCK
+		if((type === '0' || type === '\0') && name.split('/').pop() === wanted) {
+			return tar.subarray(dataStart, dataStart + size)
+		}
+
+		// Data is padded up to the next block boundary.
+		offset = dataStart + Math.ceil(size / TAR_BLOCK) * TAR_BLOCK
+	}
+
+	throw new Error(
+		`The cloudflared archive did not contain "${wanted}". Cloudflare may `
+		+ 'have changed its release layout — install cloudflared manually and '
+		+ 'set RECLAIM_CLOUDFLARED_PATH.',
+	)
+}

package/src/live/tunnel.ts

@@ -0,0 +1,208 @@
+/**
+ * Public HTTPS front door for a local port, via a `cloudflared` quick tunnel.
+ *
+ * A quick tunnel needs no Cloudflare account and no DNS: `cloudflared` dials
+ * out, Cloudflare hands back a throwaway `*.trycloudflare.com` hostname, and
+ * traffic arrives on the loopback port we pass in. That is the whole reason it
+ * is used here — the developer's machine keeps no inbound port open.
+ *
+ * **The hostname IS the secret, and it is the only one.** The tunnel carries no
+ * authorization of its own, and the browser runtime's viewer behind it has no
+ * login either — so whoever learns the URL has the browser. Cloudflare's
+ * hostnames are randomly generated, which is what makes that tolerable for a
+ * short, deliberate hand-off, and `stop_browser_view` is how it ends.
+ *
+ * That is weaker than what this file used to front. The agent's own screencast
+ * viewer added a 24-byte token to every path, so the hostname alone was not
+ * enough; the container's viewer has no equivalent. Callers must say plainly
+ * what the URL grants, and it must never front the container's CDP port, which
+ * is unauthenticated total control of the browser.
+ */
+import { spawn } from 'node:child_process'
+import { LOGGER } from '../logger.ts'
+import { type CloudflaredBinary, resolveCloudflared } from './cloudflared.ts'
+
+/** How long to wait for cloudflared to report its hostname before giving up. */
+const URL_TIMEOUT_MS = 30_000
+/** After the hostname is printed, Cloudflare needs a moment before the edge
+ *  routes to it — measured at ~3s. Wait for it so the caller never hands out a
+ *  URL that errors, but don't fail the share if it's slow. */
+const READY_TIMEOUT_MS = 20_000
+const READY_POLL_MS = 1_000
+
+/** `https://<something>.trycloudflare.com`, as printed on cloudflared's log
+ *  stream. Anchored to the scheme so a hostname inside prose can't match. */
+const QUICK_TUNNEL_URL = /https:\/\/[a-z0-9-]+\.trycloudflare\.com/i
+
+/**
+ * cloudflared says this once per edge connection it establishes, and until it
+ * does, the hostname resolves but routes nowhere.
+ *
+ * This is the signal to trust, because it is LOCAL. Asking whether we can fetch
+ * the public URL ourselves conflates two questions — is the tunnel up, and does
+ * this machine have egress to Cloudflare's edge — and reports "not ready" when
+ * only the second fails. Observed: a tunnel printed its hostname and registered
+ * nothing for 12s, which is indistinguishable from a blocked fetch unless you
+ * read cloudflared's own account of it.
+ */
+const REGISTERED = /Registered tunnel connection|connIndex=/i
+
+/** Tail of cloudflared's log kept for error messages. */
+const LOG_TAIL_BYTES = 4096
+
+export interface Tunnel {
+	/** The public HTTPS origin Cloudflare assigned. */
+	url: string
+	/** Whether the tunnel was carrying traffic before {@link READY_TIMEOUT_MS}
+	 *  elapsed. False means it may still be propagating rather than broken — say
+	 *  so rather than treating the share as failed. */
+	ready: boolean
+	/** cloudflared's own last words, for when `ready` is false and someone has
+	 *  to work out why. Absent when the tunnel came up normally. */
+	diagnostics?: string
+	/** Terminate the tunnel. Safe to call more than once. */
+	stop: () => Promise<void>
+	/** Which cloudflared ran, and whether we had to fetch it. */
+	binary: CloudflaredBinary
+}
+
+/**
+ * Wait until the tunnel is actually carrying traffic.
+ *
+ * Two independent signals, either of which is proof:
+ *  - cloudflared reported an edge registration (see {@link REGISTERED}). Local,
+ *    so it works regardless of this machine's own egress.
+ *  - the public origin answered. ANY HTTP response counts, including a 404:
+ *    the live-view server 404s unknown paths, so a status code at all proves
+ *    the hop is live. Kept as a second opinion in case a cloudflared release
+ *    rewords its log.
+ */
+async function waitUntilRouted(
+	url: string,
+	isRegistered: () => boolean,
+): Promise<boolean> {
+	const deadline = Date.now() + READY_TIMEOUT_MS
+	while(Date.now() < deadline) {
+		if(isRegistered()) {
+			return true
+		}
+
+		try {
+			await fetch(url, {
+				method: 'GET',
+				redirect: 'manual',
+				signal: AbortSignal.timeout(READY_POLL_MS * 2),
+			})
+			return true
+		} catch{
+			await new Promise((r) => setTimeout(r, READY_POLL_MS))
+		}
+	}
+
+	return isRegistered()
+}
+
+/**
+ * Open a quick tunnel to `http://127.0.0.1:<port>` and resolve once
+ * Cloudflare has assigned a hostname.
+ *
+ * Rejects with an actionable message when `cloudflared` is not installed,
+ * exits early, or never reports a hostname.
+ */
+export async function startTunnel(
+	port: number,
+	{ autoInstall = true }: { autoInstall?: boolean } = {},
+): Promise<Tunnel> {
+	// Resolves an existing cloudflared, or fetches one on first use. Throws
+	// with an actionable message when neither is possible.
+	const binary = await resolveCloudflared({ autoInstall })
+	const child = spawn(binary.path, [
+		'tunnel',
+		'--no-autoupdate',
+		'--url', `http://127.0.0.1:${port}`,
+	], { stdio: ['ignore', 'pipe', 'pipe'] })
+
+	const stop = async() => {
+		if(!child.killed) {
+			child.kill('SIGTERM')
+		}
+	}
+
+	return new Promise<Tunnel>((resolve, reject) => {
+		const timer = setTimeout(() => {
+			void stop()
+			reject(new Error(
+				'cloudflared did not report a tunnel hostname within '
+				+ `${URL_TIMEOUT_MS / 1000}s. Check network egress to Cloudflare, `
+				+ 'or run `cloudflared tunnel --url http://127.0.0.1:'
+				+ `${port}` + '` by hand to see its output.',
+			))
+		}, URL_TIMEOUT_MS)
+
+		let registered = false
+		let tail = ''
+		let settled = false
+		const finish = (fn: () => void) => {
+			if(settled) {
+				return
+			}
+
+			settled = true
+			clearTimeout(timer)
+			fn()
+		}
+
+		// cloudflared logs the assigned hostname to stderr, not stdout. Watch
+		// both so a future version that switches streams still works.
+		const onChunk = (chunk: Buffer) => {
+			const text = chunk.toString()
+			// Keep watching after the hostname: registration lands a beat later, and
+			// on a bad run it never lands at all, which is the case worth reporting.
+			tail = (tail + text).slice(-LOG_TAIL_BYTES)
+			if(REGISTERED.test(text)) {
+				registered = true
+			}
+
+			const match = QUICK_TUNNEL_URL.exec(text)
+			if(match) {
+				const url = match[0]
+				finish(() => {
+					void waitUntilRouted(url, () => registered).then((ready) => {
+						if(ready) {
+							resolve({ url, ready, stop, binary })
+							return
+						}
+
+						LOGGER.warn({ url, tail },
+							'tunnel never registered with an edge; returning anyway')
+						resolve({ url, ready, stop, binary, diagnostics: tail })
+					})
+				})
+			}
+		}
+
+		child.stdout?.on('data', onChunk)
+		child.stderr?.on('data', onChunk)
+
+		child.on('error', (err: NodeJS.ErrnoException) => {
+			// `resolveCloudflared` already proved the binary runs, so an ENOENT
+			// here means it vanished between resolving and spawning.
+			finish(() => reject(
+				err.code === 'ENOENT'
+					? new Error(
+						`cloudflared disappeared from ${binary.path} before it could `
+						+ 'start. Retry — the agent will fetch it again if needed.',
+					)
+					: err,
+			))
+		})
+
+		child.on('exit', (code) => {
+			LOGGER.debug({ code }, 'cloudflared exited')
+			finish(() => reject(new Error(
+				`cloudflared exited with code ${code} before reporting a `
+				+ 'hostname.',
+			)))
+		})
+	})
+}