membot 0.5.2 → 0.6.0

package/src/ingest/downloaders/linear.ts

@@ -0,0 +1,291 @@
+import { HelpfulError } from "../../errors.ts";
+import { sha256Hex } from "../local-reader.ts";
+import type { DownloadedRemote, Downloader } from "./index.ts";
+
+const ISSUE_PATH = /^\/([^/]+)\/issue\/([A-Z]+-\d+)(?:$|\/|#|\?)/;
+const PROJECT_PATH = /^\/([^/]+)\/project\/([^/?#]+)/;
+
+const GRAPHQL_ENDPOINT = "https://api.linear.app/graphql";
+
+/**
+ * Linear's web app uses a sophisticated cookie + signed-request scheme
+ * (`client-api.linear.app/graphql` with `useraccount`/`linear-client-id`
+ * headers) that's not realistically replayable from outside a real
+ * Linear browser session. Instead we use Linear's official API at
+ * `api.linear.app/graphql` with a personal API key — set up once via
+ * `membot config set downloaders.linear.api_key <KEY>` after creating
+ * the key at https://linear.app/settings/api.
+ *
+ * The API gives us the structured issue/project payload (title, body,
+ * comments, status, …) directly; we render it to markdown
+ * deterministically rather than scraping the rendered DOM.
+ */
+export const linearDownloader: Downloader = {
+	name: "linear",
+	description: "Linear (linear.app/<workspace>/issue/<KEY> and /project/<slug>) — uses the Linear API.",
+	logins: [
+		{
+			kind: "api_key",
+			name: "Linear",
+			url: "https://linear.app/settings/api",
+			setupCommand: "membot config set downloaders.linear.api_key <KEY>",
+			description: "create a personal API key, then run the command on the right",
+		},
+	],
+	requiresApiKey: true,
+	matches(url) {
+		return url.hostname === "linear.app" && (ISSUE_PATH.test(url.pathname) || PROJECT_PATH.test(url.pathname));
+	},
+	async download(url, ctx): Promise<DownloadedRemote> {
+		const apiKey = ctx.config.downloaders.linear.api_key.trim();
+		if (apiKey === "") {
+			throw new HelpfulError({
+				kind: "auth_error",
+				message: `Linear API key not configured.`,
+				hint: "Create a personal API key at https://linear.app/settings/api, then run `membot config set downloaders.linear.api_key <KEY>`.",
+			});
+		}
+
+		const issueMatch = url.pathname.match(ISSUE_PATH);
+		const projectMatch = url.pathname.match(PROJECT_PATH);
+		let markdown: string;
+		let downloaderArgs: Record<string, unknown>;
+
+		if (issueMatch) {
+			const identifier = issueMatch[2] as string;
+			ctx.onProgress?.(`querying issue ${identifier}`);
+			const issue = await fetchIssue(identifier, apiKey, url);
+			markdown = renderIssue(issue);
+			downloaderArgs = { kind: "issue", workspace: issueMatch[1], identifier };
+		} else if (projectMatch) {
+			const slug = projectMatch[2] as string;
+			const slugId = extractProjectSlugId(slug);
+			ctx.onProgress?.(`querying project ${slugId}`);
+			const project = await fetchProject(slugId, apiKey, url);
+			markdown = renderProject(project);
+			downloaderArgs = { kind: "project", workspace: projectMatch[1], slug, slug_id: slugId };
+		} else {
+			throw new HelpfulError({
+				kind: "input_error",
+				message: `not a Linear issue/project URL: ${url.toString()}`,
+				hint: "Pass a URL like https://linear.app/<workspace>/issue/<KEY> or .../project/<slug>.",
+			});
+		}
+
+		const bytes = new TextEncoder().encode(markdown);
+		return {
+			bytes,
+			sha256: sha256Hex(bytes),
+			mimeType: "text/markdown",
+			downloader: "linear",
+			downloaderArgs,
+			sourceUrl: url.toString(),
+		};
+	},
+};
+
+interface LinearUser {
+	name?: string | null;
+	displayName?: string | null;
+	email?: string | null;
+}
+
+interface LinearComment {
+	body: string | null;
+	createdAt: string | null;
+	user: LinearUser | null;
+}
+
+interface LinearIssue {
+	identifier: string;
+	url: string;
+	title: string;
+	description: string | null;
+	priorityLabel: string | null;
+	state: { name: string } | null;
+	assignee: LinearUser | null;
+	creator: LinearUser | null;
+	createdAt: string;
+	updatedAt: string;
+	comments: { nodes: LinearComment[] };
+}
+
+interface LinearProject {
+	id: string;
+	url: string;
+	name: string;
+	slugId: string;
+	description: string | null;
+	content: string | null;
+	state: string | null;
+	startDate: string | null;
+	targetDate: string | null;
+	createdAt: string;
+	updatedAt: string;
+	lead: LinearUser | null;
+	members: { nodes: LinearUser[] };
+}
+
+const ISSUE_QUERY = `query Issue($id: String!) {
+  issue(id: $id) {
+    identifier url title description priorityLabel
+    state { name }
+    assignee { name displayName email }
+    creator { name displayName email }
+    createdAt updatedAt
+    comments(first: 100) {
+      nodes { body createdAt user { name displayName email } }
+    }
+  }
+}`;
+
+const PROJECT_QUERY = `query ProjectBySlug($slugId: String!) {
+  projects(filter: { slugId: { eq: $slugId } }, first: 1) {
+    nodes {
+      id url name slugId description content state startDate targetDate createdAt updatedAt
+      lead { name displayName email }
+      members(first: 50) { nodes { name displayName email } }
+    }
+  }
+}`;
+
+async function fetchIssue(identifier: string, apiKey: string, url: URL): Promise<LinearIssue> {
+	const result = await graphql<{ issue: LinearIssue | null }>(apiKey, ISSUE_QUERY, { id: identifier }, url);
+	if (!result.issue) {
+		throw new HelpfulError({
+			kind: "not_found",
+			message: `Linear has no issue ${identifier} visible to this API key.`,
+			hint: "Verify the URL exists and that the API key belongs to a member of the issue's workspace.",
+		});
+	}
+	return result.issue;
+}
+
+async function fetchProject(slugId: string, apiKey: string, url: URL): Promise<LinearProject> {
+	const result = await graphql<{ projects: { nodes: LinearProject[] } }>(apiKey, PROJECT_QUERY, { slugId }, url);
+	const project = result.projects.nodes[0];
+	if (!project) {
+		throw new HelpfulError({
+			kind: "not_found",
+			message: `Linear has no project with slug ${slugId} visible to this API key.`,
+			hint: "Verify the URL exists and that the API key belongs to a member of the project's workspace.",
+		});
+	}
+	return project;
+}
+
+/**
+ * The trailing token on a Linear project URL is `<name>-<slugId>`,
+ * where `slugId` is a 12-char hex suffix. Linear's API matches by
+ * `slugId` exactly, so we slice the suffix off here.
+ */
+function extractProjectSlugId(slug: string): string {
+	const match = slug.match(/-([0-9a-f]{8,})$/i);
+	return match ? (match[1] as string) : slug;
+}
+
+async function graphql<T>(apiKey: string, query: string, variables: Record<string, unknown>, url: URL): Promise<T> {
+	const response = await fetch(GRAPHQL_ENDPOINT, {
+		method: "POST",
+		headers: {
+			"Content-Type": "application/json",
+			Authorization: apiKey,
+		},
+		body: JSON.stringify({ query, variables }),
+	});
+	if (!response.ok) {
+		throw new HelpfulError({
+			kind: response.status === 401 || response.status === 403 ? "auth_error" : "network_error",
+			message: `Linear GraphQL returned ${response.status} ${response.statusText} for ${url.toString()}.`,
+			hint:
+				response.status === 401 || response.status === 403
+					? "Re-create the API key at https://linear.app/settings/api and run `membot config set downloaders.linear.api_key <KEY>`."
+					: "Check that the URL is reachable and that the API key has access to the issue/project.",
+		});
+	}
+	const json = (await response.json()) as { data?: T; errors?: Array<{ message: string }> };
+	if (json.errors && json.errors.length > 0) {
+		const detail = json.errors.map((e) => e.message).join("; ");
+		throw new HelpfulError({
+			kind: "input_error",
+			message: `Linear GraphQL errors for ${url.toString()}: ${detail}`,
+			hint: "Verify the URL is correct and the API key has visibility into the workspace.",
+		});
+	}
+	if (!json.data) {
+		throw new HelpfulError({
+			kind: "internal_error",
+			message: `Linear GraphQL returned no data for ${url.toString()}.`,
+			hint: "Re-run with `--verbose` and report the response shape.",
+		});
+	}
+	return json.data;
+}
+
+function renderIssue(issue: LinearIssue): string {
+	const lines: string[] = [];
+	lines.push(`# ${issue.identifier}: ${issue.title}`);
+	lines.push("");
+	lines.push(`- URL: ${issue.url}`);
+	if (issue.state) lines.push(`- Status: ${issue.state.name}`);
+	if (issue.priorityLabel) lines.push(`- Priority: ${issue.priorityLabel}`);
+	if (issue.assignee) lines.push(`- Assignee: ${userLabel(issue.assignee)}`);
+	if (issue.creator) lines.push(`- Author: ${userLabel(issue.creator)}`);
+	lines.push(`- Created: ${issue.createdAt}`);
+	lines.push(`- Updated: ${issue.updatedAt}`);
+	lines.push("");
+	if (issue.description) {
+		lines.push("## Description");
+		lines.push("");
+		lines.push(issue.description.trim());
+		lines.push("");
+	}
+	const comments = issue.comments.nodes;
+	if (comments.length > 0) {
+		lines.push(`## Comments (${comments.length})`);
+		lines.push("");
+		for (const c of comments) {
+			const who = c.user ? userLabel(c.user) : "(unknown)";
+			lines.push(`### ${who} — ${c.createdAt ?? ""}`);
+			lines.push("");
+			lines.push((c.body ?? "").trim());
+			lines.push("");
+		}
+	}
+	return lines.join("\n").trim();
+}
+
+function renderProject(project: LinearProject): string {
+	const lines: string[] = [];
+	lines.push(`# ${project.name}`);
+	lines.push("");
+	lines.push(`- URL: ${project.url}`);
+	if (project.state) lines.push(`- State: ${project.state}`);
+	if (project.startDate) lines.push(`- Start: ${project.startDate}`);
+	if (project.targetDate) lines.push(`- Target: ${project.targetDate}`);
+	if (project.lead) lines.push(`- Lead: ${userLabel(project.lead)}`);
+	const members = project.members.nodes;
+	if (members.length > 0) lines.push(`- Members: ${members.map(userLabel).join(", ")}`);
+	lines.push(`- Created: ${project.createdAt}`);
+	lines.push(`- Updated: ${project.updatedAt}`);
+	lines.push("");
+	if (project.description) {
+		lines.push("## Summary");
+		lines.push("");
+		lines.push(project.description.trim());
+		lines.push("");
+	}
+	if (project.content) {
+		lines.push("## Overview");
+		lines.push("");
+		lines.push(project.content.trim());
+		lines.push("");
+	}
+	return lines.join("\n").trim();
+}
+
+function userLabel(user: LinearUser): string {
+	const name = user.displayName ?? user.name ?? "(unknown)";
+	if (user.email) return `${name} <${user.email}>`;
+	return name;
+}

package/src/ingest/fetcher.ts

@@ -1,158 +1,133 @@
-import type { LlmConfig } from "../config/schemas.ts";
-import { DEFAULTS } from "../constants.ts";
-import { asHelpful, HelpfulError } from "../errors.ts";
+import { join } from "node:path";
+import type { MembotConfig } from "../config/schemas.ts";
+import { FILES } from "../constants.ts";
+import { HelpfulError } from "../errors.ts";
 import { logger } from "../output/logger.ts";
-import type { AgentMcpxAdapter } from "./agent-fetcher.ts";
-import { agentFetch } from "./agent-fetcher.ts";
-import { sha256Hex } from "./local-reader.ts";
+import { BrowserPool } from "./downloaders/browser.ts";
+import {
+	type DownloadedRemote,
+	type Downloader,
+	type DownloaderCtx,
+	findDownloader,
+	findDownloaderByName,
+	listDownloaders,
+} from "./downloaders/index.ts";
 
-export interface FetchedRemote {
-	bytes: Uint8Array;
-	sha256: string;
-	mimeType: string;
-	fetcher: "http" | "mcpx";
-	fetcherServer: string | null;
-	fetcherTool: string | null;
-	fetcherArgs: Record<string, unknown> | null;
-	sourceUrl: string;
-}
+export type FetchedRemote = DownloadedRemote;
 
 export interface FetchOptions {
 	/**
-	 * User-provided hint. Free-form keyword (e.g. "firecrawl", "github",
-	 * "google-docs", "http"). Special-cased: "http" forces plain fetch.
-	 * Otherwise the hint is passed verbatim to the agent loop as extra
-	 * guidance about which provider to prefer.
+	 * Optional explicit downloader override. Free-form; matched
+	 * case-insensitively against `Downloader.name`. When given, skips the
+	 * URL-based matching and forces that downloader (useful for the
+	 * "use the generic-web fallback even though google-docs claimed
+	 * this URL" escape hatch).
 	 */
-	hint?: string;
-	/** Live mcpx adapter the agent loop drives via search/list/info/exec. */
-	mcpx?: AgentMcpxAdapter | null;
+	downloaderName?: string;
 	/**
-	 * LLM config. The agent loop needs an Anthropic key; without one the
-	 * mcpx path is skipped and we fall back to plain HTTP.
+	 * Override the on-disk path of the persistent chromium profile.
+	 * Defaults to `<ctx.dataDir>/auth/browser-profile`.
 	 */
-	llm?: LlmConfig;
+	userDataDir?: string;
+	/** Pre-built BrowserPool to share across many fetches (set by ingest's outer loop). */
+	pool?: BrowserPool;
 	/**
-	 * Forwarded to the agent loop so callers (e.g. the ingest progress
-	 * reporter) can drive a spinner sublabel from per-turn agent activity.
+	 * Sublabel hook forwarded to the downloader's `DownloaderCtx`.
+	 * Drives the per-entry spinner text during multi-step fetches.
 	 */
 	onProgress?: (sublabel: string) => void;
 }
 
 /**
- * Fetch a remote URL.
- *
- * - `--fetcher http` (or no mcpx, or no LLM key) → plain HTTP.
- * - Otherwise → multi-turn agent loop: Claude is given mcpx tools
- *   (search/list/info/exec) and decides how to retrieve the URL,
- *   including multi-step flows (start a job → poll → download).
- *   The agent's selected mcp_exec invocation is recorded on the
- *   returned row so refresh can replay it deterministically without
- *   another agent round-trip.
- *
- * If the agent decides plain HTTP is the right call (`request_http_fallback`,
- * no tool calls, max turns) we transparently fall through to `httpFetch`.
- * If the agent reports an actionable failure, we surface that as a
- * `HelpfulError`. If mcpx is configured but the LLM key is missing AND
- * the HTTP fallback also fails, we surface an `auth_error` naming the env
- * var so users see the real cause instead of a misleading 401.
+ * Fetch a remote URL via the per-service downloader registry. Specific
+ * downloaders (Google, GitHub, Linear) match first; the generic-web
+ * downloader is the always-matching catch-all. Every fetch authenticates
+ * via the cookies the user persisted with `membot login`. The returned
+ * shape includes the chosen downloader name and its args so refresh can
+ * replay it deterministically without involving the LLM.
  */
-export async function fetchRemote(url: string, options: FetchOptions = {}): Promise<FetchedRemote> {
-	const mcpx = options.mcpx;
-	const hint = options.hint?.trim();
-
-	if (hint === "http") return httpFetch(url);
-	if (!mcpx) return httpFetch(url);
-
-	const apiKey = options.llm?.anthropic_api_key?.trim();
-	if (!apiKey) {
-		// No way to drive the agent. Try HTTP; if that fails, the user
-		// almost certainly wanted mcpx — surface a clear key-missing error.
-		try {
-			return await httpFetch(url);
-		} catch (err) {
-			if (err instanceof HelpfulError && err.kind === "network_error") {
-				throw new HelpfulError({
-					kind: "auth_error",
-					message: `${url} couldn't be fetched directly (${err.message}). Membot has mcpx configured, but routing through it requires Claude to translate the URL into the right tool arguments — and ANTHROPIC_API_KEY isn't set.`,
-					hint: `Set ANTHROPIC_API_KEY in your environment (or under llm.anthropic_api_key in ~/.membot/config.json), then retry. To force the HTTP path explicitly, run \`membot add ${url} --fetcher http\`.`,
-				});
-			}
-			throw err;
-		}
-	}
+export async function fetchRemote(
+	url: string,
+	config: MembotConfig,
+	options: FetchOptions = {},
+	dataDir?: string,
+): Promise<FetchedRemote> {
+	const downloader = pickDownloader(url, options.downloaderName);
+	const userDataDir = options.userDataDir ?? defaultProfileDir(dataDir);
+	const ownsPool = !options.pool;
+	const headless = !downloader.requireHeaded;
+	const pool = options.pool ?? new BrowserPool({ userDataDir, headless });
+	const dctx: DownloaderCtx = { pool, logger, config, onProgress: options.onProgress };
 
-	let outcome: Awaited<ReturnType<typeof agentFetch>>;
 	try {
-		outcome = await agentFetch({ url, mcpx, llm: options.llm!, hint, onProgress: options.onProgress });
-	} catch (err) {
-		if (err instanceof HelpfulError) throw err;
-		logger.warn(`agent-fetch failed (${err instanceof Error ? err.message : String(err)}) — falling back to HTTP`);
-		return httpFetch(url);
+		// Fetches are strictly non-interactive: there's no auto-launch
+		// of a browser when auth fails. Batch ingest (`membot add` of
+		// many URLs) and the refresh daemon both run without a human
+		// available to drive a window, so any auth_error must
+		// propagate as-is. The HelpfulError's hint tells the user to
+		// `membot login` (cookie-based services) or `membot config set
+		// downloaders.<svc>.api_key` (API-key services); they fix it
+		// once and re-run.
+		return await downloader.download(new URL(url), dctx);
+	} finally {
+		if (ownsPool) await pool.dispose();
 	}
+}
 
-	if (outcome.kind === "accepted") {
-		return {
-			bytes: outcome.result.bytes,
-			sha256: outcome.result.sha256,
-			mimeType: outcome.result.mimeType,
-			fetcher: "mcpx",
-			fetcherServer: outcome.result.fetcherServer,
-			fetcherTool: outcome.result.fetcherTool,
-			fetcherArgs: outcome.result.fetcherArgs,
-			sourceUrl: url,
-		};
+/**
+ * Replay a fetch by downloader name (used by refresh). Looks up the
+ * persisted downloader by name and calls it against the original URL —
+ * deterministic, no agent loop. When the persisted downloader is no
+ * longer registered (e.g. from a prior membot version), falls back to
+ * URL-based dispatch so refresh degrades gracefully instead of erroring.
+ */
+export async function fetchRemoteByDownloader(
+	downloaderName: string | null,
+	url: string,
+	pool: BrowserPool,
+	config: MembotConfig,
+): Promise<FetchedRemote> {
+	const named = downloaderName ? findDownloaderByName(downloaderName) : null;
+	const downloader = named ?? findDownloader(url);
+	if (!downloader) {
+		throw new HelpfulError({
+			kind: "input_error",
+			message: `no downloader matches ${url}`,
+			hint: "Re-add the URL with `membot add <url>` to pick a fresh downloader.",
+		});
 	}
-	logger.info(`[fetcher] falling back to HTTP: ${outcome.reason}`);
-	return httpFetch(url);
+	const dctx: DownloaderCtx = { pool, logger, config };
+	return downloader.download(new URL(url), dctx);
 }
 
-/** Plain `fetch` fallback. Used when mcpx isn't configured or the hint says so. */
-async function httpFetch(url: string): Promise<FetchedRemote> {
-	let resp: Response;
-	try {
-		resp = await fetch(url, {
-			headers: { "User-Agent": "membot/0.1" },
-			signal: AbortSignal.timeout(DEFAULTS.HTTP_TIMEOUT_MS),
-		});
-	} catch (err) {
-		throw asHelpful(
-			err,
-			`while fetching ${url}`,
-			`Check your network and that ${url} is reachable. For mcpx-managed sources (gdocs/github/firecrawl), set ANTHROPIC_API_KEY so membot can drive an mcpx tool.`,
-			"network_error",
-		);
+function pickDownloader(url: string, override?: string): Downloader {
+	if (override) {
+		const named = findDownloaderByName(override.toLowerCase());
+		if (!named) {
+			const available = listDownloaders()
+				.map((d) => d.name)
+				.join(", ");
+			throw new HelpfulError({
+				kind: "input_error",
+				message: `unknown downloader '${override}'`,
+				hint: `Pick one of: ${available}.`,
+			});
+		}
+		return named;
 	}
-	if (!resp.ok) {
+	const matched = findDownloader(url);
+	if (!matched) {
 		throw new HelpfulError({
-			kind: "network_error",
-			message: `HTTP ${resp.status} ${resp.statusText}: ${url}`,
-			hint: "Verify the URL is reachable and not gated behind auth. For private docs use mcpx (set ANTHROPIC_API_KEY).",
+			kind: "input_error",
+			message: `not a fetchable URL: ${url}`,
+			hint: "Pass an http(s):// URL.",
 		});
 	}
-	const bytes = new Uint8Array(await resp.arrayBuffer());
-	const ct = resp.headers.get("content-type") ?? "";
-	const mime = ct.split(";")[0]?.trim() || "application/octet-stream";
-	return {
-		bytes,
-		sha256: sha256Hex(bytes),
-		mimeType: mime,
-		fetcher: "http",
-		fetcherServer: null,
-		fetcherTool: null,
-		fetcherArgs: null,
-		sourceUrl: url,
-	};
+	return matched;
 }
 
-/**
- * Detect MCP `CallToolResult` envelopes that signal tool failure. MCP
- * tool errors don't throw — they return `{ isError: true, content: [...] }`
- * — so callers must check this explicitly before treating the content
- * as a successful payload. Used by the refresh runner; the agent loop
- * has its own preview-aware check.
- */
-export function isMcpToolError(result: unknown): boolean {
-	if (!result || typeof result !== "object") return false;
-	return (result as { isError?: unknown }).isError === true;
+function defaultProfileDir(dataDir?: string): string {
+	if (dataDir) return join(dataDir, FILES.BROWSER_PROFILE);
+	const home = process.env.MEMBOT_HOME ?? `${process.env.HOME ?? "."}/.membot`;
+	return join(home, FILES.BROWSER_PROFILE);
 }