membot 0.5.2 → 0.7.0

package/src/ingest/downloaders/google-sheets.ts

@@ -0,0 +1,58 @@
+import { HelpfulError } from "../../errors.ts";
+import { sha256Hex } from "../local-reader.ts";
+import { fetchWithBrowserCookies } from "./google-shared.ts";
+import type { DownloadedRemote, Downloader } from "./index.ts";
+
+const SHEET_PATH = /^\/spreadsheets\/d\/([a-zA-Z0-9_-]+)/;
+
+const XLSX_MIME = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
+
+/**
+ * Download a Google Sheet as `.xlsx` (the workbook's native format)
+ * — the export includes **every tab** in a single file. The bytes
+ * flow through `convertXlsx`, which renders each tab as a markdown
+ * `## <tab name>` section with a real GitHub-flavored pipe table.
+ * Cleaner than the PDF route (preserves cell structure, no layout
+ * truncation) and `format=html` is no longer supported by Google.
+ */
+export const googleSheetsDownloader: Downloader = {
+	name: "google-sheets",
+	description:
+		"Google Sheets (docs.google.com/spreadsheets/d/<id>) — exports every tab as .xlsx, rendered to markdown tables locally.",
+	logins: [
+		{
+			kind: "browser",
+			name: "Google",
+			url: "https://accounts.google.com/signin",
+			description: "covers Docs, Sheets, and Slides",
+		},
+	],
+	matches(url) {
+		return url.hostname === "docs.google.com" && SHEET_PATH.test(url.pathname);
+	},
+	async download(url, ctx): Promise<DownloadedRemote> {
+		const sheetId = extractSheetId(url);
+		const exportUrl = `https://docs.google.com/spreadsheets/d/${sheetId}/export?format=xlsx`;
+		const body = await fetchWithBrowserCookies(exportUrl, ctx, "Google Sheets", url);
+		return {
+			bytes: new Uint8Array(body),
+			sha256: sha256Hex(body),
+			mimeType: XLSX_MIME,
+			downloader: "google-sheets",
+			downloaderArgs: { sheet_id: sheetId },
+			sourceUrl: url.toString(),
+		};
+	},
+};
+
+function extractSheetId(url: URL): string {
+	const match = url.pathname.match(SHEET_PATH);
+	if (!match || !match[1]) {
+		throw new HelpfulError({
+			kind: "input_error",
+			message: `not a Google Sheets URL: ${url.toString()}`,
+			hint: "Pass a URL like https://docs.google.com/spreadsheets/d/<SHEET_ID>/edit.",
+		});
+	}
+	return match[1];
+}

package/src/ingest/downloaders/google-slides.ts

@@ -0,0 +1,53 @@
+import { HelpfulError } from "../../errors.ts";
+import { sha256Hex } from "../local-reader.ts";
+import { fetchWithBrowserCookies } from "./google-shared.ts";
+import type { DownloadedRemote, Downloader } from "./index.ts";
+
+const SLIDE_PATH = /^\/presentation\/d\/([a-zA-Z0-9_-]+)/;
+
+/**
+ * Download a Google Slides deck as a PDF via the canonical export
+ * endpoint. PDF preserves layout and text-on-slides faithfully; the
+ * existing `convertPdf` pipeline (unpdf) extracts the speaker text +
+ * bullets without losing slide ordering.
+ */
+export const googleSlidesDownloader: Downloader = {
+	name: "google-slides",
+	description: "Google Slides (docs.google.com/presentation/d/<id>) — exports as PDF for layout-faithful conversion.",
+	logins: [
+		{
+			kind: "browser",
+			name: "Google",
+			url: "https://accounts.google.com/signin",
+			description: "covers Docs, Sheets, and Slides",
+		},
+	],
+	matches(url) {
+		return url.hostname === "docs.google.com" && SLIDE_PATH.test(url.pathname);
+	},
+	async download(url, ctx): Promise<DownloadedRemote> {
+		const slidesId = extractSlidesId(url);
+		const exportUrl = `https://docs.google.com/presentation/d/${slidesId}/export/pdf`;
+		const body = await fetchWithBrowserCookies(exportUrl, ctx, "Google Slides", url);
+		return {
+			bytes: new Uint8Array(body),
+			sha256: sha256Hex(body),
+			mimeType: "application/pdf",
+			downloader: "google-slides",
+			downloaderArgs: { slides_id: slidesId },
+			sourceUrl: url.toString(),
+		};
+	},
+};
+
+function extractSlidesId(url: URL): string {
+	const match = url.pathname.match(SLIDE_PATH);
+	if (!match || !match[1]) {
+		throw new HelpfulError({
+			kind: "input_error",
+			message: `not a Google Slides URL: ${url.toString()}`,
+			hint: "Pass a URL like https://docs.google.com/presentation/d/<SLIDES_ID>/edit.",
+		});
+	}
+	return match[1];
+}

package/src/ingest/downloaders/index.ts

@@ -0,0 +1,182 @@
+import type { MembotConfig } from "../../config/schemas.ts";
+import type { logger as Logger } from "../../output/logger.ts";
+import type { BrowserPool } from "./browser.ts";
+import { genericWebDownloader } from "./generic-web.ts";
+import { githubDownloader } from "./github.ts";
+import { googleDocsDownloader } from "./google-docs.ts";
+import { googleSheetsDownloader } from "./google-sheets.ts";
+import { googleSlidesDownloader } from "./google-slides.ts";
+import { linearDownloader } from "./linear.ts";
+
+/**
+ * The shape every URL fetch produces — drop-in replacement for the
+ * old `FetchedRemote` shape. `downloader` + `downloaderArgs` get
+ * persisted on the row so refresh replays the same downloader against
+ * the same URL deterministically (no LLM, no agent loop).
+ */
+export interface DownloadedRemote {
+	bytes: Uint8Array;
+	sha256: string;
+	mimeType: string;
+	downloader: string;
+	downloaderArgs: Record<string, unknown>;
+	sourceUrl: string;
+}
+
+export interface DownloaderCtx {
+	pool: BrowserPool;
+	logger: typeof Logger;
+	config: MembotConfig;
+	/**
+	 * Optional sublabel hook for the host's progress spinner. Long-running
+	 * downloaders (multi-query GraphQL, paginated REST fetches, headless
+	 * browser navigation) can call this with short status strings —
+	 * "fetching", "rendering", "parsing 3/4 pages" — and the CLI will
+	 * surface them under the per-entry progress bar. No-op when the host
+	 * doesn't supply one (e.g. MCP server, JSON-mode CLI).
+	 */
+	onProgress?: (sublabel: string) => void;
+}
+
+/**
+ * One tactic for fetching a URL. Specific downloaders (Google,
+ * GitHub, Linear) match URLs by host/pattern and hit the canonical
+ * export endpoint; the generic-web downloader is the registry's
+ * always-matching catch-all (HEADs the URL, prints to PDF if HTML,
+ * else streams the raw bytes through). Adding a 6th service is one
+ * file — implement `Downloader`, register it here.
+ *
+ * If a downloader requires a logged-in browser session, it declares
+ * one or more `LoginEntry` objects; the `membot login` page collects
+ * those across every downloader, dedupes by URL, and renders one
+ * button per service.
+ */
+export interface Downloader {
+	name: string;
+	description: string;
+	matches(url: URL): boolean;
+	download(url: URL, ctx: DownloaderCtx): Promise<DownloadedRemote>;
+	logins?: LoginEntry[];
+	/**
+	 * Force the BrowserPool into headed mode for this downloader's
+	 * fetches. Used for SPAs that detect headless Chromium and refuse
+	 * to hydrate; we don't currently use it (services that needed it
+	 * have moved to the API-key flow), but the hook remains for
+	 * future cookie-based downloaders.
+	 */
+	requireHeaded?: boolean;
+	/**
+	 * The downloader authenticates via a config-stored API key, not
+	 * browser cookies. The fetcher uses this to skip the auto-login
+	 * browser prompt on `auth_error` (opening a browser doesn't help
+	 * when the missing credential is in the config file).
+	 */
+	requiresApiKey?: boolean;
+}
+
+/**
+ * A service the user might need to set up before fetches against it
+ * succeed. Two flavors:
+ *  - `kind: "browser"` — the user clicks a link in the `membot login`
+ *    browser, signs in, and closes the window. Cookies + IndexedDB
+ *    land in the persistent profile and downloaders use them
+ *    automatically.
+ *  - `kind: "api_key"` — the user visits the service's API-key page,
+ *    copies the key, and runs the displayed `setupCommand`. The key
+ *    lives in `~/.membot/config.json` and downloaders read it from
+ *    `ctx.config`.
+ *
+ * Multiple downloaders can declare the same `LoginEntry` (e.g. all
+ * three Google downloaders share Google sign-in); the login page
+ * dedupes by `(kind, url)`.
+ */
+export type LoginEntry = BrowserLoginEntry | ApiKeyLoginEntry;
+
+export interface BrowserLoginEntry {
+	kind: "browser";
+	/** Display name (e.g. "Google"). */
+	name: string;
+	/** Login URL the button opens. */
+	url: string;
+	/** Optional one-liner shown next to the button. */
+	description?: string;
+}
+
+export interface ApiKeyLoginEntry {
+	kind: "api_key";
+	/** Display name (e.g. "Linear"). */
+	name: string;
+	/** Settings page where the user creates the key. */
+	url: string;
+	/** Shell command the user copies — e.g. `membot config set linear.api_key <KEY>`. */
+	setupCommand: string;
+	/** Optional one-liner shown next to the link. */
+	description?: string;
+}
+
+const REGISTRY: Downloader[] = [
+	googleDocsDownloader,
+	googleSheetsDownloader,
+	googleSlidesDownloader,
+	githubDownloader,
+	linearDownloader,
+	genericWebDownloader,
+];
+
+/**
+ * Find the first downloader that matches `url`. Returns `null` only
+ * if `url` doesn't parse — in normal use the generic-web downloader
+ * matches everything else, so callers can treat `findDownloader` as
+ * total over valid URLs.
+ */
+export function findDownloader(url: string | URL): Downloader | null {
+	let parsed: URL;
+	try {
+		parsed = typeof url === "string" ? new URL(url) : url;
+	} catch {
+		return null;
+	}
+	for (const d of REGISTRY) {
+		if (d.matches(parsed)) return d;
+	}
+	return null;
+}
+
+/** Lookup by name (used by refresh to replay a persisted downloader). */
+export function findDownloaderByName(name: string): Downloader | null {
+	return REGISTRY.find((d) => d.name === name) ?? null;
+}
+
+/** Read-only view of every registered downloader. */
+export function listDownloaders(): readonly Downloader[] {
+	return REGISTRY;
+}
+
+/**
+ * Collect every `LoginEntry` declared by a downloader, deduped by URL
+ * within each kind. Used by `membot login` to render one button per
+ * service (browser-login) and one set of instructions per service
+ * (api-key) even when multiple downloaders share the same setup
+ * (e.g. Google Docs / Sheets / Slides all share Google sign-in).
+ */
+export function collectLoginEntries(): { browser: BrowserLoginEntry[]; apiKey: ApiKeyLoginEntry[] } {
+	const browser = new Map<string, BrowserLoginEntry>();
+	const apiKey = new Map<string, ApiKeyLoginEntry>();
+	for (const d of REGISTRY) {
+		if (!d.logins) continue;
+		for (const entry of d.logins) {
+			if (entry.kind === "browser") {
+				if (!browser.has(entry.url)) browser.set(entry.url, entry);
+			} else {
+				if (!apiKey.has(entry.url)) apiKey.set(entry.url, entry);
+			}
+		}
+	}
+	return { browser: [...browser.values()], apiKey: [...apiKey.values()] };
+}
+
+/**
+ * Compute a stable sha256 hex digest of the bytes. Re-exposed here
+ * because every downloader uses it.
+ */
+export { sha256Hex } from "../local-reader.ts";

package/src/ingest/downloaders/linear.ts

@@ -0,0 +1,291 @@
+import { HelpfulError } from "../../errors.ts";
+import { sha256Hex } from "../local-reader.ts";
+import type { DownloadedRemote, Downloader } from "./index.ts";
+
+const ISSUE_PATH = /^\/([^/]+)\/issue\/([A-Z]+-\d+)(?:$|\/|#|\?)/;
+const PROJECT_PATH = /^\/([^/]+)\/project\/([^/?#]+)/;
+
+const GRAPHQL_ENDPOINT = "https://api.linear.app/graphql";
+
+/**
+ * Linear's web app uses a sophisticated cookie + signed-request scheme
+ * (`client-api.linear.app/graphql` with `useraccount`/`linear-client-id`
+ * headers) that's not realistically replayable from outside a real
+ * Linear browser session. Instead we use Linear's official API at
+ * `api.linear.app/graphql` with a personal API key — set up once via
+ * `membot config set downloaders.linear.api_key <KEY>` after creating
+ * the key at https://linear.app/settings/api.
+ *
+ * The API gives us the structured issue/project payload (title, body,
+ * comments, status, …) directly; we render it to markdown
+ * deterministically rather than scraping the rendered DOM.
+ */
+export const linearDownloader: Downloader = {
+	name: "linear",
+	description: "Linear (linear.app/<workspace>/issue/<KEY> and /project/<slug>) — uses the Linear API.",
+	logins: [
+		{
+			kind: "api_key",
+			name: "Linear",
+			url: "https://linear.app/settings/api",
+			setupCommand: "membot config set downloaders.linear.api_key <KEY>",
+			description: "create a personal API key, then run the command on the right",
+		},
+	],
+	requiresApiKey: true,
+	matches(url) {
+		return url.hostname === "linear.app" && (ISSUE_PATH.test(url.pathname) || PROJECT_PATH.test(url.pathname));
+	},
+	async download(url, ctx): Promise<DownloadedRemote> {
+		const apiKey = ctx.config.downloaders.linear.api_key.trim();
+		if (apiKey === "") {
+			throw new HelpfulError({
+				kind: "auth_error",
+				message: `Linear API key not configured.`,
+				hint: "Create a personal API key at https://linear.app/settings/api, then run `membot config set downloaders.linear.api_key <KEY>`.",
+			});
+		}
+
+		const issueMatch = url.pathname.match(ISSUE_PATH);
+		const projectMatch = url.pathname.match(PROJECT_PATH);
+		let markdown: string;
+		let downloaderArgs: Record<string, unknown>;
+
+		if (issueMatch) {
+			const identifier = issueMatch[2] as string;
+			ctx.onProgress?.(`querying issue ${identifier}`);
+			const issue = await fetchIssue(identifier, apiKey, url);
+			markdown = renderIssue(issue);
+			downloaderArgs = { kind: "issue", workspace: issueMatch[1], identifier };
+		} else if (projectMatch) {
+			const slug = projectMatch[2] as string;
+			const slugId = extractProjectSlugId(slug);
+			ctx.onProgress?.(`querying project ${slugId}`);
+			const project = await fetchProject(slugId, apiKey, url);
+			markdown = renderProject(project);
+			downloaderArgs = { kind: "project", workspace: projectMatch[1], slug, slug_id: slugId };
+		} else {
+			throw new HelpfulError({
+				kind: "input_error",
+				message: `not a Linear issue/project URL: ${url.toString()}`,
+				hint: "Pass a URL like https://linear.app/<workspace>/issue/<KEY> or .../project/<slug>.",
+			});
+		}
+
+		const bytes = new TextEncoder().encode(markdown);
+		return {
+			bytes,
+			sha256: sha256Hex(bytes),
+			mimeType: "text/markdown",
+			downloader: "linear",
+			downloaderArgs,
+			sourceUrl: url.toString(),
+		};
+	},
+};
+
+interface LinearUser {
+	name?: string | null;
+	displayName?: string | null;
+	email?: string | null;
+}
+
+interface LinearComment {
+	body: string | null;
+	createdAt: string | null;
+	user: LinearUser | null;
+}
+
+interface LinearIssue {
+	identifier: string;
+	url: string;
+	title: string;
+	description: string | null;
+	priorityLabel: string | null;
+	state: { name: string } | null;
+	assignee: LinearUser | null;
+	creator: LinearUser | null;
+	createdAt: string;
+	updatedAt: string;
+	comments: { nodes: LinearComment[] };
+}
+
+interface LinearProject {
+	id: string;
+	url: string;
+	name: string;
+	slugId: string;
+	description: string | null;
+	content: string | null;
+	state: string | null;
+	startDate: string | null;
+	targetDate: string | null;
+	createdAt: string;
+	updatedAt: string;
+	lead: LinearUser | null;
+	members: { nodes: LinearUser[] };
+}
+
+const ISSUE_QUERY = `query Issue($id: String!) {
+  issue(id: $id) {
+    identifier url title description priorityLabel
+    state { name }
+    assignee { name displayName email }
+    creator { name displayName email }
+    createdAt updatedAt
+    comments(first: 100) {
+      nodes { body createdAt user { name displayName email } }
+    }
+  }
+}`;
+
+const PROJECT_QUERY = `query ProjectBySlug($slugId: String!) {
+  projects(filter: { slugId: { eq: $slugId } }, first: 1) {
+    nodes {
+      id url name slugId description content state startDate targetDate createdAt updatedAt
+      lead { name displayName email }
+      members(first: 50) { nodes { name displayName email } }
+    }
+  }
+}`;
+
+async function fetchIssue(identifier: string, apiKey: string, url: URL): Promise<LinearIssue> {
+	const result = await graphql<{ issue: LinearIssue | null }>(apiKey, ISSUE_QUERY, { id: identifier }, url);
+	if (!result.issue) {
+		throw new HelpfulError({
+			kind: "not_found",
+			message: `Linear has no issue ${identifier} visible to this API key.`,
+			hint: "Verify the URL exists and that the API key belongs to a member of the issue's workspace.",
+		});
+	}
+	return result.issue;
+}
+
+async function fetchProject(slugId: string, apiKey: string, url: URL): Promise<LinearProject> {
+	const result = await graphql<{ projects: { nodes: LinearProject[] } }>(apiKey, PROJECT_QUERY, { slugId }, url);
+	const project = result.projects.nodes[0];
+	if (!project) {
+		throw new HelpfulError({
+			kind: "not_found",
+			message: `Linear has no project with slug ${slugId} visible to this API key.`,
+			hint: "Verify the URL exists and that the API key belongs to a member of the project's workspace.",
+		});
+	}
+	return project;
+}
+
+/**
+ * The trailing token on a Linear project URL is `<name>-<slugId>`,
+ * where `slugId` is a 12-char hex suffix. Linear's API matches by
+ * `slugId` exactly, so we slice the suffix off here.
+ */
+function extractProjectSlugId(slug: string): string {
+	const match = slug.match(/-([0-9a-f]{8,})$/i);
+	return match ? (match[1] as string) : slug;
+}
+
+async function graphql<T>(apiKey: string, query: string, variables: Record<string, unknown>, url: URL): Promise<T> {
+	const response = await fetch(GRAPHQL_ENDPOINT, {
+		method: "POST",
+		headers: {
+			"Content-Type": "application/json",
+			Authorization: apiKey,
+		},
+		body: JSON.stringify({ query, variables }),
+	});
+	if (!response.ok) {
+		throw new HelpfulError({
+			kind: response.status === 401 || response.status === 403 ? "auth_error" : "network_error",
+			message: `Linear GraphQL returned ${response.status} ${response.statusText} for ${url.toString()}.`,
+			hint:
+				response.status === 401 || response.status === 403
+					? "Re-create the API key at https://linear.app/settings/api and run `membot config set downloaders.linear.api_key <KEY>`."
+					: "Check that the URL is reachable and that the API key has access to the issue/project.",
+		});
+	}
+	const json = (await response.json()) as { data?: T; errors?: Array<{ message: string }> };
+	if (json.errors && json.errors.length > 0) {
+		const detail = json.errors.map((e) => e.message).join("; ");
+		throw new HelpfulError({
+			kind: "input_error",
+			message: `Linear GraphQL errors for ${url.toString()}: ${detail}`,
+			hint: "Verify the URL is correct and the API key has visibility into the workspace.",
+		});
+	}
+	if (!json.data) {
+		throw new HelpfulError({
+			kind: "internal_error",
+			message: `Linear GraphQL returned no data for ${url.toString()}.`,
+			hint: "Re-run with `--verbose` and report the response shape.",
+		});
+	}
+	return json.data;
+}
+
+function renderIssue(issue: LinearIssue): string {
+	const lines: string[] = [];
+	lines.push(`# ${issue.identifier}: ${issue.title}`);
+	lines.push("");
+	lines.push(`- URL: ${issue.url}`);
+	if (issue.state) lines.push(`- Status: ${issue.state.name}`);
+	if (issue.priorityLabel) lines.push(`- Priority: ${issue.priorityLabel}`);
+	if (issue.assignee) lines.push(`- Assignee: ${userLabel(issue.assignee)}`);
+	if (issue.creator) lines.push(`- Author: ${userLabel(issue.creator)}`);
+	lines.push(`- Created: ${issue.createdAt}`);
+	lines.push(`- Updated: ${issue.updatedAt}`);
+	lines.push("");
+	if (issue.description) {
+		lines.push("## Description");
+		lines.push("");
+		lines.push(issue.description.trim());
+		lines.push("");
+	}
+	const comments = issue.comments.nodes;
+	if (comments.length > 0) {
+		lines.push(`## Comments (${comments.length})`);
+		lines.push("");
+		for (const c of comments) {
+			const who = c.user ? userLabel(c.user) : "(unknown)";
+			lines.push(`### ${who} — ${c.createdAt ?? ""}`);
+			lines.push("");
+			lines.push((c.body ?? "").trim());
+			lines.push("");
+		}
+	}
+	return lines.join("\n").trim();
+}
+
+function renderProject(project: LinearProject): string {
+	const lines: string[] = [];
+	lines.push(`# ${project.name}`);
+	lines.push("");
+	lines.push(`- URL: ${project.url}`);
+	if (project.state) lines.push(`- State: ${project.state}`);
+	if (project.startDate) lines.push(`- Start: ${project.startDate}`);
+	if (project.targetDate) lines.push(`- Target: ${project.targetDate}`);
+	if (project.lead) lines.push(`- Lead: ${userLabel(project.lead)}`);
+	const members = project.members.nodes;
+	if (members.length > 0) lines.push(`- Members: ${members.map(userLabel).join(", ")}`);
+	lines.push(`- Created: ${project.createdAt}`);
+	lines.push(`- Updated: ${project.updatedAt}`);
+	lines.push("");
+	if (project.description) {
+		lines.push("## Summary");
+		lines.push("");
+		lines.push(project.description.trim());
+		lines.push("");
+	}
+	if (project.content) {
+		lines.push("## Overview");
+		lines.push("");
+		lines.push(project.content.trim());
+		lines.push("");
+	}
+	return lines.join("\n").trim();
+}
+
+function userLabel(user: LinearUser): string {
+	const name = user.displayName ?? user.name ?? "(unknown)";
+	if (user.email) return `${name} <${user.email}>`;
+	return name;
+}