doppelgangers 0.0.1

package/src/embed.ts

@@ -0,0 +1,182 @@
+import fs from "fs";
+import OpenAI from "openai";
+import path from "path";
+
+export interface EmbedOptions {
+	input: string;
+	output: string;
+	model: string;
+	batchSize: number;
+	maxChars: number;
+	bodyChars: number;
+	resume: boolean;
+}
+
+interface Item {
+	url: string;
+	number?: number;
+	title: string;
+	body: string | null;
+	state?: string;
+	type?: string;
+}
+
+interface EmbeddingRecord {
+	url: string;
+	number?: number;
+	title: string;
+	body: string;
+	state?: string;
+	type?: string;
+	embedding: number[];
+}
+
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+
+const buildText = (title: string, body: string | null, maxChars: number): string => {
+	const bodyText = (body || "").replace(/\r\n/g, "\n").trim();
+	const combined = `${title}\n\n${bodyText}`.trim();
+	return combined.slice(0, maxChars);
+};
+
+const buildSnippet = (body: string | null, bodyChars: number): string => {
+	if (!body) return "";
+	return body.replace(/\s+/g, " ").trim().slice(0, bodyChars);
+};
+
+export async function embed(options: EmbedOptions): Promise<void> {
+	if (!process.env.OPENAI_API_KEY) {
+		console.error("OPENAI_API_KEY is required");
+		process.exit(1);
+	}
+
+	const inputPath = path.resolve(options.input);
+	const outputPath = path.resolve(options.output);
+
+	const items: Item[] = JSON.parse(fs.readFileSync(inputPath, "utf8"));
+
+	const existing = new Map<string, EmbeddingRecord>();
+	if (options.resume && fs.existsSync(outputPath)) {
+		const lines = fs.readFileSync(outputPath, "utf8").split("\n").filter(Boolean);
+		for (const line of lines) {
+			try {
+				const item: EmbeddingRecord = JSON.parse(line);
+				if (item.url) existing.set(item.url, item);
+			} catch {
+				// skip invalid lines
+			}
+		}
+	}
+
+	const client = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });
+
+	const outputDir = path.dirname(outputPath);
+	fs.mkdirSync(outputDir, { recursive: true });
+	const outputStream = fs.createWriteStream(outputPath, { flags: options.resume ? "a" : "w" });
+
+	const pending = items.filter((item) => item?.url && !existing.has(item.url));
+	const total = pending.length;
+	let processed = 0;
+	const skipped = items.length - pending.length;
+
+	let batchInputs: string[] = [];
+	let batchMeta: { url: string; number?: number; title: string; body: string; state?: string; type?: string }[] = [];
+
+	const createEmbeddings = async (inputs: string[], attempt = 1): Promise<number[][]> => {
+		try {
+			const response = await client.embeddings.create({
+				model: options.model,
+				input: inputs,
+			});
+			return response.data.map((item) => item.embedding);
+		} catch (error) {
+			if (attempt >= 5) throw error;
+			const delay = 1000 * attempt;
+			console.warn(`Embedding request failed, retrying in ${delay}ms`, (error as Error).message);
+			await sleep(delay);
+			return createEmbeddings(inputs, attempt + 1);
+		}
+	};
+
+	const flushBatch = async () => {
+		if (!batchInputs.length) return;
+		const embeddings = await createEmbeddings(batchInputs);
+		for (let i = 0; i < embeddings.length; i += 1) {
+			const meta = batchMeta[i];
+			const record: EmbeddingRecord = {
+				url: meta.url,
+				number: meta.number,
+				title: meta.title,
+				body: meta.body,
+				state: meta.state,
+				type: meta.type,
+				embedding: embeddings[i],
+			};
+			outputStream.write(`${JSON.stringify(record)}\n`);
+			processed += 1;
+			if (processed % 50 === 0 || processed === total) {
+				console.log(`Embedded ${processed}/${total} items`);
+			}
+		}
+		batchInputs = [];
+		batchMeta = [];
+	};
+
+	for (const item of pending) {
+		const title = item.title || "";
+		const body = item.body || "";
+		const text = buildText(title, body, options.maxChars);
+		batchInputs.push(text || title || item.url);
+		batchMeta.push({
+			url: item.url,
+			number: item.number,
+			title,
+			body: buildSnippet(body, options.bodyChars),
+			state: item.state,
+			type: item.type,
+		});
+		if (batchInputs.length >= options.batchSize) {
+			await flushBatch();
+		}
+	}
+
+	await flushBatch();
+	outputStream.end();
+
+	console.log(`Done. Embedded ${processed}/${total} PRs, skipped ${skipped}.`);
+}
+
+// CLI entry point
+if (process.argv[1]?.endsWith("embed.js") || process.argv[1]?.endsWith("embed.ts")) {
+	const args = process.argv.slice(2);
+	const options: EmbedOptions = {
+		input: "prs.json",
+		output: "embeddings.jsonl",
+		model: "text-embedding-3-small",
+		batchSize: 100,
+		maxChars: 4000,
+		bodyChars: 2000,
+		resume: true,
+	};
+
+	for (let i = 0; i < args.length; i += 1) {
+		const arg = args[i];
+		if (arg === "--input") {
+			options.input = args[++i];
+		} else if (arg === "--output") {
+			options.output = args[++i];
+		} else if (arg === "--model") {
+			options.model = args[++i];
+		} else if (arg === "--batch") {
+			options.batchSize = Number(args[++i]);
+		} else if (arg === "--max-chars") {
+			options.maxChars = Number(args[++i]);
+		} else if (arg === "--body-chars") {
+			options.bodyChars = Number(args[++i]);
+		} else if (arg === "--no-resume") {
+			options.resume = false;
+		}
+	}
+
+	embed(options);
+}

package/src/triage.ts

@@ -0,0 +1,256 @@
+#!/usr/bin/env node
+
+import { spawn } from "child_process";
+import fs from "fs";
+import path from "path";
+import readline from "readline";
+import { type BuildOptions, build } from "./build.js";
+import { type EmbedOptions, embed } from "./embed.js";
+
+type ItemState = "open" | "closed" | "all";
+type ItemType = "pr" | "issue" | "all";
+
+interface TriageOptions {
+	repo: string | null;
+	state: ItemState;
+	type: ItemType;
+	output: string;
+	embeddings: string;
+	html: string;
+	model: string;
+	batch: number;
+	maxChars: number;
+	bodyChars: number;
+	neighbors: number;
+	minDist: number;
+	search: boolean;
+}
+
+function parseRepo(repo: string): { owner: string; name: string } | null {
+	const trimmed = repo.replace(/\s+/g, "");
+	const match = trimmed.match(/github\.com[/:]([^/]+)\/([^/.]+)(?:\.git)?/i);
+	if (match) {
+		return { owner: match[1], name: match[2] };
+	}
+	if (trimmed.includes("/")) {
+		const [owner, name] = trimmed.split("/");
+		if (owner && name) return { owner, name };
+	}
+	return null;
+}
+
+interface FetchResult {
+	total: number;
+	prs: number;
+	issues: number;
+}
+
+async function fetchItems(
+	owner: string,
+	name: string,
+	state: ItemState,
+	type: ItemType,
+	outputPath: string,
+): Promise<FetchResult> {
+	fs.mkdirSync(path.dirname(outputPath), { recursive: true });
+
+	const repoLabel = `${owner}/${name}`;
+	const items: object[] = [];
+	let prCount = 0;
+	let issueCount = 0;
+
+	const fetchEndpoint = async (endpoint: string, itemType: "pr" | "issue") => {
+		const jqFilter =
+			itemType === "issue"
+				? '.[] | select(.pull_request == null) | {url: .html_url, number: .number, title: .title, body: .body, state: .state, type: "issue"}'
+				: '.[] | {url: .html_url, number: .number, title: .title, body: .body, state: .state, type: "pr"}';
+
+		const gh = spawn("gh", ["api", "--paginate", endpoint, "--jq", jqFilter], {
+			stdio: ["ignore", "pipe", "inherit"],
+		});
+
+		const rl = readline.createInterface({
+			input: gh.stdout!,
+			crlfDelay: Number.POSITIVE_INFINITY,
+		});
+
+		const readPromise = (async () => {
+			for await (const line of rl) {
+				const trimmed = line.trim();
+				if (!trimmed) continue;
+				try {
+					const item = JSON.parse(trimmed);
+					items.push(item);
+					if (itemType === "pr") prCount++;
+					else issueCount++;
+					const total = prCount + issueCount;
+					if (total % 200 === 0) {
+						console.log(`[${repoLabel}] Fetched ${prCount} PRs, ${issueCount} issues`);
+					}
+				} catch {
+					// skip invalid JSON
+				}
+			}
+		})();
+
+		const exitPromise = new Promise<void>((resolve, reject) => {
+			gh.on("close", (code) => {
+				if (code === 0) resolve();
+				else reject(new Error(`gh api ${endpoint} exited with code ${code}`));
+			});
+			gh.on("error", reject);
+		});
+
+		await Promise.all([readPromise, exitPromise]);
+	};
+
+	if (type === "pr" || type === "all") {
+		const prEndpoint = `/repos/${owner}/${name}/pulls?state=${state}&per_page=100`;
+		await fetchEndpoint(prEndpoint, "pr");
+	}
+
+	if (type === "issue" || type === "all") {
+		const issueEndpoint = `/repos/${owner}/${name}/issues?state=${state}&per_page=100`;
+		await fetchEndpoint(issueEndpoint, "issue");
+	}
+
+	fs.writeFileSync(outputPath, JSON.stringify(items, null, 2));
+
+	return { total: items.length, prs: prCount, issues: issueCount };
+}
+
+async function main() {
+	const args = process.argv.slice(2);
+	const options: TriageOptions = {
+		repo: null,
+		state: "open",
+		type: "all",
+		output: "prs.json",
+		embeddings: "embeddings.jsonl",
+		html: "triage.html",
+		model: "text-embedding-3-small",
+		batch: 100,
+		maxChars: 4000,
+		bodyChars: 2000,
+		neighbors: 15,
+		minDist: 0.1,
+		search: false,
+	};
+
+	for (let i = 0; i < args.length; i += 1) {
+		const arg = args[i];
+		if (arg === "--repo") {
+			options.repo = args[++i];
+		} else if (arg === "--state") {
+			const val = args[++i];
+			if (val !== "open" && val !== "closed" && val !== "all") {
+				console.error("--state must be open, closed, or all");
+				process.exit(1);
+			}
+			options.state = val;
+		} else if (arg === "--type") {
+			const val = args[++i];
+			if (val !== "pr" && val !== "issue" && val !== "all") {
+				console.error("--type must be pr, issue, or all");
+				process.exit(1);
+			}
+			options.type = val;
+		} else if (arg === "--output") {
+			options.output = args[++i];
+		} else if (arg === "--embeddings") {
+			options.embeddings = args[++i];
+		} else if (arg === "--html") {
+			options.html = args[++i];
+		} else if (arg === "--model") {
+			options.model = args[++i];
+		} else if (arg === "--batch") {
+			options.batch = Number(args[++i]);
+		} else if (arg === "--max-chars") {
+			options.maxChars = Number(args[++i]);
+		} else if (arg === "--body-chars") {
+			options.bodyChars = Number(args[++i]);
+		} else if (arg === "--neighbors") {
+			options.neighbors = Number(args[++i]);
+		} else if (arg === "--min-dist") {
+			options.minDist = Number(args[++i]);
+		} else if (arg === "--search") {
+			options.search = true;
+		} else if (arg === "--help" || arg === "-h") {
+			console.log(`
+doppelgangers - Find duplicate PRs through embedding visualization
+
+Usage:
+  doppelgangers --repo <owner/repo>
+
+Options:
+  --repo <url|owner/repo>   GitHub repository (required)
+  --state <state>           Item state: open, closed, or all (default: open)
+  --type <type>             Item type: pr, issue, or all (default: all)
+  --output <path>           Output path for items JSON (default: prs.json)
+  --embeddings <path>       Output path for embeddings (default: embeddings.jsonl)
+  --html <path>             Output path for HTML viewer (default: triage.html)
+  --model <model>           OpenAI embedding model (default: text-embedding-3-small)
+  --batch <n>               Batch size for embeddings (default: 100)
+  --max-chars <n>           Max chars for embedding input (default: 4000)
+  --body-chars <n>          Max chars for body snippet (default: 2000)
+  --neighbors <n>           UMAP neighbors (default: 15)
+  --min-dist <n>            UMAP min distance (default: 0.1)
+  --search                  Include embeddings for semantic search (increases file size)
+
+Environment:
+  OPENAI_API_KEY            Required for embedding generation
+`);
+			process.exit(0);
+		}
+	}
+
+	if (!options.repo) {
+		console.error("--repo is required. Use --help for usage.");
+		process.exit(1);
+	}
+
+	const repoInfo = parseRepo(options.repo);
+	if (!repoInfo) {
+		console.error("Could not parse repo. Use https://github.com/org/repo or org/repo");
+		process.exit(1);
+	}
+
+	const { owner, name } = repoInfo;
+	const typeLabel = options.type === "all" ? "PRs and issues" : options.type === "pr" ? "PRs" : "issues";
+	console.log(`Fetching ${options.state} ${typeLabel} for ${owner}/${name}`);
+
+	const outputPath = path.resolve(options.output);
+	const result = await fetchItems(owner, name, options.state, options.type, outputPath);
+	console.log(`Wrote ${outputPath} (${result.prs} PRs, ${result.issues} issues)`);
+
+	const embedOptions: EmbedOptions = {
+		input: outputPath,
+		output: path.resolve(options.embeddings),
+		model: options.model,
+		batchSize: options.batch,
+		maxChars: options.maxChars,
+		bodyChars: options.bodyChars,
+		resume: false,
+	};
+	await embed(embedOptions);
+
+	const embeddingsPath = path.resolve(options.embeddings);
+	const projectionsPath = embeddingsPath.replace(/\.[^.]+$/, "-projections.json");
+
+	const buildOptions: BuildOptions = {
+		input: embeddingsPath,
+		output: path.resolve(options.html),
+		projections: projectionsPath,
+		neighbors: options.neighbors,
+		minDist: options.minDist,
+		includeEmbeddings: options.search,
+	};
+	build(buildOptions);
+
+	console.log(`Done. Open ${path.resolve(options.html)}`);
+}
+
+main().catch((error) => {
+	console.error(error);
+	process.exit(1);
+});

package/tsconfig.json

@@ -0,0 +1,21 @@
+{
+	"compilerOptions": {
+		"target": "ES2022",
+		"module": "Node16",
+		"lib": ["ES2022"],
+		"strict": true,
+		"esModuleInterop": true,
+		"skipLibCheck": true,
+		"forceConsistentCasingInFileNames": true,
+		"declaration": true,
+		"declarationMap": true,
+		"sourceMap": true,
+		"moduleResolution": "Node16",
+		"resolveJsonModule": true,
+		"outDir": "./dist",
+		"rootDir": "./src",
+		"types": ["node"]
+	},
+	"include": ["src/**/*"],
+	"exclude": ["node_modules", "dist"]
+}