@0x1f320.sh/why-did-you-render-mcp 1.0.0-dev.6 → 1.0.0-dev.8

package/README.md

@@ -126,6 +126,7 @@ Browser (project-b) ──┤
 - **Multiple MCP instances** can run simultaneously. Only the first instance starts the WebSocket server; others gracefully skip. All instances share the same JSONL data directory.
 - **Multi-project support** — Each project is identified by `location.origin`. Render data is stored in per-project JSONL files.
 - **No daemon required** — Each MCP instance is independent. The WebSocket server is opportunistically claimed by whichever instance starts first.
+- **Value dictionary deduplication** — Render reports often repeat the same `prevValue`/`nextValue` objects across thousands of entries. Each JSONL file stores a content-addressed dictionary on its first line, mapping xxhash-wasm hashes to unique values. Render lines reference them via `@@ref:<hash>` sentinels instead of inlining the full object, dramatically reducing file size. Reads hydrate refs transparently.
 
 ## Configuration
 

package/dist/server/index.js

@@ -2,14 +2,107 @@
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
 import { z } from "zod";
-import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync, unlinkSync } from "node:fs";
+import { existsSync, mkdirSync, readFileSync, readdirSync, unlinkSync, writeFileSync } from "node:fs";
 import { homedir } from "node:os";
 import { join } from "node:path";
+import xxhash from "xxhash-wasm";
 import { WebSocketServer } from "ws";
+//#region src/server/store/utils/value-dict.ts
+const DICT_KEY = "@@dict";
+const REF_PREFIX = "@@ref:";
+let h64ToString;
+const ready = xxhash().then((api) => {
+	h64ToString = api.h64ToString;
+});
+function ensureReady() {
+	return ready;
+}
+function hashValue(value) {
+	return h64ToString(JSON.stringify(value));
+}
+function shouldDehydrate(value) {
+	return typeof value === "object" && value !== null;
+}
+function dehydrateDiffs(diffs, dict) {
+	if (!diffs) return false;
+	return diffs.map((d) => {
+		let { prevValue, nextValue } = d;
+		if (shouldDehydrate(prevValue)) {
+			const hash = hashValue(prevValue);
+			dict[hash] ??= prevValue;
+			prevValue = `${REF_PREFIX}${hash}`;
+		}
+		if (shouldDehydrate(nextValue)) {
+			const hash = hashValue(nextValue);
+			dict[hash] ??= nextValue;
+			nextValue = `${REF_PREFIX}${hash}`;
+		}
+		return prevValue === d.prevValue && nextValue === d.nextValue ? d : {
+			...d,
+			prevValue,
+			nextValue
+		};
+	});
+}
+function dehydrate(render, dict) {
+	const { propsDifferences, stateDifferences, hookDifferences } = render.reason;
+	const newProps = dehydrateDiffs(propsDifferences, dict);
+	const newState = dehydrateDiffs(stateDifferences, dict);
+	const newHooks = dehydrateDiffs(hookDifferences, dict);
+	if (newProps === propsDifferences && newState === stateDifferences && newHooks === hookDifferences) return render;
+	return {
+		...render,
+		reason: {
+			propsDifferences: newProps,
+			stateDifferences: newState,
+			hookDifferences: newHooks
+		}
+	};
+}
+function hydrateDiffs(diffs, dict) {
+	if (!diffs) return false;
+	return diffs.map((d) => {
+		let { prevValue, nextValue } = d;
+		if (typeof prevValue === "string" && prevValue.startsWith(REF_PREFIX)) prevValue = dict[prevValue.slice(6)] ?? prevValue;
+		if (typeof nextValue === "string" && nextValue.startsWith(REF_PREFIX)) nextValue = dict[nextValue.slice(6)] ?? nextValue;
+		return prevValue === d.prevValue && nextValue === d.nextValue ? d : {
+			...d,
+			prevValue,
+			nextValue
+		};
+	});
+}
+function hydrate(render, dict) {
+	const { propsDifferences, stateDifferences, hookDifferences } = render.reason;
+	const newProps = hydrateDiffs(propsDifferences, dict);
+	const newState = hydrateDiffs(stateDifferences, dict);
+	const newHooks = hydrateDiffs(hookDifferences, dict);
+	if (newProps === propsDifferences && newState === stateDifferences && newHooks === hookDifferences) return render;
+	return {
+		...render,
+		reason: {
+			propsDifferences: newProps,
+			stateDifferences: newState,
+			hookDifferences: newHooks
+		}
+	};
+}
+//#endregion
 //#region src/server/store/utils/read-jsonl.ts
 function readJsonl(file) {
 	if (!existsSync(file)) return [];
-	return readFileSync(file, "utf-8").split("\n").filter(Boolean).map((line) => JSON.parse(line));
+	const lines = readFileSync(file, "utf-8").split("\n").filter(Boolean);
+	if (lines.length === 0) return [];
+	let dict;
+	let startIdx = 0;
+	const first = JSON.parse(lines[0]);
+	if ("@@dict" in first) {
+		dict = first[DICT_KEY];
+		startIdx = 1;
+	}
+	const renders = lines.slice(startIdx).map((l) => JSON.parse(l));
+	if (!dict) return renders;
+	return renders.map((r) => hydrate(r, dict));
 }
 //#endregion
 //#region src/server/store/utils/sanitize-project-id.ts
@@ -30,10 +123,13 @@ function toResult(stored) {
 //#endregion
 //#region src/server/store/render-store.ts
 const FLUSH_DELAY_MS = 200;
+const NOCOMMIT = "nocommit";
 var RenderStore = class {
 	dir;
 	buffers = /* @__PURE__ */ new Map();
 	timers = /* @__PURE__ */ new Map();
+	dicts = /* @__PURE__ */ new Map();
+	bufferMeta = /* @__PURE__ */ new Map();
 	constructor(dir) {
 		this.dir = dir ?? join(homedir(), ".wdyr-mcp", "renders");
 		mkdirSync(this.dir, { recursive: true });
@@ -44,35 +140,66 @@ var RenderStore = class {
 			projectId,
 			...commitId != null && { commitId }
 		};
-		let buf = this.buffers.get(projectId);
+		const bk = this.bufferKey(projectId, commitId);
+		let buf = this.buffers.get(bk);
 		if (!buf) {
 			buf = [];
-			this.buffers.set(projectId, buf);
+			this.buffers.set(bk, buf);
+			this.bufferMeta.set(bk, {
+				projectId,
+				commitId
+			});
 		}
 		buf.push(stored);
-		const existing = this.timers.get(projectId);
+		const existing = this.timers.get(bk);
 		if (existing) clearTimeout(existing);
-		this.timers.set(projectId, setTimeout(() => this.flush(projectId), FLUSH_DELAY_MS));
+		this.timers.set(bk, setTimeout(() => {
+			this.flushAsync(projectId, commitId).catch((err) => console.error(`[wdyr-mcp] flush error for ${bk}:`, err));
+		}, FLUSH_DELAY_MS));
+	}
+	async flushAsync(projectId, commitId) {
+		await ensureReady();
+		this.flush(projectId, commitId);
 	}
-	flush(projectId) {
-		if (projectId) this.flushProject(projectId);
-		else for (const id of this.buffers.keys()) this.flushProject(id);
+	flush(projectId, commitId) {
+		if (projectId != null && commitId !== void 0) this.flushBuffer(this.bufferKey(projectId, commitId));
+		else if (projectId != null) for (const bk of this.bufferKeysForProject(projectId)) this.flushBuffer(bk);
+		else for (const bk of [...this.buffers.keys()]) this.flushBuffer(bk);
 	}
-	flushProject(projectId) {
-		const buf = this.buffers.get(projectId);
+	flushBuffer(bk) {
+		const buf = this.buffers.get(bk);
 		if (!buf || buf.length === 0) return;
-		const lines = buf.map((s) => JSON.stringify(s)).join("\n");
-		appendFileSync(this.projectFile(projectId), `${lines}\n`);
+		const meta = this.bufferMeta.get(bk);
+		if (!meta) return;
+		let dict = this.dicts.get(bk);
+		if (!dict) {
+			dict = {};
+			this.dicts.set(bk, dict);
+		}
+		const dehydrated = buf.map((r) => dehydrate(r, dict));
+		const file = this.commitFile(meta.projectId, meta.commitId);
+		const existingLines = this.readDataLines(file);
+		const newLines = dehydrated.map((r) => JSON.stringify(r));
+		const allDataLines = [...existingLines, ...newLines];
+		writeFileSync(file, `${(Object.keys(dict).length > 0 ? [JSON.stringify({ [DICT_KEY]: dict }), ...allDataLines] : allDataLines).join("\n")}\n`);
 		buf.length = 0;
-		const timer = this.timers.get(projectId);
+		const timer = this.timers.get(bk);
 		if (timer) {
 			clearTimeout(timer);
-			this.timers.delete(projectId);
+			this.timers.delete(bk);
 		}
 	}
+	readDataLines(file) {
+		if (!existsSync(file)) return [];
+		return readFileSync(file, "utf-8").split("\n").filter((line) => {
+			if (!line) return false;
+			if (line.startsWith(`{"@@dict"`)) return false;
+			return true;
+		});
+	}
 	getAllRenders(projectId) {
 		this.flush(projectId);
-		if (projectId) return readJsonl(this.projectFile(projectId)).map(toResult);
+		if (projectId) return this.projectFiles(projectId).flatMap((f) => readJsonl(join(this.dir, f)).map(toResult));
 		return this.jsonlFiles().flatMap((f) => readJsonl(join(this.dir, f)).map(toResult));
 	}
 	getRendersByComponent(componentName, projectId) {
@@ -80,38 +207,64 @@ var RenderStore = class {
 	}
 	clearRenders(projectId) {
 		if (projectId) {
-			this.buffers.delete(projectId);
-			const timer = this.timers.get(projectId);
-			if (timer) {
-				clearTimeout(timer);
-				this.timers.delete(projectId);
+			for (const bk of this.bufferKeysForProject(projectId)) {
+				this.buffers.delete(bk);
+				this.dicts.delete(bk);
+				this.bufferMeta.delete(bk);
+				const timer = this.timers.get(bk);
+				if (timer) {
+					clearTimeout(timer);
+					this.timers.delete(bk);
+				}
 			}
-			const file = this.projectFile(projectId);
-			if (existsSync(file)) unlinkSync(file);
+			for (const f of this.projectFiles(projectId)) unlinkSync(join(this.dir, f));
 		} else {
-			for (const [id, timer] of this.timers) clearTimeout(timer);
+			for (const [, timer] of this.timers) clearTimeout(timer);
 			this.buffers.clear();
 			this.timers.clear();
+			this.dicts.clear();
+			this.bufferMeta.clear();
 			for (const f of this.jsonlFiles()) unlinkSync(join(this.dir, f));
 		}
 	}
 	getProjects() {
 		this.flush();
 		const projects = /* @__PURE__ */ new Set();
+		const seen = /* @__PURE__ */ new Set();
 		for (const f of this.jsonlFiles()) {
-			const firstLine = readFileSync(join(this.dir, f), "utf-8").split("\n")[0];
-			if (!firstLine) continue;
-			const stored = JSON.parse(firstLine);
-			projects.add(stored.projectId);
+			const parsed = this.parseFilename(f);
+			if (!parsed) continue;
+			if (seen.has(parsed.projectSanitized)) continue;
+			seen.add(parsed.projectSanitized);
+			const lines = readFileSync(join(this.dir, f), "utf-8").split("\n");
+			for (const line of lines) {
+				if (!line) continue;
+				const obj = JSON.parse(line);
+				if ("@@dict" in obj) continue;
+				projects.add(obj.projectId);
+				break;
+			}
 		}
 		return [...projects];
 	}
 	getCommitIds(projectId) {
-		const renders = this.getAllRenders(projectId);
-		return [...new Set(renders.map((r) => r.commitId).filter((id) => id != null))];
+		this.flush(projectId);
+		const files = projectId ? this.projectFiles(projectId) : this.jsonlFiles();
+		const ids = /* @__PURE__ */ new Set();
+		for (const f of files) {
+			const parsed = this.parseFilename(f);
+			if (parsed?.commitId != null) ids.add(parsed.commitId);
+		}
+		return [...ids].sort((a, b) => a - b);
 	}
 	getRendersByCommit(commitId, projectId) {
-		return this.getAllRenders(projectId).filter((r) => r.commitId === commitId);
+		if (projectId) {
+			this.flush(projectId, commitId);
+			return readJsonl(this.commitFile(projectId, commitId)).map(toResult);
+		}
+		this.flush();
+		const suffix = `_commit_${commitId}.jsonl`;
+		return this.jsonlFiles().filter((f) => f.endsWith(suffix)).flatMap((f) => readJsonl(join(this.dir, f)).map(toResult));
 	}
 	getSummary(projectId) {
 		const renders = this.getAllRenders(projectId);
@@ -123,8 +276,33 @@ var RenderStore = class {
 		}
 		return summary;
 	}
-	projectFile(projectId) {
-		return join(this.dir, `${sanitizeProjectId(projectId)}.jsonl`);
+	bufferKey(projectId, commitId) {
+		return `${projectId}\0${commitId ?? NOCOMMIT}`;
+	}
+	bufferKeysForProject(projectId) {
+		const prefix = `${projectId}\0`;
+		return [...this.buffers.keys()].filter((bk) => bk.startsWith(prefix));
+	}
+	commitFile(projectId, commitId) {
+		const sanitized = sanitizeProjectId(projectId);
+		const suffix = commitId != null ? `_commit_${commitId}` : `_${NOCOMMIT}`;
+		return join(this.dir, `${sanitized}${suffix}.jsonl`);
+	}
+	projectFiles(projectId) {
+		const prefix = sanitizeProjectId(projectId);
+		return readdirSync(this.dir).filter((f) => f.startsWith(prefix) && f.endsWith(".jsonl"));
+	}
+	parseFilename(filename) {
+		if (!filename.endsWith(".jsonl")) return null;
+		const base = filename.slice(0, -6);
+		const commitMatch = base.match(/^(.+)_commit_(\d+)$/);
+		if (commitMatch) return {
+			projectSanitized: commitMatch[1],
+			commitId: Number(commitMatch[2])
+		};
+		const nocommitMatch = base.match(/^(.+)_nocommit$/);
+		if (nocommitMatch) return { projectSanitized: nocommitMatch[1] };
+		return { projectSanitized: base };
 	}
 	jsonlFiles() {
 		return readdirSync(this.dir).filter((f) => f.endsWith(".jsonl"));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@0x1f320.sh/why-did-you-render-mcp",
-  "version": "1.0.0-dev.6",
+  "version": "1.0.0-dev.8",
   "type": "module",
   "description": "MCP server that collects why-did-you-render data from browser and exposes it to coding agents",
   "license": "MIT",
@@ -50,6 +50,7 @@
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.12.1",
     "ws": "^8.18.0",
+    "xxhash-wasm": "^1.1.0",
     "zod": "^3.24.4"
   },
   "devDependencies": {