granola-toolkit 0.4.0 → 0.5.0

package/README.md

@@ -119,9 +119,21 @@ Structured output formats are useful when you want to post-process exports in sc
 
 ### Incremental Writes
 
-Both commands are incremental. They only rewrite an export file when the source document appears newer than the file already on disk.
+Both commands keep a small hidden state file in the output directory to track:
 
-That means repeated runs are cheap, and you can safely point the CLI at the same output directory over time.
+- document id to filename
+- content hash
+- source timestamp
+- last export time
+
+That state is used to:
+
+- keep filenames stable even if a meeting title changes later
+- skip rewrites when the rendered content is unchanged
+- migrate old files cleanly when the output format changes
+- delete stale exports when a document disappears from the source data
+
+That makes repeated runs cheap and keeps long-lived export directories much cleaner.
 
 ## Config
 

package/dist/cli.js

@@ -1,8 +1,9 @@
 #!/usr/bin/env node
-import { mkdir, readFile, stat, writeFile } from "node:fs/promises";
+import { mkdir, readFile, rm, stat, writeFile } from "node:fs/promises";
 import { existsSync } from "node:fs";
 import { homedir } from "node:os";
 import { dirname, join } from "node:path";
+import { createHash } from "node:crypto";
 //#region src/utils.ts
 const INVALID_FILENAME_CHARS = /[<>:"/\\|?*]/g;
 const CONTROL_CHARACTERS = /\p{Cc}/gu;
@@ -147,16 +148,6 @@ function latestDocumentTimestamp(document) {
 	});
 	return candidates[0] ?? document.updatedAt;
 }
-async function shouldWriteFile(filePath, updatedAt) {
-	try {
-		const existing = await stat(filePath);
-		const updatedTime = new Date(updatedAt);
-		if (Number.isNaN(updatedTime.getTime())) return true;
-		return updatedTime.getTime() > existing.mtime.getTime();
-	} catch {
-		return true;
-	}
-}
 async function writeTextFile(filePath, content) {
 	await mkdir(dirname(filePath), { recursive: true });
 	await writeFile(filePath, content, "utf8");
@@ -430,6 +421,134 @@ async function loadConfig(options) {
 	};
 }
 //#endregion
+//#region src/export-state.ts
+const EXPORT_STATE_VERSION = 1;
+function exportStatePath(outputDir, kind) {
+	return join(outputDir, `.granola-toolkit-${kind}-state.json`);
+}
+function emptyExportState(kind) {
+	return {
+		entries: {},
+		kind,
+		version: EXPORT_STATE_VERSION
+	};
+}
+function normaliseExportState(parsed, kind) {
+	const record = asRecord(parsed);
+	if (!record || record.version !== EXPORT_STATE_VERSION || record.kind !== kind) return emptyExportState(kind);
+	const rawEntries = asRecord(record.entries) ?? {};
+	return {
+		entries: Object.fromEntries(Object.entries(rawEntries).map(([id, entry]) => {
+			const value = asRecord(entry);
+			if (!value) return;
+			const fileName = stringValue(value.fileName);
+			const fileStem = stringValue(value.fileStem);
+			if (!fileName || !fileStem) return;
+			return [id, {
+				contentHash: stringValue(value.contentHash),
+				exportedAt: stringValue(value.exportedAt),
+				fileName,
+				fileStem,
+				sourceUpdatedAt: stringValue(value.sourceUpdatedAt)
+			}];
+		}).filter((entry) => Boolean(entry))),
+		kind,
+		version: EXPORT_STATE_VERSION
+	};
+}
+async function loadExportState(outputDir, kind) {
+	const statePath = exportStatePath(outputDir, kind);
+	try {
+		return normaliseExportState(parseJsonString(await readUtf8(statePath)), kind);
+	} catch {
+		return emptyExportState(kind);
+	}
+}
+function hashContent(content) {
+	return createHash("sha256").update(content).digest("hex");
+}
+function reserveStem(used, preferredStem, existingStem) {
+	if (existingStem && (used.get(existingStem) ?? 0) === 0) {
+		used.set(existingStem, 1);
+		return existingStem;
+	}
+	return makeUniqueFilename(preferredStem, used);
+}
+async function fileExists(pathname) {
+	try {
+		await stat(pathname);
+		return true;
+	} catch {
+		return false;
+	}
+}
+function entryChanged(left, right) {
+	if (!left) return true;
+	return left.contentHash !== right.contentHash || left.exportedAt !== right.exportedAt || left.fileName !== right.fileName || left.fileStem !== right.fileStem || left.sourceUpdatedAt !== right.sourceUpdatedAt;
+}
+async function syncManagedExports({ items, kind, outputDir }) {
+	await ensureDirectory(outputDir);
+	const previousEntries = (await loadExportState(outputDir, kind)).entries;
+	const used = /* @__PURE__ */ new Map();
+	const plans = items.map((item) => {
+		const existing = previousEntries[item.id];
+		const fileStem = reserveStem(used, item.preferredStem, existing?.fileStem);
+		return {
+			content: item.content,
+			contentHash: hashContent(item.content),
+			existing,
+			fileName: `${fileStem}${item.extension}`,
+			fileStem,
+			id: item.id,
+			sourceUpdatedAt: item.sourceUpdatedAt
+		};
+	});
+	const activeIds = new Set(plans.map((plan) => plan.id));
+	const activeFileNames = new Set(plans.map((plan) => plan.fileName));
+	const exportedAt = (/* @__PURE__ */ new Date()).toISOString();
+	const nextEntries = {};
+	let written = 0;
+	let stateChanged = false;
+	for (const plan of plans) {
+		const filePath = join(outputDir, plan.fileName);
+		const shouldWrite = !plan.existing || plan.existing.contentHash !== plan.contentHash || plan.existing.fileName !== plan.fileName || !await fileExists(filePath);
+		if (shouldWrite) {
+			await writeTextFile(filePath, plan.content);
+			written += 1;
+		}
+		const nextEntry = {
+			contentHash: plan.contentHash,
+			exportedAt: shouldWrite ? exportedAt : plan.existing?.exportedAt ?? exportedAt,
+			fileName: plan.fileName,
+			fileStem: plan.fileStem,
+			sourceUpdatedAt: plan.sourceUpdatedAt
+		};
+		nextEntries[plan.id] = nextEntry;
+		stateChanged = stateChanged || entryChanged(plan.existing, nextEntry);
+	}
+	for (const plan of plans) {
+		const previousFileName = plan.existing?.fileName;
+		if (previousFileName && previousFileName !== plan.fileName && !activeFileNames.has(previousFileName)) {
+			await rm(join(outputDir, previousFileName), { force: true });
+			stateChanged = true;
+		}
+	}
+	for (const [id, entry] of Object.entries(previousEntries)) {
+		if (activeIds.has(id)) continue;
+		if (!activeFileNames.has(entry.fileName)) await rm(join(outputDir, entry.fileName), { force: true });
+		stateChanged = true;
+	}
+	const serialisedState = `${JSON.stringify({
+		entries: nextEntries,
+		kind,
+		version: EXPORT_STATE_VERSION
+	}, null, 2)}\n`;
+	const statePath = exportStatePath(outputDir, kind);
+	const existingState = await fileExists(statePath) ? await readUtf8(statePath) : void 0;
+	if (stateChanged || existingState !== serialisedState) await writeTextFile(statePath, serialisedState);
+	return written;
+}
+//#endregion
 //#region src/render.ts
 function formatScalar(value) {
 	if (value == null) return "null";
@@ -645,17 +764,20 @@ function noteFileExtension(format) {
 	}
 }
 async function writeNotes(documents, outputDir, format = "markdown") {
-	await ensureDirectory(outputDir);
-	const sorted = [...documents].sort((left, right) => compareStrings(left.title || left.id, right.title || right.id) || compareStrings(left.id, right.id));
-	const used = /* @__PURE__ */ new Map();
-	let written = 0;
-	for (const document of sorted) {
-		const filePath = join(outputDir, `${makeUniqueFilename(documentFilename(document), used)}${noteFileExtension(format)}`);
-		if (!await shouldWriteFile(filePath, latestDocumentTimestamp(document))) continue;
-		await writeTextFile(filePath, renderNoteExport(buildNoteExport(document), format));
-		written += 1;
-	}
-	return written;
+	return await syncManagedExports({
+		items: [...documents].sort((left, right) => compareStrings(left.title || left.id, right.title || right.id) || compareStrings(left.id, right.id)).map((document) => {
+			const note = buildNoteExport(document);
+			return {
+				content: renderNoteExport(note, format),
+				extension: noteFileExtension(format),
+				id: note.id,
+				preferredStem: documentFilename(document),
+				sourceUpdatedAt: latestDocumentTimestamp(document)
+			};
+		}),
+		kind: "notes",
+		outputDir
+	});
 }
 //#endregion
 //#region src/commands/shared.ts
@@ -853,29 +975,31 @@ function transcriptFileExtension(format) {
 	}
 }
 async function writeTranscripts(cacheData, outputDir, format = "text") {
-	await ensureDirectory(outputDir);
-	const entries = Object.entries(cacheData.transcripts).filter(([, segments]) => segments.length > 0).sort(([leftId], [rightId]) => {
-		const leftDocument = cacheData.documents[leftId];
-		const rightDocument = cacheData.documents[rightId];
-		return compareStrings(leftDocument?.title || leftId, rightDocument?.title || rightId) || compareStrings(leftId, rightId);
+	return await syncManagedExports({
+		items: Object.entries(cacheData.transcripts).filter(([, segments]) => segments.length > 0).sort(([leftId], [rightId]) => {
+			const leftDocument = cacheData.documents[leftId];
+			const rightDocument = cacheData.documents[rightId];
+			return compareStrings(leftDocument?.title || leftId, rightDocument?.title || rightId) || compareStrings(leftId, rightId);
+		}).flatMap(([documentId, segments]) => {
+			const document = cacheData.documents[documentId] ?? {
+				createdAt: "",
+				id: documentId,
+				title: documentId,
+				updatedAt: ""
+			};
+			const content = renderTranscriptExport(buildTranscriptExport(document, segments), format);
+			if (!content) return [];
+			return [{
+				content,
+				extension: transcriptFileExtension(format),
+				id: document.id,
+				preferredStem: transcriptFilename(document),
+				sourceUpdatedAt: document.updatedAt
+			}];
+		}),
+		kind: "transcripts",
+		outputDir
 	});
-	const used = /* @__PURE__ */ new Map();
-	let written = 0;
-	for (const [documentId, segments] of entries) {
-		const document = cacheData.documents[documentId] ?? {
-			createdAt: "",
-			id: documentId,
-			title: documentId,
-			updatedAt: ""
-		};
-		const filePath = join(outputDir, `${makeUniqueFilename(transcriptFilename(document), used)}${transcriptFileExtension(format)}`);
-		if (!await shouldWriteFile(filePath, document.updatedAt)) continue;
-		const content = renderTranscriptExport(buildTranscriptExport(document, segments), format);
-		if (!content) continue;
-		await writeTextFile(filePath, content);
-		written += 1;
-	}
-	return written;
 }
 //#endregion
 //#region src/commands/transcripts.ts

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "granola-toolkit",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "description": "CLI toolkit for exporting and working with Granola notes and transcripts",
   "keywords": [
     "cli",