@hank-warren/pi-stats 0.4.0 → 0.4.1

package/CHANGELOG.md

@@ -1,5 +1,15 @@
 # @hank-warren/pi-stats
 
+## 0.4.1
+
+### Patch Changes
+
+- c079c51: Scan sessions through a bounded worker pool instead of one file at a time.
+
+  The stats scan now runs an ordered pool capped at eight concurrent readers, reduced by the discovery index, which shortens a cold scan over a large session history. Results stay in discovery order regardless of completion order, and progress is reported monotonically.
+
+  Cancellation is settled rather than abandoned: an abort stops further work being assigned and publishes nothing afterwards, but waits for reads already in flight, so the caller's `catch` never runs while file reads are still landing in the result array or progress is still being reported into a dashboard that is being torn down.
+
 ## 0.4.0
 
 ### Minor Changes

package/cache.ts

@@ -240,6 +240,64 @@ export function statsCachePath(agentDir: string): string {
 	return join(agentDir, "pi-stats", "cache.json");
 }
 
+export async function mapConcurrentOrdered<T, R>(
+	items: readonly T[],
+	limit: number,
+	worker: (item: T, index: number) => Promise<R>,
+	onProgress?: (completed: number, total: number) => void,
+	signal?: AbortSignal,
+): Promise<R[]> {
+	if (!Number.isSafeInteger(limit) || limit <= 0) throw new Error("concurrency limit must be positive");
+	const results = new Array<R>(items.length);
+	let nextIndex = 0;
+	let completed = 0;
+	// The first failure (an abort, a worker throw, or a progress callback that
+	// throws) stops work being assigned and stops anything else being published,
+	// but never abandons work already in flight: the caller's `catch` must not
+	// run while file reads are still landing in `results` and progress is still
+	// being reported into a widget that is about to be torn down.
+	let failure: unknown;
+	const failed = () => failure !== undefined;
+	const run = async () => {
+		while (!failed()) {
+			try {
+				signal?.throwIfAborted();
+			} catch (error) {
+				failure ??= error;
+				return;
+			}
+			const index = nextIndex;
+			if (index >= items.length) return;
+			nextIndex += 1;
+			try {
+				const value = await worker(items[index]!, index);
+				// Re-check *after* the await: an abort during the read must not still
+				// publish a result or a progress tick into a dashboard that is being
+				// torn down. A sibling runner may not have noticed the abort yet.
+				signal?.throwIfAborted();
+				if (failed()) return;
+				results[index] = value;
+				completed += 1;
+				onProgress?.(completed, items.length);
+			} catch (error) {
+				failure ??= error;
+				return;
+			}
+		}
+	};
+	await Promise.all(Array.from({ length: Math.min(limit, items.length) }, () => run()));
+	if (failed()) throw failure;
+	return results;
+}
+
+type ScannedFile = {
+	file: string;
+	record?: CachedFileRecord;
+	reused: boolean;
+	unreadable: boolean;
+	changed: boolean;
+};
+
 export async function scanStats(options: ScanOptions): Promise<StatsIndex> {
 	const roots = await resolveSessionRoots(options);
 	const sessionFiles = await discoverJsonlFiles(roots, options.signal);
@@ -260,45 +318,65 @@ export async function scanStats(options: ScanOptions): Promise<StatsIndex> {
 	};
 	let changed = options.force === true || !loaded.valid;
 
-	for (let index = 0; index < files.length; index++) {
-		options.signal?.throwIfAborted();
-		const file = files[index]!;
-		try {
-			const metadata = await stat(file);
-			const previous = loaded.cache.files[file];
-			if (!options.force && sameFile(previous, metadata.size, metadata.mtimeMs)) {
-				next.files[file] = previous!;
-				diagnostics.reusedFiles++;
-			} else if (sidecars.has(file)) {
-				const parsed = parseUsageSidecar(file, await readFile(file, "utf8"));
-				next.files[file] = {
-					size: metadata.size,
-					mtimeMs: metadata.mtimeMs,
-					sidecar: parsed.records,
-					malformedLines: parsed.malformedLines,
-				};
-				diagnostics.parsedFiles++;
-				changed = true;
-			} else {
+	const scanned = await mapConcurrentOrdered(
+		files,
+		8,
+		async (file): Promise<ScannedFile> => {
+			options.signal?.throwIfAborted();
+			try {
+				const metadata = await stat(file);
+				const previous = loaded.cache.files[file];
+				if (!options.force && sameFile(previous, metadata.size, metadata.mtimeMs)) {
+					return { file, record: previous!, reused: true, unreadable: false, changed: false };
+				}
+				if (sidecars.has(file)) {
+					const parsed = parseUsageSidecar(file, await readFile(file, "utf8"));
+					return {
+						file,
+						record: {
+							size: metadata.size,
+							mtimeMs: metadata.mtimeMs,
+							sidecar: parsed.records,
+							malformedLines: parsed.malformedLines,
+						},
+						reused: false,
+						unreadable: false,
+						changed: true,
+					};
+				}
 				const parsed = parseSessionText(file, await readFile(file, "utf8"));
-				next.files[file] = {
-					size: metadata.size,
-					mtimeMs: metadata.mtimeMs,
-					...(parsed.ignored ? { ignored: true as const } : {}),
-					...(parsed.session ? { session: parsed.session } : {}),
-					malformedLines: parsed.malformedLines,
+				return {
+					file,
+					record: {
+						size: metadata.size,
+						mtimeMs: metadata.mtimeMs,
+						...(parsed.ignored ? { ignored: true as const } : {}),
+						...(parsed.session ? { session: parsed.session } : {}),
+						malformedLines: parsed.malformedLines,
+					},
+					reused: false,
+					unreadable: false,
+					changed: true,
 				};
-				diagnostics.parsedFiles++;
-				changed = true;
+			} catch {
+				options.signal?.throwIfAborted();
+				return { file, reused: false, unreadable: true, changed: true };
 			}
-			const current = next.files[file]!;
-			if (current.ignored) diagnostics.ignoredFiles++;
-			diagnostics.malformedLines += current.malformedLines;
-		} catch {
-			diagnostics.unreadableFiles++;
-			changed = true;
+		},
+		options.onProgress,
+		options.signal,
+	);
+
+	for (const result of scanned) {
+		if (result.record) {
+			next.files[result.file] = result.record;
+			if (result.reused) diagnostics.reusedFiles++;
+			else diagnostics.parsedFiles++;
+			if (result.record.ignored) diagnostics.ignoredFiles++;
+			diagnostics.malformedLines += result.record.malformedLines;
 		}
-		options.onProgress?.(index + 1, files.length);
+		if (result.unreadable) diagnostics.unreadableFiles++;
+		if (result.changed) changed = true;
 	}
 
 	if (Object.keys(loaded.cache.files).some((file) => !(file in next.files))) changed = true;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hank-warren/pi-stats",
-  "version": "0.4.0",
+  "version": "0.4.1",
   "description": "A compact /stats dashboard for all-time Pi token usage, including persisted subagents.",
   "type": "module",
   "keywords": [