khal-os 1.260324.2

package/src/components/apps/dev3000/service/index.ts

@@ -0,0 +1,520 @@
+import { existsSync, readdirSync, statSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { connect } from '@nats-io/transport-node';
+import type { D3kCmdRequest, D3kCmdResponse } from '../schema';
+import type { D3kLevel, D3kSource } from '../types';
+
+const startTime = Date.now();
+const D3K_BASE = join(homedir(), '.d3k');
+
+// ---------------------------------------------------------------------------
+// Binary resolution — find d3k and Chrome at runtime
+// ---------------------------------------------------------------------------
+
+function findBin(name: string, candidates: string[]): string {
+	for (const c of candidates) {
+		if (existsSync(c)) return c;
+	}
+	return name; // fallback to PATH
+}
+
+function findChromeBin(): string {
+	// Check puppeteer cache first
+	const puppeteerBase = join(homedir(), '.cache', 'puppeteer', 'chrome');
+	if (existsSync(puppeteerBase)) {
+		const versions = readdirSync(puppeteerBase).sort().reverse();
+		for (const v of versions) {
+			const bin = join(puppeteerBase, v, 'chrome-linux64', 'chrome');
+			if (existsSync(bin)) return bin;
+		}
+	}
+	return ''; // let d3k find it
+}
+
+const D3K_BIN =
+	process.env.D3K_BIN || findBin('d3k', [join(homedir(), '.npm-global', 'bin', 'd3k'), '/usr/local/bin/d3k']);
+const CHROME_BIN = process.env.CHROME_BIN || findChromeBin();
+
+// ---------------------------------------------------------------------------
+// d3k log file discovery — find the active session's log
+// ---------------------------------------------------------------------------
+
+function findD3kProjectDir(): string | null {
+	if (!existsSync(D3K_BASE)) return null;
+	const dirs = readdirSync(D3K_BASE).filter((d) => {
+		const full = join(D3K_BASE, d);
+		return statSync(full).isDirectory() && existsSync(join(full, 'logs'));
+	});
+	if (dirs.length === 0) return null;
+	// Pick the most recently modified project dir
+	dirs.sort((a, b) => {
+		const sa = statSync(join(D3K_BASE, a)).mtimeMs;
+		const sb = statSync(join(D3K_BASE, b)).mtimeMs;
+		return sb - sa;
+	});
+	return join(D3K_BASE, dirs[0]);
+}
+
+function findLatestLogFile(projectDir: string): string | null {
+	const logsDir = join(projectDir, 'logs');
+	if (!existsSync(logsDir)) return null;
+	const files = readdirSync(logsDir)
+		.filter((f) => f.endsWith('.log'))
+		.sort()
+		.reverse();
+	return files.length > 0 ? join(logsDir, files[0]) : null;
+}
+
+// Also check the d3k.log (cleared each session, always written to)
+function getD3kMainLog(projectDir: string): string | null {
+	const p = join(projectDir, 'd3k.log');
+	return existsSync(p) ? p : null;
+}
+
+// ---------------------------------------------------------------------------
+// d3k log line parser — format: "[HH:mm:ss.SSS] [SOURCE] message"
+// SOURCE can be SERVER or BROWSER, with optional sub-tags like [NETWORK], [ERROR], etc.
+// ---------------------------------------------------------------------------
+
+const D3K_LINE_RE = /^\[(\d{2}:\d{2}:\d{2}\.\d{3})\]\s+\[(\w+)]\s+(.*)$/;
+// biome-ignore lint/suspicious/noControlCharactersInRegex: stripping ANSI escape sequences
+const ANSI_RE = /\x1b\[[0-9;]*m/g;
+
+interface ParsedLogLine {
+	source: D3kSource;
+	level: D3kLevel;
+	message: string;
+	tag?: string;
+	rawTimestamp: string;
+}
+
+function classifySource(source: string, message: string): { source: D3kSource; tag?: string } {
+	const upper = source.toUpperCase();
+	if (upper === 'SERVER') return { source: 'SERVER' };
+	if (upper === 'BROWSER') {
+		// Extract sub-tag: [NETWORK], [ERROR], [NAVIGATION], [DOM], [SCREENSHOT], etc.
+		const tagMatch = message.match(/^\[(\w+(?:\.\w+)?)]/);
+		if (tagMatch) {
+			const tag = tagMatch[1].toUpperCase();
+			if (tag === 'NETWORK' || tag.startsWith('NETWORK')) return { source: 'NETWORK', tag };
+			return { source: 'BROWSER', tag };
+		}
+		return { source: 'BROWSER' };
+	}
+	return { source: 'UNKNOWN' };
+}
+
+function classifyLevel(message: string, tag?: string): D3kLevel {
+	if (tag === 'ERROR' || tag === 'RUNTIME.ERROR') return 'error';
+	if (tag === 'WARNING' || tag === 'CONSOLE WARNING') return 'warn';
+	if (tag === 'DEBUG') return 'debug';
+	const lower = message.toLowerCase();
+	if (lower.includes('error') || lower.includes('failed') || lower.includes('exception')) return 'error';
+	if (lower.includes('warn') || lower.includes('⚠')) return 'warn';
+	return 'info';
+}
+
+function parseLine(line: string): ParsedLogLine | null {
+	// Strip ANSI escape codes before parsing
+	const clean = line.replace(ANSI_RE, '');
+	const match = clean.match(D3K_LINE_RE);
+	if (!match) return null;
+	const [, rawTimestamp, rawSource, rawMessage] = match;
+	const { source, tag } = classifySource(rawSource, rawMessage);
+	const level = classifyLevel(rawMessage, tag);
+	return { source, level, message: rawMessage, tag, rawTimestamp };
+}
+
+// ---------------------------------------------------------------------------
+// File tail watcher
+// ---------------------------------------------------------------------------
+
+async function readNewLines(filePath: string, start: number, onLine: (parsed: ParsedLogLine) => void): Promise<void> {
+	const { createReadStream } = await import('node:fs');
+	const { createInterface } = await import('node:readline');
+	const stream = createReadStream(filePath, { start, encoding: 'utf8' });
+	const rl = createInterface({ input: stream });
+	for await (const line of rl) {
+		const trimmed = line.trim();
+		if (!trimmed) continue;
+		const parsed = parseLine(trimmed);
+		if (parsed) onLine(parsed);
+	}
+}
+
+async function tailFile(filePath: string, onLine: (parsed: ParsedLogLine) => void, signal: AbortSignal): Promise<void> {
+	const { watch } = await import('node:fs/promises');
+	let offset = existsSync(filePath) ? statSync(filePath).size : 0;
+
+	try {
+		const watcher = watch(filePath, { signal });
+		for await (const _event of watcher) {
+			if (signal.aborted) break;
+			const stat = statSync(filePath);
+			if (stat.size > offset) {
+				await readNewLines(filePath, offset, onLine);
+				offset = stat.size;
+			} else if (stat.size < offset) {
+				offset = 0;
+			}
+		}
+	} catch (err: unknown) {
+		if (signal.aborted) return;
+		console.error('[d3k-service] tail watcher error:', err);
+	}
+}
+
+// ---------------------------------------------------------------------------
+// Watch for new log files (d3k creates a new one each session)
+// ---------------------------------------------------------------------------
+
+async function watchForLogFiles(
+	projectDir: string,
+	onNewFile: (filePath: string) => void,
+	signal: AbortSignal
+): Promise<void> {
+	const { watch } = await import('node:fs/promises');
+	const logsDir = join(projectDir, 'logs');
+	const seen = new Set<string>();
+
+	// Track existing files
+	if (existsSync(logsDir)) {
+		for (const f of readdirSync(logsDir)) {
+			seen.add(f);
+		}
+	}
+
+	try {
+		const watcher = watch(logsDir, { signal });
+		for await (const event of watcher) {
+			if (signal.aborted) break;
+			if (event.filename && event.filename.endsWith('.log') && !seen.has(event.filename)) {
+				seen.add(event.filename);
+				const fullPath = join(logsDir, event.filename);
+				if (existsSync(fullPath)) {
+					onNewFile(fullPath);
+				}
+			}
+		}
+	} catch (err: unknown) {
+		if (signal.aborted) return;
+		console.error('[d3k-service] log dir watcher error:', err);
+	}
+}
+
+// ---------------------------------------------------------------------------
+// d3k process management
+// ---------------------------------------------------------------------------
+
+interface D3kManager {
+	get process(): import('node:child_process').ChildProcess | null;
+	get running(): boolean;
+	stop(): void;
+}
+
+function createD3kManager(signal: AbortSignal): D3kManager {
+	let child: import('node:child_process').ChildProcess | null = null;
+	let stopped = false;
+	const RESTART_DELAY = 5_000;
+
+	const doSpawn = async () => {
+		if (stopped || signal.aborted) return;
+
+		const { spawn } = await import('node:child_process');
+		const args = [
+			'--headless',
+			...(CHROME_BIN ? ['--browser', CHROME_BIN] : []),
+			'--no-agent',
+			'--no-tui',
+			'--command',
+			'bun dev',
+			'--port',
+			'8888',
+		];
+
+		console.log(`[d3k-service] spawning: ${D3K_BIN} ${args.join(' ')}`);
+
+		child = spawn(D3K_BIN, args, {
+			cwd: process.cwd(),
+			env: { ...process.env, FORCE_COLOR: '0', VERCEL_SANDBOX: '1' },
+			stdio: ['ignore', 'pipe', 'pipe'],
+		});
+
+		child.stdout?.on('data', (chunk: Buffer) => {
+			for (const line of chunk.toString().split('\n').filter(Boolean)) {
+				console.log(`[d3k] ${line}`);
+			}
+		});
+
+		child.stderr?.on('data', (chunk: Buffer) => {
+			for (const line of chunk.toString().split('\n').filter(Boolean)) {
+				console.error(`[d3k:err] ${line}`);
+			}
+		});
+
+		child.on('close', (code) => {
+			console.log(`[d3k-service] d3k exited with code ${code}`);
+			child = null;
+			if (!stopped && !signal.aborted) {
+				console.log(`[d3k-service] restarting d3k in ${RESTART_DELAY / 1000}s...`);
+				setTimeout(() => doSpawn(), RESTART_DELAY);
+			}
+		});
+	};
+
+	// Kill d3k when service shuts down
+	signal.addEventListener('abort', () => {
+		stopped = true;
+		if (child && !child.killed) child.kill('SIGTERM');
+	});
+
+	doSpawn();
+
+	return {
+		get process() {
+			return child;
+		},
+		get running() {
+			return child !== null && !child.killed;
+		},
+		stop() {
+			stopped = true;
+			if (child && !child.killed) child.kill('SIGTERM');
+		},
+	};
+}
+
+// ---------------------------------------------------------------------------
+// Command execution
+// ---------------------------------------------------------------------------
+
+async function runD3kCommand(command: string, args: string[] = []): Promise<D3kCmdResponse> {
+	const { spawn } = await import('node:child_process');
+	const CMD_TIMEOUT = 15_000;
+
+	return new Promise((resolve) => {
+		let resolved = false;
+		const done = (result: D3kCmdResponse) => {
+			if (resolved) return;
+			resolved = true;
+			clearTimeout(timer);
+			resolve(result);
+		};
+
+		const child = spawn(D3K_BIN, [command, '--json', ...args], {
+			cwd: process.cwd(),
+			env: { ...process.env, FORCE_COLOR: '0' },
+		});
+
+		const timer = setTimeout(() => {
+			child.kill('SIGTERM');
+			done({ ok: false, error: `command timed out after ${CMD_TIMEOUT / 1000}s` });
+		}, CMD_TIMEOUT);
+
+		let stdout = '';
+		let stderr = '';
+
+		child.stdout?.on('data', (chunk: Buffer) => {
+			stdout += chunk.toString();
+		});
+		child.stderr?.on('data', (chunk: Buffer) => {
+			stderr += chunk.toString();
+		});
+
+		child.on('close', (code) => {
+			if (code === 0) {
+				done({ ok: true, output: stdout.trim() || undefined });
+			} else {
+				done({ ok: false, error: stderr.trim() || `exit code ${code}`, output: stdout.trim() || undefined });
+			}
+		});
+
+		child.on('error', (err) => {
+			done({ ok: false, error: err.message });
+		});
+	});
+}
+
+// ---------------------------------------------------------------------------
+// Main
+// ---------------------------------------------------------------------------
+
+async function main() {
+	const nc = await connect({
+		servers: process.env.NATS_URL || 'nats://localhost:4222',
+	});
+
+	console.log('[d3k-service] connected to NATS');
+
+	let lineCount = 0;
+	const abortController = new AbortController();
+	const REPLAY_CAPACITY = 500;
+
+	const knownOrgIds = new Set<string>();
+	const replayBuffer: string[] = [];
+	let currentTailAbort: AbortController | null = null;
+
+	function extractOrgId(subject: string): string {
+		return subject.split('.')[1];
+	}
+
+	function makePayload(parsed: ParsedLogLine): string {
+		return JSON.stringify({
+			id: crypto.randomUUID(),
+			timestamp: Date.now(),
+			source: parsed.source,
+			level: parsed.level,
+			message: parsed.message,
+			tag: parsed.tag,
+		});
+	}
+
+	const publishEntry = (parsed: ParsedLogLine) => {
+		lineCount++;
+		const payload = makePayload(parsed);
+		replayBuffer.push(payload);
+		if (replayBuffer.length > REPLAY_CAPACITY) {
+			replayBuffer.splice(0, replayBuffer.length - REPLAY_CAPACITY);
+		}
+		for (const orgId of knownOrgIds) {
+			nc.publish(`khal.${orgId}.d3k.log`, payload);
+		}
+	};
+
+	// --- Spawn d3k process (with auto-restart) ---
+	const d3k = createD3kManager(abortController.signal);
+
+	// --- Wait for d3k to create its log, then tail it ---
+	const startTailing = async () => {
+		// Wait a bit for d3k to start and create its project dir + log
+		const waitForProjectDir = async (): Promise<string | null> => {
+			for (let i = 0; i < 30; i++) {
+				const dir = findD3kProjectDir();
+				if (dir) return dir;
+				await new Promise((r) => setTimeout(r, 1000));
+				if (abortController.signal.aborted) return null;
+			}
+			return null;
+		};
+
+		const projectDir = await waitForProjectDir();
+		if (!projectDir) {
+			console.error('[d3k-service] could not find d3k project dir after 30s');
+			return;
+		}
+		console.log(`[d3k-service] found d3k project: ${projectDir}`);
+
+		// Chain abort: when main controller aborts, also abort the current tail
+		abortController.signal.addEventListener('abort', () => {
+			if (currentTailAbort && !currentTailAbort.signal.aborted) currentTailAbort.abort();
+		});
+
+		// Tail the current latest log file
+		const tailLatest = async (logFile: string) => {
+			// Abort previous tail if any
+			if (currentTailAbort) currentTailAbort.abort();
+			currentTailAbort = new AbortController();
+
+			console.log(`[d3k-service] tailing ${logFile}`);
+			await readNewLines(logFile, 0, publishEntry);
+			await tailFile(logFile, publishEntry, currentTailAbort.signal);
+		};
+
+		// Start with current latest log
+		const initialLog = findLatestLogFile(projectDir) || getD3kMainLog(projectDir);
+		if (initialLog) {
+			tailLatest(initialLog).catch((err) => console.error('[d3k-service] tail error:', err));
+		} else {
+			console.log('[d3k-service] no log file yet, waiting...');
+		}
+
+		// Watch for new log files (d3k creates new ones on restart)
+		watchForLogFiles(
+			projectDir,
+			(newFile) => {
+				console.log(`[d3k-service] new log file detected: ${newFile}`);
+				tailLatest(newFile).catch((err) => console.error('[d3k-service] tail error:', err));
+			},
+			abortController.signal
+		).catch((err) => console.error('[d3k-service] log dir watch error:', err));
+	};
+
+	startTailing();
+
+	// --- khal.*.d3k.cmd (request-reply) ---
+	const cmdSub = nc.subscribe('khal.*.d3k.cmd');
+	(async () => {
+		for await (const msg of cmdSub) {
+			try {
+				knownOrgIds.add(extractOrgId(msg.subject));
+				let request: D3kCmdRequest = { command: 'logs' };
+				if (msg.data.length > 0) {
+					request = msg.json<D3kCmdRequest>();
+				}
+				console.log(`[d3k-service] cmd: ${request.command} ${(request.args || []).join(' ')}`);
+				const result = await runD3kCommand(request.command, request.args);
+				msg.respond(JSON.stringify(result));
+			} catch (err) {
+				console.error('[d3k-service] cmd error:', err);
+				msg.respond(JSON.stringify({ ok: false, error: (err as Error).message }));
+			}
+		}
+	})();
+
+	// --- khal.*.d3k.status (request-reply) ---
+	const statusSub = nc.subscribe('khal.*.d3k.status');
+	(async () => {
+		for await (const msg of statusSub) {
+			try {
+				const orgId = extractOrgId(msg.subject);
+				const isNew = !knownOrgIds.has(orgId);
+				knownOrgIds.add(orgId);
+
+				const entries: unknown[] = [];
+				for (const p of replayBuffer) {
+					try {
+						entries.push(JSON.parse(p));
+					} catch {
+						// skip corrupted buffer entries
+					}
+				}
+				console.log(
+					`[d3k-service] ${isNew ? 'registered' : 'status for'} orgId: ${orgId} (${entries.length} buffered)`
+				);
+				msg.respond(
+					JSON.stringify({
+						d3kRunning: d3k.running,
+						lineCount,
+						version: '0.0.167',
+						uptime: (Date.now() - startTime) / 1000,
+						entries,
+					})
+				);
+			} catch (err) {
+				console.error('[d3k-service] status error:', err);
+				msg.respond(JSON.stringify({ error: (err as Error).message }));
+			}
+		}
+	})();
+
+	// --- Graceful shutdown ---
+	const shutdown = async () => {
+		console.log('[d3k-service] shutting down...');
+		abortController.abort();
+		cmdSub.unsubscribe();
+		statusSub.unsubscribe();
+		await nc.close();
+		process.exit(0);
+	};
+
+	process.on('SIGINT', shutdown);
+	process.on('SIGTERM', shutdown);
+
+	await nc.closed();
+}
+
+main().catch((err) => {
+	console.error('[d3k-service] fatal:', err);
+	process.exit(1);
+});

package/src/components/apps/dev3000/service/runtime

@@ -0,0 +1 @@
+node

package/src/components/apps/dev3000/types.ts

@@ -0,0 +1,15 @@
+export type D3kSource = 'SERVER' | 'BROWSER' | 'D3K' | 'NETWORK' | 'UNKNOWN';
+
+export type D3kLevel = 'info' | 'warn' | 'error' | 'debug';
+
+export interface D3kLogEntry {
+	id: string;
+	timestamp: number;
+	source: D3kSource;
+	level: D3kLevel;
+	message: string;
+	/** Sub-tag from d3k log, e.g. NAVIGATION, SCREENSHOT, ERROR, NETWORK */
+	tag?: string;
+}
+
+export type D3kTab = 'timeline' | 'errors';

package/src/components/apps/dev3000/use-message-buffer.ts

@@ -0,0 +1,46 @@
+import { useCallback, useReducer } from 'react';
+import type { D3kLogEntry } from './types';
+
+const DEFAULT_CAPACITY = 1000;
+
+type Action = { type: 'push'; entry: D3kLogEntry } | { type: 'clear' } | { type: 'set-capacity'; capacity: number };
+
+interface State {
+	entries: D3kLogEntry[];
+	capacity: number;
+}
+
+function reducer(state: State, action: Action): State {
+	switch (action.type) {
+		case 'push': {
+			const next = [...state.entries, action.entry];
+			if (next.length > state.capacity) {
+				return { ...state, entries: next.slice(next.length - state.capacity) };
+			}
+			return { ...state, entries: next };
+		}
+		case 'clear':
+			return { ...state, entries: [] };
+		case 'set-capacity': {
+			const entries =
+				state.entries.length > action.capacity
+					? state.entries.slice(state.entries.length - action.capacity)
+					: state.entries;
+			return { capacity: action.capacity, entries };
+		}
+	}
+}
+
+export function useMessageBuffer(capacity = DEFAULT_CAPACITY) {
+	const [state, dispatch] = useReducer(reducer, { entries: [], capacity });
+
+	const push = useCallback((entry: D3kLogEntry) => {
+		dispatch({ type: 'push', entry });
+	}, []);
+
+	const clear = useCallback(() => {
+		dispatch({ type: 'clear' });
+	}, []);
+
+	return { entries: state.entries, push, clear };
+}