@myrialabs/clopen 0.0.1

package/shared/utils/file-type-detection.ts

@@ -0,0 +1,166 @@
+import { fileTypeFromBuffer } from 'file-type';
+import isTextPath from 'is-text-path';
+
+import { debug } from '$shared/utils/logger';
+/**
+ * Simple and reliable text file detection using external libraries
+ */
+export async function isTextFile(filePath: string): Promise<boolean> {
+	try {
+		// Fast path: check by extension first (covers 90% of cases)
+		if (isTextPath(filePath)) {
+			return true;
+		}
+		
+		const file = Bun.file(filePath);
+		const buffer = Buffer.from(await file.arrayBuffer());
+		
+		// Empty files are text
+		if (buffer.length === 0) {
+			return true;
+		}
+		
+		// Let file-type check for binary formats
+		const detectedType = await fileTypeFromBuffer(buffer);
+		if (detectedType) {
+			// If it detected something, check if content is actually text
+			return isContentText(buffer);
+		}
+		
+		// Unknown format - analyze content
+		return isContentText(buffer);
+		
+	} catch (error) {
+		debug.error('file', 'Error checking if file is text:', error);
+		return false;
+	}
+}
+
+/**
+ * Simple content analysis for text detection
+ */
+function isContentText(buffer: Buffer): boolean {
+	const sample = buffer.slice(0, Math.min(8192, buffer.length));
+	let textBytes = 0;
+	let nullBytes = 0;
+	
+	for (let i = 0; i < sample.length; i++) {
+		const byte = sample[i];
+		
+		if (byte === 0) {
+			nullBytes++;
+		} else if (
+			(byte >= 0x20 && byte <= 0x7E) || // Printable ASCII
+			byte === 0x09 || byte === 0x0A || byte === 0x0D || // Whitespace
+			byte >= 0x80 // UTF-8 or other text encodings
+		) {
+			textBytes++;
+		}
+	}
+	
+	const textRatio = textBytes / sample.length;
+	const nullRatio = nullBytes / sample.length;
+	
+	// Handle UTF-16 (has many nulls but in pattern)
+	if (nullRatio > 0.3) {
+		return isLikelyUTF16(sample);
+	}
+	
+	// If mostly text characters, it's text
+	return textRatio > 0.7;
+}
+
+/**
+ * Check if buffer might be UTF-16
+ */
+function isLikelyUTF16(buffer: Buffer): boolean {
+	// Check for BOM
+	if (buffer.length >= 2) {
+		if ((buffer[0] === 0xFF && buffer[1] === 0xFE) || 
+			(buffer[0] === 0xFE && buffer[1] === 0xFF)) {
+			return true;
+		}
+	}
+	
+	// Check for alternating null pattern
+	let evenNulls = 0;
+	let oddNulls = 0;
+	
+	for (let i = 0; i < Math.min(200, buffer.length); i++) {
+		if (buffer[i] === 0) {
+			if (i % 2 === 0) evenNulls++;
+			else oddNulls++;
+		}
+	}
+	
+	// UTF-16 has nulls mostly on one side
+	return Math.abs(evenNulls - oddNulls) > (evenNulls + oddNulls) * 0.6;
+}
+
+/**
+ * Auto-detect encoding and read file content
+ */
+export async function readFileWithEncoding(filePath: string): Promise<{ content: string; detectedEncoding: string }> {
+	try {
+		const file = Bun.file(filePath);
+		const buffer = Buffer.from(await file.arrayBuffer());
+		
+		// Check BOM for encoding
+		if (buffer.length >= 3 && buffer[0] === 0xEF && buffer[1] === 0xBB && buffer[2] === 0xBF) {
+			return {
+				content: buffer.subarray(3).toString('utf8'),
+				detectedEncoding: 'utf-8'
+			};
+		}
+		
+		if (buffer.length >= 2) {
+			if (buffer[0] === 0xFF && buffer[1] === 0xFE) {
+				return {
+					content: buffer.subarray(2).toString('utf16le'),
+					detectedEncoding: 'utf16le'
+				};
+			}
+			if (buffer[0] === 0xFE && buffer[1] === 0xFF) {
+				// Convert UTF-16 BE to LE
+				const swapped = Buffer.alloc(buffer.length - 2);
+				for (let i = 2; i < buffer.length - 1; i += 2) {
+					swapped[i - 2] = buffer[i + 1];
+					swapped[i - 1] = buffer[i];
+				}
+				return {
+					content: swapped.toString('utf16le'),
+					detectedEncoding: 'utf16be'
+				};
+			}
+		}
+		
+		// Try UTF-8, fallback to latin1 if invalid
+		const utf8Content = buffer.toString('utf8');
+		if (!utf8Content.includes('\ufffd')) {
+			return { content: utf8Content, detectedEncoding: 'utf-8' };
+		}
+		
+		// Check if it's UTF-16 without BOM
+		if (isLikelyUTF16(buffer)) {
+			return {
+				content: buffer.toString('utf16le'),
+				detectedEncoding: 'utf16le'
+			};
+		}
+		
+		// Fallback to latin1
+		return {
+			content: buffer.toString('latin1'),
+			detectedEncoding: 'latin1'
+		};
+		
+	} catch (error) {
+		debug.error('file', 'Error reading file with encoding:', error);
+		const file = Bun.file(filePath);
+		const buffer = Buffer.from(await file.arrayBuffer());
+		return {
+			content: buffer.toString('latin1'),
+			detectedEncoding: 'latin1'
+		};
+	}
+}

package/shared/utils/logger.ts

@@ -0,0 +1,158 @@
+/**
+ * Custom Logger Module - debug
+ *
+ * Provides filtered logging capabilities with label-based categorization.
+ * All logs can be filtered by label, method type, and text content.
+ *
+ * Label Categories (22 total):
+ * - Communication & Real-time: WebSocket, MCP operations
+ * - File Operations: File I/O operations
+ * - Chat & Notifications: Messages and notifications
+ * - Terminal: Shell/PTY operations
+ * - Preview: Browser preview and video encoding
+ * - Data Persistence: Database, migrations, snapshots, checkpoints
+ * - Engine & Processing: Engine operations
+ * - User & Session: User and session management
+ * - Configuration & Settings: Settings
+ * - Infrastructure & Utilities: Server, git, project ops
+ * - Frontend/UI State: Workspace and checkpoint UI state
+ */
+
+export type LogLabel =
+	// Chat
+	| 'chat'
+	| 'checkpoint'
+	| 'snapshot'
+	
+	// Preview
+	| 'preview'
+	| 'webcodecs'
+
+	// File
+	| 'file'
+	
+	// Terminal
+	| 'terminal'
+	
+	// Git
+	| 'git'
+	
+	// Communication
+	| 'websocket'
+	| 'mcp'
+	| 'notification'
+	
+	// Configuration
+	| 'project'
+	| 'workspace'
+	| 'settings'
+	| 'engine'
+	| 'tunnel'
+	
+	// User
+	| 'user'
+	| 'session'
+	
+	// System
+	| 'server'
+	| 'database'
+	| 'migration'
+	| 'seeder';
+export type LogMethod = 'log' | 'info' | 'warn' | 'error' | 'debug' | 'trace';
+
+interface LoggerConfig {
+	enabled: boolean; // Global enable/disable based on NODE_ENV (production = disabled)
+	filterLabels: LogLabel[] | null; // null = show all, array = only show specified labels
+	filterMethods: LogMethod[] | null; // null = show all, array = only show specified methods
+	filterText: string | null; // null = no text filter, string = only show logs containing this text (case-sensitive)
+}
+
+// Default configuration - null means no filtering
+// enabled = true in development, false in production
+const config: LoggerConfig = {
+	enabled: process.env.NODE_ENV !== 'production',
+	filterLabels: ['snapshot', 'checkpoint', 'chat'],
+	// filterLabels: ['file'],
+	filterMethods: null,
+	filterText: null
+};
+
+/**
+ * Check if a log should be displayed based on current filters
+ */
+function shouldLog(label: LogLabel, method: LogMethod, args: any[]): boolean {
+	// Global enable/disable check first
+	if (!config.enabled) {
+		return false;
+	}
+
+	// Filter by label
+	if (config.filterLabels !== null && !config.filterLabels.includes(label)) {
+		return false;
+	}
+
+	// Filter by method
+	if (config.filterMethods !== null && !config.filterMethods.includes(method)) {
+		return false;
+	}
+
+	// Filter by text (case-sensitive)
+	if (config.filterText !== null) {
+		// Convert all arguments to string for searching
+		const argsString = args
+			.map((arg) => {
+				if (typeof arg === 'object') {
+					try {
+						return JSON.stringify(arg);
+					} catch {
+						return String(arg);
+					}
+				}
+				return String(arg);
+			})
+			.join(' ');
+
+		// Case-sensitive search
+		if (!argsString.includes(config.filterText)) {
+			return false;
+		}
+	}
+
+	return true;
+}
+
+/**
+ * Format log output with label prefix
+ */
+function formatOutput(label: LogLabel, args: any[]): any[] {
+	const d = new Date();
+	const timestamp =
+		`${String(d.getHours()).padStart(2, '0')}:` +
+		`${String(d.getMinutes()).padStart(2, '0')}:` +
+		`${String(d.getSeconds()).padStart(2, '0')}.` +
+		`${String(d.getMilliseconds()).padStart(3, '0')}`;
+
+	return [`[${timestamp}] [${label}]`, ...args];
+}
+
+/**
+ * Core logging function
+ */
+function createLogMethod(method: LogMethod) {
+	return (label: LogLabel, ...args: any[]): void => {
+		if (shouldLog(label, method, args)) {
+			const output = formatOutput(label, args);
+			console[method](...output);
+		}
+	};
+}
+
+// Export the debug logger object
+export const debug = {
+	log: createLogMethod('log'),
+	info: createLogMethod('info'),
+	warn: createLogMethod('warn'),
+	error: createLogMethod('error'),
+	debug: createLogMethod('debug'),
+	trace: createLogMethod('trace'),
+};

package/shared/utils/message-formatter.ts

@@ -0,0 +1,79 @@
+/**
+ * Message Formatter - Single source of truth for SDK message formatting
+ *
+ * Converts raw data → SDKMessageFormatter with metadata.
+ * Used by ALL paths that return messages to the frontend:
+ * - Backend: message-queries.ts, messages/crud.ts
+ * - Frontend: chat.service.ts (stream transport → metadata)
+ *
+ * This ensures stream response and DB response produce identical structure.
+ */
+
+import type { DatabaseMessage, SDKMessageFormatter } from '$shared/types/database/schema';
+import type { EngineSDKMessage } from '$shared/types/messaging';
+
+/**
+ * Format a DatabaseMessage into SDKMessageFormatter
+ * Parses sdk_message JSON and attaches system metadata
+ */
+export function formatDatabaseMessage(
+	msg: DatabaseMessage,
+	overrides?: {
+		sender_id?: string | null;
+		sender_name?: string | null;
+	}
+): SDKMessageFormatter {
+	const sdkMessage = JSON.parse(msg.sdk_message) as EngineSDKMessage;
+
+	return {
+		...sdkMessage,
+		metadata: {
+			...buildMetadataFromDb(msg, overrides),
+			...(sdkMessage.metadata?.reasoning && { reasoning: true }),
+		}
+	};
+}
+
+/**
+ * Build metadata object from DatabaseMessage fields
+ * This is the SINGLE definition of what metadata contains from DB
+ */
+export function buildMetadataFromDb(
+	msg: DatabaseMessage,
+	overrides?: {
+		sender_id?: string | null;
+		sender_name?: string | null;
+	}
+): NonNullable<SDKMessageFormatter['metadata']> {
+	return {
+		message_id: msg.id,
+		created_at: msg.timestamp,
+		parent_message_id: msg.parent_message_id || null,
+		sender_id: overrides?.sender_id ?? msg.sender_id ?? null,
+		sender_name: overrides?.sender_name ?? msg.sender_name ?? null,
+	};
+}
+
+/**
+ * Build metadata from stream transport fields (used by frontend during stream)
+ * Produces identical structure as buildMetadataFromDb
+ */
+export function buildMetadataFromTransport(transport: {
+	message_id?: string;
+	timestamp?: string;
+	parent_message_id?: string | null;
+	sender_id?: string | null;
+	sender_name?: string | null;
+	engine?: string;
+	reasoning?: boolean;
+}): NonNullable<SDKMessageFormatter['metadata']> {
+	return {
+		message_id: transport.message_id || undefined,
+		created_at: transport.timestamp || new Date().toISOString(),
+		parent_message_id: transport.parent_message_id ?? null,
+		sender_id: transport.sender_id ?? null,
+		sender_name: transport.sender_name ?? null,
+		engine: transport.engine,
+		...(transport.reasoning && { reasoning: true }),
+	};
+}

package/shared/utils/path.ts

@@ -0,0 +1,47 @@
+/**
+ * Shared path normalization utilities.
+ * Normalizes path separators to forward slashes for consistent display and comparison.
+ */
+
+/**
+ * Normalize path separators to forward slash and strip trailing slashes.
+ * Keeps trailing slash only for drive roots like "C:/".
+ */
+export function normalizePath(p: string): string {
+	let n = p.replace(/\\/g, '/');
+	// Keep trailing slash only for drive roots like "C:/"
+	if (n.length > 1 && !n.match(/^[A-Za-z]:\/$/)) {
+		n = n.replace(/\/+$/, '');
+	}
+	return n;
+}
+
+/**
+ * Compare two paths, case-insensitively on Windows paths.
+ */
+export function pathsEqual(a: string, b: string): boolean {
+	const na = normalizePath(a);
+	const nb = normalizePath(b);
+	if (/^[A-Za-z]:/.test(na) || /^[A-Za-z]:/.test(nb)) {
+		return na.toLowerCase() === nb.toLowerCase();
+	}
+	return na === nb;
+}
+
+/**
+ * Get relative path from a base path, normalized with forward slashes.
+ */
+export function getRelativePath(fullPath: string, basePath: string): string {
+	const normalizedFull = normalizePath(fullPath);
+	const normalizedBase = normalizePath(basePath);
+
+	if (normalizedFull.startsWith(normalizedBase)) {
+		let rel = normalizedFull.slice(normalizedBase.length);
+		if (rel.startsWith('/')) {
+			rel = rel.slice(1);
+		}
+		return rel;
+	}
+
+	return normalizePath(fullPath);
+}