khal-os 1.260324.2

package/packages/os-sdk/src/config.ts

@@ -0,0 +1,68 @@
+/**
+ * OS Config — source of truth for all environment variables.
+ *
+ * All values are resolved eagerly at module import time.
+ * Required variables that are missing will throw immediately (fail-fast).
+ *
+ * Server-side exports: NATS_URL, getDatabaseUrl()
+ * Path config exports (server-only, no default applied): FILES_ROOT_ENV, AGENT_REPO_ROOT_ENV, AGENT_WORKTREE_ROOT_ENV
+ * getDatabaseUrl() is lazy (not resolved at import) so client components can safely
+ * import this module without triggering a browser crash.
+ */
+
+/**
+ * URL of the NATS server.
+ *
+ * Resolution order: NATS_URL → 'nats://localhost:4222'
+ * Never throws — falls back to localhost.
+ */
+export const NATS_URL: string = process.env.NATS_URL ?? 'nats://localhost:4222';
+
+/**
+ * PostgreSQL connection string.
+ *
+ * Lazy — resolved on first access, not at import time.
+ * This allows client components to safely import this module without triggering
+ * a DATABASE_URL crash in the browser.
+ *
+ * Throws if accessed from the browser or if DATABASE_URL is not set.
+ */
+let _databaseUrl: string | undefined;
+
+export function getDatabaseUrl(): string {
+	if (typeof window !== 'undefined') {
+		throw new Error('[config] DATABASE_URL must not be accessed from the browser.');
+	}
+	if (_databaseUrl === undefined) {
+		const url = process.env.DATABASE_URL;
+		if (!url) {
+			throw new Error('[config] DATABASE_URL is required but not set. Set it in your .env file.');
+		}
+		_databaseUrl = url;
+	}
+	return _databaseUrl;
+}
+
+/**
+ * Raw env var for the files root directory.
+ * Use getFilesRoot() from @/lib/files/safe-path which applies the ~/genie-os-files default.
+ *
+ * Server-only path config — safe to read (just a string), but the default requires node:os.
+ */
+export const FILES_ROOT_ENV: string | undefined = process.env.OS_FILES_ROOT;
+
+/**
+ * Raw env var for the agent git repository root directory.
+ * Use getAgentRepoRoot() from @/lib/git/agent-repo which applies the default.
+ *
+ * Server-only path config.
+ */
+export const AGENT_REPO_ROOT_ENV: string | undefined = process.env.OS_AGENT_REPO_ROOT;
+
+/**
+ * Raw env var for the agent git worktree root directory.
+ * Use createWorktreeManager() from @/lib/git/worktree which applies the default.
+ *
+ * Server-only path config.
+ */
+export const AGENT_WORKTREE_ROOT_ENV: string | undefined = process.env.OS_AGENT_WORKTREE_ROOT;

package/packages/os-sdk/src/db/factory.test.ts

@@ -0,0 +1,42 @@
+// @ts-nocheck
+import { beforeEach, describe, expect, test } from 'bun:test';
+import { closeDb, getDb, initDb, isDbInitialized } from './factory';
+
+describe('db factory', () => {
+	beforeEach(async () => {
+		await closeDb();
+	});
+
+	test('throws if getDb() called before initDb()', () => {
+		expect(() => getDb()).toThrow('Call initDb() during host bootstrap');
+	});
+
+	test('singleton: getDb() returns same instance', () => {
+		initDb({
+			schema: {},
+			getDatabaseUrl: () => 'postgres://localhost:5432/test',
+		});
+		expect(isDbInitialized()).toBe(true);
+		const db1 = getDb();
+		const db2 = getDb();
+		expect(db1).toBe(db2);
+	});
+
+	test('closeDb resets db + config; getDb requires init again', async () => {
+		initDb({
+			schema: {},
+			getDatabaseUrl: () => 'postgres://localhost:5432/test',
+		});
+		const db1 = getDb();
+		await closeDb();
+		expect(isDbInitialized()).toBe(false);
+		expect(() => getDb()).toThrow('Call initDb() during host bootstrap');
+
+		initDb({
+			schema: {},
+			getDatabaseUrl: () => 'postgres://localhost:5432/test',
+		});
+		const db2 = getDb();
+		expect(db2).not.toBe(db1);
+	});
+});

package/packages/os-sdk/src/db/factory.ts

@@ -0,0 +1,72 @@
+import { drizzle } from 'drizzle-orm/postgres-js';
+import postgres from 'postgres';
+
+export interface DbConfig {
+	schema: Record<string, unknown>;
+	getDatabaseUrl: () => string;
+	appId?: string;
+	searchPath?: string; // e.g. 'myapp,public'
+}
+
+let _config: DbConfig | null = null;
+let _db: ReturnType<typeof drizzle> | null = null;
+let _client: ReturnType<typeof postgres> | null = null;
+
+/**
+ * Register database configuration during host bootstrap.
+ * Connection remains lazy and is opened on first getDb() call.
+ */
+export function initDb(config: DbConfig): void {
+	if (_config) return;
+	_config = config;
+}
+
+export function isDbInitialized(): boolean {
+	return _config !== null;
+}
+
+export function assertDbInitialized(): void {
+	if (!_config) {
+		throw new Error('[os-sdk] Database not initialized. Call initDb() during host bootstrap.');
+	}
+}
+
+/**
+ * Return the appId that was passed to initDb(), if any.
+ */
+export function getAppId(): string | undefined {
+	return _config?.appId;
+}
+
+/**
+ * Return the Drizzle DB instance, creating the connection on first call.
+ * Throws if initDb() has not been called yet.
+ */
+export function getDb() {
+	assertDbInitialized();
+	const config = _config;
+	if (!config) {
+		throw new Error('[os-sdk] Database not initialized. Call initDb() during host bootstrap.');
+	}
+	if (!_db) {
+		const url = config.getDatabaseUrl();
+		const opts: Record<string, unknown> = { max: 10, idle_timeout: 20, connect_timeout: 10 };
+		if (config.searchPath) {
+			opts.connection = { search_path: config.searchPath };
+		}
+		_client = postgres(url, opts);
+		_db = drizzle(_client, { schema: config.schema });
+	}
+	return _db;
+}
+
+export type Database = ReturnType<typeof getDb>;
+
+export async function closeDb(): Promise<void> {
+	if (_client) {
+		await _client.end();
+	}
+	_client = null;
+	_db = null;
+	_config = null;
+}

package/packages/os-sdk/src/db/migrate.ts

@@ -0,0 +1,140 @@
+import { existsSync, readdirSync, readFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { drizzle } from 'drizzle-orm/postgres-js';
+import { migrate } from 'drizzle-orm/postgres-js/migrator';
+import postgres from 'postgres';
+import { getDatabaseUrl } from '../config';
+
+const HOST_MIGRATIONS_FOLDER = './drizzle';
+
+/**
+ * Run host-only migrations from ./drizzle (backwards-compatible).
+ */
+export async function runMigrations(): Promise<void> {
+	const url = getDatabaseUrl();
+
+	// Drizzle migrator requires meta/_journal.json — skip if no migrations exist yet
+	if (!existsSync(`${HOST_MIGRATIONS_FOLDER}/meta/_journal.json`)) {
+		return;
+	}
+
+	const searchPath = process.env.GENIE_OS_SEARCH_PATH;
+	const opts: Record<string, unknown> = { max: 1 };
+	if (searchPath) {
+		opts.connection = { search_path: searchPath };
+	}
+	const migrationClient = postgres(url, opts);
+	const db = drizzle(migrationClient);
+
+	try {
+		await migrate(db, { migrationsFolder: HOST_MIGRATIONS_FOLDER });
+	} finally {
+		await migrationClient.end();
+	}
+}
+
+interface AppMigrationTarget {
+	packageName: string;
+	schemaName: string;
+	migrationsFolder: string;
+}
+
+/**
+ * Scan packages/* /drizzle/ for app migration folders.
+ * Returns a list of migration targets with their schema names.
+ */
+function discoverAppMigrations(): AppMigrationTarget[] {
+	const targets: AppMigrationTarget[] = [];
+	const packagesDir = resolve(process.cwd(), 'packages');
+
+	if (!existsSync(packagesDir)) return targets;
+
+	const entries = readdirSync(packagesDir, { withFileTypes: true });
+	for (const entry of entries) {
+		if (!entry.isDirectory()) continue;
+
+		const pkgDir = resolve(packagesDir, entry.name);
+		const journalPath = resolve(pkgDir, 'drizzle', 'meta', '_journal.json');
+
+		// Skip packages without drizzle/meta/_journal.json
+		if (!existsSync(journalPath)) continue;
+
+		const pkgJsonPath = resolve(pkgDir, 'package.json');
+		if (!existsSync(pkgJsonPath)) continue;
+
+		try {
+			const pkg = JSON.parse(readFileSync(pkgJsonPath, 'utf8'));
+			const schemaName = pkg.genieOs?.schema?.schemaName;
+
+			if (!schemaName) {
+				// biome-ignore lint/suspicious/noConsole: migration logging
+				console.warn(
+					`[db:migrate] package ${entry.name} has drizzle/ but no genieOs.schema.schemaName in package.json — skipping`
+				);
+				continue;
+			}
+
+			targets.push({
+				packageName: entry.name,
+				schemaName,
+				migrationsFolder: resolve(pkgDir, 'drizzle'),
+			});
+		} catch {
+			// biome-ignore lint/suspicious/noConsole: migration logging
+			console.warn(`[db:migrate] failed to read package.json for ${entry.name} — skipping`);
+		}
+	}
+
+	return targets;
+}
+
+/**
+ * Run migrations for a single app package with search_path isolation.
+ */
+async function runAppMigrations(target: AppMigrationTarget): Promise<void> {
+	const url = getDatabaseUrl();
+	const searchPath = `${target.schemaName},public`;
+
+	const migrationClient = postgres(url, {
+		max: 1,
+		connection: { search_path: searchPath },
+	});
+	const db = drizzle(migrationClient);
+
+	try {
+		// biome-ignore lint/suspicious/noConsole: migration logging
+		console.log(`[db:migrate] running migrations for ${target.packageName} (schema: ${target.schemaName})`);
+		await migrate(db, { migrationsFolder: target.migrationsFolder });
+		// biome-ignore lint/suspicious/noConsole: migration logging
+		console.log(`[db:migrate] migrations complete for ${target.packageName}`);
+	} finally {
+		await migrationClient.end();
+	}
+}
+
+/**
+ * Run all migrations: host first, then per-app.
+ * Each app's migrations run with search_path = <appSchema>,public.
+ * Apps without drizzle/ are silently skipped.
+ * Migration errors are fatal (not swallowed).
+ */
+export async function runAllMigrations(): Promise<void> {
+	// 1. Host migrations (OS core tables)
+	await runMigrations();
+
+	// 2. Per-app migrations
+	const appTargets = discoverAppMigrations();
+	for (const target of appTargets) {
+		await runAppMigrations(target);
+	}
+}
+
+if ((import.meta as { main?: boolean }).main) {
+	runAllMigrations()
+		.then(() => process.exit(0))
+		.catch((err) => {
+			// biome-ignore lint/suspicious/noConsole: CLI entrypoint must log errors
+			console.error('[db:migrate] failed:', err);
+			process.exit(1);
+		});
+}

package/packages/os-sdk/src/db/provision.ts

@@ -0,0 +1,44 @@
+/**
+ * App schema provisioning — core OS capability.
+ *
+ * Each app calls provisionAppSchema() on boot to create its isolated
+ * PostgreSQL schema + role via pgserve. Idempotent, safe for restarts.
+ *
+ * Requires Bun runtime (uses Bun.SQL for the pgserve admin connection).
+ */
+
+import { getDatabaseUrl } from '../config';
+
+export interface AppSchemaConfig {
+	name: string;
+	schemaName: string;
+	roleName: string;
+}
+
+export async function provisionAppSchema(config: AppSchemaConfig): Promise<{ created: boolean }> {
+	// pgserve is an optional runtime dependency — indirect import to skip tsc module resolution
+	const pgserveModule = 'pgserve';
+	// biome-ignore lint/suspicious/noExplicitAny: optional runtime dependency, no types guaranteed
+	const { initCatalog, provisionSchema } = (await import(pgserveModule)) as any;
+	// Bun.SQL is only available at runtime under Bun — indirect import to skip tsc module resolution
+	const bunModule = 'bun';
+	// biome-ignore lint/suspicious/noExplicitAny: Bun-only runtime API, no type declarations available
+	const { SQL } = (await import(bunModule)) as any;
+
+	const url = new URL(getDatabaseUrl());
+	const adminSql = new SQL({
+		hostname: url.hostname,
+		port: Number(url.port),
+		database: url.pathname.slice(1),
+		username: url.username,
+		password: url.password,
+	});
+
+	try {
+		await initCatalog(adminSql);
+		const result = await provisionSchema(adminSql, config);
+		return { created: result.created };
+	} finally {
+		await adminSql.close();
+	}
+}

package/packages/os-sdk/src/index.ts

@@ -0,0 +1,36 @@
+export type { ApiContext, ApiContextWithDb } from './api/handler';
+export { apiHandler, apiHandlerWithDb } from './api/handler';
+export {
+	AGENT_REPO_ROOT_ENV,
+	AGENT_WORKTREE_ROOT_ENV,
+	FILES_ROOT_ENV,
+	getDatabaseUrl,
+	NATS_URL,
+} from './config';
+export type { Database, DbConfig } from './db/factory';
+export {
+	assertDbInitialized,
+	closeDb,
+	getAppId,
+	getDb,
+	initDb,
+	initDb as createDb,
+	isDbInitialized,
+} from './db/factory';
+export { runMigrations } from './db/migrate';
+export { interceptConsole, restoreConsole } from './service/console-intercept';
+export type { LogEntry, Logger } from './service/logger';
+export { createLogger } from './service/logger';
+export {
+	ensureO11yStreams,
+	O11Y_STREAM_EVENTS,
+	O11Y_STREAM_LOGS,
+	O11Y_STREAM_TRACES,
+	O11Y_SUBJECT_EVENTS,
+	O11Y_SUBJECT_LOGS,
+	O11Y_SUBJECT_TRACES,
+} from './service/o11y-streams';
+export type { Msg, NatsConnection, ObserveConfig, ServiceConfig, ServiceHandler } from './service/runtime';
+export { createService } from './service/runtime';
+export type { TraceContext } from './service/trace';
+export { extractTrace, injectTrace, newSpan, PARENT_SPAN_HEADER, SPAN_HEADER, TRACE_HEADER } from './service/trace';

package/packages/os-sdk/src/service/console-intercept.ts

@@ -0,0 +1,60 @@
+import type { Logger } from './logger';
+
+interface OriginalConsole {
+	log: typeof console.log;
+	info: typeof console.info;
+	warn: typeof console.warn;
+	error: typeof console.error;
+	debug: typeof console.debug;
+}
+
+let originals: OriginalConsole | null = null;
+
+function formatArgs(args: unknown[]): string {
+	return args
+		.map((arg) => {
+			if (typeof arg === 'string') return arg;
+			if (arg instanceof Error) return `${arg.message}\n${arg.stack ?? ''}`;
+			try {
+				return JSON.stringify(arg);
+			} catch {
+				return String(arg);
+			}
+		})
+		.join(' ');
+}
+
+export function interceptConsole(log: Logger): () => void {
+	if (originals) {
+		// Already intercepted — avoid double-wrapping
+		return restoreConsole;
+	}
+
+	originals = {
+		log: console.log,
+		info: console.info,
+		warn: console.warn,
+		error: console.error,
+		debug: console.debug,
+	};
+
+	console.log = (...args: unknown[]) => log.info(formatArgs(args));
+	console.info = (...args: unknown[]) => log.info(formatArgs(args));
+	console.warn = (...args: unknown[]) => log.warn(formatArgs(args));
+	console.error = (...args: unknown[]) => log.error(formatArgs(args));
+	console.debug = (...args: unknown[]) => log.debug(formatArgs(args));
+
+	return restoreConsole;
+}
+
+export function restoreConsole(): void {
+	if (!originals) return;
+
+	console.log = originals.log;
+	console.info = originals.info;
+	console.warn = originals.warn;
+	console.error = originals.error;
+	console.debug = originals.debug;
+
+	originals = null;
+}

package/packages/os-sdk/src/service/logger.ts

@@ -0,0 +1,88 @@
+import type { NatsConnection } from '@nats-io/transport-node';
+
+export interface LogEntry {
+	ts: string;
+	level: 'debug' | 'info' | 'warn' | 'error';
+	service: string;
+	msg: string;
+	error?: { message: string; stack?: string };
+	trace_id?: string;
+	span_id?: string;
+	req_id?: string;
+	meta?: Record<string, unknown>;
+}
+
+export interface Logger {
+	debug(msg: string, meta?: Record<string, unknown>): void;
+	info(msg: string, meta?: Record<string, unknown>): void;
+	warn(msg: string, meta?: Record<string, unknown>): void;
+	error(msg: string, meta?: Record<string, unknown>): void;
+	child(defaultMeta: Record<string, unknown>): Logger;
+}
+
+const originalStdoutWrite = process.stdout.write.bind(process.stdout);
+
+function writeStdout(line: string): void {
+	originalStdoutWrite(`${line}\n`);
+}
+
+function applyMeta(entry: LogEntry, merged: Record<string, unknown>): void {
+	if (merged.error instanceof Error) {
+		entry.error = { message: merged.error.message, stack: merged.error.stack };
+		const { error: _, ...rest } = merged;
+		if (Object.keys(rest).length > 0) entry.meta = rest;
+		return;
+	}
+
+	const rest: Record<string, unknown> = {};
+	for (const [key, value] of Object.entries(merged)) {
+		if (key === 'trace_id' && value) entry.trace_id = value as string;
+		else if (key === 'span_id' && value) entry.span_id = value as string;
+		else if (key === 'req_id' && value) entry.req_id = value as string;
+		else rest[key] = value;
+	}
+	if (Object.keys(rest).length > 0) entry.meta = rest;
+}
+
+function publishToNats(nc: NatsConnection | null, subject: string, json: string): void {
+	if (!nc) return;
+	try {
+		nc.publish(subject, json);
+	} catch {
+		// Silently ignore publish failures
+	}
+}
+
+export function createLogger(
+	serviceName: string,
+	nc: NatsConnection | null,
+	defaultMeta?: Record<string, unknown>
+): Logger {
+	const subject = `os.o11y.logs.${serviceName}`;
+
+	function emit(level: LogEntry['level'], msg: string, meta?: Record<string, unknown>): void {
+		const entry: LogEntry = {
+			ts: new Date().toISOString(),
+			level,
+			service: serviceName,
+			msg,
+		};
+
+		const merged = defaultMeta ? { ...defaultMeta, ...meta } : meta;
+		if (merged) applyMeta(entry, merged);
+
+		const json = JSON.stringify(entry);
+		writeStdout(json);
+		publishToNats(nc, subject, json);
+	}
+
+	return {
+		debug: (msg, meta) => emit('debug', msg, meta),
+		info: (msg, meta) => emit('info', msg, meta),
+		warn: (msg, meta) => emit('warn', msg, meta),
+		error: (msg, meta) => emit('error', msg, meta),
+		child(childMeta) {
+			return createLogger(serviceName, nc, { ...defaultMeta, ...childMeta });
+		},
+	};
+}

package/packages/os-sdk/src/service/o11y-streams.ts

@@ -0,0 +1,88 @@
+/**
+ * JetStream stream provisioning for observability data.
+ *
+ * Creates ring buffer streams for logs, events, and traces on startup.
+ * Idempotent — safe to call on every service-loader boot.
+ */
+
+import { DiscardPolicy, jetstreamManager, RetentionPolicy, StorageType } from '@nats-io/jetstream';
+import { type NatsConnection } from '@nats-io/transport-node';
+
+// ── Stream names ────────────────────────────────────────────────────
+export const O11Y_STREAM_LOGS = 'OS_O11Y_LOGS';
+export const O11Y_STREAM_EVENTS = 'OS_O11Y_EVENTS';
+export const O11Y_STREAM_TRACES = 'OS_O11Y_TRACES';
+
+// ── Subject prefixes ────────────────────────────────────────────────
+export const O11Y_SUBJECT_LOGS = 'os.o11y.logs.>';
+export const O11Y_SUBJECT_EVENTS = 'os.o11y.events.>';
+export const O11Y_SUBJECT_TRACES = 'os.o11y.traces.>';
+
+const MB = 1024 * 1024;
+
+interface StreamDef {
+	name: string;
+	subjects: string[];
+	max_bytes: number;
+}
+
+const STREAMS: StreamDef[] = [
+	{ name: O11Y_STREAM_LOGS, subjects: [O11Y_SUBJECT_LOGS], max_bytes: 20 * MB },
+	{ name: O11Y_STREAM_EVENTS, subjects: [O11Y_SUBJECT_EVENTS], max_bytes: 10 * MB },
+	{ name: O11Y_STREAM_TRACES, subjects: [O11Y_SUBJECT_TRACES], max_bytes: 5 * MB },
+];
+
+/**
+ * Ensure all observability JetStream streams exist with the correct config.
+ * Creates streams that don't exist; updates streams whose config has drifted.
+ */
+export async function ensureO11yStreams(nc: NatsConnection): Promise<void> {
+	const jsm = await jetstreamManager(nc);
+
+	for (const def of STREAMS) {
+		try {
+			const info = await jsm.streams.info(def.name);
+
+			// Check if config needs updating
+			const cfg = info.config;
+			if (
+				cfg.max_bytes !== def.max_bytes ||
+				cfg.retention !== RetentionPolicy.Limits ||
+				JSON.stringify(cfg.subjects) !== JSON.stringify(def.subjects)
+			) {
+				await jsm.streams.update(def.name, {
+					subjects: def.subjects,
+					max_bytes: def.max_bytes,
+				});
+				console.log(`[o11y] updated stream ${def.name}`);
+			}
+		} catch (err: unknown) {
+			// Stream doesn't exist — create it
+			if (isStreamNotFound(err)) {
+				await jsm.streams.add({
+					name: def.name,
+					subjects: def.subjects,
+					retention: RetentionPolicy.Limits,
+					storage: StorageType.File,
+					max_bytes: def.max_bytes,
+					discard: DiscardPolicy.Old,
+				});
+				console.log(`[o11y] created stream ${def.name} (max ${def.max_bytes / MB}MB)`);
+			} else {
+				throw err;
+			}
+		}
+	}
+}
+
+function isStreamNotFound(err: unknown): boolean {
+	if (err && typeof err === 'object') {
+		// JetStreamApiError has a code property; 404 = stream not found
+		if ('code' in err && (err as { code: number }).code === 404) return true;
+		// Fallback: check message
+		if ('message' in err && typeof (err as { message: string }).message === 'string') {
+			return (err as { message: string }).message.includes('stream not found');
+		}
+	}
+	return false;
+}