@db3.ai/app 0.1.0-beta.1 → 0.1.0-beta.4

package/src/logging/contracts/LoggingOptions.ts

@@ -1,3 +1,4 @@
+import type { LogTransportOptions } from './LogTransportOptions';
 import type { LoggerDriver } from './LoggerDriver';
 import type { LogBindings, LogLevel } from './Logger';
 
@@ -25,6 +26,10 @@ export interface DevtoolsLogOptions {
  * Application logging configuration.
  */
 export interface LoggingOptions {
+	/** Explicit destinations replace default console/file/devtools selection. An empty array disables output. */
+	transports?: LogTransportOptions[];
+	/** Optional JSON log file retained outside the process/container. The operator owns rotation and retention. */
+	file?: string;
 	/** Injectable logging driver used by tests or alternate implementations. */
 	driver?: LoggerDriver;
 	/** Minimum emitted severity. Defaults to info, or silent in tests. */
@@ -39,8 +44,14 @@ export interface LoggingOptions {
 	bindings?: LogBindings;
 	/** Sensitive field paths removed before records reach any destination. */
 	redact?: false | string[];
-	/** Writes newline-delimited JSON to standard output. Defaults to true. */
+	/** Writes logs to standard output. Defaults to true. */
 	console?: boolean;
+	/**
+	 * Console presentation only; files and devtools always retain structured JSON.
+	 * Auto (the default) uses pretty output in development terminals and JSON
+	 * elsewhere. Set PLATFORM_LOG_FORMAT to override the default process-wide.
+	 */
+	consoleFormat?: 'auto' | 'pretty' | 'json';
 	/**
 	 * Streams logs to Platform devtools.
 	 *

package/src/logging/contracts/index.ts

@@ -2,3 +2,7 @@ export * from './HttpExchange';
 export * from './Logger';
 export * from './LoggerDriver';
 export * from './LoggingOptions';
+export * from './EmailLogOptions';
+export * from './DevelopmentConsoleOptions';
+export * from './HttpExchangeMonitorOptions';
+export * from './LogTransportOptions';

package/src/logging/development/README.md

@@ -0,0 +1,88 @@
+# Development logging
+
+Development terminals show a small live waiting area immediately on request arrival.
+Completed requests move into permanent blocks in completion order, paired by
+process, source and request ID. Each block contains method, URL, status, duration
+and readable request/response bodies when capture is enabled:
+
+```text
+14:50:00 [http] POST → /api/v1/user  201 Created · 842 ms  #req-1
+  > request
+    {"user":1234}
+  < response
+    {"id":1234,"created":true}
+```
+
+The waiting area displays elapsed time, labels requests slow after five seconds,
+and shows at most five rows. It contracts on narrow terminals. Timeouts,
+disconnects and process shutdown produce explicit terminal outcomes. Up to 1,000
+pending requests retain context; eviction is reported and later responses still
+print. Normal console/stdout/stderr messages in the same process appear above
+pending rows. Closing the last logger restores the original stream methods,
+removes timers/listeners and clears the waiting area without clearing history.
+
+Install the shared capture adapter **before routes** on a Fastify server using
+`app().log.logger` as its `loggerInstance`:
+
+```ts
+import { registerHttpExchangeMonitor } from '@db3.ai/app/logging';
+
+registerHttpExchangeMonitor(server, {
+	exclude: request => request.url.startsWith('/private-reports/'),
+});
+```
+
+The adapter only captures in development. `enabled: false` disables capture;
+`maxBodyBytes` defaults to 16 KiB and is capped at 64 KiB per body. Credential
+fields are recursively redacted. Streams and binary bodies show metadata and
+are never consumed for logging. Exclusion controls body/header capture; disable
+route logging separately if even its URL must not appear. Existing JSON and
+devtools destinations retain the captured records.
+
+Small JSON values stay inline; larger values are indented. Terminal previews
+show at most 20 lines and 160 characters per line, with explicit omission markers.
+Errors retain bounded stacks and causes. Headers and routine process metadata
+stay out of the terminal presentation. Complete *captured* details remain in
+configured JSON files and devtools; capture limits still apply there.
+
+Production, tests and redirected/piped stdout keep newline-delimited JSON.
+Override this with `log: { consoleFormat: 'pretty' }` or
+`PLATFORM_LOG_FORMAT=pretty`; use `json` for raw records. Explicit app options take
+precedence. `auto` restores environment/terminal detection. `NO_COLOR` or
+`FORCE_COLOR=0` disables colours; `TERM=dumb` disables live redraws. Explicit
+pretty output to a pipe is append-only. Files and devtools remain structured.
+
+For a multi-process launcher, create **one** `DevelopmentConsole` from
+`@db3.ai/app/logging`, pipe each child's stdout/stderr into separate
+`console.createInput()` streams, and set child `PLATFORM_LOG_FORMAT=json`.
+Call `endSource(child.pid)` after a direct child closes, then `close()` and
+`await flush()` when all streams finish. The renderer cannot coordinate another
+process writing directly to the same terminal. It does not close caller-owned
+streams. Do not create a separate renderer for each child.
+
+The shipped `examples/runDevelopmentLogs.ts` demonstrates concurrent requests,
+payloads and interleaved client output without opening a listening socket.
+Interactive development also enables the devtools transport automatically;
+tests only enable network delivery when explicitly configured.
+
+
+## Structured logging lab
+
+From your independent app after [Installation](https://db3.ai/docs/installation):
+
+```sh
+mkdir -p examples
+cp -R node_modules/@db3.ai/app/src/logging/examples/. examples/
+npx tsx examples/runNoteLogs.ts
+```
+
+This captures the real Pino driver's output in a writable stream, with devtools HTTP delivery disabled. Expect three records: `Note saved`, `Summary failed`, and `Diagnostics enabled`. The first debug message is filtered, then changing `log.level` enables debug output. Changing it to silent suppresses later errors. The output omits unstable metadata and stack text only for readability.
+
+The [Logging guide](https://db3.ai/docs/logging#testing) contains the exact consumer test. Save it as `tests/logging/runNoteLogs.test.ts` and run:
+
+```sh
+npx vitest run tests/logging/runNoteLogs.test.ts
+npx tsc --noEmit --target ES2022 --module ESNext --moduleResolution Bundler --types node --skipLibCheck examples/*.ts
+```
+
+The test asserts structured context, error serialization, level changes and the absence of synthetic password/token/provider-key values. It uses the real logger, not a framework mock. This does not prove remote log ingestion or retention.

package/src/logging/drivers/PinoLoggerDriver.ts

@@ -1,15 +1,14 @@
 import { once } from 'node:events';
 import { fileURLToPath } from 'node:url';
 import { hostname } from 'node:os';
-import pino, {
-	type DestinationStream,
-	type Logger as PinoLogger,
-	type TransportTargetOptions,
-} from 'pino';
+import pino, { type DestinationStream, type Logger as PinoLogger, type TransportTargetOptions } from 'pino';
 import type ThreadStream from 'thread-stream';
 
 import { isInteractiveDevelopmentEnvironment } from '../../devtools';
 import type * as logging from '../contracts';
+import type { Mail } from '../../mail';
+import { EmailLogTransport } from '../transports/EmailLogTransport';
+import { acquireDevelopmentConsole } from '../acquireDevelopmentConsole';
 
 /**
  * Pino worker transport lifecycle members present at runtime but omitted from
@@ -32,6 +31,8 @@ interface LoggingEnvironment {
 	PLATFORM_LOG_DEVTOOLS?: string;
 	PLATFORM_LOG_LEVEL?: string;
 	PLATFORM_LOG_SOURCE?: string;
+	PLATFORM_LOG_FILE?: string;
+	PLATFORM_LOG_FORMAT?: string;
 	npm_package_name?: string;
 }
 
@@ -57,6 +58,8 @@ const defaultRedactions = [
 export class PinoLoggerDriver implements logging.LoggerDriver {
 	readonly logger: PinoLogger;
 	readonly #transport: ManagedPinoTransport | null;
+	readonly #emails: { level: logging.LogLevel; stream: EmailLogTransport }[] = [];
+	readonly #console?: ReturnType<typeof acquireDevelopmentConsole>;
 	#closed = false;
 	#closing = false;
 	#closePromise: Promise<void> | null = null;
@@ -66,13 +69,25 @@ export class PinoLoggerDriver implements logging.LoggerDriver {
 	 *
 	 * @param options - Framework logging configuration.
 	 * @param destination - Optional direct destination used by focused tests.
+	 * @param mail - Application-owned mail service used by the email destination.
 	 */
 	constructor(
 		options: logging.LoggingOptions = {},
 		destination?: DestinationStream,
+		mail?: () => Mail,
 	) {
 		const environment = options.environment ?? process.env.NODE_ENV ?? 'development';
 		const level = resolveLogLevel(options, process.env);
+		if ((options.transports?.filter(transport => transport.type === 'console').length ?? 0) > 1) throw new Error('Configure at most one console log transport.');
+		for (const transport of options.transports ?? []) {
+			if (!['console', 'file', 'email', 'devtools'].includes(transport.type)) throw new Error('Unknown logging transport type.');
+			if (transport.level !== undefined && !isLogLevel(transport.level)) throw new Error('Invalid logging transport level.');
+			if (transport.type === 'email' && transport.level !== 'silent') {
+				if (!mail) throw new Error('Email logging requires an application Mail resolver.');
+				this.#emails.push({ level: transport.level ?? 'error', stream: new EmailLogTransport(transport, mail) });
+			}
+		}
+
 		const source = options.source
 			?? process.env.PLATFORM_LOG_SOURCE
 			?? process.env.npm_package_name
@@ -96,24 +111,38 @@ export class PinoLoggerDriver implements logging.LoggerDriver {
 
 		if (destination) {
 			this.#transport = null;
-			this.logger = pino(pinoOptions, destination);
+			this.logger = pino(pinoOptions, this.#destinations(destination));
 			return;
 		}
 
-		const targets = transportTargets(options, level, environment, process.env);
+		const format = options.consoleFormat ?? process.env.PLATFORM_LOG_FORMAT ?? 'auto';
+		if (!['auto', 'pretty', 'json'].includes(format)) throw new Error(`Invalid PLATFORM_LOG_FORMAT value "${format}". Use auto, pretty, or json.`);
+		const consoleTransport = options.transports?.find(transport => transport.type === 'console');
+		const consoleEnabled = options.transports ? Boolean(consoleTransport) : options.console !== false;
+		const pretty = consoleEnabled && consoleTransport?.level !== 'silent' && level !== 'silent' && (format === 'pretty' || (format === 'auto' && isInteractiveDevelopmentEnvironment(environment) && Boolean(process.stdout.isTTY)));
+		const targets = transportTargets(pretty ? { ...options, console: false, transports: options.transports?.filter(transport => transport.type !== 'console') } : options, level, environment, process.env);
+		if (pretty) {
+			this.#transport = targets.length ? pino.transport({ targets }) as ManagedPinoTransport : null;
+			this.#transport?.on('error', reportTransportError);
+			this.#console = acquireDevelopmentConsole();
+			const streams: pino.StreamEntry[] = [{ level: (consoleTransport?.level === 'silent' ? 'trace' : consoleTransport?.level) ?? level, stream: this.#console.console }];
+			if (this.#transport) streams.push({ level: 'trace', stream: this.#transport });
+			this.logger = pino(pinoOptions, this.#destinations(pino.multistream(streams)));
+			return;
+		}
 
 		if (targets.length === 0) {
 			this.#transport = null;
 			this.logger = pino({
 				...pinoOptions,
-				level: 'silent',
-			});
+				level: this.#emails.length ? level : 'silent',
+			}, this.#emails.length ? this.#destinations({ /** Discards the disabled console destination. */ write() {} }) : undefined);
 			return;
 		}
 
-		if (targets.length === 1 && targets[0]?.target === 'pino/file') {
+		if (targets.length === 1 && targets[0]?.target === 'pino/file' && targets[0]?.options?.destination === 1 && targets[0]?.level === level) {
 			this.#transport = null;
-			this.logger = pino(pinoOptions);
+			this.logger = pino(pinoOptions, this.#destinations(process.stdout));
 			return;
 		}
 
@@ -121,7 +150,15 @@ export class PinoLoggerDriver implements logging.LoggerDriver {
 			targets,
 		}) as ManagedPinoTransport;
 		this.#transport.on('error', reportTransportError);
-		this.logger = pino(pinoOptions, this.#transport);
+		this.logger = pino(pinoOptions, this.#destinations(this.#transport));
+	}
+
+	/** Fans already serialized and redacted records out to ordinary destinations and operator email. */
+	#destinations(destination: DestinationStream): DestinationStream {
+		return this.#emails.length ? pino.multistream([
+			{ level: 'trace', stream: destination },
+			...this.#emails,
+		]) : destination;
 	}
 
 	/**
@@ -139,7 +176,8 @@ export class PinoLoggerDriver implements logging.LoggerDriver {
 			}
 
 			await new Promise<void>((resolve, reject) => {
-				this.logger.flush((error?: Error) => {
+				const destination = this.#transport ?? this.logger;
+				destination.flush((error?: Error) => {
 					if (error) {
 						reject(error);
 						return;
@@ -148,6 +186,8 @@ export class PinoLoggerDriver implements logging.LoggerDriver {
 					resolve();
 				});
 			});
+			await this.#console?.console.flush();
+			await Promise.all(this.#emails.map(email => email.stream.flush()));
 		} finally {
 			if (!this.#closing) {
 				this.#transport?.unref();
@@ -170,6 +210,7 @@ export class PinoLoggerDriver implements logging.LoggerDriver {
 		if (this.#closed) return;
 
 		this.#closing = true;
+		this.#console?.release();
 		await this.flush();
 		this.#closed = true;
 
@@ -224,17 +265,28 @@ function transportTargets(
 	if (level === 'silent') return [];
 
 	const targets: TransportTargetOptions[] = [];
+	if (options.transports) {
+		for (const transport of options.transports) {
+			const minimum = transport.level ?? level;
+			if (minimum === 'silent') continue;
+			if (transport.type === 'console') targets.push({ target: 'pino/file', level: minimum, options: { destination: 1 } });
+			if (transport.type === 'file') {
+				if (!transport.destination) throw new Error('File logging requires a destination.');
+				targets.push({ target: 'pino/file', level: minimum, options: { destination: transport.destination, mkdir: true } });
+			}
+			if (transport.type === 'devtools') targets.push({ target: devtoolsTransportTarget(), level: minimum, options: transport.options ?? {} });
+		}
+		return targets;
+	}
+
 
 	if (options.console !== false) {
-		targets.push({
-			target: 'pino/file',
-			level,
-			options: {
-				destination: 1,
-			},
-		});
+		targets.push({ target: 'pino/file', level, options: { destination: 1 } });
 	}
 
+	const file = options.file ?? env.PLATFORM_LOG_FILE;
+	if (file) targets.push({ target: 'pino/file', level, options: { destination: file, mkdir: true } });
+
 	if (devtoolsEnabled(options.devtools, environment, env)) {
 		targets.push({
 			target: devtoolsTransportTarget(),

package/src/logging/examples/runDevelopmentLogs.ts

@@ -0,0 +1,36 @@
+import { pathToFileURL } from 'node:url';
+import { setTimeout as delay } from 'node:timers/promises';
+import Fastify, { type FastifyBaseLogger } from 'fastify';
+import { Log, registerHttpExchangeMonitor } from '@db3.ai/app/logging';
+
+/**
+ * Demonstrates overlapping requests without a listening socket, database or provider.
+ * Run with tsx in a terminal to see pending rows turn into completed payload blocks.
+ */
+export async function runDevelopmentLogs(): Promise<void> {
+	const log = new Log({ environment: 'development', consoleFormat: 'pretty', devtools: false, source: 'demo' });
+	const server = Fastify({ loggerInstance: log.logger as FastifyBaseLogger });
+	registerHttpExchangeMonitor(server, { environment: 'development' });
+	server.post('/api/v1/user', async (request, reply) => {
+		await delay(1200);
+		reply.code(201);
+		return { id: 1234, name: 'Steve', created: true, input: request.body };
+	});
+	server.get('/api/v1/projects', async () => {
+		await delay(300);
+		return { projects: [{ id: 1, name: 'Example project' }], total: 1 };
+	});
+	try {
+		log.info('Development console ready');
+		await Promise.all([
+			server.inject({ method: 'POST', url: '/api/v1/user', payload: { user: 1234 } }),
+			server.inject('/api/v1/projects'),
+			delay(600).then(() => { console.log('Client rebuilt successfully'); }),
+		]);
+	} finally {
+		await server.close();
+		await log.close();
+	}
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) await runDevelopmentLogs();

package/src/logging/formatLogEmail.ts

@@ -0,0 +1,24 @@
+import type { MailMessage } from '../mail';
+import type { EmailLogOptions } from './contracts';
+import { redactLogEmail } from './redactLogEmail';
+
+/** Formats a redacted log record without copying arbitrary nested payloads or exception properties into email. */
+export function formatLogEmail(record: Record<string, unknown>, options: EmailLogOptions): MailMessage {
+	const context = Object.fromEntries(Object.entries(record).filter(([key, value]) => key !== 'err' && (value === null || ['string', 'number', 'boolean'].includes(typeof value))));
+	const sections = [`Log context (JSON):\n${JSON.stringify(context, null, 2)}`];
+	let error = record.err;
+	if (typeof error === 'string') sections.push(`Exception: ${error}`);
+	for (let depth = 0; error && typeof error === 'object' && depth < 4; depth++) {
+		const value = error as Record<string, unknown>;
+		sections.push(`${depth ? 'Caused by' : 'Exception'}: ${String(value.type ?? value.name ?? 'Error')}\nMessage: ${String(value.message ?? 'Unavailable')}\nStack trace:\n${String(value.stack ?? 'Unavailable')}`);
+		error = value.cause;
+	}
+	const text = redactLogEmail(sections.join('\n\n'));
+	const subject = redactLogEmail(`${options.subjectPrefix ?? 'Application error'}: ${String(record.msg ?? 'Exception logged')}`).replace(/[\r\n]/g, ' ').slice(0, 200);
+	return { to: options.to, subject, text, html: `<pre style="white-space:pre-wrap">${escapeHtml(text)}</pre>` };
+}
+
+/** Escapes log text as inert HTML content in the email's preformatted body. */
+function escapeHtml(text: string): string {
+	return text.replace(/[&<>"']/g, character => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&#39;' })[character]!);
+}

package/src/logging/highlightQueueMessage.ts

@@ -0,0 +1,38 @@
+/** Terminal colour for each queue state verb at the start of a `[queue]` message. */
+const STATES: Array<[RegExp, string]> = [
+	[/^(Claimed|Running|Dispatched)\b/, '34'],
+	[/^(Processed|Retried)\b/, '32'],
+	[/^(Released|Deferred)\b/, '33'],
+	[/^(Failed|Lost lease|Stopped fenced)\b/, '31'],
+];
+
+/** A job label such as `CollectKeywordRanksJob#5249`. */
+const JOB_LABEL = /\b[A-Z][A-Za-z0-9_]*#\d+\b/g;
+
+/**
+ * Colours the parts of a queue log message that are scanned most: the state
+ * verb after `[queue] ` (blue in progress, green done, yellow retrying, red
+ * failed) and job labels (bold magenta). Other messages are returned unchanged
+ * apart from job labels.
+ *
+ * @param message - Plain log message.
+ * @param paint - Wraps text in an SGR code, such as `'32'` or `'1;35'`.
+ * @returns The message with colour codes.
+ */
+export function highlightQueueMessage(message: string, paint: (text: string, code: string) => string): string {
+	let prefix = '';
+	let rest = message;
+	if (rest.startsWith('[queue] ')) {
+		prefix = '[queue] ';
+		rest = rest.slice(prefix.length);
+		for (const [pattern, code] of STATES) {
+			const match = pattern.exec(rest);
+			if (match) {
+				prefix += paint(match[0], code);
+				rest = rest.slice(match[0].length);
+				break;
+			}
+		}
+	}
+	return prefix + rest.replace(JOB_LABEL, label => paint(label, '1;35'));
+}