@orbitai/orbitrace 1.1.12 → 1.2.0

package/README.md

@@ -77,12 +77,66 @@ In short-lived processes (scripts, CI jobs, serverless functions), call `flush()
 await orbitrace.flush();
 ```
 
+### 5. Automatic Capture
+
+Pass `autoCapture: true` to `getInstance` (SDK 1.2.0 or newer) to capture what nobody handled and record breadcrumbs automatically. It is off unless you turn it on, so upgrading never changes how an application crashes or logs.
+
+```javascript
+orbitrace.getInstance({ ...config, autoCapture: true });
+// or choose parts:
+orbitrace.getInstance({ ...config, autoCapture: { exceptions: true, console: true, fetch: false } });
+```
+
+- **Uncaught exceptions and unhandled rejections** are captured. The process still crashes as Node would: the SDK waits up to 2 seconds for the event to send, prints the error and exits with code 1, and respects `--unhandled-rejections`. If the application registers its own `uncaughtException` or `unhandledRejection` handlers, they decide instead.
+- **Breadcrumbs** are recorded from `console` calls and `fetch` requests (URLs without credentials, query string or fragment). The SDK's own logs and requests are skipped.
+- `orbitrace.close()` removes everything automatic capture installed.
+
+### 6. Breadcrumbs
+
+Breadcrumbs are the last 30 things that happened before an event, sent with it. Add your own at important steps:
+
+```javascript
+orbitrace.addBreadcrumb({
+	category: "checkout",
+	message: "Payment authorized",
+	level: "info", // debug, info, warning or error
+	data: { orderId: "ord_1" },
+});
+```
+
+Breadcrumb data is filtered like metadata. Work that runs concurrently should have its own breadcrumbs: use the Express handlers or `wrap()` below, or `orbitrace.withScope(() => work())`.
+
+### 7. Express
+
+```javascript
+app.use(orbitrace.expressRequestHandler()); // before the routes: breadcrumbs per request
+app.post("/checkout", async (req, res, next) => {
+	try {
+		res.json(await checkout(req.body));
+	} catch (error) {
+		next(error); // Express 4 does not pass errors from async routes on by itself
+	}
+});
+app.use(orbitrace.expressErrorHandler()); // after the routes: captures with method, path and status, then calls next(error)
+```
+
+### 8. Cloud Functions and Other Serverless Handlers
+
+The runtime may freeze as soon as a handler returns. `wrap()` gives each call its own breadcrumbs, captures a thrown error (and rethrows it), and sends pending events before returning:
+
+```javascript
+exports.checkout = onRequest(orbitrace.wrap(async (req, res) => {
+	res.json(await checkout(req.body));
+}));
+```
+
 ## Data Collected
 
 Each event stores:
 
 - The error message (truncated to 200 characters) and stack trace, or the message text.
 - The `metadata` object you pass, plus `env`, `version` and `service` from your configuration.
+- Breadcrumbs, when there are any: the last 30 recorded before the event.
 - The IP address and user agent of the server that sent the request, with an approximate location derived from that IP.
 
 Before sending, metadata values whose key contains `password`, `passwd`, `secret`, `token`, `authorization`, `cookie`, `apikey`, `privatekey`, `credential` or `session` (ignoring case, `-` and `_`) are replaced with `[Filtered]`, at any depth. Error messages and stack traces are sent as they are.

package/dist/index.js

@@ -1,3 +1,6 @@
+const { AsyncLocalStorage } = require("node:async_hooks");
+const util = require("node:util");
+
 // Reporting must never slow down or break the host application, so every failure is logged and swallowed.
 const REQUEST_TIMEOUT_MS = 5000;
 // Repeats of the same event inside this window are counted locally and sent once as a single request.
@@ -5,6 +8,13 @@ const DEDUPE_WINDOW_MS = 10000;
 // Caps concurrent requests so a loop of distinct errors cannot exhaust the host's sockets or memory.
 const MAX_IN_FLIGHT = 10;
 const DEFAULT_RETRY_AFTER_SECONDS = 60;
+// Breadcrumbs kept per scope; the oldest is dropped when a new one arrives.
+const MAX_BREADCRUMBS = 30;
+const MAX_BREADCRUMB_MESSAGE_LENGTH = 200;
+const BREADCRUMB_LEVELS = ["debug", "info", "warning", "error"];
+// How long a crashing process waits for its last event before exiting.
+const CRASH_FLUSH_TIMEOUT_MS = 2000;
+const CONSOLE_LEVELS = { debug: "debug", log: "info", info: "info", warn: "warning", error: "error" };
 // Metadata keys containing any of these (ignoring case, "-" and "_") have their values replaced before sending.
 const SENSITIVE_KEY_PARTS = [
 	"password",
@@ -20,6 +30,9 @@ const SENSITIVE_KEY_PARTS = [
 ];
 const FILTERED_VALUE = "[Filtered]";
 
+// Each request or wrapped invocation gets its own breadcrumbs, so concurrent work never mixes histories.
+const scopes = new AsyncLocalStorage();
+
 function isSensitiveKey(key) {
 	const normalized = key.toLowerCase().replace(/[-_]/g, "");
 	return SENSITIVE_KEY_PARTS.some((part) => normalized.includes(part));
@@ -57,15 +70,41 @@ function filterSensitive(value, ancestors = new WeakSet()) {
 	return filtered;
 }
 
+function truncate(value, maxLength) {
+	return value.length > maxLength ? `${value.slice(0, maxLength)}...` : value;
+}
+
+// URLs are recorded without credentials, query string or fragment, which often carry tokens or personal data.
+function cleanUrl(url) {
+	try {
+		const parsed = new URL(url);
+		return `${parsed.origin}${parsed.pathname}`;
+	} catch {
+		return String(url).split(/[?#]/)[0];
+	}
+}
+
+// What Node does with an unhandled rejection when nothing else handles it (--unhandled-rejections).
+function unhandledRejectionMode() {
+	const flags = [...process.execArgv, ...(process.env.NODE_OPTIONS || "").split(/\s+/)].join(" ");
+	return flags.match(/--unhandled-rejections[= ](\S+)/)?.[1] || "throw";
+}
+
 class Orbitrace {
 	static #config = null;
 	static #pending = new Map();
 	static #inFlight = new Set();
 	static #pausedUntil = 0;
+	static #breadcrumbs = [];
+	// Undo functions for everything automatic capture installed; close() runs them.
+	static #restore = [];
+	static #fetch = null;
+	static #handlingCrash = false;
 
 	static getInstance(config) {
 		this.assertServerRuntime();
 		this.validateConfig(config);
+		this.close();
 
 		this.#config = {
 			apiKey: config.apiKey,
@@ -77,6 +116,12 @@ class Orbitrace {
 			service: config.service || "not-specified",
 			disabled: config.disabled || false,
 		};
+
+		const autoCapture = this.#autoCaptureOptions(config.autoCapture);
+
+		if (autoCapture && !this.#config.disabled) {
+			this.#installAutoCapture(autoCapture);
+		}
 	}
 
 	static validateConfig(config) {
@@ -112,6 +157,71 @@ class Orbitrace {
 		return this.#capture("capture_message", { message, metadata });
 	}
 
+	// Records something that happened, sent with the next event of the current scope.
+	static addBreadcrumb({ category = "manual", message = "", level = "info", data } = {}) {
+		try {
+			const breadcrumbs = scopes.getStore()?.breadcrumbs ?? this.#breadcrumbs;
+
+			breadcrumbs.push({
+				timestamp: Date.now(),
+				category: String(category).slice(0, 64),
+				level: BREADCRUMB_LEVELS.includes(level) ? level : "info",
+				message: truncate(String(message), MAX_BREADCRUMB_MESSAGE_LENGTH),
+				...(data !== undefined && { data: filterSensitive(data) }),
+			});
+
+			if (breadcrumbs.length > MAX_BREADCRUMBS) {
+				breadcrumbs.shift();
+			}
+		} catch {
+			// A breadcrumb that cannot be recorded is skipped.
+		}
+	}
+
+	// Runs callback with its own breadcrumbs, for work that is not an Express request or wrapped function.
+	static withScope(callback) {
+		return scopes.run({ breadcrumbs: [] }, callback);
+	}
+
+	// Wraps a serverless handler: each invocation gets its own breadcrumbs, a thrown error is captured, and
+	// pending events are sent before the handler returns, because the runtime may freeze right after.
+	static wrap(handler) {
+		return function orbitraceWrapped(...args) {
+			return Orbitrace.withScope(async () => {
+				try {
+					return await handler.apply(this, args);
+				} catch (error) {
+					await Orbitrace.captureException(error, { mechanism: "wrap" });
+					throw error;
+				} finally {
+					await Orbitrace.flush();
+				}
+			});
+		};
+	}
+
+	// Express middleware, registered before the routes, that gives each request its own breadcrumbs.
+	static expressRequestHandler() {
+		return function orbitraceRequestHandler(req, res, next) {
+			Orbitrace.withScope(() => next());
+		};
+	}
+
+	// Express error middleware, registered after the routes: captures the error and passes it on unchanged.
+	static expressErrorHandler() {
+		return function orbitraceErrorHandler(error, req, res, next) {
+			Orbitrace.captureException(error, {
+				mechanism: "express",
+				request: {
+					method: req.method,
+					path: String(req.originalUrl || req.url || "").split(/[?#]/)[0],
+					statusCode: error?.status || error?.statusCode || 500,
+				},
+			});
+			next(error);
+		};
+	}
+
 	// Sends counts still held by the dedupe window and waits for in-flight requests.
 	// Call it before a short-lived process or serverless invocation ends.
 	static async flush() {
@@ -119,6 +229,175 @@ class Orbitrace {
 		await Promise.all(this.#inFlight);
 	}
 
+	// Removes the handlers and wrappers automatic capture installed and forgets the global breadcrumbs.
+	static close() {
+		for (const undo of this.#restore.splice(0).reverse()) {
+			try {
+				undo();
+			} catch {
+				// Restoring one hook must not stop the others.
+			}
+		}
+
+		this.#fetch = null;
+		this.#breadcrumbs = [];
+		this.#handlingCrash = false;
+	}
+
+	// autoCapture: true, or { exceptions, console, fetch } to switch parts off. Off unless asked for, so
+	// upgrading never changes how an existing application handles crashes or logs.
+	static #autoCaptureOptions(value) {
+		if (!value) {
+			return null;
+		}
+
+		const options = value === true ? {} : value;
+
+		return {
+			exceptions: options.exceptions !== false,
+			console: options.console !== false,
+			fetch: options.fetch !== false,
+		};
+	}
+
+	static #installAutoCapture(options) {
+		if (options.exceptions) {
+			const onUncaughtException = (error) => this.#handleCrash(error, "uncaughtException");
+			const onUnhandledRejection = (reason) => this.#handleCrash(reason, "unhandledRejection");
+
+			process.on("uncaughtException", onUncaughtException);
+			process.on("unhandledRejection", onUnhandledRejection);
+			this.#restore.push(() => {
+				process.removeListener("uncaughtException", onUncaughtException);
+				process.removeListener("unhandledRejection", onUnhandledRejection);
+			});
+		}
+
+		if (options.console) {
+			let recording = false;
+
+			for (const [method, level] of Object.entries(CONSOLE_LEVELS)) {
+				const original = console[method];
+
+				if (typeof original !== "function") {
+					continue;
+				}
+
+				console[method] = function orbitraceConsole(...args) {
+					// The guard stops a breadcrumb that logs from recording itself again.
+					if (!recording) {
+						recording = true;
+						try {
+							const message = util.formatWithOptions(
+								{
+									depth: 1,
+									maxArrayLength: 10,
+									maxStringLength: MAX_BREADCRUMB_MESSAGE_LENGTH,
+									breakLength: Infinity,
+								},
+								...args
+							);
+
+							if (!message.startsWith("Orbitrace:")) {
+								Orbitrace.addBreadcrumb({ category: "console", level, message });
+							}
+						} catch {
+							// Unformattable arguments only lose their breadcrumb.
+						} finally {
+							recording = false;
+						}
+					}
+
+					return original.apply(this, args);
+				};
+				this.#restore.push(() => {
+					console[method] = original;
+				});
+			}
+		}
+
+		if (options.fetch && typeof globalThis.fetch === "function") {
+			const originalFetch = globalThis.fetch;
+			const endpoint = this.#config.endpoint;
+
+			// Orbitrace's own requests use the original fetch, so they are never recorded.
+			this.#fetch = originalFetch;
+			globalThis.fetch = async function orbitraceFetch(input, init) {
+				const url = typeof input === "string" || input instanceof URL ? String(input) : input?.url;
+
+				if (!url || url.startsWith(endpoint)) {
+					return originalFetch.call(this, input, init);
+				}
+
+				const method = String(init?.method || input?.method || "GET").toUpperCase();
+				const started = Date.now();
+				const data = { method, url: cleanUrl(url) };
+
+				try {
+					const response = await originalFetch.call(this, input, init);
+
+					Orbitrace.addBreadcrumb({
+						category: "http",
+						level: response.status >= 500 ? "error" : response.status >= 400 ? "warning" : "info",
+						message: `${method} ${data.url} ${response.status}`,
+						data: { ...data, status: response.status, durationMs: Date.now() - started },
+					});
+
+					return response;
+				} catch (error) {
+					Orbitrace.addBreadcrumb({
+						category: "http",
+						level: "error",
+						message: `${method} ${data.url} failed`,
+						data: { ...data, error: error?.message, durationMs: Date.now() - started },
+					});
+
+					throw error;
+				}
+			};
+			this.#restore.push(() => {
+				globalThis.fetch = originalFetch;
+			});
+		}
+	}
+
+	// Captures a crash, then does what Node would have done without this handler: exit with code 1, or follow
+	// --unhandled-rejections. When the application registered its own handlers, it decides instead.
+	static #handleCrash(reason, kind) {
+		if (this.#handlingCrash) {
+			return;
+		}
+
+		const decidesExit = process.listenerCount(kind) === 1;
+		const mode = kind === "unhandledRejection" ? unhandledRejectionMode() : "throw";
+		const error = reason instanceof Error ? reason : new Error(`Unhandled rejection: ${util.inspect(reason)}`);
+		const exits = decidesExit && ["throw", "strict"].includes(mode);
+
+		if (exits) {
+			this.#handlingCrash = true;
+		}
+
+		const delivered = Promise.race([
+			this.captureException(error, { mechanism: kind }).then(() => this.flush()),
+			// Not unref'd: this timer keeps a crashing process alive until it exits with code 1.
+			new Promise((resolve) => setTimeout(resolve, CRASH_FLUSH_TIMEOUT_MS)),
+		]).catch(() => {});
+
+		if (!decidesExit || mode === "none") {
+			return;
+		}
+
+		process.stderr.write(`${error.stack || error.message}\n`);
+
+		if (mode === "warn-with-error-code") {
+			process.exitCode = 1;
+		}
+
+		if (exits) {
+			delivered.finally(() => process.exit(1));
+		}
+	}
+
 	static async #capture(event, { metadata, ...fields }) {
 		if (!this.#config) {
 			console.error("Orbitrace: call Orbitrace.getInstance() before capturing events");
@@ -130,6 +409,7 @@ class Orbitrace {
 			return;
 		}
 
+		const breadcrumbs = [...(scopes.getStore()?.breadcrumbs ?? this.#breadcrumbs)];
 		const payload = {
 			...fields,
 			metadata: {
@@ -138,6 +418,7 @@ class Orbitrace {
 				version: this.#config.version,
 				service: this.#config.service,
 			},
+			...(breadcrumbs.length > 0 && { breadcrumbs }),
 		};
 		const key = JSON.stringify([event, payload.message, payload.stack ?? null]);
 		const pending = this.#pending.get(key);
@@ -187,7 +468,7 @@ class Orbitrace {
 
 	static async #post(event, payload, count) {
 		try {
-			const response = await fetch(this.#config.endpoint, {
+			const response = await (this.#fetch || fetch)(this.#config.endpoint, {
 				method: "POST",
 				headers: {
 					"Content-Type": "application/json",

package/package.json

@@ -1,12 +1,13 @@
 {
 	"name": "@orbitai/orbitrace",
-	"version": "1.1.12",
-	"description": "Error tracking and monitoring for backend Node.js applications",
+	"version": "1.2.0",
+	"description": "Error tracking for Node.js services: manual and automatic capture with breadcrumbs",
 	"main": "dist/index.js",
 	"files": [
 		"dist"
 	],
 	"scripts": {
+		"test": "node --test test/*.test.js",
 		"build": "mkdir -p dist && cp src/index.js dist/index.js",
 		"deploy": "npm version patch && npm run build && npm publish --access public"
 	},