@comprehend/telemetry-node 0.1.0

package/src/ComprehendDevSpanProcessor.ts

@@ -0,0 +1,563 @@
+import {sha256} from "@noble/hashes/sha2";
+import {utf8ToBytes} from "@noble/hashes/utils";
+import {Context} from "@opentelemetry/api";
+import {ReadableSpan, Span, SpanProcessor} from "@opentelemetry/sdk-trace-base";
+import {
+    DatabaseQueryObservation,
+    HttpClientObservation,
+    HttpServerObservation, NewObservedDatabaseConnectionMessage,
+    NewObservedDatabaseMessage, NewObservedDatabaseQueryMessage,
+    NewObservedHttpRequestMessage,
+    NewObservedHttpRouteMessage,
+    NewObservedHttpServiceMessage,
+    NewObservedServiceMessage,
+    ObservationInputMessage
+} from "./wire-protocol";
+import {analyzeSQL} from "./sql-analyzer";
+import {WebSocketConnection} from "./WebSocketConnection";
+
+interface ObservedService {
+    name: string;
+    namespace?: string;
+    environment?: string;
+    hash: string;
+    httpRoutes: ObservedHTTPRoute[];
+}
+
+interface ObservedHTTPRoute {
+    method: string;
+    route: string;
+    hash: string;
+}
+
+interface ObservedDatabase {
+    system: string;
+    connection?: string; // Connection string, hopefully scrubbed of user info
+    host?: string;
+    port?: number;
+    name?: string;
+    hash: string;
+}
+
+interface ObservedHttpService {
+    protocol: string;
+    host: string;
+    port?: number;
+    hash: string;
+}
+
+interface ObservedHttpRequest {
+    hash: string;
+}
+
+interface ObservedDatabaseConnection {
+    hash: string;
+    connection?: string;
+    user?: string;
+}
+
+interface ObservedDatabaseQuery {
+    hash: string;
+    query: string;
+}
+
+const sqlDbSystems = new Set([
+    'mysql', 'postgresql', 'mssql', 'oracle', 'db2', 'sqlite', 'hsqldb', 'h2',
+    'informix', 'cockroachdb', 'redshift', 'tidb', 'trino', 'greenplum'
+]);
+
+export class ComprehendDevSpanProcessor implements SpanProcessor {
+    private readonly connection: WebSocketConnection;
+    private observedServices: ObservedService[] = [];
+    private observedDatabases: ObservedDatabase[] = [];
+    private observedHttpServices: ObservedHttpService[] = [];
+    private observedInteractions: Map<string, Map<string, {
+        dbConnections: ObservedDatabaseConnection[],
+        dbQueries: ObservedDatabaseQuery[],
+        httpRequest?: ObservedHttpRequest
+    }>> = new Map();
+    private observationsSeq = 1;
+
+    constructor(options: { organization: string, token: string, debug?: boolean | ((message: string) => void) }) {
+        this.connection = new WebSocketConnection(options.organization, options.token,
+            options.debug === true ? console.log : options.debug === false ? undefined : options.debug);
+    }
+
+    onStart(span: Span, parentContext: Context): void {
+    }
+
+    onEnd(span: ReadableSpan): void {
+        const currentService = this.discoverService(span);
+        if (!currentService)
+            return;
+
+        const attrs = span.attributes;
+        if (span.kind === 1) {
+            // Server span, see if it's something to ingest.
+            if (attrs['http.route'] && attrs['http.method']) {
+                this.processHTTPRoute(currentService, attrs['http.route'] as string, attrs['http.method'] as string, span);
+            }
+        }
+        else if (span.kind === 2) {
+            // Client span, see if it's something to ingest.
+            if (attrs['http.url']) {
+                this.processHttpRequest(currentService, attrs['http.url'] as string, span);
+            }
+        }
+        if (attrs['db.system']) {
+            this.processDatabaseOperation(currentService, span);
+        }
+    }
+
+    private discoverService(span: ReadableSpan): ObservedService | undefined {
+        // Look for an existing matching entry.
+        const resAttrs = span.resource.attributes;
+        const name = resAttrs['service.name'] as string | undefined;
+        if (!name)
+            return;
+        const namespace = resAttrs['service.namespace'] as string | undefined;
+        const environment = resAttrs['deployment.environment'] as string | undefined;
+        const existing = this.observedServices.find(s =>
+            s.name === name &&
+            s.namespace === namespace &&
+            s.environment === environment
+        );
+        if (existing)
+            return existing;
+
+        // New; hash it and add it to the observed services.
+        const idString = `service:${name}:${namespace ?? ''}:${environment ?? ''}`;
+        const hash = hashIdString(idString);
+        const newService: ObservedService = {
+            name, namespace, environment, hash,
+            httpRoutes: []
+        };
+        this.observedServices.push(newService);
+
+        // Ingest its existence.
+        const message: NewObservedServiceMessage = {
+            event: "new-entity",
+            type: "service",
+            hash,
+            name,
+            ...(namespace ? { namespace } : {}),
+            ...(environment ? { environment } : {})
+        };
+        this.ingestMessage(message);
+    }
+
+    private processHTTPRoute(service: ObservedService, route: string, method: string, span: ReadableSpan): void {
+        // Check if this route+method already exists under the service
+        let observedRoute = service.httpRoutes.find(r =>
+            r.route === route && r.method === method
+        );
+        if (!observedRoute) {
+            // It's new; hash it and add it to our collection.
+            const idString = `http-route:${service.hash}:${method}:${route}`;
+            const hash = hashIdString(idString);
+            observedRoute = { method, route, hash };
+            service.httpRoutes.push(observedRoute);
+
+            // Emit observation message
+            const message: NewObservedHttpRouteMessage = {
+                event: "new-entity",
+                type: "http-route",
+                hash,
+                parent: service.hash,
+                method, route
+            };
+            this.ingestMessage(message);
+        }
+
+        // Extract the request path, making sure we only get that, without any query string.
+        const attrs = span.attributes;
+        let path: string;
+        if (attrs['http.target']) {
+            try {
+                // This might be just a path like "/search?q=foo"
+                const rawTarget = attrs['http.target'] as string;
+                const fakeUrl = new URL(rawTarget, 'http://placeholder'); // placeholder base
+                path = fakeUrl.pathname;
+            } catch {
+                path = '/';
+            }
+        }
+        else if (attrs['http.url']) {
+            try {
+                const rawUrl = attrs['http.url'] as string;
+                const parsed = new URL(rawUrl);
+                path = parsed.pathname;
+            } catch {
+                path = '/';
+            }
+        }
+        else {
+            path = '/';
+        }
+
+        // Build and ingest observation.
+        const status = attrs['http.status_code'] as number | undefined ?? 0;
+        const duration = span.duration;
+        const httpVersion = attrs['http.flavor'] as string | undefined;
+        const userAgent = attrs['http.user_agent'] as string | undefined;
+        const requestBytes = attrs['http.request_content_length'] as number | undefined;
+        const responseBytes = attrs['http.response_content_length'] as number | undefined;
+        const { message: errorMessage, type: errorType, stack: stack } = extractErrorInfo(span);
+        const observation: HttpServerObservation = {
+            type: 'http-server',
+            subject: observedRoute.hash,
+            timestamp: span.startTime,
+            path,
+            status,
+            duration,
+            ...(httpVersion ? { httpVersion } : {}),
+            ...(userAgent ? { userAgent } : {}),
+            ...(requestBytes !== undefined ? { requestBytes } : {}),
+            ...(responseBytes !== undefined ? { responseBytes } : {}),
+            ...(errorMessage ? { errorMessage } : {}),
+            ...(errorType ? { errorType } : {}),
+            ...(stack ? { stack } : {})
+        };
+        this.ingestMessage({
+            event: "observations",
+            seq: this.observationsSeq++,
+            observations: [observation]
+        });
+    }
+
+    private processDatabaseOperation(currentService: ObservedService, span: ReadableSpan): void {
+        // Parse the connection string.
+        const attrs = span.attributes;
+        const system = attrs['db.system'] as string;
+        const rawConnection = attrs['db.connection_string'] as string | undefined;
+        const parsed = rawConnection
+            ? parseDatabaseConnectionStringRaw(rawConnection)
+            : {
+                scrubbed: '',
+                user: undefined,
+                host: (attrs['net.peer.name'] ?? attrs['net.peer.ip']) as string | undefined,
+                port: (attrs['net.peer.port'] as number | undefined)?.toString(),
+                name: attrs['db.name'] as string | undefined,
+            };
+
+        // See if we already have an entry for this database.
+        const hash = hashIdString(`database:${system}:${parsed.host ?? ''}:${parsed.port ?? ''}:${parsed.name ?? ''}`);
+        let observedDatabase = this.observedDatabases.find(db => db.hash === hash);
+
+        // If we see it for the first time, add and ingest it.
+        if (!observedDatabase) {
+            observedDatabase = {
+                system,
+                host: parsed.host,
+                port: parsed.port ? parseInt(parsed.port) : undefined,
+                name: parsed.name,
+                hash
+            };
+            this.observedDatabases.push(observedDatabase);
+
+            // The existence of the database.
+            const message: NewObservedDatabaseMessage = {
+                event: "new-entity",
+                type: "database",
+                hash,
+                system,
+                ...(parsed.name ? { name: parsed.name } : {}),
+                ...(parsed.scrubbed ? { connection: parsed.scrubbed } : {}),
+                ...(parsed.host ? { host: parsed.host } : {}),
+                ...(parsed.port ? { port: parseInt(parsed.port) } : {}),
+                ...(parsed.user ? { user: parsed.user } : {})
+            }
+            this.ingestMessage(message);
+        }
+
+        // The connection to the database should have an interaction.
+        const interactions = this.getInteractions(currentService.hash, observedDatabase.hash);
+        let connectionInteraction = interactions.dbConnections.find(c => c.connection === parsed.scrubbed && c.user === parsed.user);
+        if (!connectionInteraction) {
+            connectionInteraction = {
+                hash: hashIdString(`db-connection:${currentService.hash}:${observedDatabase.hash}:${parsed.scrubbed ?? ''}:${parsed.user ?? ''}`),
+                connection: parsed.scrubbed,
+                user: parsed.user
+            };
+            interactions.dbConnections.push(connectionInteraction);
+            const message: NewObservedDatabaseConnectionMessage = {
+                event: "new-interaction",
+                type: "db-connection",
+                hash: connectionInteraction.hash,
+                from: currentService.hash,
+                to: observedDatabase.hash,
+                ...(connectionInteraction.connection ? { connection: connectionInteraction.connection } : {}),
+                ...(connectionInteraction.user ? { user: connectionInteraction.user } : {})
+            };
+            this.ingestMessage(message);
+        }
+
+        // The query of the database from the service (only for SQL for now).
+        if (sqlDbSystems.has(system) && attrs['db.statement']) {
+            // The interaction, based upon the normalized query.
+            let queryInfo = analyzeSQL(attrs['db.statement'] as string);
+            let queryInteraction = interactions.dbQueries.find(q => q.query === queryInfo.normalizedQuery);
+            if (!queryInteraction) {
+                queryInteraction = {
+                    hash: hashIdString(`db-query:${currentService.hash}:${observedDatabase.hash}:${queryInfo.normalizedQuery}`),
+                    query: queryInfo.normalizedQuery
+                };
+                interactions.dbQueries.push(queryInteraction);
+                const selects = getTablesWithOperation(queryInfo.tableOperations, 'SELECT');
+                const inserts = getTablesWithOperation(queryInfo.tableOperations, 'INSERT');
+                const updates = getTablesWithOperation(queryInfo.tableOperations, 'UPDATE');
+                const deletes = getTablesWithOperation(queryInfo.tableOperations, 'DELETE');
+                const message: NewObservedDatabaseQueryMessage = {
+                    event: "new-interaction",
+                    type: "db-query",
+                    hash: queryInteraction.hash,
+                    from: currentService.hash,
+                    to: observedDatabase.hash,
+                    query: queryInfo.presentableQuery,
+                    ...(selects ? { selects } : {}),
+                    ...(inserts ? { inserts } : {}),
+                    ...(updates ? { updates } : {}),
+                    ...(deletes ? { deletes } : {}),
+                };
+                this.ingestMessage(message);
+            }
+
+            // Build and ingest observation.
+            const duration = span.duration;
+            const returnedRows = (attrs['db.response.returned_rows'] as number | undefined)
+                ?? (attrs['db.sql.rows'] as number | undefined);
+            const { message: errorMessage, type: errorType, stack } = extractErrorInfo(span);
+            const observation: DatabaseQueryObservation = {
+                type: "db-query",
+                subject: queryInteraction.hash,
+                timestamp: span.startTime,
+                duration,
+                ...(errorMessage ? { errorMessage } : {}),
+                ...(errorType ? {  errorType } : {}),
+                ...(stack ? { stack } : {}),
+                ...(returnedRows !== undefined ? { returnedRows } : {})
+            };
+            this.ingestMessage({
+                event: "observations",
+                seq: this.observationsSeq++,
+                observations: [observation]
+            });
+        }
+    }
+
+    private processHttpRequest(currentService: ObservedService, url: string, span: ReadableSpan): void {
+        // Build identity based upon protocol, host, and port.
+        const parsed = new URL(url);
+        const protocol = parsed.protocol.replace(':', ''); // Remove trailing colon
+        const host = parsed.hostname;
+        const port = parsed.port ? parseInt(parsed.port) : (protocol === 'https' ? 443 : 80);
+        const idString = `http-service:${protocol}:${host}:${port}`;
+        const hash = hashIdString(idString);
+
+        // Ingest it if it's not already observed.
+        let observedHttpService = this.observedHttpServices.find(s =>
+            s.protocol === protocol && s.host === host && s.port === port
+        );
+        if (!observedHttpService) {
+            observedHttpService = { protocol, host, port, hash };
+            this.observedHttpServices.push(observedHttpService);
+
+            // The existence of the service.
+            const message: NewObservedHttpServiceMessage = {
+                event: "new-entity",
+                type: "http-service",
+                hash,
+                protocol,
+                host,
+                port
+            };
+            this.ingestMessage(message);
+        }
+
+        // Ingest the interaction if first observed.
+        const interactions = this.getInteractions(currentService.hash, observedHttpService.hash);
+        if (!interactions.httpRequest) {
+            const idString = `http-request:${currentService.hash}:${observedHttpService.hash}`;
+            const hash = hashIdString(idString);
+            interactions.httpRequest = { hash };
+            const message: NewObservedHttpRequestMessage = {
+                event: "new-interaction",
+                type: "http-request",
+                hash,
+                from: currentService.hash,
+                to: observedHttpService.hash
+            };
+            this.ingestMessage(message);
+        }
+
+        // Build and ingest observation.
+        const attrs = span.attributes;
+        const path = parsed.pathname || '/';
+        const method = span.attributes['http.method'] as string;
+        if (!method) // Really should always be there
+            return;
+        const status = attrs['http.status_code'] as number | undefined;
+        const duration = span.duration;
+        const httpVersion = span.attributes['http.flavor'] as string | undefined;
+        const requestBytes = attrs['http.request_content_length'] as number | undefined;
+        const responseBytes = attrs['http.response_content_length'] as number | undefined;
+        const { message: errorMessage, type: errorType, stack } = extractErrorInfo(span);
+        const observation: HttpClientObservation = {
+            type: "http-client",
+            subject: interactions.httpRequest.hash,
+            timestamp: span.startTime,
+            path,
+            method,
+            ...(status !== undefined ? { status } : {}),
+            duration,
+            ...(httpVersion !== undefined ? { httpVersion } : {}),
+            ...(requestBytes !== undefined ? { requestBytes } : {}),
+            ...(responseBytes !== undefined ? { responseBytes } : {}),
+            ...(errorMessage ? { errorMessage } : {}),
+            ...(errorType ? {  errorType } : {}),
+            ...(stack ? { stack } : {})
+        };
+        this.ingestMessage({
+            event: "observations",
+            seq: this.observationsSeq++,
+            observations: [observation]
+        });
+    }
+
+    private getInteractions(from: string, to: string) {
+        let fromMap = this.observedInteractions.get(from);
+        if (!fromMap) {
+            fromMap = new Map();
+            this.observedInteractions.set(from, fromMap);
+        }
+        let interactions = fromMap.get(to);
+        if (!interactions) {
+            interactions = { httpRequest: undefined, dbConnections: [], dbQueries: [] };
+            fromMap.set(to, interactions);
+        }
+        return interactions;
+    }
+
+    private ingestMessage(message: ObservationInputMessage) {
+        this.connection.sendMessage(message);
+    }
+
+    async forceFlush(): Promise<void> {
+    }
+
+    async shutdown(): Promise<void> {
+        this.connection.close()
+    }
+}
+
+function hashIdString(idString: string) {
+    return Array.from(sha256(utf8ToBytes(idString)))
+        .map(b => b.toString(16).padStart(2, '0'))
+        .join('');
+}
+
+/** Try to extract data from the database connection string. Many are URL based, fall back on
+ * some kind of comma-seperated style for the rest. */
+function parseDatabaseConnectionStringRaw(conn: string): {
+    scrubbed: string;
+    user?: string;
+    host?: string;
+    port?: string;
+    name?: string;
+} {
+    try {
+        // Try URL-style parsing first; scrub the user details if present.
+        const url = new URL(conn);
+        const user = url.username || undefined;
+        const host = url.hostname || undefined;
+        const port = url.port || undefined;
+        const dbName = url.pathname?.replace(/^\//, '') || undefined;
+        url.username = '';
+        url.password = '';
+        return {
+            scrubbed: url.toString(),
+            user,
+            host,
+            port,
+            name: dbName
+        };
+    } catch {
+        // Not a URL-style string; try semi-structured parsing
+        const parts = conn.split(';');
+        const kv: Record<string, string> = {};
+
+        for (const part of parts) {
+            const [k, v] = part.split('=');
+            if (k && v) kv[k.trim().toLowerCase()] = v.trim();
+        }
+
+        const user = kv['user id'] || kv['uid'];
+        const host = kv['server'] || kv['data source'] || kv['address'];
+        const port = kv['port'];
+        const name = kv['database'] || kv['initial catalog'];
+
+        // Reconstruct a scrubbed connection string without credentials
+        const scrubbed = Object.entries(kv)
+            .filter(([key]) => !['user id', 'uid', 'password', 'pwd'].includes(key))
+            .map(([k, v]) => `${k}=${v}`)
+            .join(';');
+
+        return {
+            scrubbed,
+            user,
+            host,
+            port,
+            name
+        };
+    }
+}
+
+/** Try pretty hard to get error info out of a span, if it has any. Handles it being there as an event,
+ * directly on the span with error semantics, and some other more ad-hoc cases. */
+function extractErrorInfo(span: ReadableSpan): {
+    message?: string;
+    type?: string;
+    stack?: string;
+} {
+    const attrs = span.attributes;
+
+    // Try to extract from a structured 'exception' event, as it should have more detail
+    const exceptionEvent = span.events.find(e => e.name === 'exception');
+    if (exceptionEvent?.attributes) {
+        const message = exceptionEvent.attributes['exception.message'] as string | undefined;
+        const type = exceptionEvent.attributes['exception.type'] as string | undefined;
+        const stack = exceptionEvent.attributes['exception.stacktrace'] as string | undefined;
+        if (message || type || stack) {
+            return { message, type, stack };
+        }
+    }
+
+    // Fallback to attributes directly on the span.
+    const isError = span.status.code === 2;
+    const message =
+        (attrs['exception.message'] as string | undefined) ??
+        (attrs['http.error_message'] as string | undefined) ??
+        (attrs['db.response.status_code'] as string | undefined) ??
+        (isError ? (attrs['otel.status_description'] as string | undefined) : undefined);
+    const type =
+        (attrs['exception.type'] as string | undefined) ??
+        (attrs['error.type'] as string | undefined) ??
+        (attrs['http.error_name'] as string | undefined);
+    const stack = attrs['exception.stacktrace'] as string | undefined;
+    return {
+        message,
+        type,
+        stack
+    };
+}
+
+function getTablesWithOperation(
+    tableOps: Record<string, string[]>,
+    operation: string
+): string[] | undefined {
+    const op = operation.toUpperCase();
+    const result = Object.entries(tableOps)
+        .filter(([_, ops]) => ops.includes(op))
+        .map(([table]) => table);
+    return result.length > 0 ? result : undefined;
+}

package/src/WebSocketConnection.ts

@@ -0,0 +1,121 @@
+import WebSocket from 'ws';
+import {
+    InitMessage,
+    NewObservedEntityMessage,
+    NewObservedInteractionMessage,
+    ObservationInputMessage,
+    ObservationMessage,
+    ObservationOutputMessage,
+} from './wire-protocol';
+
+const INGESTION_ENDPOINT = 'wss://ingestion.comprehend.dev';
+
+export class WebSocketConnection {
+    private readonly organization: string;
+    private readonly token: string;
+    private readonly logger?: (message: string) => void;
+    private readonly unacknowledgedObserved = new Map<string, NewObservedEntityMessage | NewObservedInteractionMessage>();
+    private readonly unacknowledgedObservations = new Map<number, ObservationMessage>();
+    private socket: WebSocket | null = null;
+    private reconnectDelay = 1000;
+    private shouldReconnect = true;
+    private authorized = false;
+
+    constructor(organization: string, token: string, logger?: (message: string) => void) {
+        this.organization = organization;
+        this.token = token;
+        this.logger = logger;
+        this.connect();
+    }
+
+    private log(message: string) {
+        if (this.logger) {
+            this.logger(message);
+        }
+    }
+
+    private connect(): void {
+        const webSocketUrl = `${INGESTION_ENDPOINT}/${this.organization}/observations`;
+        this.log(`Attempting to connect to ${webSocketUrl}...`);
+        this.socket = new WebSocket(webSocketUrl);
+
+        this.socket.on('open', () => this.onOpen());
+        this.socket.on('message', (data) => this.onMessage(data));
+        this.socket.on('close', (code, reason) => this.onClose(code, reason.toString()));
+        this.socket.on('error', (err) => this.onError(err));
+    }
+
+    private onOpen(): void {
+        this.log('WebSocket connected. Sending init/auth message.');
+        const init: InitMessage = {
+            event: 'init',
+            protocolVersion: 1,
+            token: this.token,
+        };
+        this.sendRaw(init);
+    }
+
+    private onMessage(data: WebSocket.Data): void {
+        try {
+            const msg: ObservationOutputMessage = JSON.parse(data.toString());
+
+            if (msg.type === 'ack-authorized') {
+                this.authorized = true;
+                this.log('Authorization acknowledged by server.');
+
+                for (const message of this.unacknowledgedObserved.values()) {
+                    this.sendRaw(message);
+                }
+                for (const message of this.unacknowledgedObservations.values()) {
+                    this.sendRaw(message);
+                }
+            }
+            else if (msg.type === 'ack-observed') {
+                this.unacknowledgedObserved.delete(msg.hash);
+            }
+            else if (msg.type === 'ack-observations') {
+                this.unacknowledgedObservations.delete(msg.seq);
+            }
+        } catch (e) {
+            this.log('Error parsing message from server: ' + (e instanceof Error ? e.message : String(e)));
+        }
+    }
+
+    private onClose(code: number, reason: string): void {
+        this.log(`WebSocket disconnected. Code: ${code}, Reason: ${reason}`);
+        this.authorized = false;
+        if (this.shouldReconnect) {
+            setTimeout(() => this.connect(), this.reconnectDelay);
+        }
+    }
+
+    private onError(error: Error): void {
+        this.log(`WebSocket encountered an error: ${error.message}`);
+    }
+
+    private sendRaw(message: ObservationInputMessage): void {
+        if (this.socket && this.socket.readyState === WebSocket.OPEN) {
+            this.socket.send(JSON.stringify(message));
+        }
+    }
+
+    public sendMessage(message: ObservationInputMessage): void {
+        if (message.event === 'new-entity' || message.event === 'new-interaction') {
+            this.unacknowledgedObserved.set(message.hash, message);
+        }
+        else if (message.event === 'observations') {
+            this.unacknowledgedObservations.set(message.seq, message);
+        }
+
+        if (this.authorized) {
+            this.sendRaw(message);
+        }
+    }
+
+    public close(): void {
+        this.shouldReconnect = false;
+        if (this.socket) {
+            this.socket.close();
+        }
+    }
+}

package/src/index.ts

@@ -0,0 +1,2 @@
+export {ComprehendDevSpanProcessor} from "./ComprehendDevSpanProcessor";
+