@comprehend/telemetry-node 0.1.0

package/src/sql-analyzer.ts

@@ -0,0 +1,316 @@
+type Token = { type: 'keyword' | 'identifier' | 'id-quote' | 'string' | 'comment' | 'punct' | 'operator' | 'whitespace' | 'unknown'; value: string };
+
+const KEYWORDS = new Set([
+    'SELECT', 'FROM', 'WHERE', 'INSERT', 'REPLACE', 'INTO', 'VALUES', 'DELETE', 'UPDATE',
+    'MERGE', 'SET', 'JOIN', 'LEFT', 'RIGHT', 'FULL', 'OUTER', 'INNER', 'ON', 'AS', 'AND', 'OR',
+    'NOT', 'IS', 'NULL', 'IN', 'WITH', 'RECURSIVE', 'UNION', 'ALL',
+    'GROUP', 'BY', 'HAVING', 'ORDER', 'LIMIT', 'OFFSET', 'LATERAL', 'USING'
+]);
+
+export interface SQLAnalysisResult {
+    tableOperations: Record<string, string[]>;
+    normalizedQuery: string;
+    presentableQuery: string;
+}
+
+/** Performs a rough tokenization of the SQL, extracts the tables involved and the operations on them, and
+ * produces two versions of the query:
+ * - A normalized version for hashing purposes that does not account for whitespace, comments, and collapses
+ *   IN clauses that might cause a cardinality explosion.
+ * - A presentable version that only does the IN clause collapsing */
+export function analyzeSQL(sql: string): SQLAnalysisResult {
+    let semanticTokens = new Array<Token>();
+    let presentableTokens = new Array<Token>();
+    let seekingInParen = false;
+    let analyzingIn = false;
+    let skippingIn = false;
+    for (let token of tokenizeSQL(sql)) {
+        switch (token.type) {
+            case "whitespace":
+            case "comment":
+            case "id-quote":
+                // Skip
+                break;
+            case "keyword":
+                // Normalize to uppercase.
+                semanticTokens.push({ type: "keyword", value: token.value.toUpperCase() });
+                break;
+            case "identifier":
+                // Normalize to lowercase.
+                semanticTokens.push({ type: "identifier", value: token.value.toLowerCase() });
+                break;
+            default:
+                semanticTokens.push(token);
+                break;
+        }
+
+        if (seekingInParen) {
+            // We saw IN, and now look for an opening (. Skip whitespace/comments, bail if anything else.
+            presentableTokens.push(token);
+            switch (token.type) {
+                case "comment":
+                case "whitespace":
+                    break;
+                case "punct":
+                    seekingInParen = false;
+                    analyzingIn = token.value === "(";
+                    break;
+                default:
+                    seekingInParen = false;
+                    break;
+            }
+        }
+        else if (analyzingIn) {
+            // We saw the opening paren of an IN. Pass over whitespace and comments. If we see a
+            // keyword we know it's not something to collapse, it's a sub-query. Otherwise, we
+            // enter skipping mode.
+            switch (token.type) {
+                case "comment":
+                case "whitespace":
+                    presentableTokens.push(token);
+                    break;
+                case "keyword":
+                case "punct": // maybe immediate ), certainly not a value
+                    presentableTokens.push(token);
+                    analyzingIn = false;
+                    break;
+                default:
+                    analyzingIn = false;
+                    skippingIn = true;
+                    presentableTokens.push({ type: "unknown", value: "..." })
+            }
+        }
+        else if (skippingIn) {
+            // Omit tokens until a closing ).
+            if (token.type === "punct" && token.value === ")") {
+                presentableTokens.push(token);
+                skippingIn = false;
+            }
+        }
+        else {
+            presentableTokens.push(token);
+            seekingInParen = token.type === "keyword" && token.value.toUpperCase() === "IN";
+        }
+    }
+    return {
+        ...analyzeSQLTokens(semanticTokens),
+        presentableQuery: presentableTokens.map(t => t.value).join("")
+    };
+}
+
+function* tokenizeSQL(sql: string): Generator<Token> {
+    const regex = /(--[^\n]*|\/\*[\s\S]*?\*\/)|('[^']*')|("(?:[^"]*)")|(`[^`]*`)|(\[[^\]]+\])|(\b[a-zA-Z_][\w$]*\b)|([(),;])|(<=|>=|<>|!=|=|<|>)|(\s+)|(\S)/g;
+    let match: RegExpExecArray | null;
+
+    while ((match = regex.exec(sql))) {
+        const [
+            full,
+            comment,         // 1
+            singleQuoted,    // 2: string literal
+            doubleQuoted,    // 3: "identifier"
+            backtickQuoted,  // 4: `identifier`
+            bracketQuoted,   // 5: [identifier]
+            word,            // 6
+            punct,           // 7
+            operator,        // 8
+            ws,              // 9
+            unknown          // 10
+        ] = match;
+        if (comment) {
+            yield { type: 'comment', value: comment };
+        }
+        else if (singleQuoted) {
+            yield { type: 'string', value: singleQuoted };
+        }
+        else if (doubleQuoted) {
+            yield { type: 'id-quote', value: '"' };
+            yield { type: 'identifier', value: doubleQuoted.slice(1, -1) };
+            yield { type: 'id-quote', value: '"' };
+        }
+        else if (backtickQuoted) {
+            yield { type: 'id-quote', value: '`' };
+            yield { type: 'identifier', value: backtickQuoted.slice(1, -1) };
+            yield { type: 'id-quote', value: '`' };
+        }
+        else if (bracketQuoted) {
+            yield { type: 'id-quote', value: '[' };
+            yield { type: 'identifier', value: bracketQuoted.slice(1, -1) };
+            yield { type: 'id-quote', value: ']' };
+        }
+        else if (word) {
+            yield { type: KEYWORDS.has(word.toUpperCase()) ? 'keyword' : "identifier", value: word };
+        }
+        else if (punct) {
+            yield { type: 'punct', value: punct };
+        }
+        else if (operator) {
+            yield {type: 'operator', value: operator};
+        }
+        else if (ws) {
+            yield { type: 'whitespace', value: ws };
+        }
+        else if (unknown) {
+            yield { type: 'unknown', value: unknown };
+        }
+    }
+}
+
+export function analyzeSQLTokens(tokens: Token[]) {
+    const aliasNames = new Set<string>();
+    const tableOps: Record<string, Set<string>> = {};
+    const normalizedTokens: string[] = [];
+
+    let currentOp: { ops: string[], at: number } | null = null;
+    let lastTokenType: string | null = null;
+
+    function appendToken(val: string, type: string) {
+        if (normalizedTokens.length && type !== 'punct' && lastTokenType !== 'punct') {
+            normalizedTokens.push(' ');
+        }
+        normalizedTokens.push(val);
+        lastTokenType = type;
+    }
+
+    for (let i = 0; i < tokens.length; ) {
+        const token = tokens[i];
+
+        // Record operation context
+        if (token.type === 'keyword' && ['SELECT', 'INSERT', 'UPDATE', 'DELETE'].includes(token.value)) {
+            currentOp = { ops: [token.value.toUpperCase()], at: i };
+        }
+        else if (token.type === "keyword" && token.value === 'USING') {
+            currentOp = { ops: ["SELECT"], at: i };
+        }
+        else if (token.type === "keyword" && token.value === 'REPLACE') {
+            currentOp = { ops: ["INSERT", "UPDATE"], at: i };
+        }
+        else if (token.type === "keyword" && token.value === 'MERGE') {
+            let sawInsert = false;
+            let sawUpdate = false;
+            let sawDelete = false;
+            for (let j = i + 1; j < tokens.length; j++) {
+                if (tokens[j].type === "keyword") {
+                    if (tokens[j].value === "INSERT")
+                        sawInsert = true;
+                    if (tokens[j].value === "UPDATE")
+                        sawUpdate = true;
+                    if (tokens[j].value === "DELETE")
+                        sawDelete = true;
+                }
+            }
+            currentOp = { ops: [], at: i };
+            if (sawInsert)
+                currentOp.ops.push("INSERT");
+            if (sawUpdate)
+                currentOp.ops.push("UPDATE");
+            if (sawDelete)
+                currentOp.ops.push("DELETE");
+        }
+
+        // Detect CTE-style alias: <identifier> AS (
+        if (
+            token.type === 'identifier' &&
+            tokens[i + 1]?.type === 'keyword' &&
+            tokens[i + 1].value === 'AS' &&
+            tokens[i + 2]?.value === '('
+        ) {
+            const alias = token.value.toLowerCase();
+            aliasNames.add(alias);
+            appendToken(token.value, token.type);
+            appendToken('AS', 'keyword');
+            appendToken('(', 'punct');
+            i += 3;
+            continue;
+        }
+
+        // Detect AS <alias> (table aliases, subquery aliases, etc.)
+        if (
+            token.type === 'keyword' &&
+            token.value.toUpperCase() === 'AS' &&
+            tokens[i + 1]?.type === 'identifier'
+        ) {
+            const alias = tokens[i + 1].value.toLowerCase();
+            aliasNames.add(alias);
+            appendToken(token.value, token.type);
+            appendToken(tokens[i + 1].value, tokens[i + 1].type);
+            i += 2;
+            continue;
+        }
+
+        // Record table name if in FROM, JOIN, INTO, UPDATE
+        if (
+            token.type === 'keyword' &&
+            ['FROM', 'JOIN', 'INTO', 'UPDATE', 'USING'].includes(token.value) &&
+            tokens[i + 1]?.type === 'identifier' &&
+            !(['FROM', 'JOIN', 'USING'].includes(token.value) && tokens[i + 2]?.value === "(") // functions
+        ) {
+            const table = tokens[i + 1].value.toLowerCase();
+            if (currentOp && !aliasNames.has(table) && hasBalancedParens(tokens, currentOp.at, i)) {
+                tableOps[table] ||= new Set();
+                for (let op of currentOp.ops) {
+                    tableOps[table].add(op);
+                }
+            }
+        }
+
+        // Normalize IN (...) clauses
+        if (token.type === 'keyword' && token.value === 'IN') {
+            if (
+                tokens[i + 1]?.value === '(' &&
+                tokens[i + 2] // make sure something exists inside
+            ) {
+                appendToken('IN', 'keyword');
+                appendToken('(', 'punct');
+
+                const firstInside = tokens[i + 2];
+                if (firstInside.type === 'keyword') {
+                    // Subquery → parse normally
+                    i += 2;
+                    continue;
+                }
+                else {
+                    // Literal list → collapse
+                    appendToken('...', 'identifier');
+
+                    // Skip until matching ')'
+                    let depth = 1;
+                    let j = i + 3;
+                    while (j < tokens.length && depth > 0) {
+                        if (tokens[j].value === '(') depth++;
+                        else if (tokens[j].value === ')') depth--;
+                        j++;
+                    }
+
+                    appendToken(')', 'punct');
+                    i = j;
+                    continue;
+                }
+            }
+        }
+
+        appendToken(token.value, token.type);
+        i++;
+    }
+
+    return {
+        tableOperations: Object.fromEntries(
+            Object.entries(tableOps).map(([k, v]) => [k, Array.from(v)])
+        ),
+        normalizedQuery: normalizedTokens.join('')
+    };
+}
+
+function hasBalancedParens(tokens: Token[], start: number, end: number): boolean {
+    let balance = 0;
+    for (let i = start; i < end; i++) {
+        const token = tokens[i];
+        if (token.type === 'punct') {
+            if (token.value === '(') balance++;
+            else if (token.value === ')') balance--;
+        }
+
+        // Early exit: unbalanced in wrong direction
+        if (balance < 0) return false;
+    }
+    return balance === 0;
+}

package/src/wire-protocol.ts

@@ -0,0 +1,134 @@
+import {HrTime} from "@opentelemetry/api";
+
+export type ObservationInputMessage = InitMessage | NewObservedEntityMessage | NewObservedInteractionMessage | ObservationMessage;
+export type ObservationOutputMessage = InitAck | ObservedAck | ObservationsAck;
+
+
+export interface InitMessage {
+    event: "init";
+    protocolVersion: 1;
+    token: string;
+}
+
+
+export interface NewObservedEntityMessage {
+    event: "new-entity";
+    type: string;
+    hash: string;
+}
+
+export interface NewObservedServiceMessage extends NewObservedEntityMessage {
+    type: "service";
+    name: string;
+    namespace?: string;
+    environment?: string;
+}
+
+export interface NewObservedHttpRouteMessage extends NewObservedEntityMessage {
+    type: "http-route";
+    parent: string;
+    method: string;
+    route: string;
+}
+
+export interface NewObservedDatabaseMessage extends NewObservedEntityMessage {
+    type: "database";
+    system: string;
+    name?: string;
+    host?: string;
+    port?: number;
+}
+
+export interface NewObservedHttpServiceMessage extends NewObservedEntityMessage {
+    type: "http-service";
+    protocol: string;
+    host: string;
+    port: number;
+}
+
+
+export interface NewObservedInteractionMessage {
+    event: "new-interaction";
+    type: string;
+    hash: string;
+    from: string;
+    to: string;
+}
+
+export interface NewObservedHttpRequestMessage extends NewObservedInteractionMessage {
+    type: "http-request";
+}
+
+export interface NewObservedDatabaseConnectionMessage extends NewObservedInteractionMessage {
+    type: "db-connection";
+    connection?: string;
+    user?: string;
+}
+
+export interface NewObservedDatabaseQueryMessage extends NewObservedInteractionMessage {
+    type: "db-query";
+    query: string;
+    selects?: string[];
+    inserts?: string[];
+    updates?: string[];
+    deletes?: string[];
+}
+
+
+export interface ObservationMessage {
+    event: "observations";
+    seq: number;
+    observations: Array<Observation>;
+}
+
+export interface Observation {
+    type: string;
+    subject: string; // Hash of the entity or interaction the observation relates to
+    timestamp: HrTime;
+    errorMessage?: string;
+    errorType?: string;
+    stack?: string;
+}
+
+export interface HttpClientObservation extends Observation {
+    type: "http-client";
+    path: string;
+    method: string;
+    duration: HrTime;
+    status?: number;
+    httpVersion?: string;
+    requestBytes?: number;
+    responseBytes?: number;
+}
+
+export interface HttpServerObservation extends Observation {
+    type: "http-server";
+    path: string;
+    status: number;
+    duration: HrTime;
+    httpVersion?: string;
+    requestBytes?: number;
+    responseBytes?: number;
+    userAgent?: string;
+}
+
+export interface DatabaseQueryObservation extends Observation {
+    type: "db-query";
+    duration: HrTime;
+    returnedRows?: number;
+}
+
+
+export interface InitAck {
+    type: "ack-authorized";
+}
+
+export interface ObservedAck {
+    type: "ack-observed";
+    hash: string;
+}
+
+export interface ObservationsAck {
+    type: "ack-observations";
+    seq: number;
+}

package/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "compilerOptions": {
+    "target": "ES2020",
+    "module": "CommonJS",
+    "moduleResolution": "Node",
+    "declaration": true,
+    "outDir": "dist",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true
+  },
+  "include": ["src"]
+}