@vscode/telemetry-extractor 1.20.4 → 1.20.5

package/src/lib/ts-parser.ts

@@ -1,22 +1,79 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
-import { Project, SyntaxKind, Symbol, Node, CallExpression, Type } from "ts-morph";
+import { Project, SyntaxKind, Symbol, Node, CallExpression, Type, ts, CompilerOptions } from "ts-morph";
 import * as fs from 'fs';
 import * as cp from 'child_process';
 import * as path from 'path';
+import { Worker } from 'worker_threads';
 import { rgPath } from "@vscode/ripgrep";
 import { makeExclusionsRelativeToSource } from "./operations";
 import { Event, Metadata } from './events';
 import { Property } from "./common-properties";
+import { parseRipgrepFilePaths } from './ripgrep';
+import { EventDefinition } from './event-definition';
 
-interface EventPropertySignature {
-    classification: string;
-    purpose: string;
+const sourceFilesPerBatch = 64;
+type ParsedEvents = Record<string, Record<string, unknown>>;
+
+interface TelemetryCall {
+    start: number;
+    width: number;
 }
 
-interface EventDefinition {
-    properties: Record<string, EventPropertySignature>;
-    location: string;
+interface TelemetryCalls {
+    filePath: string;
+    calls: TelemetryCall[];
+}
+
+export type ParserWorkerRequest = {
+    kind: 'prepare';
+    sourceFiles: string[];
+    compilerOptions: CompilerOptions;
+} | {
+    kind: 'parse';
+    calls: TelemetryCalls[];
+    sharedSourceFiles: string[];
+    compilerOptions: CompilerOptions;
+    applyEndpoints: boolean;
+    lowerCaseEvents: boolean;
+    events: ParsedEvents;
+    definitions: [string, EventDefinition[]][];
+};
+
+export type ParserWorkerResult = {
+    kind: 'prepared';
+    calls: TelemetryCalls[];
+    sharedSourceFiles: string[];
+} | {
+    kind: 'parsed';
+    events: ParsedEvents;
+    definitions: [string, EventDefinition[]][];
+};
+
+export function runParserWorker(request: ParserWorkerRequest): Promise<ParserWorkerResult> {
+    return new Promise((resolve, reject) => {
+        const worker = new Worker(path.join(__dirname, 'ts-parser-worker.js'), { workerData: request });
+        let result: ParserWorkerResult | undefined;
+        let error: Error | undefined;
+        worker.once('message', (message: ParserWorkerResult) => {
+            result = message;
+        });
+        worker.once('error', workerError => {
+            error = workerError;
+        });
+        // Do not start another compiler until this worker has released its heap.
+        worker.once('exit', code => {
+            if (error) {
+                reject(error);
+            } else if (code !== 0) {
+                reject(new Error(`Telemetry parser worker exited with code ${code}`));
+            } else if (!result) {
+                reject(new Error('Telemetry parser worker exited without a result'));
+            } else {
+                resolve(result);
+            }
+        });
+    });
 }
 
 function isMeasurement(type: Type) {
@@ -115,7 +172,7 @@ class NodeVisitor {
 
     private visitNode(currentNode: Symbol, previousNode?: Symbol) {
         let type = currentNode.getTypeAtLocation(this.pl_node);
-        // If we mark a property as optional then it is nullable, however we want all properties 
+        // If we mark a property as optional then it is nullable, however we want all properties
         // So we want its non nullable type tl;dr this chops off the | undefined
         if (type.isNullable()) {
             type = type.getNonNullableType();
@@ -140,10 +197,23 @@ class NodeVisitor {
             }
             return;
         }
-        const properties = type.getProperties();
-        properties.forEach((prop) => {
-            this.visitNode(prop, currentNode);
-        });
+        const nodeName = currentNode.getEscapedName();
+        if (nodeName !== 'column') {
+            const properties = type.getProperties();
+            properties.forEach((prop) => {
+                this.visitNode(prop, currentNode);
+            });
+        } else {
+            const properties = type.getProperties();
+            const value = Object.create(null);
+            properties.forEach((prop) => {
+                const propType = prop.getTypeAtLocation(this.pl_node);
+                if (propType.isStringLiteral()) {
+                    value[prop.getEscapedName()] = propType.getText().substring(1, propType.getText().length - 1);
+                }
+            });
+            this.resolved_property['column'] = value;
+        }
         // 95% of the time there is only one property in this array but inlines allow
         // for the number of properties found to be unpredictable so we must return an array
         if (this.inline && this.prop_name === this.original_prop_name) {
@@ -161,7 +231,7 @@ class NodeVisitor {
 
     private visitMetadataNode(currentNode: Symbol) {
         let type = currentNode.getTypeAtLocation(this.pl_node);
-        // If we mark a property as optional then it is nullable, however we want all properties 
+        // If we mark a property as optional then it is nullable, however we want all properties
         // So we want its non nullable type tl;dr this chops off the | undefined
         if (type.isNullable()) {
             type = type.getNonNullableType();
@@ -176,6 +246,9 @@ class NodeVisitor {
     }
 
     public resolveProperties(currentNode: Symbol): Array<Property | Metadata> {
+        // @lramos15 Actually this.properties is of type any[] and the property data pushing into
+        // the array is not an instance of Property.
+
         // It could be a complex node with nested types or a simple node with a string literal
         // representing some kind of metadata, so we try both visitors.
         this.visitMetadataNode(currentNode);
@@ -190,6 +263,7 @@ export class TsParser {
     private applyEndpoints: boolean;
     private lowerCaseEvents: boolean;
     private project: Project;
+    private sourceFiles: string[] = [];
     private eventDefinitions: Map<string, EventDefinition[]>;
     constructor(sourceDir: string, excludedDirs: string[], applyEndpoints: boolean, lowerCaseEvents: boolean) {
         this.sourceDir = sourceDir;
@@ -226,18 +300,69 @@ export class TsParser {
 
         const ripgrepArgs = ['--files-with-matches', ...rgGlobs, '--no-ignore', 'publicLog2|publicLogError2', this.sourceDir]
         try {
-            const retrieved_paths = cp.execFileSync(rgPath, ripgrepArgs, { encoding: 'ascii' });
-            // Split the paths into an array
-            retrieved_paths.split(/(?:\r\n|\r|\n)/g).filter(path => path && path.length > 0).map((f) => {
-                this.project.addSourceFileAtPathIfExists(f);
-                return f;
-            });
+            const retrievedPaths = cp.execFileSync(rgPath, ripgrepArgs, { encoding: 'ascii' });
+            this.sourceFiles = parseRipgrepFilePaths(retrievedPaths);
             // Empty catch because this fails when there are no typescript annotations which causes weird error messages
         } catch {
             // No-op
         }
     }
 
+    public async parseFiles() {
+        if (this.sourceFiles.length <= sourceFilesPerBatch) {
+            for (const file of this.sourceFiles) {
+                this.project.addSourceFileAtPathIfExists(file);
+            }
+            const parser = new TsProjectParser(this.project, this.applyEndpoints, this.lowerCaseEvents, [...this.eventDefinitions]);
+            const events = parser.parseFiles();
+            this.eventDefinitions = parser.getEventDefinitions();
+            return events;
+        }
+
+        const compilerOptions = this.project.getCompilerOptions();
+        const prepared = await runParserWorker({ kind: 'prepare', sourceFiles: this.sourceFiles, compilerOptions });
+        if (prepared.kind !== 'prepared') {
+            throw new Error('Telemetry parser worker did not return call locations');
+        }
+
+        let events: ParsedEvents = Object.create(null);
+        for (let index = 0; index < prepared.calls.length; index += sourceFilesPerBatch) {
+            const parsed = await runParserWorker({
+                kind: 'parse',
+                calls: prepared.calls.slice(index, index + sourceFilesPerBatch),
+                sharedSourceFiles: prepared.sharedSourceFiles,
+                compilerOptions,
+                applyEndpoints: this.applyEndpoints,
+                lowerCaseEvents: this.lowerCaseEvents,
+                events,
+                definitions: [...this.eventDefinitions]
+            });
+            if (parsed.kind !== 'parsed') {
+                throw new Error('Telemetry parser worker did not return declarations');
+            }
+            events = Object.assign(Object.create(null), parsed.events);
+            this.eventDefinitions = new Map(parsed.definitions);
+        }
+        return events;
+    }
+
+    public getEventDefinitions() {
+        return new Map([...this.eventDefinitions].map(([event, entries]) => [event, [...entries]]));
+    }
+}
+
+export class TsProjectParser {
+    private eventDefinitions: Map<string, EventDefinition[]>;
+
+    constructor(
+        private readonly project: Project,
+        private readonly applyEndpoints: boolean,
+        private readonly lowerCaseEvents: boolean,
+        definitions: [string, EventDefinition[]][] = []
+    ) {
+        this.eventDefinitions = new Map(definitions);
+    }
+
     public getEventDefinitions() {
         const definitions = new Map<string, EventDefinition[]>();
         for (const [eventName, entries] of this.eventDefinitions.entries()) {
@@ -246,35 +371,91 @@ export class TsParser {
         return definitions;
     }
 
-    private addEventDefinition(eventName: string, properties: Record<string, EventPropertySignature>, location: string) {
+    private addEventDefinition(eventName: string, properties: Record<string, unknown>, location: string) {
         const existing = this.eventDefinitions.get(eventName) ?? [];
         existing.push({ properties, location });
         this.eventDefinitions.set(eventName, existing);
     }
 
-    private extractConflictProperties(eventProperties: Record<string, unknown>): Record<string, EventPropertySignature> {
-        const result: Record<string, EventPropertySignature> = {};
-        for (const [key, value] of Object.entries(eventProperties)) {
-            if (value && typeof value === 'object' && !Array.isArray(value)) {
-                const obj = value as Record<string, unknown>;
-                if (typeof obj.classification === 'string' && typeof obj.purpose === 'string') {
-                    result[key] = { classification: obj.classification, purpose: obj.purpose };
-                }
-            }
-        }
-        return result;
+    private extractConflictProperties(eventProperties: Record<string, unknown>): Record<string, unknown> {
+        return { ...eventProperties };
     }
 
-    public parseFiles() {
-        let publicLogUse: Array<CallExpression> = [];
+    private collectCalls(): TelemetryCalls[] {
+        const publicLogCalls: TelemetryCalls[] = [];
+        const publicLogErrorCalls: TelemetryCalls[] = [];
         this.project.getSourceFiles().forEach((source) => {
-            const descendants = source.getDescendantsOfKind(SyntaxKind.CallExpression).filter((c) => c.getExpression().getText().includes('publicLog2') && c.getArguments().length > 0);
-            const descendants2 = source.getDescendantsOfKind(SyntaxKind.CallExpression).filter((c) => c.getExpression().getText().includes('publicLogError2') && c.getArguments().length > 0);
-            publicLogUse = descendants.concat(publicLogUse, descendants2);
+            const calls: TelemetryCall[] = [];
+            const errorCalls: TelemetryCall[] = [];
+            const sourceFile = source.compilerNode;
+            const visit = (node: ts.Node): void => {
+                if (ts.isCallExpression(node) && node.arguments.length > 0) {
+                    const expression = node.expression.getText(sourceFile);
+                    const isPublicLog = expression.includes('publicLog2');
+                    const isPublicLogError = expression.includes('publicLogError2');
+                    if ((isPublicLog || isPublicLogError) && node.arguments[0].getText(sourceFile) !== 'eventName') {
+                        const call = { start: node.getStart(sourceFile), width: node.getWidth(sourceFile) };
+                        if (isPublicLog) {
+                            calls.push(call);
+                        }
+                        if (isPublicLogError) {
+                            errorCalls.push(call);
+                        }
+                    }
+                }
+                ts.forEachChild(node, visit);
+            };
+            ts.forEachChild(sourceFile, visit);
+            if (calls.length > 0) {
+                publicLogCalls.unshift({ filePath: source.getFilePath(), calls });
+            }
+            if (errorCalls.length > 0) {
+                publicLogErrorCalls.push({ filePath: source.getFilePath(), calls: errorCalls });
+            }
         });
+        return publicLogCalls.concat(publicLogErrorCalls);
+    }
+
+    private getSharedSourceFiles(): string[] {
+        const program = this.project.getProgram().compilerObject;
+        return program.getSourceFiles().filter(source =>
+            !ts.isExternalModule(source) || source.statements.some(statement =>
+                ts.isModuleDeclaration(statement) &&
+                (ts.isStringLiteral(statement.name) || (statement.flags & ts.NodeFlags.GlobalAugmentation) !== 0))
+        ).map(source => source.fileName);
+    }
 
+    public prepare(): ParserWorkerResult {
+        const calls = this.collectCalls();
+        return {
+            kind: 'prepared',
+            calls,
+            sharedSourceFiles: calls.length > 0 ? this.getSharedSourceFiles() : []
+        };
+    }
+
+    private parseCalls(groups: TelemetryCalls[], parseCall: (call: CallExpression) => void): void {
+        for (const group of groups) {
+            const source = this.project.getSourceFileOrThrow(group.filePath);
+            for (const call of group.calls) {
+                const node = source.getDescendantAtStartWithWidth(call.start, call.width);
+                if (!node || !Node.isCallExpression(node)) {
+                    throw new Error(`Could not locate telemetry call in ${group.filePath} at ${call.start}`);
+                }
+                parseCall(node);
+            }
+        }
+    }
+
+    public parseFiles(calls = this.collectCalls(), previousEvents?: ParsedEvents) {
         const events = Object.create(null);
-        publicLogUse.forEach((pl) => {
+        if (previousEvents) {
+            // Structured cloning does not preserve dictionary prototypes.
+            for (const [name, properties] of Object.entries(previousEvents)) {
+                events[name] = Object.assign(Object.create(null), properties);
+            }
+        }
+        const parseCall = (pl: CallExpression): void => {
             try {
                 const typeArgs = pl.getTypeArguments();
                 if (typeArgs.length != 2) {
@@ -303,7 +484,18 @@ export class TsParser {
                 type_properties.forEach((prop) => {
                     const propName = prop.getEscapedName().toLowerCase();
                     const node_visitor = new NodeVisitor(pl, propName, this.applyEndpoints);
-                    created_event.properties = created_event.properties.concat(node_visitor.resolveProperties(prop));
+                    const resolved_properties = node_visitor.resolveProperties(prop);
+                    for (const rp of resolved_properties) {
+                        if (!(rp instanceof Metadata)) {
+                            // This cast is necessary since rp is not of type Property although
+                            // the resolveProperties claims it to be.
+                            const propInfo = (rp as unknown as { [key: string]: object })[propName];
+                            if (propInfo !== undefined) {
+                                this.captureOriginalPropNameForColumnInformation(prop.getEscapedName(), propInfo);
+                            }
+                        }
+                    }
+                    created_event.properties = created_event.properties.concat(resolved_properties);
                 });
                 created_event.properties.forEach((prop) => {
                     Object.assign(events[event_name], prop);
@@ -344,7 +536,18 @@ export class TsParser {
                 events[event_name] = {};
                 this.addEventDefinition(event_name, this.extractConflictProperties(events[event_name]), `${pl.getSourceFile().getFilePath()}:${pl.getStartLineNumber()}`);
             }
-        });
+        };
+
+        this.parseCalls(calls, parseCall);
         return events;
     }
+
+    private captureOriginalPropNameForColumnInformation(propName: string, property: { type?: string; column?: { name?: string; type: string } }) {
+        if (property.column && property.column.name === undefined) {
+            property.column.name = propName;
+        } else if (typeof property.type === 'string') {
+            property.column = { name: propName, type: property.type };
+            delete property.type;
+        }
+    }
 }

package/tsconfig.json

@@ -62,6 +62,7 @@
   "exclude": [
     "./src/telemetry-sources",
     "./src/tests/mocha/resources",
+    "./src/tests/mocha/tableResources",
     "**/*.d.ts"
   ]
 }

package/vscode-telemetry-extractor.d.ts

@@ -48,13 +48,33 @@ export interface Property {
     purpose: string;
     endPoint?: string;
     isMeasurement?: boolean;
+    column?: { name?: string; type: string };
+}
+
+export type ColumnType =
+    'bool' |
+    'int' |
+    'long' |
+    'real' |
+    'decimal' |
+    'dynamic' |
+    'guid' |
+    'string' |
+    'datetime' |
+    'timespan';
+
+export interface TableInfo {
+    name: string;
+    commonProperties: 'standard';
+    backfill: boolean | string;
+    columns: { name: string; type: ColumnType; bag: { store: 'Measures' | 'Properties'; name: string }}[];
 }
 
 /**
  * Extracts and resolves all typescript declarations from a series of different sources into a formatted object
  * @param sourceSpecs The various sources and their options which you would like to extract from
  */
-export declare function extractAndResolveDeclarations(sourceSpecs: Array<SourceSpec>): Promise<{ events: Events, commonProperties: CommonProperties }>;
+export declare function extractAndResolveDeclarations(sourceSpecs: Array<SourceSpec>): Promise<{ events: Events, commonProperties: CommonProperties, tableInfos: TableInfo[] }>;
 
 /**
  * Parses a valid extractor config file into an array of sourceSpecs that can be passed into an extract function