@vscode/telemetry-extractor 1.9.5

package/src/lib/ts-parser.ts

@@ -0,0 +1,206 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+import { Project, SyntaxKind, Symbol, Node, CallExpression } from "ts-morph";
+import * as fs from 'fs';
+import * as cp from 'child_process';
+import * as path from 'path';
+import { rgPath } from "vscode-ripgrep";
+import { makeExclusionsRelativeToSource } from "./operations";
+
+interface IGDPRProperty {
+    propName: string;
+    classification: 'SystemMetaData' | 'CallstackOrException';
+    purpose: 'PerformanceAndHealth' | 'FeatureInsight';
+    expiration?: string;
+    owner?: string;
+    comment?: string;
+    endpoint?: string;
+    isMeasurement?: boolean;
+}
+class GDPREvent {
+    public eventName: string;
+    public properties: Array<IGDPRProperty>;
+    constructor(name: string) {
+        this.eventName = name;
+        this.properties = [];
+    }
+}
+
+class NodeVisitor {
+
+    private pl_node: Node;
+    private prop_name: string;
+    private inline: boolean = false;
+    private original_prop_name: string;
+    private applyEndpoints: boolean;
+    public properties: Array<any> = [];
+    private resolved_property: any = Object.create(null);
+    constructor(callexpress_node: Node, prop_name: string, applyEndpoints: boolean) {
+        this.pl_node = callexpress_node;
+        this.prop_name = prop_name;
+        this.original_prop_name = prop_name;
+        this.applyEndpoints = applyEndpoints;
+    }
+
+    private visitNode(currentNode: Symbol, previousNode?: Symbol) {
+        let type = currentNode.getTypeAtLocation(this.pl_node);
+        // If we mark a property as optional then it is nullable, however we want all properties 
+        // So we want its non nullable type tl;dr this chops off the | undefined
+        if (type.isNullable()) {
+            type = type.getNonNullableType();
+        }
+        if (type.isStringLiteral() || type.isBooleanLiteral()) {
+            if (previousNode) {
+                // This means it is an inline because we had to recurse deeper than the first level to find the properties
+                if (this.prop_name !== previousNode.getEscapedName().toLowerCase() && !this.prop_name.includes(`.${previousNode.getEscapedName().toLowerCase()}`)) {
+                    this.prop_name = `${this.prop_name}.${previousNode.getEscapedName().toLowerCase()}`;
+                    this.inline = true;
+                }
+            }
+            // If we don't want endpoints skip them
+            if (currentNode.getEscapedName().toLowerCase() === "endpoint" && !this.applyEndpoints) return;
+
+            // If it's a string we strip the quotes
+            if (type.isStringLiteral()) {
+                this.resolved_property[currentNode.getEscapedName()] = type.getText().substring(1, type.getText().length - 1);
+            } else {
+                this.resolved_property[currentNode.getEscapedName()] = type.getText() === 'true';
+            }
+            return;
+        }
+        const properties = type.getProperties();
+        properties.forEach((prop) => {
+            this.visitNode(prop, currentNode);
+        });
+        // 95% of the time there is only one property in this array but inlines allow
+        // for the number of properties found to be unpredictable so we must return an array
+        if (this.inline && this.prop_name === this.original_prop_name) {
+            // This handles the case where the recursion will cause the inline to be counted one too many times
+            return;
+        }
+        const resolved = Object.create(null);
+        if (this.applyEndpoints) {
+            this.resolved_property['endPoint'] = this.resolved_property['endPoint'] ? this.resolved_property['endPoint'] : 'none';
+        }
+        resolved[this.prop_name] = this.resolved_property;
+        this.properties.push(resolved);
+        this.prop_name = this.original_prop_name;
+    }
+
+    public resolveProperties(currentNode: Symbol) {
+        this.visitNode(currentNode);
+        return this.properties;
+    }
+}
+
+export class TsParser {
+    private sourceDir: string;
+    private excludedDirs: string[];
+    private applyEndpoints: boolean;
+    private lowerCaseEvents: boolean;
+    private project: Project;
+    constructor(sourceDir: string, excludedDirs: string[], applyEndpoints: boolean, lowerCaseEvents: boolean) {
+        this.sourceDir = sourceDir;
+        this.excludedDirs = excludedDirs;
+        this.applyEndpoints = applyEndpoints;
+        this.lowerCaseEvents = lowerCaseEvents;
+        // We search for a TS config as that allows the language service to handle weird imports
+        if (fs.existsSync(path.join(this.sourceDir, 'src/tsconfig.json'))) {
+            this.project = new Project({
+                tsConfigFilePath: path.join(this.sourceDir, 'src/tsconfig.json'),
+                skipAddingFilesFromTsConfig: true
+            });
+        } else if (fs.existsSync(path.join(this.sourceDir, 'tsconfig.json'))) {
+            this.project = new Project({
+                tsConfigFilePath: path.join(this.sourceDir, 'tsconfig.json'),
+                skipAddingFilesFromTsConfig: true
+            });
+        } else {
+            this.project = new Project({});
+        }
+        const fileGlobs: string[] = [];
+        fileGlobs.push(`'**/*.ts'`);
+        // Excluded added lasts because order determines what takes effect
+        this.excludedDirs = makeExclusionsRelativeToSource(this.sourceDir, this.excludedDirs);
+        this.excludedDirs.forEach((dir) => {
+            fileGlobs.push(`'!${dir}/**'`);
+        });
+        let rg_glob = '';
+        for (const fg of fileGlobs) {
+            rg_glob += ` --glob ${fg}`;
+        }
+        const cmd = `${rgPath} --files-with-matches ${rg_glob} --no-ignore 'publicLog2|publicLogError2' ${this.sourceDir}`;
+        try {
+            const retrieved_paths = cp.execSync(cmd, { encoding: 'ascii' });
+            // Split the paths into an array
+            retrieved_paths.split(/(?:\r\n|\r|\n)/g).filter(path => path && path.length > 0).map((f) => {
+                this.project.addSourceFileAtPathIfExists(f);
+                return f;
+            });
+        // Empty catch because this fails when there are no typescript annotations which causes weird error messages
+        } catch { }
+    }
+
+    public parseFiles() {
+        let publicLogUse: Array<CallExpression> = [];
+        this.project.getSourceFiles().forEach((source) => {
+            const descendants = source.getDescendantsOfKind(SyntaxKind.CallExpression).filter((c) => c.getExpression().getText().includes('publicLog2'));
+            const descendants2 = source.getDescendantsOfKind(SyntaxKind.CallExpression).filter((c) => c.getExpression().getText().includes('publicLogError2'));
+            publicLogUse = descendants.concat(publicLogUse, descendants2);
+        });
+
+        const events = Object.create(null);
+        publicLogUse.forEach((pl) => {
+            try {
+                const typeArgs = pl.getTypeArguments();
+                if (typeArgs.length != 2) {
+                    throw new Error(`Missing generic arguments on public log call ${pl}`);
+                }
+                if (pl.getArguments()[0].getText() === "eventName") {
+                    return;
+                }
+                // Create an event from the name of the first argument passed in
+                let event_name = pl.getArguments()[0].getType().isStringLiteral() ? pl.getArguments()[0].getType().getText() : '';
+                // If we can't resolve the event_name there is no use continuing
+                if (event_name === '') {
+                    console.error('Unable to resolve event name, skipping....');
+                    return;
+                } else {
+                    event_name = event_name.substring(1, event_name.length - 1);
+                }
+                event_name = this.lowerCaseEvents ? event_name.toLowerCase() : event_name;
+                const created_event = new GDPREvent(event_name);
+                // We want the second one because public log is in the form <Event, Classification> and we care about the classification
+                const type_properties = typeArgs[1].getType().getProperties();
+                type_properties.forEach((prop) => {
+                    const propName = prop.getEscapedName().toLowerCase();
+                    const node_visitor = new NodeVisitor(pl, propName, this.applyEndpoints);
+                    created_event.properties = created_event.properties.concat(node_visitor.resolveProperties(prop));
+                });
+                // We don't want to overwrite an event if we have already defined it, we just want to add to it
+                if (!events[event_name]) {
+                    events[event_name] = Object.create(null);
+                }
+                created_event.properties.forEach((prop) => {
+                    Object.assign(events[event_name], prop);
+                });
+            } catch (err) {
+                if (pl.getArguments()[0].getText() === "eventName") {
+                    return;
+                }
+                // If the publicLog call isn't generic that means we're just sending an event name with no classifications
+                // that are unique to that event (it just has common properties)
+                let event_name = pl.getArguments()[0].getType().isStringLiteral() ? pl.getArguments()[0].getType().getText() : '';
+                // If we can't resolve the event_name this is most likely because it is not a public log call and therefore we skip it
+                if (event_name === '') {
+                    return;
+                } else {
+                    event_name = event_name.substring(1, event_name.length - 1);
+                }
+                event_name = this.lowerCaseEvents ? event_name.toLowerCase() : event_name;
+                events[event_name] = {};
+            }
+        });
+        return events;
+    }
+}

package/tsconfig.json

@@ -0,0 +1,67 @@
+{
+  "compilerOptions": {
+    /* Basic Options */
+    "target": "esnext",                          /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019' or 'ESNEXT'. */
+    "module": "commonjs",                     /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */
+    // "lib": [],                             /* Specify library files to be included in the compilation. */
+    // "allowJs": true,                       /* Allow javascript files to be compiled. */
+    // "checkJs": true,                       /* Report errors in .js files. */
+    // "jsx": "preserve",                     /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */
+    //"declaration": false,                   /* Generates corresponding '.d.ts' file. */
+    // "declarationMap": true,                /* Generates a sourcemap for each corresponding '.d.ts' file. */
+    "sourceMap": true,                     /* Generates corresponding '.map' file. */
+    // "outFile": "./",                       /* Concatenate and emit output to single file. */
+    "outDir": "./out",                        /* Redirect output structure to the directory. */
+    "rootDir": "./src",                       /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */
+    // "composite": true,                     /* Enable project compilation */
+    // "incremental": true,                   /* Enable incremental compilation */
+    // "tsBuildInfoFile": "./",               /* Specify file to store incremental compilation information */
+    // "removeComments": true,                /* Do not emit comments to output. */
+    // "noEmit": true,                        /* Do not emit outputs. */
+    // "importHelpers": true,                 /* Import emit helpers from 'tslib'. */
+    // "downlevelIteration": true,            /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */
+    // "isolatedModules": true,               /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */
+
+    /* Strict Type-Checking Options */
+    "strict": true,                           /* Enable all strict type-checking options. */
+    // "noImplicitAny": true,                 /* Raise error on expressions and declarations with an implied 'any' type. */
+    // "strictNullChecks": true,              /* Enable strict null checks. */
+    // "strictFunctionTypes": true,           /* Enable strict checking of function types. */
+    // "strictBindCallApply": true,           /* Enable strict 'bind', 'call', and 'apply' methods on functions. */
+    // "strictPropertyInitialization": true,  /* Enable strict checking of property initialization in classes. */
+    // "noImplicitThis": true,                /* Raise error on 'this' expressions with an implied 'any' type. */
+    // "alwaysStrict": true,                  /* Parse in strict mode and emit "use strict" for each source file. */
+
+    /* Additional Checks */
+    // "noUnusedLocals": true,                /* Report errors on unused locals. */
+    // "noUnusedParameters": true,            /* Report errors on unused parameters. */
+    // "noImplicitReturns": true,             /* Report error when not all code paths in function return a value. */
+    // "noFallthroughCasesInSwitch": true,    /* Report errors for fallthrough cases in switch statement. */
+
+    /* Module Resolution Options */
+    // "moduleResolution": "node",            /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */
+    // "baseUrl": "./",                       /* Base directory to resolve non-absolute module names. */
+    // "paths": {},                           /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */
+    // "rootDirs": [],                        /* List of root folders whose combined content represents the structure of the project at runtime. */
+    // "typeRoots": [],                       /* List of folders to include type definitions from. */
+    // "types": [],                           /* Type declaration files to be included in compilation. */
+    // "allowSyntheticDefaultImports": true,  /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */
+    "esModuleInterop": true                   /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */
+    // "preserveSymlinks": true,              /* Do not resolve the real path of symlinks. */
+
+    /* Source Map Options */
+    // "sourceRoot": "",                      /* Specify the location where debugger should locate TypeScript files instead of source locations. */
+    // "mapRoot": "",                         /* Specify the location where debugger should locate map files instead of generated locations. */
+    // "inlineSourceMap": true,               /* Emit a single file with source maps instead of having a separate file. */
+    // "inlineSources": true,                 /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */
+
+    /* Experimental Options */
+    // "experimentalDecorators": true,        /* Enables experimental support for ES7 decorators. */
+    // "emitDecoratorMetadata": true,         /* Enables experimental support for emitting type metadata for decorators. */
+  },
+  "exclude": [
+    "./src/telemetry-sources",
+    "./src/tests/mocha/resources",
+    "**/*.d.ts"
+  ]
+}

package/vscode-telemetry-extractor.d.ts

@@ -0,0 +1,72 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+import { PathLike } from "fs";
+/**
+ * The options which the parser takes in
+ * evenetPrefix: The prefix to append to all telemetry event names
+ * applyEndpoints: Whether to include the endPoints property on events in the final report
+ */
+export interface ParserOptions {
+    eventPrefix: string;
+    applyEndpoints: boolean;
+    patchDebugEvents: boolean;
+    lowerCaseEvents: boolean;
+    silenceOutput: boolean;
+    verbose: boolean;
+}
+
+/**
+ * Allows specifying different options for groups of sources
+ * sourceDirs: The directories to extract from
+ * excludedDirs: The sub directories to exclude from the telemetry extraction
+ * parserOptions: The parser options to apply to these set of directories
+ */
+export interface SourceSpec {
+    sourceDirs: string[],
+    excludedDirs: string[],
+    parserOptions: ParserOptions
+}
+
+export interface CommonProperties {
+    [key: string]: CommonProperty;
+}
+
+export interface CommonProperty {
+    name: string;
+    classification: string;
+    purpose: string;
+    endPoint?: string;
+    isMeasurement?: boolean;
+}
+
+export interface Events {
+    [key: string]: Event;
+}
+
+export interface Event {
+    [key: string]: Properties;
+}
+
+export interface Properties {
+    [key: string]: Property;
+}
+
+export interface Property {
+    name: string;
+    classification: string;
+    purpose: string;
+    endPoint?: string;
+    isMeasurement?: boolean;
+}
+
+/**
+ * Extracts and resolves all typescript declarations from a series of different sources into a formatted object
+ * @param sourceSpecs The various sources and their options which you would like to extract from
+ */
+export declare function extractAndResolveDeclarations(sourceSpecs: Array<SourceSpec>): Promise<{ events: Events, commonProperties: CommonProperties }>;
+
+/**
+ * Parses a valid extractor config file into an array of sourceSpecs that can be passed into an extract function
+ * @param file The path to the configuration file
+ */
+export declare function convertConfigToSourceSpecs(file: PathLike): SourceSpec[]