@vscode/telemetry-extractor 1.9.5

package/out/lib/ts-parser.js

@@ -0,0 +1,219 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TsParser = void 0;
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+const ts_morph_1 = require("ts-morph");
+const fs = __importStar(require("fs"));
+const cp = __importStar(require("child_process"));
+const path = __importStar(require("path"));
+const vscode_ripgrep_1 = require("vscode-ripgrep");
+const operations_1 = require("./operations");
+class GDPREvent {
+    eventName;
+    properties;
+    constructor(name) {
+        this.eventName = name;
+        this.properties = [];
+    }
+}
+class NodeVisitor {
+    pl_node;
+    prop_name;
+    inline = false;
+    original_prop_name;
+    applyEndpoints;
+    properties = [];
+    resolved_property = Object.create(null);
+    constructor(callexpress_node, prop_name, applyEndpoints) {
+        this.pl_node = callexpress_node;
+        this.prop_name = prop_name;
+        this.original_prop_name = prop_name;
+        this.applyEndpoints = applyEndpoints;
+    }
+    visitNode(currentNode, previousNode) {
+        let type = currentNode.getTypeAtLocation(this.pl_node);
+        // If we mark a property as optional then it is nullable, however we want all properties 
+        // So we want its non nullable type tl;dr this chops off the | undefined
+        if (type.isNullable()) {
+            type = type.getNonNullableType();
+        }
+        if (type.isStringLiteral() || type.isBooleanLiteral()) {
+            if (previousNode) {
+                // This means it is an inline because we had to recurse deeper than the first level to find the properties
+                if (this.prop_name !== previousNode.getEscapedName().toLowerCase() && !this.prop_name.includes(`.${previousNode.getEscapedName().toLowerCase()}`)) {
+                    this.prop_name = `${this.prop_name}.${previousNode.getEscapedName().toLowerCase()}`;
+                    this.inline = true;
+                }
+            }
+            // If we don't want endpoints skip them
+            if (currentNode.getEscapedName().toLowerCase() === "endpoint" && !this.applyEndpoints)
+                return;
+            // If it's a string we strip the quotes
+            if (type.isStringLiteral()) {
+                this.resolved_property[currentNode.getEscapedName()] = type.getText().substring(1, type.getText().length - 1);
+            }
+            else {
+                this.resolved_property[currentNode.getEscapedName()] = type.getText() === 'true';
+            }
+            return;
+        }
+        const properties = type.getProperties();
+        properties.forEach((prop) => {
+            this.visitNode(prop, currentNode);
+        });
+        // 95% of the time there is only one property in this array but inlines allow
+        // for the number of properties found to be unpredictable so we must return an array
+        if (this.inline && this.prop_name === this.original_prop_name) {
+            // This handles the case where the recursion will cause the inline to be counted one too many times
+            return;
+        }
+        const resolved = Object.create(null);
+        if (this.applyEndpoints) {
+            this.resolved_property['endPoint'] = this.resolved_property['endPoint'] ? this.resolved_property['endPoint'] : 'none';
+        }
+        resolved[this.prop_name] = this.resolved_property;
+        this.properties.push(resolved);
+        this.prop_name = this.original_prop_name;
+    }
+    resolveProperties(currentNode) {
+        this.visitNode(currentNode);
+        return this.properties;
+    }
+}
+class TsParser {
+    sourceDir;
+    excludedDirs;
+    applyEndpoints;
+    lowerCaseEvents;
+    project;
+    constructor(sourceDir, excludedDirs, applyEndpoints, lowerCaseEvents) {
+        this.sourceDir = sourceDir;
+        this.excludedDirs = excludedDirs;
+        this.applyEndpoints = applyEndpoints;
+        this.lowerCaseEvents = lowerCaseEvents;
+        // We search for a TS config as that allows the language service to handle weird imports
+        if (fs.existsSync(path.join(this.sourceDir, 'src/tsconfig.json'))) {
+            this.project = new ts_morph_1.Project({
+                tsConfigFilePath: path.join(this.sourceDir, 'src/tsconfig.json'),
+                skipAddingFilesFromTsConfig: true
+            });
+        }
+        else if (fs.existsSync(path.join(this.sourceDir, 'tsconfig.json'))) {
+            this.project = new ts_morph_1.Project({
+                tsConfigFilePath: path.join(this.sourceDir, 'tsconfig.json'),
+                skipAddingFilesFromTsConfig: true
+            });
+        }
+        else {
+            this.project = new ts_morph_1.Project({});
+        }
+        const fileGlobs = [];
+        fileGlobs.push(`'**/*.ts'`);
+        // Excluded added lasts because order determines what takes effect
+        this.excludedDirs = (0, operations_1.makeExclusionsRelativeToSource)(this.sourceDir, this.excludedDirs);
+        this.excludedDirs.forEach((dir) => {
+            fileGlobs.push(`'!${dir}/**'`);
+        });
+        let rg_glob = '';
+        for (const fg of fileGlobs) {
+            rg_glob += ` --glob ${fg}`;
+        }
+        const cmd = `${vscode_ripgrep_1.rgPath} --files-with-matches ${rg_glob} --no-ignore 'publicLog2|publicLogError2' ${this.sourceDir}`;
+        try {
+            const retrieved_paths = cp.execSync(cmd, { encoding: 'ascii' });
+            // Split the paths into an array
+            retrieved_paths.split(/(?:\r\n|\r|\n)/g).filter(path => path && path.length > 0).map((f) => {
+                this.project.addSourceFileAtPathIfExists(f);
+                return f;
+            });
+            // Empty catch because this fails when there are no typescript annotations which causes weird error messages
+        }
+        catch { }
+    }
+    parseFiles() {
+        let publicLogUse = [];
+        this.project.getSourceFiles().forEach((source) => {
+            const descendants = source.getDescendantsOfKind(ts_morph_1.SyntaxKind.CallExpression).filter((c) => c.getExpression().getText().includes('publicLog2'));
+            const descendants2 = source.getDescendantsOfKind(ts_morph_1.SyntaxKind.CallExpression).filter((c) => c.getExpression().getText().includes('publicLogError2'));
+            publicLogUse = descendants.concat(publicLogUse, descendants2);
+        });
+        const events = Object.create(null);
+        publicLogUse.forEach((pl) => {
+            try {
+                const typeArgs = pl.getTypeArguments();
+                if (typeArgs.length != 2) {
+                    throw new Error(`Missing generic arguments on public log call ${pl}`);
+                }
+                if (pl.getArguments()[0].getText() === "eventName") {
+                    return;
+                }
+                // Create an event from the name of the first argument passed in
+                let event_name = pl.getArguments()[0].getType().isStringLiteral() ? pl.getArguments()[0].getType().getText() : '';
+                // If we can't resolve the event_name there is no use continuing
+                if (event_name === '') {
+                    console.error('Unable to resolve event name, skipping....');
+                    return;
+                }
+                else {
+                    event_name = event_name.substring(1, event_name.length - 1);
+                }
+                event_name = this.lowerCaseEvents ? event_name.toLowerCase() : event_name;
+                const created_event = new GDPREvent(event_name);
+                // We want the second one because public log is in the form <Event, Classification> and we care about the classification
+                const type_properties = typeArgs[1].getType().getProperties();
+                type_properties.forEach((prop) => {
+                    const propName = prop.getEscapedName().toLowerCase();
+                    const node_visitor = new NodeVisitor(pl, propName, this.applyEndpoints);
+                    created_event.properties = created_event.properties.concat(node_visitor.resolveProperties(prop));
+                });
+                // We don't want to overwrite an event if we have already defined it, we just want to add to it
+                if (!events[event_name]) {
+                    events[event_name] = Object.create(null);
+                }
+                created_event.properties.forEach((prop) => {
+                    Object.assign(events[event_name], prop);
+                });
+            }
+            catch (err) {
+                if (pl.getArguments()[0].getText() === "eventName") {
+                    return;
+                }
+                // If the publicLog call isn't generic that means we're just sending an event name with no classifications
+                // that are unique to that event (it just has common properties)
+                let event_name = pl.getArguments()[0].getType().isStringLiteral() ? pl.getArguments()[0].getType().getText() : '';
+                // If we can't resolve the event_name this is most likely because it is not a public log call and therefore we skip it
+                if (event_name === '') {
+                    return;
+                }
+                else {
+                    event_name = event_name.substring(1, event_name.length - 1);
+                }
+                event_name = this.lowerCaseEvents ? event_name.toLowerCase() : event_name;
+                events[event_name] = {};
+            }
+        });
+        return events;
+    }
+}
+exports.TsParser = TsParser;
+//# sourceMappingURL=ts-parser.js.map

package/out/lib/ts-parser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ts-parser.js","sourceRoot":"","sources":["../../src/lib/ts-parser.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;AAAA,uCAAuC;AACvC,kCAAkC;AAClC,uCAA6E;AAC7E,uCAAyB;AACzB,kDAAoC;AACpC,2CAA6B;AAC7B,mDAAwC;AACxC,6CAA8D;AAY9D,MAAM,SAAS;IACJ,SAAS,CAAS;IAClB,UAAU,CAAuB;IACxC,YAAY,IAAY;QACpB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;IACzB,CAAC;CACJ;AAED,MAAM,WAAW;IAEL,OAAO,CAAO;IACd,SAAS,CAAS;IAClB,MAAM,GAAY,KAAK,CAAC;IACxB,kBAAkB,CAAS;IAC3B,cAAc,CAAU;IACzB,UAAU,GAAe,EAAE,CAAC;IAC3B,iBAAiB,GAAQ,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACrD,YAAY,gBAAsB,EAAE,SAAiB,EAAE,cAAuB;QAC1E,IAAI,CAAC,OAAO,GAAG,gBAAgB,CAAC;QAChC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,kBAAkB,GAAG,SAAS,CAAC;QACpC,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;IACzC,CAAC;IAEO,SAAS,CAAC,WAAmB,EAAE,YAAqB;QACxD,IAAI,IAAI,GAAG,WAAW,CAAC,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACvD,yFAAyF;QACzF,wEAAwE;QACxE,IAAI,IAAI,CAAC,UAAU,EAAE,EAAE;YACnB,IAAI,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;SACpC;QACD,IAAI,IAAI,CAAC,eAAe,EAAE,IAAI,IAAI,CAAC,gBAAgB,EAAE,EAAE;YACnD,IAAI,YAAY,EAAE;gBACd,0GAA0G;gBAC1G,IAAI,IAAI,CAAC,SAAS,KAAK,YAAY,CAAC,cAAc,EAAE,CAAC,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,YAAY,CAAC,cAAc,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC,EAAE;oBAC/I,IAAI,CAAC,SAAS,GAAG,GAAG,IAAI,CAAC,SAAS,IAAI,YAAY,CAAC,cAAc,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;oBACpF,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;iBACtB;aACJ;YACD,uCAAuC;YACvC,IAAI,WAAW,CAAC,cAAc,EAAE,CAAC,WAAW,EAAE,KAAK,UAAU,IAAI,CAAC,IAAI,CAAC,cAAc;gBAAE,OAAO;YAE9F,uCAAuC;YACvC,IAAI,IAAI,CAAC,eAAe,EAAE,EAAE;gBACxB,IAAI,CAAC,iBAAiB,CAAC,WAAW,CAAC,cAAc,EAAE,CAAC,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;aACjH;iBAAM;gBACH,IAAI,CAAC,iBAAiB,CAAC,WAAW,CAAC,cAAc,EAAE,CAAC,GAAG,IAAI,CAAC,OAAO,EAAE,KAAK,MAAM,CAAC;aACpF;YACD,OAAO;SACV;QACD,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QACxC,UAAU,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YACxB,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QACH,6EAA6E;QAC7E,oFAAoF;QACpF,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,KAAK,IAAI,CAAC,kBAAkB,EAAE;YAC3D,mGAAmG;YACnG,OAAO;SACV;QACD,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACrC,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;SACzH;QACD,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,iBAAiB,CAAC;QAClD,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC/B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC;IAC7C,CAAC;IAEM,iBAAiB,CAAC,WAAmB;QACxC,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;QAC5B,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;CACJ;AAED,MAAa,QAAQ;IACT,SAAS,CAAS;IAClB,YAAY,CAAW;IACvB,cAAc,CAAU;IACxB,eAAe,CAAU;IACzB,OAAO,CAAU;IACzB,YAAY,SAAiB,EAAE,YAAsB,EAAE,cAAuB,EAAE,eAAwB;QACpG,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,wFAAwF;QACxF,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,mBAAmB,CAAC,CAAC,EAAE;YAC/D,IAAI,CAAC,OAAO,GAAG,IAAI,kBAAO,CAAC;gBACvB,gBAAgB,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,mBAAmB,CAAC;gBAChE,2BAA2B,EAAE,IAAI;aACpC,CAAC,CAAC;SACN;aAAM,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC,EAAE;YAClE,IAAI,CAAC,OAAO,GAAG,IAAI,kBAAO,CAAC;gBACvB,gBAAgB,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC;gBAC5D,2BAA2B,EAAE,IAAI;aACpC,CAAC,CAAC;SACN;aAAM;YACH,IAAI,CAAC,OAAO,GAAG,IAAI,kBAAO,CAAC,EAAE,CAAC,CAAC;SAClC;QACD,MAAM,SAAS,GAAa,EAAE,CAAC;QAC/B,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC5B,kEAAkE;QAClE,IAAI,CAAC,YAAY,GAAG,IAAA,2CAA8B,EAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;QACtF,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;YAC9B,SAAS,CAAC,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC;QACnC,CAAC,CAAC,CAAC;QACH,IAAI,OAAO,GAAG,EAAE,CAAC;QACjB,KAAK,MAAM,EAAE,IAAI,SAAS,EAAE;YACxB,OAAO,IAAI,WAAW,EAAE,EAAE,CAAC;SAC9B;QACD,MAAM,GAAG,GAAG,GAAG,uBAAM,yBAAyB,OAAO,6CAA6C,IAAI,CAAC,SAAS,EAAE,CAAC;QACnH,IAAI;YACA,MAAM,eAAe,GAAG,EAAE,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;YAChE,gCAAgC;YAChC,eAAe,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;gBACvF,IAAI,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC,CAAC,CAAC;gBAC5C,OAAO,CAAC,CAAC;YACb,CAAC,CAAC,CAAC;YACP,4GAA4G;SAC3G;QAAC,MAAM,GAAG;IACf,CAAC;IAEM,UAAU;QACb,IAAI,YAAY,GAA0B,EAAE,CAAC;QAC7C,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;YAC7C,MAAM,WAAW,GAAG,MAAM,CAAC,oBAAoB,CAAC,qBAAU,CAAC,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,EAAE,CAAC,OAAO,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC;YAC7I,MAAM,YAAY,GAAG,MAAM,CAAC,oBAAoB,CAAC,qBAAU,CAAC,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,EAAE,CAAC,OAAO,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC,CAAC;YACnJ,YAAY,GAAG,WAAW,CAAC,MAAM,CAAC,YAAY,EAAE,YAAY,CAAC,CAAC;QAClE,CAAC,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACnC,YAAY,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE;YACxB,IAAI;gBACA,MAAM,QAAQ,GAAG,EAAE,CAAC,gBAAgB,EAAE,CAAC;gBACvC,IAAI,QAAQ,CAAC,MAAM,IAAI,CAAC,EAAE;oBACtB,MAAM,IAAI,KAAK,CAAC,gDAAgD,EAAE,EAAE,CAAC,CAAC;iBACzE;gBACD,IAAI,EAAE,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,KAAK,WAAW,EAAE;oBAChD,OAAO;iBACV;gBACD,gEAAgE;gBAChE,IAAI,UAAU,GAAG,EAAE,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAClH,gEAAgE;gBAChE,IAAI,UAAU,KAAK,EAAE,EAAE;oBACnB,OAAO,CAAC,KAAK,CAAC,4CAA4C,CAAC,CAAC;oBAC5D,OAAO;iBACV;qBAAM;oBACH,UAAU,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,EAAE,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;iBAC/D;gBACD,UAAU,GAAG,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,UAAU,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC;gBAC1E,MAAM,aAAa,GAAG,IAAI,SAAS,CAAC,UAAU,CAAC,CAAC;gBAChD,wHAAwH;gBACxH,MAAM,eAAe,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,aAAa,EAAE,CAAC;gBAC9D,eAAe,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;oBAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC,WAAW,EAAE,CAAC;oBACrD,MAAM,YAAY,GAAG,IAAI,WAAW,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC;oBACxE,aAAa,CAAC,UAAU,GAAG,aAAa,CAAC,UAAU,CAAC,MAAM,CAAC,YAAY,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC;gBACrG,CAAC,CAAC,CAAC;gBACH,+FAA+F;gBAC/F,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE;oBACrB,MAAM,CAAC,UAAU,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;iBAC5C;gBACD,aAAa,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;oBACtC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,IAAI,CAAC,CAAC;gBAC5C,CAAC,CAAC,CAAC;aACN;YAAC,OAAO,GAAG,EAAE;gBACV,IAAI,EAAE,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,KAAK,WAAW,EAAE;oBAChD,OAAO;iBACV;gBACD,0GAA0G;gBAC1G,gEAAgE;gBAChE,IAAI,UAAU,GAAG,EAAE,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAClH,sHAAsH;gBACtH,IAAI,UAAU,KAAK,EAAE,EAAE;oBACnB,OAAO;iBACV;qBAAM;oBACH,UAAU,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,EAAE,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;iBAC/D;gBACD,UAAU,GAAG,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,UAAU,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC;gBAC1E,MAAM,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC;aAC3B;QACL,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC;IAClB,CAAC;CACJ;AA9GD,4BA8GC"}

package/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "@vscode/telemetry-extractor",
+  "version": "1.9.5",
+  "description": "Extracts telemetry from VS Code",
+  "main": "out/index.js",
+  "typings": "vscode-telemetry-extractor.d.ts",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/microsoft/vscode-telemetry-extractor"
+  },
+  "scripts": {
+    "test": "mocha out/tests/mocha/*.js",
+    "coverage": "nyc mocha out/tests/mocha/*.js",
+    "extract-core": "node ./out/extractor.js --sourceDir src/telemetry-sources/vscode --excludedDir src/telemetry-sources/vscode/extensions --eventPrefix monacoworkbench/  --outputDir . --applyEndpoints",
+    "extract-core-verbose": "node ./out/extractor.js --sourceDir src/telemetry-sources/vscode --excludedDir src/telemetry-sources/vscode/extensions --eventPrefix monacoworkbench/  --outputDir . --applyEndpoints --silent --verbose",
+    "compile": "tsc",
+    "watch": "tsc --watch"
+  },
+  "bin": {
+    "vscode-telemetry-extractor": "./out/extractor.js"
+  },
+  "author": "lramos15",
+  "license": "MIT",
+  "dependencies": {
+    "command-line-args": "^5.2.0",
+    "ts-morph": "^12.2.0",
+    "vscode-ripgrep": "^1.12.1"
+  },
+  "devDependencies": {
+    "@types/command-line-args": "^5.2.0",
+    "@types/mocha": "^9.0.0",
+    "@types/node": "^15.12.2",
+    "mocha": "^9.1.3",
+    "nyc": "^15.1.0",
+    "source-map-support": "^0.5.20",
+    "ts-node": "^10.4.0",
+    "typescript": "^4.4.4"
+  }
+}

package/src/cli-help.ts

@@ -0,0 +1,29 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+import { optionDefinitions } from './cli-options';
+
+function buildHelpMessage() {
+    console.log('Allows the extraction of telemetry annotation from code. For more details please read: https://github.com/microsoft/vscode-telemetry-extractor/blob/master/README.md\n');
+    const optionLengths = optionDefinitions.map((opt) => {
+        if (opt.description) {
+            return opt.name.length
+        }
+        return -1;
+    });
+    const maxLength = Math.max(...optionLengths);
+    for (const opt of optionDefinitions) {
+        if (opt.description) {
+            let outputString = '';
+            if (opt.alias) {
+                outputString += `-${opt.alias} `;
+            }
+            // Extra padding if you don't have an alias
+            const extraPadding = opt.alias ? 0 : 3;
+            outputString += `--${opt.name.padEnd(maxLength + extraPadding)}\t\t\t`;
+            outputString += `${opt.description}`;
+            console.log(outputString);
+        }
+    }
+}
+
+buildHelpMessage();

package/src/cli-options.ts

@@ -0,0 +1,31 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+import commandLineArgs from 'command-line-args';
+import * as path from 'path';
+
+function resolveDirectories(options: commandLineArgs.CommandLineOptions): commandLineArgs.CommandLineOptions {
+    const cwd = process.cwd();
+
+    if (options.sourceDir) {
+        options.sourceDir = (options.sourceDir as string[]).map(d => path.resolve(cwd, d));
+    }
+    if (options.outputDir) {
+        options.outputDir = path.resolve(cwd, options.outputDir);
+    }
+
+    return options;
+}
+
+export const optionDefinitions = [
+    { name: 'sourceDir', alias: 's', description: 'The folder which you want to extract telemetry from (relative to the CWD)', type: String, multiple: true },
+    { name: 'excludedDir', alias: 'x', description: 'A subdirectory which you would like to exclude from the extraction (relative to the CWD)', type: String, multiple: true, defaultValue: [] },
+    { name: 'config', alias: 'c', description: 'A JSON Configuration file containing extraction details', type: String },
+    { name: 'outputDir', alias: 'o', description: 'The directory which you would like the outputted JSON file to be placed in', type: String },
+    { name: 'eventPrefix', alias: 'p', type: String, description: 'The string you wish to prepend to every telemetry event.', defaultValue: '' },
+    { name: 'help', alias: 'h', type: Boolean, description: 'Displays the help dialog which provides more information on how to use the tool', defaultValue: false },
+    { name: 'applyEndpoints', alias: 'e', type: Boolean, defaultValue: false },
+    { name: 'silenceOutput', type: Boolean, description: 'Silences all progress messages.', defaultValue: false },
+    { name: 'verbose', alias: 'v', type: Boolean, defaultValue: false }
+];
+
+export const options = resolveDirectories(commandLineArgs(optionDefinitions, { partial: true }));

package/src/extractor.ts

@@ -0,0 +1,57 @@
+#!/usr/bin/env node
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+import { options } from './cli-options';
+import * as path from 'path';
+import { cwd } from 'process';
+import { writeToFile, extractAndResolveDeclarations } from './lib/save-declarations';
+import { convertConfigToSourceSpecs, ParserOptions, SourceSpec } from './lib/source-spec';
+import { logMessage } from './lib/logger';
+
+if (options.config) {
+    const sourceSpecs = convertConfigToSourceSpecs(options.config);
+    extractAndResolveDeclarations(sourceSpecs).then((declarations) => {
+        if (options.outputDir) {
+            writeToFile(options.outputDir, declarations, 'config-resolved', !options.silent);
+        } else {
+            console.log(JSON.stringify(declarations));
+        }
+    });
+} else if (options.help || !options.sourceDir) {
+    require('./cli-help');
+} else {
+    const parserOptions: ParserOptions = {
+        eventPrefix: options.eventPrefix,
+        applyEndpoints: options.applyEndpoints,
+        patchDebugEvents: false,
+        lowerCaseEvents: false,
+        silenceOutput: options.silenceOutput,
+        verbose: options.verbose
+    };
+    logMessage('....running.', parserOptions.silenceOutput);
+    const sourceSpec: SourceSpec = {
+        sourceDirs: resolveDirectories(options.sourceDir),
+        excludedDirs: options.excludedDir ? resolveDirectories(options.excludedDir) : [],
+        parserOptions: parserOptions
+    };
+    extractAndResolveDeclarations([sourceSpec]).then((declarations) => {
+        if (options.outputDir) {
+            writeToFile(options.outputDir, declarations, 'declarations-resolved', !options.silent);
+        } else {
+            console.log(JSON.stringify(declarations));
+        }
+    });
+}
+
+// Resolves an array of paths
+function resolveDirectories(dirs: string[], workingDir?: string): string[] {
+    if (workingDir) {
+        if (path.isAbsolute(workingDir)) {
+            return dirs.map(s => path.resolve(workingDir, s));
+        } else {
+            return dirs.map(s => path.resolve(cwd(), workingDir, s));
+        }
+    } else {
+        return dirs.map(s => path.resolve(cwd(), s));
+    }
+}

package/src/index.ts

@@ -0,0 +1,4 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+export { extractAndResolveDeclarations } from './lib/save-declarations';
+export { convertConfigToSourceSpecs, SourceSpec } from './lib/source-spec';

package/src/lib/common-properties.ts

@@ -0,0 +1,41 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+import { IProperty } from "./telemetry-interfaces";
+
+export class CommonProperties {
+    public properties: Array<Property>;
+    constructor () {
+        this.properties = [];
+    }
+}
+
+export class Property implements IProperty {
+    public name: string;
+    public classification: string;
+    public purpose: string;
+    public endPoint?: string;
+    public isMeasurement?: boolean;
+    public expiration?: string;
+    public owner?: string;
+    public comment?: string;
+    
+    constructor (
+        name: string, 
+        classification: string,
+        purpose: string, 
+        expiration?: string,
+        owner?: string,
+        comment?: string,
+        endpoint?: string, 
+        isMeasurement?: boolean
+    ) {
+        this.name = name;
+        this.classification = classification;
+        this.purpose = purpose;
+        this.endPoint = endpoint;
+        this.isMeasurement = isMeasurement;
+        this.expiration = expiration;
+        this.owner = owner;
+        this.comment = comment;
+    }
+}

package/src/lib/debug-patch.ts

@@ -0,0 +1,13 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+import { Events, Event } from "./events";
+import { Property } from "./common-properties";
+
+export function patchDebugEvents(currentEvents: Events, eventPrefix: string) {
+    const debugProtocolErrorResponse = new Event(eventPrefix + 'debugProtocolErrorResponse');
+    debugProtocolErrorResponse.properties.push(new Property('error', 'CallstackOrException', 'PerformanceAndHealth'));
+    const optin = new Event(eventPrefix + 'optinstatus');
+    optin.properties.push(new Property('optin', 'SystemMetaData', 'BusinessInsight', undefined, undefined, undefined, undefined, true));
+    currentEvents.dataPoints.push(debugProtocolErrorResponse);
+    currentEvents.dataPoints.push(optin);
+}

package/src/lib/declarations.ts

@@ -0,0 +1,131 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+import { Fragments } from "./fragments";
+import { Events, Include, Inline } from "./events";
+import { CommonProperties, Property } from "./common-properties";
+
+export interface Declarations {
+    fragments: Fragments;
+    events: Events;
+    commonProperties: CommonProperties;
+}
+
+// We don't output Fragments as they don't really provide much context on their own
+export interface OutputtedDeclarations {
+    events: Events;
+    commonProperties: CommonProperties;
+}
+
+function resolveIncludes(target: Events | Fragments, fragments: Fragments) {
+    const notResolved: Array<String> = [];
+    const usedFragments: Array<String> = [];
+    for (const item of target.dataPoints) {
+        for (let i = 0; i < item.properties.length; i++) {
+            const property = item.properties[i];
+            if (property instanceof Include) {
+                // Removes the current include from the array since we have filled it in
+                item.properties = item.properties.filter((val) => {
+                    return val != property;
+                });
+                for (let reference of property.includeNames) {
+                    reference = reference.substring(2, reference.length - 1);
+                    const fragment = fragments.dataPoints.find((f) => {
+                        return f.name === reference;
+                    });
+                    if (fragment) {
+                        item.properties = item.properties.concat(fragment.properties);
+                        usedFragments.push(fragment.name);
+                    } else {
+                        notResolved.push(reference);
+                    }
+                    // This is an inefficient solution, but since adding and removing from the array
+                    // messes up the iteration we just restart it until we can get through the whole loop without any includes
+                    i = 0;
+                }
+            }
+        }
+    }
+    return { target, notResolved, usedFragments };
+}
+
+function resolveInlines(target: Events | Fragments, fragments: Fragments) {
+    const notResolved: Array<String> = [];
+    const usedFragments: Array<String> = [];
+    for (const item of target.dataPoints) {
+        for (let i = 0; i < item.properties.length; i++) {
+            const property = item.properties[i];
+            if (property instanceof Inline) {
+                // Removes the current inline from the array since we are resolving it
+                item.properties = item.properties.filter((val) => {
+                    return val !== property;
+                });
+                for (let reference of property.inlines) {
+                    // Gets rid of the ${}
+                    reference = reference.substring(2, reference.length - 1);
+                    const fragment = fragments.dataPoints.find((f) => {
+                        return f.name === reference;
+                    });
+                    if (fragment) {
+                        usedFragments.push(fragment.name);
+                        fragment.properties.forEach((prop) => {
+                            if (prop instanceof Property) {
+                                // We create the new property in the format inlineName.propName keeping the rest the same
+                                let currentProp = new Property(`${property.inlineName}.${prop.name}`, prop.classification, prop.purpose, prop.expiration, prop.owner, prop.comment);
+                                if (prop.endPoint) {
+                                    currentProp.endPoint = prop.endPoint;
+                                }
+                                if (prop.isMeasurement) {
+                                    currentProp.isMeasurement = prop.isMeasurement;
+                                }
+                                item.properties.push(currentProp);
+                            }
+                        });
+                    } else {
+                        notResolved.push(reference);
+                    }
+                }
+            }
+        }
+    }
+    return { target, notResolved, usedFragments };
+}
+
+export function resolveDeclarations(declarations: Declarations, verbose: boolean) {
+    let notResolved: Array<String> = [];
+    let usedFragments: Array<String> = [];
+    if (verbose) {
+        const fragmentsResolveInlines = resolveInlines(declarations.fragments, declarations.fragments);
+        const eventsResolveInlines = resolveInlines(declarations.events, declarations.fragments);
+        const fragmentsResolveIncludes = resolveIncludes(declarations.fragments, declarations.fragments);
+        const eventsResolveIncludes = resolveIncludes(declarations.events, declarations.fragments);
+        notResolved = notResolved.concat(fragmentsResolveIncludes.notResolved);
+        notResolved = notResolved.concat(fragmentsResolveInlines.notResolved);
+        notResolved = notResolved.concat(eventsResolveIncludes.notResolved);
+        notResolved = notResolved.concat(eventsResolveInlines.notResolved);
+        // Remove duplicates
+        notResolved = [...new Set(notResolved)];
+        usedFragments = usedFragments.concat(fragmentsResolveIncludes.usedFragments);
+        usedFragments = usedFragments.concat(fragmentsResolveInlines.usedFragments);
+        usedFragments = usedFragments.concat(eventsResolveIncludes.usedFragments);
+        usedFragments = usedFragments.concat(eventsResolveInlines.usedFragments);
+        // Remove duplicates
+        const usedFragmentsSet = new Set(usedFragments);
+        const allFragments = declarations.fragments.dataPoints.map(f => f.name);
+        // The Symmetric difference of a set
+        let unusedFragments = [...new Set(allFragments.filter(x => !usedFragmentsSet.has(x)))];
+        notResolved = notResolved.length > 0 ? notResolved : ['None'];
+        unusedFragments = unusedFragments.length > 0 ? unusedFragments : ['None'];
+        console.log(`Unresolved References: ${notResolved}`);
+        console.log(`Unused Fragments: ${unusedFragments}`);
+        declarations.fragments = fragmentsResolveInlines.target;
+        declarations.events = eventsResolveInlines.target;
+        declarations.fragments = fragmentsResolveIncludes.target;
+        declarations.events = eventsResolveIncludes.target;
+    } else {
+        declarations.fragments = resolveInlines(declarations.fragments, declarations.fragments).target;
+        declarations.events = resolveInlines(declarations.events, declarations.fragments).target;
+        declarations.fragments = resolveIncludes(declarations.fragments, declarations.fragments).target;
+        declarations.events = resolveIncludes(declarations.events, declarations.fragments).target;
+    }
+    return { events: declarations.events, commonProperties: declarations.commonProperties, fragments: declarations.fragments };
+}

package/src/lib/events.ts

@@ -0,0 +1,56 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+import { IProperty, IInclude, ITelemetryDataPoint, ITelemetryData, IInline, IWildcard, IWildcardEntry } from './telemetry-interfaces';
+import { Property } from './common-properties';
+
+
+export class Events implements ITelemetryData {
+    public dataPoints: Array<Event>;
+    constructor () {
+        this.dataPoints = [];
+    }
+}
+
+export class Event implements ITelemetryDataPoint {
+    public name: string;
+    // It gets a little more complicated here as events can have a bunch of different things
+    public properties: Array<Property | Include | Inline | Wildcard>;
+    constructor (name: string) {
+        this.name =  name;
+        this.properties = [];
+    } 
+}
+
+export class Include implements IInclude {
+    public includeNames: Array<string>;
+    constructor (includeNames: Array<string>) {
+        this.includeNames = includeNames;
+    }
+}
+
+export class Inline implements IInline {
+    public inlineName: string;
+    public inlines: Array<string>;
+    constructor (name: string, inlines: Array<string>) {
+        this.inlineName = name;
+        this.inlines = inlines;
+    }
+}
+
+export class Wildcard implements IWildcard {
+    public entries: Array<WildcardEntry>;
+    constructor () {
+        this.entries = [];
+    }
+}
+
+export class WildcardEntry implements IWildcardEntry {
+    public prefix: string;
+    public classification: {classification: string, purpose: string};
+    public endpoint: string | undefined;
+    constructor (prefix: string, classification: {classification: string, purpose: string}, endPoint?: string | undefined) {
+        this.prefix = prefix;
+        this.classification = classification;
+        this.endpoint = endPoint;
+    }
+}