@vscode/telemetry-extractor 1.20.2 → 1.20.5

package/src/lib/object-converter.ts

@@ -1,25 +1,33 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 import { Declarations, OutputtedDeclarations } from "./declarations";
-import { Property } from "./common-properties";
-import { Metadata, Wildcard } from "./events";
+import { ColumnType, Property, ColumnInfo } from "./common-properties";
+import { Metadata, Wildcard, TableInfo as JsonTableInfo } from "./events";
 import * as keywords from './keywords';
 
 // Converts the declarations array to an object format for easy readability.
 
+type TableInfo = OutputtedDeclarations['tableInfos'][string];
+type BagInfo = TableInfo['columns'][number]['bag'];
+
 export async function transformOutput(output: Declarations): Promise<OutputtedDeclarations> {
     // If there's no events or common properties, we emit a null object
     if (output.events.dataPoints.length === 0 && output.commonProperties.properties.length === 0) {
-        return { events: Object.create(null), commonProperties: Object.create(null) };
+        return { events: Object.create(null), commonProperties: Object.create(null), tableInfos: {} };
     }
     const newEvents = Object.create(null);
     const oldEvents = output.events.dataPoints;
+    const tableInfos: { [key: string]: TableInfo } = {};
     for (const event of oldEvents) {
         // Check if event.name ends with a number, if so throw an error because we don't support event names which end with numbers
         if (/\d$/.test(event.name)) {
             throw new Error(`Event name ${event.name} ends with a number. Event names cannot end with numbers.`);
         }
         newEvents[event.name] = Object.create(null);
+        const tableInfo: TableInfo | undefined = event.tableInfo ? { ...event.tableInfo, columns: [] } : undefined;
+        if (tableInfo) {
+            tableInfos[tableInfo.name] = tableInfo;
+        }
         //We know there won't be anymore includes or inlines because we have resolved them
         for (const property of event.properties as Array<Property | Wildcard | Metadata>) {
             if (property instanceof Wildcard) {
@@ -66,11 +74,18 @@ export async function transformOutput(output: Declarations): Promise<OutputtedDe
                 if (property.isMeasurement) {
                     newEvents[event.name][propetyNameChanger(property.name)]['isMeasurement'] = property.isMeasurement;
                 }
+                if (tableInfo !== undefined && property.column) {
+                    const name = property.column.name ?? property.name;
+                    tableInfo.columns.push({ name, type: property.column.type, bag: { name: property.name.toLowerCase(), store: property.isMeasurement === true ? 'Measures' : 'Properties' } });
+                }
             } else {
                 // Comments, expiration, and owner metadata are handled here
                 newEvents[event.name][propetyNameChanger(property.name)] = property.value;
             }
         }
+        if (tableInfo !== undefined) {
+            tableInfos[tableInfo.name] = tableInfo;
+        }
     }
     const newCommonProperties = Object.create(null);
     const oldCommonProperties = output.commonProperties.properties;
@@ -89,8 +104,62 @@ export async function transformOutput(output: Declarations): Promise<OutputtedDe
             newCommonProperties[propetyNameChanger(property.name)]['isMeasurement'] = property.isMeasurement;
         }
     }
-    return { events: newEvents, commonProperties: newCommonProperties };
+    return { events: newEvents, commonProperties: newCommonProperties, tableInfos };
+}
+
+type TypeScriptPropertyDeclaration = {
+    name?: string;
+    isMeasurement?: boolean;
+    type?: string;
+    column?: { name?: string; type?: string }
 }
+type TypeScriptEventDeclaration = {
+    $tableInfo?: unknown;
+    [name: string]: TypeScriptPropertyDeclaration | unknown;
+};
+
+export function transformTypeScriptDeclaration(declaration: TypeScriptEventDeclaration): { declaration: object, tableInfo: TableInfo | undefined} {
+    // The property names in the TS events are already all lowercased.
+    const tableInfoProperty = declaration['$tableinfo'];
+    let tableInfo: TableInfo | undefined = undefined;
+    if (tableInfoProperty !== undefined) {
+        delete declaration['$tableinfo'];
+        const json = JsonTableInfo.fromObject(tableInfoProperty);
+        if (json !== undefined) {
+            tableInfo = { name: json.name, commonProperties: json.commonProperties, backfill: json.backfill, columns: [] };
+        }
+    }
+    for (const name of Object.keys(declaration)) {
+        if (name === '$tableinfo') {
+            continue;
+        }
+        const property = declaration[name] as TypeScriptPropertyDeclaration;
+        if (!property || typeof property !== 'object') {
+            continue;
+        }
+        if (tableInfo !== undefined) {
+            if (property.column !== undefined) {
+                if (ColumnInfo.is(property.column)) {
+                    const type = ColumnType.fromString(property.column.type);
+                    if (type !== undefined) {
+                        const bag: BagInfo = { store: property.isMeasurement === true ? 'Measures' : 'Properties', name: name };
+                        tableInfo.columns.push({ name: property.column.name ?? name, type: type, bag });
+                    }
+                }
+            } else if (typeof property.type === 'string') {
+                const columnType = ColumnType.fromString(property.type);
+                if (columnType) {
+                    const bag: BagInfo = { store: property.isMeasurement === true ? 'Measures' : 'Properties', name: name };
+                    tableInfo.columns.push({ name: name, type: columnType, bag });
+                }
+            }
+        }
+        delete property.type;
+        delete property.column;
+    }
+    return { declaration, tableInfo };
+}
+
 
 function propetyNameChanger(name: string) {
     name = name.toLowerCase();

package/src/lib/operations.ts

@@ -2,7 +2,7 @@
 // Licensed under the MIT license.
 import { Fragments, Fragment } from "./fragments";
 import { Events, Event, Include, Inline, Wildcard, WildcardEntry, Metadata } from "./events";
-import { Property } from "./common-properties";
+import { ColumnType, ColumnInfo, Property } from "./common-properties";
 import * as keywords from './keywords';
 import * as path from 'path';
 
@@ -14,9 +14,54 @@ export function merge(target: Fragments | Events, source: Fragments | Events) {
         // We combine their properties together if the event already exists
         if (found) {
             if (target instanceof Events && source instanceof Events && found instanceof Event && item instanceof Event) {
-                if (!sameEventDefinition(found, item)) {
+                if (!eventsAreCompatible(found, item)) {
                     continue;
                 }
+                if (found.tableInfo === undefined) {
+                    found.tableInfo = item.tableInfo;
+                }
+                // Merge unique properties from source into target
+                for (const prop of item.properties) {
+                    if (prop instanceof Property) {
+                        const exists = found.properties.some(p => p instanceof Property && p.name === prop.name);
+                        if (!exists) {
+                            found.properties.push(prop);
+                        }
+                    } else if (prop instanceof Metadata) {
+                        const exists = found.properties.some(p => p instanceof Metadata && p.name === prop.name);
+                        if (!exists) {
+                            found.properties.push(prop);
+                        }
+                    } else if (prop instanceof Include) {
+                        const existingInclude = found.properties.find(p => p instanceof Include) as Include | undefined;
+                        if (existingInclude) {
+                            for (const name of prop.includeNames) {
+                                if (!existingInclude.includeNames.includes(name)) {
+                                    existingInclude.includeNames.push(name);
+                                }
+                            }
+                        } else {
+                            found.properties.push(new Include([...prop.includeNames]));
+                        }
+                    } else if (prop instanceof Inline) {
+                        const exists = found.properties.some(p => p instanceof Inline && p.inlineName === prop.inlineName);
+                        if (!exists) {
+                            found.properties.push(prop);
+                        }
+                    } else if (prop instanceof Wildcard) {
+                        const existingWildcard = found.properties.find(p => p instanceof Wildcard) as Wildcard | undefined;
+                        if (existingWildcard) {
+                            for (const entry of prop.entries) {
+                                const entryExists = existingWildcard.entries.some(e => e.prefix === entry.prefix);
+                                if (!entryExists) {
+                                    existingWildcard.entries.push(entry);
+                                }
+                            }
+                        } else {
+                            found.properties.push(prop);
+                        }
+                    }
+                }
                 continue;
             }
             found.properties = found.properties.concat(item.properties);
@@ -26,79 +71,22 @@ export function merge(target: Fragments | Events, source: Fragments | Events) {
     }
 }
 
-function sameEventDefinition(left: Event, right: Event) {
-    return stableSerialize(eventToComparable(left)) === stableSerialize(eventToComparable(right));
-}
-
-function eventToComparable(event: Event) {
-    const properties = event.properties.map(propertyToComparable);
-    properties.sort((a, b) => stableSerialize(a).localeCompare(stableSerialize(b)));
-    return {
-        name: event.name,
-        properties
-    };
-}
-
-function propertyToComparable(property: Property | Metadata | Include | Inline | Wildcard) {
-    if (property instanceof Property) {
-        return {
-            type: 'property',
-            name: property.name,
-            classification: property.classification,
-            purpose: property.purpose,
-            expiration: property.expiration,
-            owner: property.owner,
-            comment: property.comment,
-            endPoint: property.endPoint,
-            isMeasurement: property.isMeasurement
-        };
-    }
-    if (property instanceof Metadata) {
-        return {
-            type: 'metadata',
-            name: property.name,
-            value: property.value
-        };
-    }
-    if (property instanceof Include) {
-        return {
-            type: 'include',
-            includeNames: [...property.includeNames].sort()
-        };
-    }
-    if (property instanceof Inline) {
-        return {
-            type: 'inline',
-            inlineName: property.inlineName,
-            inlines: [...property.inlines].sort()
-        };
-    }
-    return {
-        type: 'wildcard',
-        entries: property.entries.map(entry => ({
-            prefix: entry.prefix,
-            classification: entry.classification,
-            endpoint: entry.endpoint
-        })).sort((a, b) => stableSerialize(a).localeCompare(stableSerialize(b)))
-    };
-}
-
-function stableSerialize(value: unknown): string {
-    if (Array.isArray(value)) {
-        return `[${value.map(stableSerialize).join(',')}]`;
-    }
-
-    if (value && typeof value === 'object') {
-        const entries = Object.entries(value as Record<string, unknown>)
-            .sort(([left], [right]) => left.localeCompare(right))
-            .map(([key, entryValue]) => `${JSON.stringify(key)}:${stableSerialize(entryValue)}`);
-        return `{${entries.join(',')}}`;
+function eventsAreCompatible(left: Event, right: Event): boolean {
+    for (const leftProp of left.properties) {
+        if (!(leftProp instanceof Property)) continue;
+        for (const rightProp of right.properties) {
+            if (!(rightProp instanceof Property)) continue;
+            if (leftProp.name === rightProp.name) {
+                if (leftProp.classification !== rightProp.classification || leftProp.purpose !== rightProp.purpose) {
+                    return false;
+                }
+            }
+        }
     }
-
-    return JSON.stringify(value);
+    return true;
 }
 
-// Searches the object for an event or fragment of the specific name 
+// Searches the object for an event or fragment of the specific name
 // If found returns, if not found creates it, places it in the array, and then returns
 export function findOrCreate(searchTarget: Events | Fragments, name: string) {
     let found = searchTarget.dataPoints.find((item) => {
@@ -137,6 +125,10 @@ export function mergeWildcards(wildcard: any[], target: Event | Fragment, applyE
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 export function populateProperties(properties: any, target: Event | Fragment, applyEndpoints = false) {
     for (const propertyName in properties) {
+        // Skip the `$tableInfo` property since it is an artificial property for the flat table support.
+        if (propertyName === '$tableInfo') {
+            continue;
+        }
         const currentProperty = properties[propertyName];
         if (propertyName === keywords.include) {
             target.properties.push(new Include(currentProperty));
@@ -157,6 +149,17 @@ export function populateProperties(properties: any, target: Event | Fragment, ap
             if (currentProperty.isMeasurement) {
                 prop.isMeasurement = currentProperty.isMeasurement;
             }
+            if (typeof currentProperty.type === 'string') {
+                const columnType = ColumnType.fromString(currentProperty.type);
+                if (columnType) {
+                    prop.column = { type: columnType };
+                }
+            } else if (ColumnInfo.is(currentProperty.column)) {
+                const columnType = ColumnType.fromString(currentProperty.column.type);
+                if (columnType) {
+                    prop.column = { name: currentProperty.column.name, type: columnType };
+                }
+            }
             target.properties.push(prop);
         }
     }

package/src/lib/parser.ts

@@ -7,14 +7,11 @@ import * as fs from 'fs';
 // Not importing 'process' as tsc claims `process.exitCode` is read-only when it actually is not.
 import { Fragments } from './fragments';
 import { Property, CommonProperties } from './common-properties';
-import { Events } from './events';
+import { Events, Event, TableInfo } from './events';
 import { Declarations } from './declarations';
 import { merge, findOrCreate, populateProperties, makeExclusionsRelativeToSource } from './operations';
-
-export interface EventDefinition {
-    signature: string;
-    location: string;
-}
+import { parseRipgrepFilePaths } from './ripgrep';
+import { EventDefinition } from './event-definition';
 
 export class Parser {
 
@@ -132,7 +129,6 @@ export class Parser {
 
     private findEvents(sourceDir: string) {
         const filesWithEvents = this.asAbsoluteFilePaths(this.findFilesWithEvents(sourceDir));
-        const eventSignatures = new Map<string, string>();
 
         // Using [\s\S]* instead of .* since the latter does not match when using /m option
         const eventMatcher = /\/\*\s*__GDPR__\b([\s\S]*?)\*\//mg;
@@ -143,19 +139,22 @@ export class Parser {
                 let eventName = Object.keys(eventDeclaration)[0];
                 eventName = this.lowerCaseEvents ? eventName.toLowerCase() : eventName;
                 const eventProperties = eventDeclaration[Object.keys(eventDeclaration)[0]];
-                const currentSignature = this.stableSerialize(eventProperties);
+                const conflictProperties = this.extractConflictProperties(eventProperties);
                 const lineNumber = this.getLineNumber(fileContents, match.index);
-                this.addEventDefinition(eventName, currentSignature, `${filePath}:${lineNumber}`);
-                const existingSignature = eventSignatures.get(eventName);
-
-                if (existingSignature) {
-                    return;
+                this.addEventDefinition(eventName, conflictProperties, `${filePath}:${lineNumber}`);
+
+                const event = new Event(eventName);
+                if (event instanceof Event && eventProperties['$tableInfo'] !== undefined) {
+                    const tableInfo: TableInfo | undefined = TableInfo.fromObject(eventProperties['$tableInfo']);
+                    if (tableInfo) {
+                        event.tableInfo = tableInfo;
+                    }
                 }
-
-                eventSignatures.set(eventName, currentSignature);
-                const event = findOrCreate(eventDeclarations, eventName);
                 // Get the propeties which the event possesses
                 populateProperties(eventProperties, event, this.applyEndpoints);
+                const currentDeclaration = new Events();
+                currentDeclaration.dataPoints.push(event);
+                merge(eventDeclarations, currentDeclaration);
             } catch (error) {
                 console.error(`Event Declaration Error: ${error} in file ${filePath}`);
                 console.error(`Source comment:\n${match[0]}`);
@@ -173,12 +172,16 @@ export class Parser {
         return definitions;
     }
 
-    private addEventDefinition(eventName: string, signature: string, location: string) {
+    private addEventDefinition(eventName: string, properties: Record<string, unknown>, location: string) {
         const existing = this.eventDefinitions.get(eventName) ?? [];
-        existing.push({ signature, location });
+        existing.push({ properties, location });
         this.eventDefinitions.set(eventName, existing);
     }
 
+    private extractConflictProperties(eventProperties: Record<string, unknown>): Record<string, unknown> {
+        return { ...eventProperties };
+    }
+
     private getLineNumber(fileContents: string, index: number | undefined) {
         if (index === undefined) {
             return 1;
@@ -186,21 +189,6 @@ export class Parser {
         return fileContents.slice(0, index).split(/\r\n|\r|\n/).length;
     }
 
-    private stableSerialize(value: unknown): string {
-        if (Array.isArray(value)) {
-            return `[${value.map((entry) => this.stableSerialize(entry)).join(',')}]`;
-        }
-
-        if (value && typeof value === 'object') {
-            const entries = Object.entries(value as Record<string, unknown>)
-                .sort(([left], [right]) => left.localeCompare(right))
-                .map(([key, entryValue]) => `${JSON.stringify(key)}:${this.stableSerialize(entryValue)}`);
-            return `{${entries.join(',')}}`;
-        }
-
-        return JSON.stringify(value);
-    }
-
     // Utilizes a regex to find the files containing the specific pattern
     private findFiles(ripgrepPattern: string, sourceDir: string) {
         const relativeExclusions = makeExclusionsRelativeToSource(sourceDir, this.excludedDirs);
@@ -208,7 +196,7 @@ export class Parser {
         const ripgrepArgs = ['--files-with-matches', '--glob', '*.ts', '--glob', '*.tsx', '--glob', '*.cs', ...exclusions, '--regexp', ripgrepPattern, '--', sourceDir];
         try {
             const filePaths = cp.execFileSync(rgPath, ripgrepArgs, { encoding: 'ascii', cwd: `${sourceDir}` });
-            return filePaths.split(/(?:\r\n|\r|\n)/g).filter(path => path && path.length > 0);
+            return parseRipgrepFilePaths(filePaths);
         } catch {
             // ripgrep's return code != 0 if there are no matches
             return [];

package/src/lib/ripgrep.ts

@@ -0,0 +1,10 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+export function parseRipgrepFilePaths(output: string): string[] {
+    // Sorting after ripgrep exits preserves its parallel filesystem traversal.
+    return output
+        .split(/(?:\r\n|\r|\n)/g)
+        .filter(filePath => filePath.length > 0)
+        .sort();
+}

package/src/lib/save-declarations.ts

@@ -4,7 +4,7 @@ import * as path from 'path';
 import { Parser } from './parser';
 import * as fileWriter from './file-writer';
 import { resolveDeclarations, OutputtedDeclarations, Declarations } from './declarations';
-import { transformOutput } from './object-converter';
+import { transformOutput, transformTypeScriptDeclaration } from './object-converter';
 import { Events } from './events';
 import { CommonProperties } from './common-properties';
 import { TsParser } from './ts-parser';
@@ -12,10 +12,10 @@ import { patchDebugEvents } from './debug-patch';
 import { ParserOptions, SourceSpec } from './source-spec';
 import { logMessage } from './logger';
 import { Fragments } from './fragments';
-import { EventDefinition } from './parser';
+import { EventDefinition } from './event-definition';
 
 interface EventConflictEntry {
-    signature: string;
+    properties: Record<string, unknown>;
     location: string;
     source: 'GDPR' | 'TS';
 }
@@ -92,15 +92,33 @@ function mergeEventDefinitions(
 function reportDuplicateEventConflicts(definitions: Map<string, EventConflictEntry[]>) {
     let hasConflicts = false;
     for (const [eventName, entries] of definitions.entries()) {
-        const signatures = new Set(entries.map(entry => entry.signature));
-        if (signatures.size <= 1) {
+        if (entries.length <= 1) {
             continue;
         }
 
-        hasConflicts = true;
-        const uniqueLocations = [...new Set(entries.map(entry => `${entry.location.replace(/\\/g, '/')} (${entry.source})`))];
-        console.error(`Duplicate telemetry event declaration '${eventName}' has conflicting details at:`);
-        uniqueLocations.forEach(location => console.error(` - ${location}`));
+        // Compatible declarations may add properties, but must agree on every overlapping output field.
+        let eventHasConflict = false;
+        for (let i = 0; i < entries.length && !eventHasConflict; i++) {
+            for (let j = i + 1; j < entries.length && !eventHasConflict; j++) {
+                for (const propName of Object.keys(entries[i].properties)) {
+                    if (propName in entries[j].properties) {
+                        const a = entries[i].properties[propName];
+                        const b = entries[j].properties[propName];
+                        if (JSON.stringify(deepSortKeys(a)) !== JSON.stringify(deepSortKeys(b))) {
+                            eventHasConflict = true;
+                            break;
+                        }
+                    }
+                }
+            }
+        }
+
+        if (eventHasConflict) {
+            hasConflicts = true;
+            const uniqueLocations = [...new Set(entries.map(entry => `${entry.location.replace(/\\/g, '/')} (${entry.source})`))];
+            console.error(`Duplicate telemetry event declaration '${eventName}' has conflicting details at:`);
+            uniqueLocations.forEach(location => console.error(` - ${location}`));
+        }
     }
     return hasConflicts;
 }
@@ -117,11 +135,11 @@ export async function extractAndResolveDeclarations(sourceSpecs: Array<SourceSpe
             declarations = resolveDeclarations(declarations, spec.parserOptions.verbose);
             let typescriptDeclarations = Object.create(null);
             // The parser does not know how to handle multiple source directories due to different TS configs, so we manually have to parse each source dir
-            spec.sourceDirs.forEach((dir) => {
+            for (const dir of spec.sourceDirs) {
                 const tsParser = new TsParser(dir, spec.excludedDirs, spec.parserOptions.applyEndpoints, spec.parserOptions.lowerCaseEvents);
-                Object.assign(typescriptDeclarations, tsParser.parseFiles());
+                Object.assign(typescriptDeclarations, await tsParser.parseFiles());
                 mergeEventDefinitions(allEventDefinitions, tsParser.getEventDefinitions(), 'TS', spec.parserOptions.eventPrefix);
-            });
+            }
             if (spec.parserOptions.eventPrefix !== '') {
                 declarations.events.dataPoints = declarations.events.dataPoints.map((event) => {
                     event.name = spec.parserOptions.eventPrefix + event.name;
@@ -152,7 +170,12 @@ export async function extractAndResolveDeclarations(sourceSpecs: Array<SourceSpe
             if (formattedDeclarations.events === undefined) {
                 formattedDeclarations.events = Object.create(null);
             }
-            formattedDeclarations.events[dec] = allTypeScriptDeclarations[dec];
+            const transformed = transformTypeScriptDeclaration(allTypeScriptDeclarations[dec]);
+            formattedDeclarations.events[dec] = transformed.declaration as OutputtedDeclarations['events'][string];
+            if (transformed.tableInfo) {
+                const tableInfo = transformed.tableInfo;
+                formattedDeclarations.tableInfos[tableInfo.name] = tableInfo;
+            }
         }
         const hasPropertyValidationErrors = validateOutputtedDeclarations(formattedDeclarations);
         if (hasDuplicateEventConflicts || hasPropertyValidationErrors || process.exitCode === 1) {
@@ -164,3 +187,4 @@ export async function extractAndResolveDeclarations(sourceSpecs: Array<SourceSpe
         return Promise.reject(error);
     }
 }
+

package/src/lib/ts-parser-worker.ts

@@ -0,0 +1,38 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+import { parentPort, workerData } from 'worker_threads';
+import { Project } from 'ts-morph';
+import { TsProjectParser, ParserWorkerRequest, ParserWorkerResult } from './ts-parser';
+
+if (!parentPort) {
+    throw new Error('Telemetry parser worker requires a parent');
+}
+
+const request: ParserWorkerRequest = workerData;
+const project = new Project({ compilerOptions: request.compilerOptions });
+let result: ParserWorkerResult;
+
+if (request.kind === 'prepare') {
+    for (const file of request.sourceFiles) {
+        project.addSourceFileAtPathIfExists(file);
+    }
+    result = new TsProjectParser(project, false, false).prepare();
+} else if (request.kind === 'parse') {
+    // Keep global declarations and module augmentations visible in every batch.
+    for (const file of request.sharedSourceFiles) {
+        project.addSourceFileAtPath(file);
+    }
+    for (const group of request.calls) {
+        project.addSourceFileAtPath(group.filePath);
+    }
+    const parser = new TsProjectParser(project, request.applyEndpoints, request.lowerCaseEvents, request.definitions);
+    result = {
+        kind: 'parsed',
+        events: parser.parseFiles(request.calls, request.events),
+        definitions: [...parser.getEventDefinitions()]
+    };
+} else {
+    throw new Error('Unknown telemetry parser worker request');
+}
+
+parentPort.postMessage(result);