@vscode/telemetry-extractor 1.20.4 → 1.20.5

package/src/lib/common-properties.ts

@@ -9,6 +9,67 @@ export class CommonProperties {
     }
 }
 
+export type ColumnType =
+    'bool' |
+    'int' |
+    'long' |
+    'real' |
+    'decimal' |
+    'dynamic' |
+    'guid' |
+    'string' |
+    'datetime' |
+    'timespan';
+
+export namespace ColumnType {
+    export function fromString(type: string): ColumnType | undefined{
+        switch (type.toLowerCase()) {
+            case 'bool':
+            case 'boolean':
+                return 'bool';
+            case 'int':
+                return 'int';
+            case 'long':
+                return 'long';
+            case 'real':
+            case 'double':
+                return 'real';
+            case 'decimal':
+                return 'decimal';
+            case 'dynamic':
+                return 'dynamic';
+            case 'guid':
+            case 'uuid':
+            case 'uniqueid':
+                return 'guid';
+            case 'string':
+                return 'string';
+            case 'datetime':
+            case 'date':
+                return 'datetime';
+            case 'timespan':
+            case 'time':
+                return 'timespan';
+            default:
+                return undefined;
+        }
+    }
+}
+
+export type ColumnInfo = {
+    name?: string;
+    type: ColumnType;
+}
+
+export namespace ColumnInfo {
+    export function is(obj: unknown): obj is ColumnInfo {
+        const candidate = obj as ColumnInfo;
+        return !!candidate &&
+            (candidate.name === undefined || typeof candidate.name === 'string') &&
+            typeof candidate.type === 'string' && ColumnType.fromString(candidate.type) !== undefined;
+    }
+}
+
 export class Property implements IProperty {
     public name: string;
     public classification: string;
@@ -18,16 +79,21 @@ export class Property implements IProperty {
     public expiration?: string;
     public owner?: string;
     public comment?: string;
-    
+
+    // The name and type of the property if mirrored into a flat table in Kusto.
+    // Needs to be removed when generating the patch for the GDPR catalog as
+    // flat table properties are not allowed in the catalog.
+    public column?: ColumnInfo;
+
     constructor (
-        name: string, 
+        name: string,
         classification: string,
-        purpose: string, 
+        purpose: string,
         expiration?: string,
         owner?: string,
         comment?: string,
-        endpoint?: string, 
-        isMeasurement?: boolean
+        endpoint?: string,
+        isMeasurement?: boolean,
     ) {
         this.name = name;
         this.classification = classification;

package/src/lib/declarations.ts

@@ -3,7 +3,7 @@
 import { Fragments } from "./fragments";
 import { Events, Include, Inline } from "./events";
 import { CommonProperties, Property } from "./common-properties";
-import type { Events as OutputEvents, CommonProperties as OutputCommonProperties } from "../../vscode-telemetry-extractor";
+import type { Events as OutputEvents, CommonProperties as OutputCommonProperties , TableInfo as OutputTableInfo } from "../../vscode-telemetry-extractor";
 
 export interface Declarations {
     fragments: Fragments;
@@ -15,6 +15,7 @@ export interface Declarations {
 export interface OutputtedDeclarations {
     events: OutputEvents;
     commonProperties: OutputCommonProperties;
+    tableInfos: { [key: string]: OutputTableInfo };
 }
 
 function resolveIncludes(target: Events | Fragments, fragments: Fragments) {

package/src/lib/event-definition.ts

@@ -0,0 +1,7 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+export interface EventDefinition {
+    properties: Record<string, unknown>;
+    location: string;
+}

package/src/lib/events.ts

@@ -11,14 +11,46 @@ export class Events implements ITelemetryData {
     }
 }
 
+export type TableInfo = {
+    name: string;
+    commonProperties: 'standard';
+    backfill: boolean | string;
+}
+export namespace TableInfo {
+    export function fromObject(obj: unknown): TableInfo | undefined {
+        if (typeof obj !== 'object' || obj === null) {
+            return undefined;
+        }
+        const candidate: Partial<TableInfo> = obj as TableInfo;
+        if (typeof candidate.name !== 'string') {
+            return undefined;
+        }
+        const result: TableInfo = {
+            name: candidate.name,
+            commonProperties: 'standard',
+            backfill: false
+
+        };
+        if (candidate.commonProperties === 'standard') {
+            result.commonProperties = 'standard';
+        }
+        if (typeof candidate.backfill === 'boolean' || typeof candidate.backfill === 'string') {
+            result.backfill = candidate.backfill;
+        }
+        return result;
+    }
+}
+
 export class Event implements ITelemetryDataPoint {
     public name: string;
     // It gets a little more complicated here as events can have a bunch of different things
     public properties: Array<Property | Metadata | Include | Inline | Wildcard>;
+    public tableInfo?: TableInfo;
     constructor (name: string) {
         this.name =  name;
         this.properties = [];
-    } 
+        this.tableInfo = undefined;
+    }
 }
 
 export class Include implements IInclude {

package/src/lib/object-converter.ts

@@ -1,25 +1,33 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 import { Declarations, OutputtedDeclarations } from "./declarations";
-import { Property } from "./common-properties";
-import { Metadata, Wildcard } from "./events";
+import { ColumnType, Property, ColumnInfo } from "./common-properties";
+import { Metadata, Wildcard, TableInfo as JsonTableInfo } from "./events";
 import * as keywords from './keywords';
 
 // Converts the declarations array to an object format for easy readability.
 
+type TableInfo = OutputtedDeclarations['tableInfos'][string];
+type BagInfo = TableInfo['columns'][number]['bag'];
+
 export async function transformOutput(output: Declarations): Promise<OutputtedDeclarations> {
     // If there's no events or common properties, we emit a null object
     if (output.events.dataPoints.length === 0 && output.commonProperties.properties.length === 0) {
-        return { events: Object.create(null), commonProperties: Object.create(null) };
+        return { events: Object.create(null), commonProperties: Object.create(null), tableInfos: {} };
     }
     const newEvents = Object.create(null);
     const oldEvents = output.events.dataPoints;
+    const tableInfos: { [key: string]: TableInfo } = {};
     for (const event of oldEvents) {
         // Check if event.name ends with a number, if so throw an error because we don't support event names which end with numbers
         if (/\d$/.test(event.name)) {
             throw new Error(`Event name ${event.name} ends with a number. Event names cannot end with numbers.`);
         }
         newEvents[event.name] = Object.create(null);
+        const tableInfo: TableInfo | undefined = event.tableInfo ? { ...event.tableInfo, columns: [] } : undefined;
+        if (tableInfo) {
+            tableInfos[tableInfo.name] = tableInfo;
+        }
         //We know there won't be anymore includes or inlines because we have resolved them
         for (const property of event.properties as Array<Property | Wildcard | Metadata>) {
             if (property instanceof Wildcard) {
@@ -66,11 +74,18 @@ export async function transformOutput(output: Declarations): Promise<OutputtedDe
                 if (property.isMeasurement) {
                     newEvents[event.name][propetyNameChanger(property.name)]['isMeasurement'] = property.isMeasurement;
                 }
+                if (tableInfo !== undefined && property.column) {
+                    const name = property.column.name ?? property.name;
+                    tableInfo.columns.push({ name, type: property.column.type, bag: { name: property.name.toLowerCase(), store: property.isMeasurement === true ? 'Measures' : 'Properties' } });
+                }
             } else {
                 // Comments, expiration, and owner metadata are handled here
                 newEvents[event.name][propetyNameChanger(property.name)] = property.value;
             }
         }
+        if (tableInfo !== undefined) {
+            tableInfos[tableInfo.name] = tableInfo;
+        }
     }
     const newCommonProperties = Object.create(null);
     const oldCommonProperties = output.commonProperties.properties;
@@ -89,8 +104,62 @@ export async function transformOutput(output: Declarations): Promise<OutputtedDe
             newCommonProperties[propetyNameChanger(property.name)]['isMeasurement'] = property.isMeasurement;
         }
     }
-    return { events: newEvents, commonProperties: newCommonProperties };
+    return { events: newEvents, commonProperties: newCommonProperties, tableInfos };
+}
+
+type TypeScriptPropertyDeclaration = {
+    name?: string;
+    isMeasurement?: boolean;
+    type?: string;
+    column?: { name?: string; type?: string }
 }
+type TypeScriptEventDeclaration = {
+    $tableInfo?: unknown;
+    [name: string]: TypeScriptPropertyDeclaration | unknown;
+};
+
+export function transformTypeScriptDeclaration(declaration: TypeScriptEventDeclaration): { declaration: object, tableInfo: TableInfo | undefined} {
+    // The property names in the TS events are already all lowercased.
+    const tableInfoProperty = declaration['$tableinfo'];
+    let tableInfo: TableInfo | undefined = undefined;
+    if (tableInfoProperty !== undefined) {
+        delete declaration['$tableinfo'];
+        const json = JsonTableInfo.fromObject(tableInfoProperty);
+        if (json !== undefined) {
+            tableInfo = { name: json.name, commonProperties: json.commonProperties, backfill: json.backfill, columns: [] };
+        }
+    }
+    for (const name of Object.keys(declaration)) {
+        if (name === '$tableinfo') {
+            continue;
+        }
+        const property = declaration[name] as TypeScriptPropertyDeclaration;
+        if (!property || typeof property !== 'object') {
+            continue;
+        }
+        if (tableInfo !== undefined) {
+            if (property.column !== undefined) {
+                if (ColumnInfo.is(property.column)) {
+                    const type = ColumnType.fromString(property.column.type);
+                    if (type !== undefined) {
+                        const bag: BagInfo = { store: property.isMeasurement === true ? 'Measures' : 'Properties', name: name };
+                        tableInfo.columns.push({ name: property.column.name ?? name, type: type, bag });
+                    }
+                }
+            } else if (typeof property.type === 'string') {
+                const columnType = ColumnType.fromString(property.type);
+                if (columnType) {
+                    const bag: BagInfo = { store: property.isMeasurement === true ? 'Measures' : 'Properties', name: name };
+                    tableInfo.columns.push({ name: name, type: columnType, bag });
+                }
+            }
+        }
+        delete property.type;
+        delete property.column;
+    }
+    return { declaration, tableInfo };
+}
+
 
 function propetyNameChanger(name: string) {
     name = name.toLowerCase();

package/src/lib/operations.ts

@@ -2,7 +2,7 @@
 // Licensed under the MIT license.
 import { Fragments, Fragment } from "./fragments";
 import { Events, Event, Include, Inline, Wildcard, WildcardEntry, Metadata } from "./events";
-import { Property } from "./common-properties";
+import { ColumnType, ColumnInfo, Property } from "./common-properties";
 import * as keywords from './keywords';
 import * as path from 'path';
 
@@ -17,6 +17,9 @@ export function merge(target: Fragments | Events, source: Fragments | Events) {
                 if (!eventsAreCompatible(found, item)) {
                     continue;
                 }
+                if (found.tableInfo === undefined) {
+                    found.tableInfo = item.tableInfo;
+                }
                 // Merge unique properties from source into target
                 for (const prop of item.properties) {
                     if (prop instanceof Property) {
@@ -83,7 +86,7 @@ function eventsAreCompatible(left: Event, right: Event): boolean {
     return true;
 }
 
-// Searches the object for an event or fragment of the specific name 
+// Searches the object for an event or fragment of the specific name
 // If found returns, if not found creates it, places it in the array, and then returns
 export function findOrCreate(searchTarget: Events | Fragments, name: string) {
     let found = searchTarget.dataPoints.find((item) => {
@@ -122,6 +125,10 @@ export function mergeWildcards(wildcard: any[], target: Event | Fragment, applyE
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 export function populateProperties(properties: any, target: Event | Fragment, applyEndpoints = false) {
     for (const propertyName in properties) {
+        // Skip the `$tableInfo` property since it is an artificial property for the flat table support.
+        if (propertyName === '$tableInfo') {
+            continue;
+        }
         const currentProperty = properties[propertyName];
         if (propertyName === keywords.include) {
             target.properties.push(new Include(currentProperty));
@@ -142,6 +149,17 @@ export function populateProperties(properties: any, target: Event | Fragment, ap
             if (currentProperty.isMeasurement) {
                 prop.isMeasurement = currentProperty.isMeasurement;
             }
+            if (typeof currentProperty.type === 'string') {
+                const columnType = ColumnType.fromString(currentProperty.type);
+                if (columnType) {
+                    prop.column = { type: columnType };
+                }
+            } else if (ColumnInfo.is(currentProperty.column)) {
+                const columnType = ColumnType.fromString(currentProperty.column.type);
+                if (columnType) {
+                    prop.column = { name: currentProperty.column.name, type: columnType };
+                }
+            }
             target.properties.push(prop);
         }
     }

package/src/lib/parser.ts

@@ -7,19 +7,11 @@ import * as fs from 'fs';
 // Not importing 'process' as tsc claims `process.exitCode` is read-only when it actually is not.
 import { Fragments } from './fragments';
 import { Property, CommonProperties } from './common-properties';
-import { Events } from './events';
+import { Events, Event, TableInfo } from './events';
 import { Declarations } from './declarations';
 import { merge, findOrCreate, populateProperties, makeExclusionsRelativeToSource } from './operations';
-
-export interface EventPropertySignature {
-    classification: string;
-    purpose: string;
-}
-
-export interface EventDefinition {
-    properties: Record<string, EventPropertySignature>;
-    location: string;
-}
+import { parseRipgrepFilePaths } from './ripgrep';
+import { EventDefinition } from './event-definition';
 
 export class Parser {
 
@@ -137,7 +129,6 @@ export class Parser {
 
     private findEvents(sourceDir: string) {
         const filesWithEvents = this.asAbsoluteFilePaths(this.findFilesWithEvents(sourceDir));
-        const seenEvents = new Set<string>();
 
         // Using [\s\S]* instead of .* since the latter does not match when using /m option
         const eventMatcher = /\/\*\s*__GDPR__\b([\s\S]*?)\*\//mg;
@@ -152,14 +143,18 @@ export class Parser {
                 const lineNumber = this.getLineNumber(fileContents, match.index);
                 this.addEventDefinition(eventName, conflictProperties, `${filePath}:${lineNumber}`);
 
-                if (seenEvents.has(eventName)) {
-                    return;
+                const event = new Event(eventName);
+                if (event instanceof Event && eventProperties['$tableInfo'] !== undefined) {
+                    const tableInfo: TableInfo | undefined = TableInfo.fromObject(eventProperties['$tableInfo']);
+                    if (tableInfo) {
+                        event.tableInfo = tableInfo;
+                    }
                 }
-
-                seenEvents.add(eventName);
-                const event = findOrCreate(eventDeclarations, eventName);
                 // Get the propeties which the event possesses
                 populateProperties(eventProperties, event, this.applyEndpoints);
+                const currentDeclaration = new Events();
+                currentDeclaration.dataPoints.push(event);
+                merge(eventDeclarations, currentDeclaration);
             } catch (error) {
                 console.error(`Event Declaration Error: ${error} in file ${filePath}`);
                 console.error(`Source comment:\n${match[0]}`);
@@ -177,23 +172,14 @@ export class Parser {
         return definitions;
     }
 
-    private addEventDefinition(eventName: string, properties: Record<string, EventPropertySignature>, location: string) {
+    private addEventDefinition(eventName: string, properties: Record<string, unknown>, location: string) {
         const existing = this.eventDefinitions.get(eventName) ?? [];
         existing.push({ properties, location });
         this.eventDefinitions.set(eventName, existing);
     }
 
-    private extractConflictProperties(eventProperties: Record<string, unknown>): Record<string, EventPropertySignature> {
-        const result: Record<string, EventPropertySignature> = {};
-        for (const [key, value] of Object.entries(eventProperties)) {
-            if (value && typeof value === 'object' && !Array.isArray(value)) {
-                const obj = value as Record<string, unknown>;
-                if (typeof obj.classification === 'string' && typeof obj.purpose === 'string') {
-                    result[key] = { classification: obj.classification, purpose: obj.purpose };
-                }
-            }
-        }
-        return result;
+    private extractConflictProperties(eventProperties: Record<string, unknown>): Record<string, unknown> {
+        return { ...eventProperties };
     }
 
     private getLineNumber(fileContents: string, index: number | undefined) {
@@ -210,7 +196,7 @@ export class Parser {
         const ripgrepArgs = ['--files-with-matches', '--glob', '*.ts', '--glob', '*.tsx', '--glob', '*.cs', ...exclusions, '--regexp', ripgrepPattern, '--', sourceDir];
         try {
             const filePaths = cp.execFileSync(rgPath, ripgrepArgs, { encoding: 'ascii', cwd: `${sourceDir}` });
-            return filePaths.split(/(?:\r\n|\r|\n)/g).filter(path => path && path.length > 0);
+            return parseRipgrepFilePaths(filePaths);
         } catch {
             // ripgrep's return code != 0 if there are no matches
             return [];

package/src/lib/ripgrep.ts

@@ -0,0 +1,10 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+export function parseRipgrepFilePaths(output: string): string[] {
+    // Sorting after ripgrep exits preserves its parallel filesystem traversal.
+    return output
+        .split(/(?:\r\n|\r|\n)/g)
+        .filter(filePath => filePath.length > 0)
+        .sort();
+}

package/src/lib/save-declarations.ts

@@ -4,7 +4,7 @@ import * as path from 'path';
 import { Parser } from './parser';
 import * as fileWriter from './file-writer';
 import { resolveDeclarations, OutputtedDeclarations, Declarations } from './declarations';
-import { transformOutput } from './object-converter';
+import { transformOutput, transformTypeScriptDeclaration } from './object-converter';
 import { Events } from './events';
 import { CommonProperties } from './common-properties';
 import { TsParser } from './ts-parser';
@@ -12,10 +12,10 @@ import { patchDebugEvents } from './debug-patch';
 import { ParserOptions, SourceSpec } from './source-spec';
 import { logMessage } from './logger';
 import { Fragments } from './fragments';
-import { EventDefinition, EventPropertySignature } from './parser';
+import { EventDefinition } from './event-definition';
 
 interface EventConflictEntry {
-    properties: Record<string, EventPropertySignature>;
+    properties: Record<string, unknown>;
     location: string;
     source: 'GDPR' | 'TS';
 }
@@ -96,7 +96,7 @@ function reportDuplicateEventConflicts(definitions: Map<string, EventConflictEnt
             continue;
         }
 
-        // Check if any overlapping properties have different classification/purpose
+        // Compatible declarations may add properties, but must agree on every overlapping output field.
         let eventHasConflict = false;
         for (let i = 0; i < entries.length && !eventHasConflict; i++) {
             for (let j = i + 1; j < entries.length && !eventHasConflict; j++) {
@@ -104,7 +104,7 @@ function reportDuplicateEventConflicts(definitions: Map<string, EventConflictEnt
                     if (propName in entries[j].properties) {
                         const a = entries[i].properties[propName];
                         const b = entries[j].properties[propName];
-                        if (a.classification !== b.classification || a.purpose !== b.purpose) {
+                        if (JSON.stringify(deepSortKeys(a)) !== JSON.stringify(deepSortKeys(b))) {
                             eventHasConflict = true;
                             break;
                         }
@@ -135,11 +135,11 @@ export async function extractAndResolveDeclarations(sourceSpecs: Array<SourceSpe
             declarations = resolveDeclarations(declarations, spec.parserOptions.verbose);
             let typescriptDeclarations = Object.create(null);
             // The parser does not know how to handle multiple source directories due to different TS configs, so we manually have to parse each source dir
-            spec.sourceDirs.forEach((dir) => {
+            for (const dir of spec.sourceDirs) {
                 const tsParser = new TsParser(dir, spec.excludedDirs, spec.parserOptions.applyEndpoints, spec.parserOptions.lowerCaseEvents);
-                Object.assign(typescriptDeclarations, tsParser.parseFiles());
+                Object.assign(typescriptDeclarations, await tsParser.parseFiles());
                 mergeEventDefinitions(allEventDefinitions, tsParser.getEventDefinitions(), 'TS', spec.parserOptions.eventPrefix);
-            });
+            }
             if (spec.parserOptions.eventPrefix !== '') {
                 declarations.events.dataPoints = declarations.events.dataPoints.map((event) => {
                     event.name = spec.parserOptions.eventPrefix + event.name;
@@ -170,7 +170,12 @@ export async function extractAndResolveDeclarations(sourceSpecs: Array<SourceSpe
             if (formattedDeclarations.events === undefined) {
                 formattedDeclarations.events = Object.create(null);
             }
-            formattedDeclarations.events[dec] = allTypeScriptDeclarations[dec];
+            const transformed = transformTypeScriptDeclaration(allTypeScriptDeclarations[dec]);
+            formattedDeclarations.events[dec] = transformed.declaration as OutputtedDeclarations['events'][string];
+            if (transformed.tableInfo) {
+                const tableInfo = transformed.tableInfo;
+                formattedDeclarations.tableInfos[tableInfo.name] = tableInfo;
+            }
         }
         const hasPropertyValidationErrors = validateOutputtedDeclarations(formattedDeclarations);
         if (hasDuplicateEventConflicts || hasPropertyValidationErrors || process.exitCode === 1) {
@@ -182,3 +187,4 @@ export async function extractAndResolveDeclarations(sourceSpecs: Array<SourceSpe
         return Promise.reject(error);
     }
 }
+

package/src/lib/ts-parser-worker.ts

@@ -0,0 +1,38 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+import { parentPort, workerData } from 'worker_threads';
+import { Project } from 'ts-morph';
+import { TsProjectParser, ParserWorkerRequest, ParserWorkerResult } from './ts-parser';
+
+if (!parentPort) {
+    throw new Error('Telemetry parser worker requires a parent');
+}
+
+const request: ParserWorkerRequest = workerData;
+const project = new Project({ compilerOptions: request.compilerOptions });
+let result: ParserWorkerResult;
+
+if (request.kind === 'prepare') {
+    for (const file of request.sourceFiles) {
+        project.addSourceFileAtPathIfExists(file);
+    }
+    result = new TsProjectParser(project, false, false).prepare();
+} else if (request.kind === 'parse') {
+    // Keep global declarations and module augmentations visible in every batch.
+    for (const file of request.sharedSourceFiles) {
+        project.addSourceFileAtPath(file);
+    }
+    for (const group of request.calls) {
+        project.addSourceFileAtPath(group.filePath);
+    }
+    const parser = new TsProjectParser(project, request.applyEndpoints, request.lowerCaseEvents, request.definitions);
+    result = {
+        kind: 'parsed',
+        events: parser.parseFiles(request.calls, request.events),
+        definitions: [...parser.getEventDefinitions()]
+    };
+} else {
+    throw new Error('Unknown telemetry parser worker request');
+}
+
+parentPort.postMessage(result);