@zzzen/pyright-internal 1.2.0-dev.20260422 → 1.2.0-dev.20260503

package/dist/tests/benchmarks/parserBenchmark.test.js

@@ -0,0 +1,220 @@
+"use strict";
+/*
+ * parserBenchmark.test.ts
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Microbenchmark for the Python parser.
+ * Measures nodes/sec, parse time, AST node count across representative corpora.
+ *
+ * Run with:
+ *   cd packages/pyright/packages/pyright-internal
+ *   node node_modules\jest\bin\jest parserBenchmark.test --runInBand --detectOpenHandles --forceExit --testTimeout=300000
+ *
+ * Results are written as JSON to:
+ *   src/tests/benchmarks/.generated/benchmark-results/parser/
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const fs = __importStar(require("fs"));
+const os = __importStar(require("os"));
+const path = __importStar(require("path"));
+const diagnosticSink_1 = require("../../common/diagnosticSink");
+const parser_1 = require("../../parser/parser");
+// --- Configuration ---
+const WARMUP_ITERATIONS = 3;
+const BENCHMARK_ITERATIONS = 10;
+const BENCHMARK_OUTPUT_DIR = path.join(__dirname, '.generated', 'benchmark-results', 'parser');
+// --- Helpers ---
+function calculateStats(times) {
+    const sorted = [...times].sort((a, b) => a - b);
+    const len = sorted.length;
+    const median = len % 2 === 0 ? (sorted[len / 2 - 1] + sorted[len / 2]) / 2 : sorted[Math.floor(len / 2)];
+    const p95Index = Math.ceil(len * 0.95) - 1;
+    const p95 = sorted[Math.min(p95Index, len - 1)];
+    const min = sorted[0];
+    const max = sorted[len - 1];
+    const avg = times.reduce((a, b) => a + b, 0) / len;
+    return { median, p95, min, max, avg };
+}
+function loadCorpus(filename) {
+    const filePath = path.resolve(__dirname, '..', 'benchmarkData', filename);
+    return fs.readFileSync(filePath, 'utf-8');
+}
+function getSystemInfo() {
+    const cpus = os.cpus();
+    return {
+        platform: os.platform(),
+        arch: os.arch(),
+        cpus: cpus[0]?.model ?? 'unknown',
+        cpuCount: cpus.length,
+        totalMemoryMB: Math.round(os.totalmem() / (1024 * 1024)),
+        nodeVersion: process.version,
+    };
+}
+function writeReport(report) {
+    fs.mkdirSync(BENCHMARK_OUTPUT_DIR, { recursive: true });
+    const filename = `parser-benchmark-${new Date().toISOString().replace(/[:.]/g, '-')}.json`;
+    const outputPath = path.join(BENCHMARK_OUTPUT_DIR, filename);
+    fs.writeFileSync(outputPath, JSON.stringify(report, undefined, 2), 'utf-8');
+    console.log(`\nBenchmark results written to: ${outputPath}`);
+}
+function printResultTable(results) {
+    console.log('\n=== Parser Benchmark Results ===\n');
+    console.log(`${'Corpus'.padEnd(25)} ${'Size'.padStart(8)} ${'Nodes'.padStart(8)} ${'Stmts'.padStart(7)} ${'Errors'.padStart(7)} ${'Median'.padStart(10)} ${'Min'.padStart(10)} ${'Max'.padStart(10)} ${'Avg'.padStart(10)} ${'Nodes/s'.padStart(12)}`);
+    console.log('-'.repeat(117));
+    for (const r of results) {
+        const sizeKB = `${(r.fileSizeBytes / 1024).toFixed(1)}KB`;
+        console.log(`${r.corpus.padEnd(25)} ${sizeKB.padStart(8)} ${String(r.nodeCount).padStart(8)} ${String(r.statementCount).padStart(7)} ${String(r.errorCount).padStart(7)} ${r.medianMs.toFixed(2).padStart(10)} ${r.minMs
+            .toFixed(2)
+            .padStart(10)} ${r.maxMs.toFixed(2).padStart(10)} ${r.avgMs.toFixed(2).padStart(10)} ${Math.round(r.nodesPerSec)
+            .toLocaleString()
+            .padStart(12)}`);
+    }
+    console.log('');
+}
+/**
+ * Count all AST nodes by walking the tree recursively.
+ * Pyright parse nodes have: { nodeType, d: { ...children }, ... }
+ */
+function countNodes(node) {
+    if (!node || typeof node !== 'object' || !('nodeType' in node)) {
+        return 0;
+    }
+    let count = 1;
+    // Walk the .d data bag where child nodes live
+    const data = node.d;
+    if (data && typeof data === 'object') {
+        for (const key of Object.keys(data)) {
+            const val = data[key];
+            if (val && typeof val === 'object') {
+                if ('nodeType' in val) {
+                    count += countNodes(val);
+                }
+                else if (Array.isArray(val)) {
+                    for (const item of val) {
+                        if (item && typeof item === 'object' && 'nodeType' in item) {
+                            count += countNodes(item);
+                        }
+                    }
+                }
+            }
+        }
+    }
+    return count;
+}
+function benchmarkParse(corpusName, code) {
+    const times = [];
+    let nodeCount = 0;
+    let statementCount = 0;
+    let errorCount = 0;
+    const parseOptions = new parser_1.ParseOptions();
+    // Warmup
+    for (let i = 0; i < WARMUP_ITERATIONS; i++) {
+        const parser = new parser_1.Parser();
+        const diagSink = new diagnosticSink_1.DiagnosticSink();
+        parser.parseSourceFile(code, parseOptions, diagSink);
+    }
+    // Benchmark
+    for (let i = 0; i < BENCHMARK_ITERATIONS; i++) {
+        const parser = new parser_1.Parser();
+        const diagSink = new diagnosticSink_1.DiagnosticSink();
+        const start = performance.now();
+        const result = parser.parseSourceFile(code, parseOptions, diagSink);
+        const elapsed = performance.now() - start;
+        times.push(elapsed);
+        statementCount = result.parserOutput.parseTree.d.statements.length;
+        errorCount = diagSink.getErrors().length;
+        // Count nodes on the last iteration only (it's expensive)
+        if (i === BENCHMARK_ITERATIONS - 1) {
+            nodeCount = countNodes(result.parserOutput.parseTree);
+        }
+    }
+    const stats = calculateStats(times);
+    return {
+        corpus: corpusName,
+        fileSizeBytes: Buffer.byteLength(code, 'utf-8'),
+        iterations: BENCHMARK_ITERATIONS,
+        timesMs: times,
+        medianMs: stats.median,
+        p95Ms: stats.p95,
+        minMs: stats.min,
+        maxMs: stats.max,
+        avgMs: stats.avg,
+        nodeCount,
+        nodesPerSec: nodeCount / (stats.median / 1000),
+        statementCount,
+        errorCount,
+    };
+}
+// --- Corpus definitions ---
+const corpora = [
+    { name: 'large_stdlib', file: 'large_stdlib.py' },
+    { name: 'fstring_heavy', file: 'fstring_heavy.py' },
+    { name: 'comment_heavy', file: 'comment_heavy.py' },
+    { name: 'large_class', file: 'large_class.py' },
+    { name: 'import_heavy', file: 'import_heavy.py' },
+    { name: 'union_heavy', file: 'union_heavy.py' },
+];
+// --- Tests ---
+describe('Parser Benchmark', () => {
+    const allResults = [];
+    for (const { name, file } of corpora) {
+        test(`parse ${name}`, () => {
+            const code = loadCorpus(file);
+            const result = benchmarkParse(name, code);
+            allResults.push(result);
+            console.log(`  ${name}: median=${result.medianMs.toFixed(2)}ms, nodes=${result.nodeCount}, stmts=${result.statementCount}, nodes/sec=${Math.round(result.nodesPerSec).toLocaleString()}`);
+            // Sanity: parser should produce statements
+            expect(result.statementCount).toBeGreaterThan(0);
+            // Sanity: should complete in reasonable time (< 10s per file)
+            expect(result.medianMs).toBeLessThan(10000);
+        });
+    }
+    test('scaled corpus (10x large_stdlib)', () => {
+        const base = loadCorpus('large_stdlib.py');
+        const scaled = Array(10).fill(base).join('\n');
+        const result = benchmarkParse('large_stdlib_10x', scaled);
+        allResults.push(result);
+        console.log(`  large_stdlib_10x: median=${result.medianMs.toFixed(2)}ms, nodes=${result.nodeCount}, nodes/sec=${Math.round(result.nodesPerSec).toLocaleString()}`);
+        expect(result.statementCount).toBeGreaterThan(0);
+    });
+    afterAll(() => {
+        if (allResults.length === 0) {
+            return;
+        }
+        printResultTable(allResults);
+        const report = {
+            timestamp: new Date().toISOString(),
+            system: getSystemInfo(),
+            config: {
+                warmupIterations: WARMUP_ITERATIONS,
+                benchmarkIterations: BENCHMARK_ITERATIONS,
+            },
+            results: allResults,
+        };
+        writeReport(report);
+    });
+});
+//# sourceMappingURL=parserBenchmark.test.js.map

package/dist/tests/benchmarks/parserBenchmark.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parserBenchmark.test.js","sourceRoot":"","sources":["../../../../../../src/tests/benchmarks/parserBenchmark.test.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;GAaG;;;;;;;;;;;;;;;;;;;;;;;;;AAEH,uCAAyB;AACzB,uCAAyB;AACzB,2CAA6B;AAE7B,gEAA6D;AAC7D,gDAA2D;AAE3D,wBAAwB;AAExB,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAC5B,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAEhC,MAAM,oBAAoB,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,EAAE,mBAAmB,EAAE,QAAQ,CAAC,CAAC;AAqC/F,kBAAkB;AAElB,SAAS,cAAc,CAAC,KAA4B;IAOhD,MAAM,MAAM,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAChD,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC;IAE1B,MAAM,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;IACzG,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;IAC3C,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;IAChD,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IACtB,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;IAC5B,MAAM,GAAG,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC;IAEnD,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC;AAC1C,CAAC;AAED,SAAS,UAAU,CAAC,QAAgB;IAChC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,EAAE,eAAe,EAAE,QAAQ,CAAC,CAAC;IAC1E,OAAO,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;AAC9C,CAAC;AAED,SAAS,aAAa;IAClB,MAAM,IAAI,GAAG,EAAE,CAAC,IAAI,EAAE,CAAC;IACvB,OAAO;QACH,QAAQ,EAAE,EAAE,CAAC,QAAQ,EAAE;QACvB,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE;QACf,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,SAAS;QACjC,QAAQ,EAAE,IAAI,CAAC,MAAM;QACrB,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;QACxD,WAAW,EAAE,OAAO,CAAC,OAAO;KAC/B,CAAC;AACN,CAAC;AAED,SAAS,WAAW,CAAC,MAAuB;IACxC,EAAE,CAAC,SAAS,CAAC,oBAAoB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACxD,MAAM,QAAQ,GAAG,oBAAoB,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC;IAC3F,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,QAAQ,CAAC,CAAC;IAC7D,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IAC5E,OAAO,CAAC,GAAG,CAAC,mCAAmC,UAAU,EAAE,CAAC,CAAC;AACjE,CAAC;AAED,SAAS,gBAAgB,CAAC,OAAuC;IAC7D,OAAO,CAAC,GAAG,CAAC,sCAAsC,CAAC,CAAC;IACpD,OAAO,CAAC,GAAG,CACP,GAAG,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAC3G,CAAC,CACJ,IAAI,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,KAAK,CAAC,QAAQ,CACpF,EAAE,CACL,IAAI,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAChC,CAAC;IACF,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IAE7B,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACtB,MAAM,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,aAAa,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;QAC1D,OAAO,CAAC,GAAG,CACP,GAAG,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,MAAM,CACrF,CAAC,CAAC,cAAc,CACnB,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,KAAK;aAC7F,OAAO,CAAC,CAAC,CAAC;aACV,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,KAAK,CACjG,CAAC,CAAC,WAAW,CAChB;aACI,cAAc,EAAE;aAChB,QAAQ,CAAC,EAAE,CAAC,EAAE,CACtB,CAAC;IACN,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;AACpB,CAAC;AAED;;;GAGG;AACH,SAAS,UAAU,CAAC,IAAS;IACzB,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,CAAC,UAAU,IAAI,IAAI,CAAC,EAAE,CAAC;QAC7D,OAAO,CAAC,CAAC;IACb,CAAC;IAED,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,8CAA8C;IAC9C,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC;IACpB,IAAI,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QACnC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAClC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;YACtB,IAAI,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;gBACjC,IAAI,UAAU,IAAI,GAAG,EAAE,CAAC;oBACpB,KAAK,IAAI,UAAU,CAAC,GAAG,CAAC,CAAC;gBAC7B,CAAC;qBAAM,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC5B,KAAK,MAAM,IAAI,IAAI,GAAG,EAAE,CAAC;wBACrB,IAAI,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;4BACzD,KAAK,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC;wBAC9B,CAAC;oBACL,CAAC;gBACL,CAAC;YACL,CAAC;QACL,CAAC;IACL,CAAC;IAED,OAAO,KAAK,CAAC;AACjB,CAAC;AAED,SAAS,cAAc,CAAC,UAAkB,EAAE,IAAY;IACpD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,MAAM,YAAY,GAAG,IAAI,qBAAY,EAAE,CAAC;IAExC,SAAS;IACT,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,iBAAiB,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,MAAM,MAAM,GAAG,IAAI,eAAM,EAAE,CAAC;QAC5B,MAAM,QAAQ,GAAG,IAAI,+BAAc,EAAE,CAAC;QACtC,MAAM,CAAC,eAAe,CAAC,IAAI,EAAE,YAAY,EAAE,QAAQ,CAAC,CAAC;IACzD,CAAC;IAED,YAAY;IACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,oBAAoB,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5C,MAAM,MAAM,GAAG,IAAI,eAAM,EAAE,CAAC;QAC5B,MAAM,QAAQ,GAAG,IAAI,+BAAc,EAAE,CAAC;QAEtC,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,MAAM,CAAC,eAAe,CAAC,IAAI,EAAE,YAAY,EAAE,QAAQ,CAAC,CAAC;QACpE,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QAE1C,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACpB,cAAc,GAAG,MAAM,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;QACnE,UAAU,GAAG,QAAQ,CAAC,SAAS,EAAE,CAAC,MAAM,CAAC;QAEzC,0DAA0D;QAC1D,IAAI,CAAC,KAAK,oBAAoB,GAAG,CAAC,EAAE,CAAC;YACjC,SAAS,GAAG,UAAU,CAAC,MAAM,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;QAC1D,CAAC;IACL,CAAC;IAED,MAAM,KAAK,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;IAEpC,OAAO;QACH,MAAM,EAAE,UAAU;QAClB,aAAa,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC;QAC/C,UAAU,EAAE,oBAAoB;QAChC,OAAO,EAAE,KAAK;QACd,QAAQ,EAAE,KAAK,CAAC,MAAM;QACtB,KAAK,EAAE,KAAK,CAAC,GAAG;QAChB,KAAK,EAAE,KAAK,CAAC,GAAG;QAChB,KAAK,EAAE,KAAK,CAAC,GAAG;QAChB,KAAK,EAAE,KAAK,CAAC,GAAG;QAChB,SAAS;QACT,WAAW,EAAE,SAAS,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC;QAC9C,cAAc;QACd,UAAU;KACb,CAAC;AACN,CAAC;AAED,6BAA6B;AAE7B,MAAM,OAAO,GAAqC;IAC9C,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,iBAAiB,EAAE;IACjD,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,kBAAkB,EAAE;IACnD,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,kBAAkB,EAAE;IACnD,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,gBAAgB,EAAE;IAC/C,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,iBAAiB,EAAE;IACjD,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,gBAAgB,EAAE;CAClD,CAAC;AAEF,gBAAgB;AAEhB,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;IAC9B,MAAM,UAAU,GAAsB,EAAE,CAAC;IAEzC,KAAK,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,OAAO,EAAE,CAAC;QACnC,IAAI,CAAC,SAAS,IAAI,EAAE,EAAE,GAAG,EAAE;YACvB,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;YAC9B,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YAC1C,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAExB,OAAO,CAAC,GAAG,CACP,KAAK,IAAI,YAAY,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,MAAM,CAAC,SAAS,WACxE,MAAM,CAAC,cACX,eAAe,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,cAAc,EAAE,EAAE,CACnE,CAAC;YAEF,2CAA2C;YAC3C,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YACjD,8DAA8D;YAC9D,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;IACP,CAAC;IAED,IAAI,CAAC,kCAAkC,EAAE,GAAG,EAAE;QAC1C,MAAM,IAAI,GAAG,UAAU,CAAC,iBAAiB,CAAC,CAAC;QAC3C,MAAM,MAAM,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE/C,MAAM,MAAM,GAAG,cAAc,CAAC,kBAAkB,EAAE,MAAM,CAAC,CAAC;QAC1D,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAExB,OAAO,CAAC,GAAG,CACP,8BAA8B,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,aACpD,MAAM,CAAC,SACX,eAAe,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,cAAc,EAAE,EAAE,CACnE,CAAC;QAEF,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,GAAG,EAAE;QACV,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,OAAO;QACX,CAAC;QAED,gBAAgB,CAAC,UAAU,CAAC,CAAC;QAE7B,MAAM,MAAM,GAAoB;YAC5B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,MAAM,EAAE,aAAa,EAAE;YACvB,MAAM,EAAE;gBACJ,gBAAgB,EAAE,iBAAiB;gBACnC,mBAAmB,EAAE,oBAAoB;aAC5C;YACD,OAAO,EAAE,UAAU;SACtB,CAAC;QAEF,WAAW,CAAC,MAAM,CAAC,CAAC;IACxB,CAAC,CAAC,CAAC;AACP,CAAC,CAAC,CAAC"}

package/dist/tests/benchmarks/tokenizerBenchmark.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/tests/benchmarks/tokenizerBenchmark.test.js

@@ -0,0 +1,236 @@
+"use strict";
+/*
+ * tokenizerBenchmark.test.ts
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Microbenchmark for the Python tokenizer.
+ * Measures tokens/sec and time-to-tokenize across representative corpora.
+ *
+ * Run with:
+ *   cd packages/pyright-internal
+ *   node node_modules\jest\bin\jest tokenizerBenchmark.test --runInBand --detectOpenHandles --forceExit --testTimeout=300000
+ *
+ * Results are written as JSON to:
+ *   src/tests/benchmarks/.generated/benchmark-results/tokenizer/
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const child_process_1 = require("child_process");
+const fs = __importStar(require("fs"));
+const os = __importStar(require("os"));
+const path = __importStar(require("path"));
+const tokenizer_1 = require("../../parser/tokenizer");
+// --- Configuration ---
+const WARMUP_ITERATIONS = 3;
+const BENCHMARK_ITERATIONS = 10;
+const BENCHMARK_OUTPUT_DIR = path.join(__dirname, '.generated', 'benchmark-results', 'tokenizer');
+const JEST_BIN_PATH = path.resolve(__dirname, '..', '..', '..', 'node_modules', 'jest', 'bin', 'jest.js');
+const CHILD_RESULT_PREFIX = '__TOKENIZER_BENCHMARK_RESULT__';
+const CHILD_MODE_ENV = 'PYRIGHT_TOKENIZER_BENCH_CHILD';
+const RUN_BENCHMARKS_ENV = 'PYRIGHT_RUN_BENCHMARKS';
+// --- Helpers ---
+function calculateStats(times) {
+    const sorted = [...times].sort((a, b) => a - b);
+    const len = sorted.length;
+    const median = len % 2 === 0 ? (sorted[len / 2 - 1] + sorted[len / 2]) / 2 : sorted[Math.floor(len / 2)];
+    const p95Index = Math.ceil(len * 0.95) - 1;
+    const p95 = sorted[Math.min(p95Index, len - 1)];
+    const min = sorted[0];
+    const max = sorted[len - 1];
+    const avg = times.reduce((a, b) => a + b, 0) / len;
+    return { median, p95, min, max, avg };
+}
+function loadCorpus(filename) {
+    const filePath = path.resolve(__dirname, '..', 'benchmarkData', filename);
+    return fs.readFileSync(filePath, 'utf-8');
+}
+function getSystemInfo() {
+    const cpus = os.cpus();
+    return {
+        platform: os.platform(),
+        arch: os.arch(),
+        cpus: cpus[0]?.model ?? 'unknown',
+        cpuCount: cpus.length,
+        totalMemoryMB: Math.round(os.totalmem() / (1024 * 1024)),
+        nodeVersion: process.version,
+    };
+}
+function writeReport(report) {
+    fs.mkdirSync(BENCHMARK_OUTPUT_DIR, { recursive: true });
+    const filename = `tokenizer-benchmark-${new Date().toISOString().replace(/[:.]/g, '-')}.json`;
+    const outputPath = path.join(BENCHMARK_OUTPUT_DIR, filename);
+    fs.writeFileSync(outputPath, JSON.stringify(report, undefined, 2), 'utf-8');
+    console.log(`\nBenchmark results written to: ${outputPath}`);
+}
+function printResultTable(results) {
+    console.log('\n=== Tokenizer Benchmark Results ===\n');
+    console.log(`${'Corpus'.padEnd(25)} ${'Size'.padStart(8)} ${'Tokens'.padStart(8)} ${'Median'.padStart(10)} ${'Min'.padStart(10)} ${'Max'.padStart(10)} ${'Avg'.padStart(10)} ${'p95'.padStart(10)} ${'Tok/sec'.padStart(12)}`);
+    console.log('-'.repeat(113));
+    for (const result of results) {
+        const sizeKB = `${(result.fileSizeBytes / 1024).toFixed(1)}KB`;
+        console.log(`${result.corpus.padEnd(25)} ${sizeKB.padStart(8)} ${String(result.tokenCount).padStart(8)} ${result.medianMs.toFixed(2).padStart(10)} ${result.minMs.toFixed(2).padStart(10)} ${result.maxMs
+            .toFixed(2)
+            .padStart(10)} ${result.avgMs.toFixed(2).padStart(10)} ${result.p95Ms
+            .toFixed(2)
+            .padStart(10)} ${Math.round(result.tokensPerSec).toLocaleString().padStart(12)}`);
+    }
+    console.log('');
+}
+function emitChildResult(result) {
+    process.stdout.write(`${CHILD_RESULT_PREFIX}${JSON.stringify(result)}\n`);
+}
+function getChildOutput(error) {
+    if (!(error instanceof Error)) {
+        return '';
+    }
+    const stdout = 'stdout' in error && typeof error.stdout === 'string' ? error.stdout : '';
+    const stderr = 'stderr' in error && typeof error.stderr === 'string' ? error.stderr : '';
+    return [stdout, stderr].filter((part) => part.length > 0).join('\n');
+}
+function escapeRegExp(text) {
+    return text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+function runBenchmarkInFreshProcess(testName) {
+    try {
+        const output = (0, child_process_1.execFileSync)(process.execPath, [
+            JEST_BIN_PATH,
+            __filename,
+            '--runInBand',
+            '--forceExit',
+            '--testTimeout=300000',
+            '--testNamePattern',
+            `^Tokenizer Benchmark ${escapeRegExp(testName)}$`,
+        ], {
+            cwd: path.resolve(__dirname, '..', '..', '..'),
+            encoding: 'utf-8',
+            env: {
+                ...process.env,
+                [CHILD_MODE_ENV]: '1',
+            },
+        });
+        const resultLine = output.split(/\r?\n/).find((line) => line.startsWith(CHILD_RESULT_PREFIX));
+        if (!resultLine) {
+            throw new Error(`Child benchmark for "${testName}" did not emit a result.\n${output}`);
+        }
+        return JSON.parse(resultLine.slice(CHILD_RESULT_PREFIX.length));
+    }
+    catch (error) {
+        const output = getChildOutput(error);
+        const message = error instanceof Error ? error.message : String(error);
+        throw new Error(`Child benchmark for "${testName}" failed.\n${message}${output ? `\n${output}` : ''}`);
+    }
+}
+function benchmarkTokenize(corpusName, code) {
+    const times = [];
+    let tokenCount = 0;
+    for (let i = 0; i < WARMUP_ITERATIONS; i++) {
+        const tokenizer = new tokenizer_1.Tokenizer();
+        tokenizer.tokenize(code);
+    }
+    for (let i = 0; i < BENCHMARK_ITERATIONS; i++) {
+        const tokenizer = new tokenizer_1.Tokenizer();
+        const start = performance.now();
+        const results = tokenizer.tokenize(code);
+        const elapsed = performance.now() - start;
+        times.push(elapsed);
+        tokenCount = results.tokens.count;
+    }
+    const stats = calculateStats(times);
+    return {
+        corpus: corpusName,
+        fileSizeBytes: Buffer.byteLength(code, 'utf-8'),
+        iterations: BENCHMARK_ITERATIONS,
+        timesMs: times,
+        medianMs: stats.median,
+        p95Ms: stats.p95,
+        minMs: stats.min,
+        maxMs: stats.max,
+        avgMs: stats.avg,
+        tokenCount,
+        tokensPerSec: tokenCount / (stats.median / 1000),
+    };
+}
+// --- Corpus definitions ---
+const corpora = [
+    { name: 'large_stdlib', file: 'large_stdlib.py' },
+    { name: 'fstring_heavy', file: 'fstring_heavy.py' },
+    { name: 'comment_heavy', file: 'comment_heavy.py' },
+    { name: 'large_class', file: 'large_class.py' },
+    { name: 'import_heavy', file: 'import_heavy.py' },
+    { name: 'union_heavy', file: 'union_heavy.py' },
+    { name: 'repetitive_identifiers', file: 'repetitive_identifiers.py' },
+];
+// --- Tests ---
+const benchmarkSuite = process.env[RUN_BENCHMARKS_ENV] === '1' ? describe : describe.skip;
+benchmarkSuite('Tokenizer Benchmark', () => {
+    const allResults = [];
+    const isChildProcess = process.env[CHILD_MODE_ENV] === '1';
+    for (const { name, file } of corpora) {
+        test(`tokenize ${name}`, () => {
+            const result = isChildProcess
+                ? benchmarkTokenize(name, loadCorpus(file))
+                : runBenchmarkInFreshProcess(`tokenize ${name}`);
+            if (!isChildProcess) {
+                allResults.push(result);
+            }
+            console.log(`  ${name}: median=${result.medianMs.toFixed(2)}ms, tokens=${result.tokenCount}, tok/sec=${Math.round(result.tokensPerSec).toLocaleString()}`);
+            if (isChildProcess) {
+                emitChildResult(result);
+            }
+            expect(result.tokenCount).toBeGreaterThan(0);
+            expect(result.medianMs).toBeLessThan(5000);
+        });
+    }
+    test('scaled corpus (10x large_stdlib)', () => {
+        const result = isChildProcess
+            ? benchmarkTokenize('large_stdlib_10x', Array(10).fill(loadCorpus('large_stdlib.py')).join('\n'))
+            : runBenchmarkInFreshProcess('scaled corpus (10x large_stdlib)');
+        if (!isChildProcess) {
+            allResults.push(result);
+        }
+        console.log(`  large_stdlib_10x: median=${result.medianMs.toFixed(2)}ms, tokens=${result.tokenCount}, tok/sec=${Math.round(result.tokensPerSec).toLocaleString()}`);
+        if (isChildProcess) {
+            emitChildResult(result);
+        }
+        expect(result.tokenCount).toBeGreaterThan(0);
+    });
+    afterAll(() => {
+        if (isChildProcess || allResults.length === 0) {
+            return;
+        }
+        printResultTable(allResults);
+        const report = {
+            timestamp: new Date().toISOString(),
+            system: getSystemInfo(),
+            config: {
+                warmupIterations: WARMUP_ITERATIONS,
+                benchmarkIterations: BENCHMARK_ITERATIONS,
+            },
+            results: allResults,
+        };
+        writeReport(report);
+    });
+});
+//# sourceMappingURL=tokenizerBenchmark.test.js.map

package/dist/tests/benchmarks/tokenizerBenchmark.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizerBenchmark.test.js","sourceRoot":"","sources":["../../../../../../src/tests/benchmarks/tokenizerBenchmark.test.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;GAaG;;;;;;;;;;;;;;;;;;;;;;;;;AAEH,iDAA6C;AAC7C,uCAAyB;AACzB,uCAAyB;AACzB,2CAA6B;AAE7B,sDAAmD;AAEnD,wBAAwB;AAExB,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAC5B,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAEhC,MAAM,oBAAoB,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,EAAE,mBAAmB,EAAE,WAAW,CAAC,CAAC;AAClG,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;AAC1G,MAAM,mBAAmB,GAAG,gCAAgC,CAAC;AAC7D,MAAM,cAAc,GAAG,+BAA+B,CAAC;AACvD,MAAM,kBAAkB,GAAG,wBAAwB,CAAC;AAmCpD,kBAAkB;AAElB,SAAS,cAAc,CAAC,KAA4B;IAOhD,MAAM,MAAM,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAChD,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC;IAE1B,MAAM,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;IACzG,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;IAC3C,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;IAChD,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IACtB,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;IAC5B,MAAM,GAAG,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC;IAEnD,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC;AAC1C,CAAC;AAED,SAAS,UAAU,CAAC,QAAgB;IAChC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,EAAE,eAAe,EAAE,QAAQ,CAAC,CAAC;IAC1E,OAAO,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;AAC9C,CAAC;AAED,SAAS,aAAa;IAClB,MAAM,IAAI,GAAG,EAAE,CAAC,IAAI,EAAE,CAAC;IACvB,OAAO;QACH,QAAQ,EAAE,EAAE,CAAC,QAAQ,EAAE;QACvB,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE;QACf,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,SAAS;QACjC,QAAQ,EAAE,IAAI,CAAC,MAAM;QACrB,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;QACxD,WAAW,EAAE,OAAO,CAAC,OAAO;KAC/B,CAAC;AACN,CAAC;AAED,SAAS,WAAW,CAAC,MAAuB;IACxC,EAAE,CAAC,SAAS,CAAC,oBAAoB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACxD,MAAM,QAAQ,GAAG,uBAAuB,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC;IAC9F,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,QAAQ,CAAC,CAAC;IAC7D,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IAC5E,OAAO,CAAC,GAAG,CAAC,mCAAmC,UAAU,EAAE,CAAC,CAAC;AACjE,CAAC;AAED,SAAS,gBAAgB,CAAC,OAAuC;IAC7D,OAAO,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAC;IACvD,OAAO,CAAC,GAAG,CACP,GAAG,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,KAAK,CAAC,QAAQ,CAC3G,EAAE,CACL,IAAI,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAClG,CAAC;IACF,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IAE7B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,aAAa,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;QAC/D,OAAO,CAAC,GAAG,CACP,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,QAAQ,CACnF,CAAC,CACJ,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,KAAK;aAC/F,OAAO,CAAC,CAAC,CAAC;aACV,QAAQ,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,KAAK;aACpE,OAAO,CAAC,CAAC,CAAC;aACV,QAAQ,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,cAAc,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CACvF,CAAC;IACN,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;AACpB,CAAC;AAED,SAAS,eAAe,CAAC,MAAuB;IAC5C,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,mBAAmB,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;AAC9E,CAAC;AAED,SAAS,cAAc,CAAC,KAAc;IAClC,IAAI,CAAC,CAAC,KAAK,YAAY,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO,EAAE,CAAC;IACd,CAAC;IAED,MAAM,MAAM,GAAG,QAAQ,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;IACzF,MAAM,MAAM,GAAG,QAAQ,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;IACzF,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACzE,CAAC;AAED,SAAS,YAAY,CAAC,IAAY;IAC9B,OAAO,IAAI,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAC;AACvD,CAAC;AAED,SAAS,0BAA0B,CAAC,QAAgB;IAChD,IAAI,CAAC;QACD,MAAM,MAAM,GAAG,IAAA,4BAAY,EACvB,OAAO,CAAC,QAAQ,EAChB;YACI,aAAa;YACb,UAAU;YACV,aAAa;YACb,aAAa;YACb,sBAAsB;YACtB,mBAAmB;YACnB,wBAAwB,YAAY,CAAC,QAAQ,CAAC,GAAG;SACpD,EACD;YACI,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC;YAC9C,QAAQ,EAAE,OAAO;YACjB,GAAG,EAAE;gBACD,GAAG,OAAO,CAAC,GAAG;gBACd,CAAC,cAAc,CAAC,EAAE,GAAG;aACxB;SACJ,CACJ,CAAC;QAEF,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC,CAAC;QAE9F,IAAI,CAAC,UAAU,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,wBAAwB,QAAQ,6BAA6B,MAAM,EAAE,CAAC,CAAC;QAC3F,CAAC;QAED,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAoB,CAAC;IACvF,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,MAAM,MAAM,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;QACrC,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACvE,MAAM,IAAI,KAAK,CAAC,wBAAwB,QAAQ,cAAc,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC3G,CAAC;AACL,CAAC;AAED,SAAS,iBAAiB,CAAC,UAAkB,EAAE,IAAY;IACvD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,iBAAiB,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,MAAM,SAAS,GAAG,IAAI,qBAAS,EAAE,CAAC;QAClC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,oBAAoB,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5C,MAAM,SAAS,GAAG,IAAI,qBAAS,EAAE,CAAC;QAElC,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAChC,MAAM,OAAO,GAAG,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACzC,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QAE1C,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACpB,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC;IACtC,CAAC;IAED,MAAM,KAAK,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;IAEpC,OAAO;QACH,MAAM,EAAE,UAAU;QAClB,aAAa,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC;QAC/C,UAAU,EAAE,oBAAoB;QAChC,OAAO,EAAE,KAAK;QACd,QAAQ,EAAE,KAAK,CAAC,MAAM;QACtB,KAAK,EAAE,KAAK,CAAC,GAAG;QAChB,KAAK,EAAE,KAAK,CAAC,GAAG;QAChB,KAAK,EAAE,KAAK,CAAC,GAAG;QAChB,KAAK,EAAE,KAAK,CAAC,GAAG;QAChB,UAAU;QACV,YAAY,EAAE,UAAU,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC;KACnD,CAAC;AACN,CAAC;AAED,6BAA6B;AAE7B,MAAM,OAAO,GAAqC;IAC9C,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,iBAAiB,EAAE;IACjD,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,kBAAkB,EAAE;IACnD,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,kBAAkB,EAAE;IACnD,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,gBAAgB,EAAE;IAC/C,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,iBAAiB,EAAE;IACjD,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,gBAAgB,EAAE;IAC/C,EAAE,IAAI,EAAE,wBAAwB,EAAE,IAAI,EAAE,2BAA2B,EAAE;CACxE,CAAC;AAEF,gBAAgB;AAEhB,MAAM,cAAc,GAAG,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC;AAE1F,cAAc,CAAC,qBAAqB,EAAE,GAAG,EAAE;IACvC,MAAM,UAAU,GAAsB,EAAE,CAAC;IACzC,MAAM,cAAc,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,KAAK,GAAG,CAAC;IAE3D,KAAK,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,OAAO,EAAE,CAAC;QACnC,IAAI,CAAC,YAAY,IAAI,EAAE,EAAE,GAAG,EAAE;YAC1B,MAAM,MAAM,GAAG,cAAc;gBACzB,CAAC,CAAC,iBAAiB,CAAC,IAAI,EAAE,UAAU,CAAC,IAAI,CAAC,CAAC;gBAC3C,CAAC,CAAC,0BAA0B,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC;YAErD,IAAI,CAAC,cAAc,EAAE,CAAC;gBAClB,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC5B,CAAC;YAED,OAAO,CAAC,GAAG,CACP,KAAK,IAAI,YAAY,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,cAAc,MAAM,CAAC,UAAU,aAAa,IAAI,CAAC,KAAK,CACjG,MAAM,CAAC,YAAY,CACtB,CAAC,cAAc,EAAE,EAAE,CACvB,CAAC;YAEF,IAAI,cAAc,EAAE,CAAC;gBACjB,eAAe,CAAC,MAAM,CAAC,CAAC;YAC5B,CAAC;YAED,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YAC7C,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;IACP,CAAC;IAED,IAAI,CAAC,kCAAkC,EAAE,GAAG,EAAE;QAC1C,MAAM,MAAM,GAAG,cAAc;YACzB,CAAC,CAAC,iBAAiB,CAAC,kBAAkB,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjG,CAAC,CAAC,0BAA0B,CAAC,kCAAkC,CAAC,CAAC;QAErE,IAAI,CAAC,cAAc,EAAE,CAAC;YAClB,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC5B,CAAC;QAED,OAAO,CAAC,GAAG,CACP,8BAA8B,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,cACpD,MAAM,CAAC,UACX,aAAa,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,cAAc,EAAE,EAAE,CAClE,CAAC;QAEF,IAAI,cAAc,EAAE,CAAC;YACjB,eAAe,CAAC,MAAM,CAAC,CAAC;QAC5B,CAAC;QAED,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,GAAG,EAAE;QACV,IAAI,cAAc,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5C,OAAO;QACX,CAAC;QAED,gBAAgB,CAAC,UAAU,CAAC,CAAC;QAE7B,MAAM,MAAM,GAAoB;YAC5B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,MAAM,EAAE,aAAa,EAAE;YACvB,MAAM,EAAE;gBACJ,gBAAgB,EAAE,iBAAiB;gBACnC,mBAAmB,EAAE,oBAAoB;aAC5C;YACD,OAAO,EAAE,UAAU;SACtB,CAAC;QAEF,WAAW,CAAC,MAAM,CAAC,CAAC;IACxB,CAAC,CAAC,CAAC;AACP,CAAC,CAAC,CAAC"}

package/dist/tests/tokenizer.test.js

@@ -1410,7 +1410,8 @@ test('Lines1', () => {
 });
 test('Comments1', () => {
     const t = new tokenizer_1.Tokenizer();
-    const results = t.tokenize('# hello\n# good bye\n\n\n""" test """ # another\n\n\npass');
+    const text = '# hello\n# good bye\n\n\n""" test """ # another\n\n\npass';
+    const results = t.tokenize(text);
     assert_1.default.equal(results.tokens.count, 4 + _implicitTokenCount);
     const token0 = results.tokens.getItemAt(0);
     assert_1.default.equal(token0.type, 2 /* TokenType.NewLine */);
@@ -1439,7 +1440,8 @@ test('Comments1', () => {
 });
 test('Comments2', () => {
     const t = new tokenizer_1.Tokenizer();
-    const results = t.tokenize('class A:\n    def func(self):\n        pass\n        # comment\n    ');
+    const text = 'class A:\n    def func(self):\n        pass\n        # comment\n    ';
+    const results = t.tokenize(text);
     assert_1.default.equal(results.tokens.count, 16 + _implicitTokenCount);
     const token17 = results.tokens.getItemAt(17);
     assert_1.default.equal(token17.type, 1 /* TokenType.EndOfStream */);
@@ -1524,6 +1526,19 @@ test('TypeIgnoreLine2', () => {
     assert_1.default.equal(results.tokens.contains(41), true);
     assert_1.default.equal(results.tokens.contains(42), false);
 });
+test('TypeIgnoreLineMalformedBracket', () => {
+    const t = new tokenizer_1.Tokenizer();
+    const results = t.tokenize('a = 3 # type: ignore[broken');
+    assert_1.default.equal(results.typeIgnoreLines.size, 0);
+});
+// A space-separated unclosed bracket (e.g. `# type: ignore [broken`) is also
+// rejected entirely. The tokenizer does not fall back to treating the
+// directive as "ignore all" when the bracket list is present but malformed.
+test('TypeIgnoreLineMalformedBracketWithSpace', () => {
+    const t = new tokenizer_1.Tokenizer();
+    const results = t.tokenize('a = 3 # type: ignore [broken');
+    assert_1.default.equal(results.typeIgnoreLines.size, 0);
+});
 // Regression test for https://github.com/microsoft/pyright/issues/11345.
 // type: ignore comments containing tool-namespaced codes (e.g. "ty:rule-name")
 // must be recognised as type: ignore comments.