@zzzen/pyright-internal 1.2.0-dev.20230514 → 1.2.0-dev.20230521

package/dist/tests/textRange.test.js

@@ -0,0 +1,45 @@
+"use strict";
+/*
+ * textRange.test.ts
+ * Copyright (c) Microsoft Corporation.
+ * Licensed under the MIT license.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const assert = __importStar(require("assert"));
+const textRange_1 = require("../common/textRange");
+test('textRange combine', () => {
+    const range1 = textRange_1.TextRange.create(10, 2);
+    const range2 = textRange_1.TextRange.create(12, 2);
+    const range3 = textRange_1.TextRange.create(8, 2);
+    const combined = textRange_1.TextRange.combine([range1, range2, range3]);
+    assert.ok(combined);
+    assert.equal(combined.start, 8);
+    assert.equal(combined.length, 6);
+    // Ensure input ranges are unchanged
+    assert.equal(range1.start, 10);
+    assert.equal(range1.length, 2);
+    assert.equal(range2.start, 12);
+    assert.equal(range2.length, 2);
+    assert.equal(range3.start, 8);
+    assert.equal(range3.length, 2);
+});
+//# sourceMappingURL=textRange.test.js.map

package/dist/tests/textRange.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"textRange.test.js","sourceRoot":"","sources":["../../../src/tests/textRange.test.ts"],"names":[],"mappings":";AAAA;;;;GAIG;;;;;;;;;;;;;;;;;;;;;AAEH,+CAAiC;AAEjC,mDAAgD;AAEhD,IAAI,CAAC,mBAAmB,EAAE,GAAG,EAAE;IAC3B,MAAM,MAAM,GAAG,qBAAS,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IACvC,MAAM,MAAM,GAAG,qBAAS,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IACvC,MAAM,MAAM,GAAG,qBAAS,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAEtC,MAAM,QAAQ,GAAG,qBAAS,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAE7D,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC;IACpB,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IAChC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAEjC,oCAAoC;IACpC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAC/B,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC/B,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAC/B,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC/B,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IAC9B,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;AACnC,CAAC,CAAC,CAAC"}

package/dist/tests/tokenizer.test.js

@@ -449,90 +449,304 @@ test('Strings: triplicate double quote escape', () => {
 test('Strings: single quoted f-string', () => {
     const t = new tokenizer_1.Tokenizer();
     const results = t.tokenize("a+f'quoted'");
-    assert_1.default.equal(results.tokens.count, 3 + _implicitTokenCount);
+    assert_1.default.equal(results.tokens.count, 5 + _implicitTokenCount);
     assert_1.default.equal(results.tokens.getItemAt(0).type, 7 /* Identifier */);
     assert_1.default.equal(results.tokens.getItemAt(1).type, 9 /* Operator */);
-    const stringToken = results.tokens.getItemAt(2);
-    assert_1.default.equal(stringToken.type, 5 /* String */);
-    assert_1.default.equal(stringToken.flags, 1 /* SingleQuote */ | 64 /* Format */);
-    assert_1.default.equal(stringToken.length, 9);
-    assert_1.default.equal(stringToken.escapedValue, 'quoted');
+    const fStringStartToken = results.tokens.getItemAt(2);
+    assert_1.default.equal(fStringStartToken.type, 24 /* FStringStart */);
+    assert_1.default.equal(fStringStartToken.flags, 1 /* SingleQuote */ | 64 /* Format */);
+    assert_1.default.equal(fStringStartToken.length, 2);
+    const fStringMiddleToken = results.tokens.getItemAt(3);
+    assert_1.default.equal(fStringMiddleToken.type, 25 /* FStringMiddle */);
+    assert_1.default.equal(fStringMiddleToken.flags, 1 /* SingleQuote */ | 64 /* Format */);
+    assert_1.default.equal(fStringMiddleToken.length, 6);
+    assert_1.default.equal(fStringMiddleToken.escapedValue, 'quoted');
+    const fStringEndToken = results.tokens.getItemAt(4);
+    assert_1.default.equal(fStringEndToken.type, 26 /* FStringEnd */);
+    assert_1.default.equal(fStringEndToken.flags, 1 /* SingleQuote */ | 64 /* Format */);
+    assert_1.default.equal(fStringEndToken.length, 1);
 });
 test('Strings: double quoted f-string', () => {
     const t = new tokenizer_1.Tokenizer();
     const results = t.tokenize('x(1,f"quoted")');
-    assert_1.default.equal(results.tokens.count, 6 + _implicitTokenCount);
+    assert_1.default.equal(results.tokens.count, 8 + _implicitTokenCount);
     assert_1.default.equal(results.tokens.getItemAt(0).type, 7 /* Identifier */);
     assert_1.default.equal(results.tokens.getItemAt(1).type, 13 /* OpenParenthesis */);
     assert_1.default.equal(results.tokens.getItemAt(2).type, 6 /* Number */);
     assert_1.default.equal(results.tokens.getItemAt(3).type, 12 /* Comma */);
-    assert_1.default.equal(results.tokens.getItemAt(5).type, 14 /* CloseParenthesis */);
-    const stringToken = results.tokens.getItemAt(4);
-    assert_1.default.equal(stringToken.type, 5 /* String */);
-    assert_1.default.equal(stringToken.flags, 2 /* DoubleQuote */ | 64 /* Format */);
-    assert_1.default.equal(stringToken.length, 9);
-    assert_1.default.equal(stringToken.escapedValue, 'quoted');
+    assert_1.default.equal(results.tokens.getItemAt(7).type, 14 /* CloseParenthesis */);
+    const fStringStartToken = results.tokens.getItemAt(4);
+    assert_1.default.equal(fStringStartToken.type, 24 /* FStringStart */);
+    assert_1.default.equal(fStringStartToken.flags, 2 /* DoubleQuote */ | 64 /* Format */);
+    assert_1.default.equal(fStringStartToken.length, 2);
+    const fStringMiddleToken = results.tokens.getItemAt(5);
+    assert_1.default.equal(fStringMiddleToken.type, 25 /* FStringMiddle */);
+    assert_1.default.equal(fStringMiddleToken.flags, 2 /* DoubleQuote */ | 64 /* Format */);
+    assert_1.default.equal(fStringMiddleToken.length, 6);
+    assert_1.default.equal(fStringMiddleToken.escapedValue, 'quoted');
+    const fStringEndToken = results.tokens.getItemAt(6);
+    assert_1.default.equal(fStringEndToken.type, 26 /* FStringEnd */);
+    assert_1.default.equal(fStringEndToken.flags, 2 /* DoubleQuote */ | 64 /* Format */);
+    assert_1.default.equal(fStringEndToken.length, 1);
 });
 test('Strings: single quoted multiline f-string', () => {
     const t = new tokenizer_1.Tokenizer();
     const results = t.tokenize("f'''quoted'''");
-    assert_1.default.equal(results.tokens.count, 1 + _implicitTokenCount);
-    const stringToken = results.tokens.getItemAt(0);
-    assert_1.default.equal(stringToken.type, 5 /* String */);
-    assert_1.default.equal(stringToken.flags, 1 /* SingleQuote */ | 4 /* Triplicate */ | 64 /* Format */);
-    assert_1.default.equal(stringToken.length, 13);
-    assert_1.default.equal(stringToken.escapedValue, 'quoted');
+    assert_1.default.equal(results.tokens.count, 3 + _implicitTokenCount);
+    const fStringStartToken = results.tokens.getItemAt(0);
+    assert_1.default.equal(fStringStartToken.type, 24 /* FStringStart */);
+    assert_1.default.equal(fStringStartToken.flags, 1 /* SingleQuote */ | 4 /* Triplicate */ | 64 /* Format */);
+    assert_1.default.equal(fStringStartToken.length, 4);
+    const fStringMiddleToken = results.tokens.getItemAt(1);
+    assert_1.default.equal(fStringMiddleToken.type, 25 /* FStringMiddle */);
+    assert_1.default.equal(fStringMiddleToken.flags, 1 /* SingleQuote */ | 4 /* Triplicate */ | 64 /* Format */);
+    assert_1.default.equal(fStringMiddleToken.length, 6);
+    assert_1.default.equal(fStringMiddleToken.escapedValue, 'quoted');
+    const fStringEndToken = results.tokens.getItemAt(2);
+    assert_1.default.equal(fStringEndToken.type, 26 /* FStringEnd */);
+    assert_1.default.equal(fStringEndToken.flags, 1 /* SingleQuote */ | 4 /* Triplicate */ | 64 /* Format */);
+    assert_1.default.equal(fStringEndToken.length, 3);
 });
 test('Strings: double quoted multiline f-string', () => {
     const t = new tokenizer_1.Tokenizer();
     const results = t.tokenize('f"""quoted """');
-    assert_1.default.equal(results.tokens.count, 1 + _implicitTokenCount);
-    const stringToken = results.tokens.getItemAt(0);
-    assert_1.default.equal(stringToken.type, 5 /* String */);
-    assert_1.default.equal(stringToken.flags, 2 /* DoubleQuote */ | 4 /* Triplicate */ | 64 /* Format */);
-    assert_1.default.equal(stringToken.length, 14);
-    assert_1.default.equal(stringToken.escapedValue, 'quoted ');
+    assert_1.default.equal(results.tokens.count, 3 + _implicitTokenCount);
+    const fStringStartToken = results.tokens.getItemAt(0);
+    assert_1.default.equal(fStringStartToken.type, 24 /* FStringStart */);
+    assert_1.default.equal(fStringStartToken.flags, 2 /* DoubleQuote */ | 4 /* Triplicate */ | 64 /* Format */);
+    assert_1.default.equal(fStringStartToken.length, 4);
+    const fStringMiddleToken = results.tokens.getItemAt(1);
+    assert_1.default.equal(fStringMiddleToken.type, 25 /* FStringMiddle */);
+    assert_1.default.equal(fStringMiddleToken.flags, 2 /* DoubleQuote */ | 4 /* Triplicate */ | 64 /* Format */);
+    assert_1.default.equal(fStringMiddleToken.length, 7);
+    assert_1.default.equal(fStringMiddleToken.escapedValue, 'quoted ');
+    const fStringEndToken = results.tokens.getItemAt(2);
+    assert_1.default.equal(fStringEndToken.type, 26 /* FStringEnd */);
+    assert_1.default.equal(fStringEndToken.flags, 2 /* DoubleQuote */ | 4 /* Triplicate */ | 64 /* Format */);
+    assert_1.default.equal(fStringEndToken.length, 3);
 });
 test('Strings: f-string with single right brace', () => {
     const t = new tokenizer_1.Tokenizer();
     const results = t.tokenize("f'hello}'");
-    assert_1.default.equal(results.tokens.count, 1 + _implicitTokenCount);
-    const stringToken = results.tokens.getItemAt(0);
-    const unescapedValue = StringTokenUtils.getUnescapedString(stringToken);
-    assert_1.default.equal(stringToken.type, 5 /* String */);
-    assert_1.default.equal(stringToken.flags, 1 /* SingleQuote */ | 64 /* Format */);
-    assert_1.default.equal(unescapedValue.formatStringSegments.length, 1);
-    assert_1.default.equal(unescapedValue.unescapeErrors.length, 1);
-    assert_1.default.equal(unescapedValue.unescapeErrors[0].offset, 5);
-    assert_1.default.equal(unescapedValue.unescapeErrors[0].length, 1);
-    assert_1.default.equal(unescapedValue.unescapeErrors[0].errorType, 2 /* SingleCloseBraceWithinFormatLiteral */);
+    assert_1.default.equal(results.tokens.count, 4 + _implicitTokenCount);
+    const fStringStartToken = results.tokens.getItemAt(0);
+    assert_1.default.equal(fStringStartToken.type, 24 /* FStringStart */);
+    assert_1.default.equal(fStringStartToken.length, 2);
+    assert_1.default.equal(fStringStartToken.flags, 1 /* SingleQuote */ | 64 /* Format */);
+    const fStringMiddleToken = results.tokens.getItemAt(1);
+    assert_1.default.equal(fStringMiddleToken.type, 25 /* FStringMiddle */);
+    assert_1.default.equal(fStringMiddleToken.length, 5);
+    assert_1.default.equal(fStringMiddleToken.flags, 1 /* SingleQuote */ | 64 /* Format */ | 256 /* ReplacementFieldEnd */);
+    const braceToken = results.tokens.getItemAt(2).type;
+    assert_1.default.equal(braceToken, 18 /* CloseCurlyBrace */);
+    const fStringEndToken = results.tokens.getItemAt(3);
+    assert_1.default.equal(fStringEndToken.type, 26 /* FStringEnd */);
+    assert_1.default.equal(fStringEndToken.flags, 1 /* SingleQuote */ | 64 /* Format */);
+    assert_1.default.equal(fStringEndToken.length, 1);
 });
 test('Strings: f-string with escape in expression', () => {
     const t = new tokenizer_1.Tokenizer();
-    const results = t.tokenize("f'hello { \\t }'");
-    assert_1.default.equal(results.tokens.count, 1 + _implicitTokenCount);
-    const stringToken = results.tokens.getItemAt(0);
-    const unescapedValue = StringTokenUtils.getUnescapedString(stringToken);
-    assert_1.default.equal(stringToken.type, 5 /* String */);
-    assert_1.default.equal(stringToken.flags, 1 /* SingleQuote */ | 64 /* Format */);
-    assert_1.default.equal(unescapedValue.formatStringSegments.length, 2);
-    assert_1.default.equal(unescapedValue.unescapeErrors.length, 1);
-    assert_1.default.equal(unescapedValue.unescapeErrors[0].offset, 8);
-    assert_1.default.equal(unescapedValue.unescapeErrors[0].length, 1);
-    assert_1.default.equal(unescapedValue.unescapeErrors[0].errorType, 1 /* EscapeWithinFormatExpression */);
+    const results = t.tokenize(`f'hello { "\\t" }'`);
+    assert_1.default.equal(results.tokens.count, 6 + _implicitTokenCount);
+    assert_1.default.equal(results.tokens.getItemAt(0).type, 24 /* FStringStart */);
+    assert_1.default.equal(results.tokens.getItemAt(1).type, 25 /* FStringMiddle */);
+    assert_1.default.equal(results.tokens.getItemAt(2).type, 17 /* OpenCurlyBrace */);
+    assert_1.default.equal(results.tokens.getItemAt(3).type, 5 /* String */);
+    assert_1.default.equal(results.tokens.getItemAt(4).type, 18 /* CloseCurlyBrace */);
+    assert_1.default.equal(results.tokens.getItemAt(5).type, 26 /* FStringEnd */);
+});
+test('Strings: f-string with escape in format string 1', () => {
+    const t = new tokenizer_1.Tokenizer();
+    const results = t.tokenize("f'he\\{ 1 }lo'");
+    assert_1.default.equal(results.tokens.count, 7 + _implicitTokenCount);
+    assert_1.default.equal(results.tokens.getItemAt(0).type, 24 /* FStringStart */);
+    const middleFString = results.tokens.getItemAt(1);
+    assert_1.default.equal(middleFString.type, 25 /* FStringMiddle */);
+    assert_1.default.equal(middleFString.escapedValue.length, 3);
+    assert_1.default.equal(results.tokens.getItemAt(2).type, 17 /* OpenCurlyBrace */);
+    assert_1.default.equal(results.tokens.getItemAt(3).type, 6 /* Number */);
+    assert_1.default.equal(results.tokens.getItemAt(4).type, 18 /* CloseCurlyBrace */);
+    assert_1.default.equal(results.tokens.getItemAt(5).type, 25 /* FStringMiddle */);
+    assert_1.default.equal(results.tokens.getItemAt(6).type, 26 /* FStringEnd */);
+});
+test('Strings: f-string with escape in format string 2', () => {
+    const t = new tokenizer_1.Tokenizer();
+    const results = t.tokenize(`f"'{{\\"{0}\\": {0}}}'"`);
+    assert_1.default.equal(results.tokens.count, 11 + _implicitTokenCount);
+    assert_1.default.equal(results.tokens.getItemAt(0).type, 24 /* FStringStart */);
+    const middleFString = results.tokens.getItemAt(1);
+    assert_1.default.equal(middleFString.type, 25 /* FStringMiddle */);
+    assert_1.default.equal(middleFString.escapedValue.length, 5);
+    assert_1.default.equal(results.tokens.getItemAt(2).type, 17 /* OpenCurlyBrace */);
+    assert_1.default.equal(results.tokens.getItemAt(3).type, 6 /* Number */);
+    assert_1.default.equal(results.tokens.getItemAt(4).type, 18 /* CloseCurlyBrace */);
+    assert_1.default.equal(results.tokens.getItemAt(5).type, 25 /* FStringMiddle */);
+    assert_1.default.equal(results.tokens.getItemAt(6).type, 17 /* OpenCurlyBrace */);
+    assert_1.default.equal(results.tokens.getItemAt(7).type, 6 /* Number */);
+    assert_1.default.equal(results.tokens.getItemAt(8).type, 18 /* CloseCurlyBrace */);
+    assert_1.default.equal(results.tokens.getItemAt(9).type, 25 /* FStringMiddle */);
+    assert_1.default.equal(results.tokens.getItemAt(10).type, 26 /* FStringEnd */);
+});
+test('Strings: f-string with double brace', () => {
+    const t = new tokenizer_1.Tokenizer();
+    const results = t.tokenize(`f"hello {{{0==0}}}"`);
+    assert_1.default.equal(results.tokens.count, 9 + _implicitTokenCount);
+    assert_1.default.equal(results.tokens.getItemAt(0).type, 24 /* FStringStart */);
+    assert_1.default.equal(results.tokens.getItemAt(1).type, 25 /* FStringMiddle */);
+    assert_1.default.equal(results.tokens.getItemAt(2).type, 17 /* OpenCurlyBrace */);
+    assert_1.default.equal(results.tokens.getItemAt(3).type, 6 /* Number */);
+    assert_1.default.equal(results.tokens.getItemAt(4).type, 9 /* Operator */);
+    assert_1.default.equal(results.tokens.getItemAt(5).type, 6 /* Number */);
+    assert_1.default.equal(results.tokens.getItemAt(6).type, 18 /* CloseCurlyBrace */);
+    assert_1.default.equal(results.tokens.getItemAt(7).type, 25 /* FStringMiddle */);
+    assert_1.default.equal(results.tokens.getItemAt(8).type, 26 /* FStringEnd */);
+});
+test('Strings: f-string with walrus operator', () => {
+    const t = new tokenizer_1.Tokenizer();
+    const results = t.tokenize(`f"{(x:=0)}"`);
+    assert_1.default.equal(results.tokens.count, 9 + _implicitTokenCount);
+    assert_1.default.equal(results.tokens.getItemAt(0).type, 24 /* FStringStart */);
+    assert_1.default.equal(results.tokens.getItemAt(1).type, 17 /* OpenCurlyBrace */);
+    assert_1.default.equal(results.tokens.getItemAt(2).type, 13 /* OpenParenthesis */);
+    assert_1.default.equal(results.tokens.getItemAt(3).type, 7 /* Identifier */);
+    assert_1.default.equal(results.tokens.getItemAt(4).type, 9 /* Operator */);
+    assert_1.default.equal(results.tokens.getItemAt(5).type, 6 /* Number */);
+    assert_1.default.equal(results.tokens.getItemAt(6).type, 14 /* CloseParenthesis */);
+    assert_1.default.equal(results.tokens.getItemAt(7).type, 18 /* CloseCurlyBrace */);
+    assert_1.default.equal(results.tokens.getItemAt(8).type, 26 /* FStringEnd */);
+});
+test('Strings: f-string with single right brace', () => {
+    const t = new tokenizer_1.Tokenizer();
+    const results = t.tokenize(`f"}"`);
+    assert_1.default.equal(results.tokens.count, 3 + _implicitTokenCount);
+    assert_1.default.equal(results.tokens.getItemAt(0).type, 24 /* FStringStart */);
+    assert_1.default.equal(results.tokens.getItemAt(1).type, 18 /* CloseCurlyBrace */);
+    assert_1.default.equal(results.tokens.getItemAt(2).type, 26 /* FStringEnd */);
+});
+test('Strings: f-string with comment', () => {
+    const t = new tokenizer_1.Tokenizer();
+    const results = t.tokenize(`f'''hello{\nx # comment\n}'''`);
+    assert_1.default.equal(results.tokens.count, 6 + _implicitTokenCount);
+    assert_1.default.equal(results.tokens.getItemAt(0).type, 24 /* FStringStart */);
+    assert_1.default.equal(results.tokens.getItemAt(1).type, 25 /* FStringMiddle */);
+    assert_1.default.equal(results.tokens.getItemAt(2).type, 17 /* OpenCurlyBrace */);
+    assert_1.default.equal(results.tokens.getItemAt(3).type, 7 /* Identifier */);
+    const closeBraceToken = results.tokens.getItemAt(4);
+    assert_1.default.equal(closeBraceToken.type, 18 /* CloseCurlyBrace */);
+    assert_1.default.deepEqual(closeBraceToken.comments, [
+        { type: 0 /* Regular */, value: ' comment', start: 14, length: 8 },
+    ]);
+    assert_1.default.equal(results.tokens.getItemAt(5).type, 26 /* FStringEnd */);
 });
 test('Strings: f-string with unterminated expression', () => {
     const t = new tokenizer_1.Tokenizer();
-    const results = t.tokenize("f'hello { a + b'");
-    assert_1.default.equal(results.tokens.count, 1 + _implicitTokenCount);
-    const stringToken = results.tokens.getItemAt(0);
-    const unescapedValue = StringTokenUtils.getUnescapedString(stringToken);
-    assert_1.default.equal(stringToken.type, 5 /* String */);
-    assert_1.default.equal(stringToken.flags, 1 /* SingleQuote */ | 64 /* Format */);
-    assert_1.default.equal(unescapedValue.formatStringSegments.length, 2);
-    assert_1.default.equal(unescapedValue.unescapeErrors.length, 1);
-    assert_1.default.equal(unescapedValue.unescapeErrors[0].offset, 7);
-    assert_1.default.equal(unescapedValue.unescapeErrors[0].errorType, 3 /* UnterminatedFormatExpression */);
+    const results = t.tokenize("f'hello { a'");
+    assert_1.default.equal(results.tokens.count, 5 + _implicitTokenCount);
+    assert_1.default.equal(results.tokens.getItemAt(0).type, 24 /* FStringStart */);
+    assert_1.default.equal(results.tokens.getItemAt(1).type, 25 /* FStringMiddle */);
+    assert_1.default.equal(results.tokens.getItemAt(2).type, 17 /* OpenCurlyBrace */);
+    assert_1.default.equal(results.tokens.getItemAt(3).type, 7 /* Identifier */);
+    const fStringEnd = results.tokens.getItemAt(4);
+    assert_1.default.equal(fStringEnd.type, 26 /* FStringEnd */);
+    assert_1.default.equal(fStringEnd.flags, 64 /* Format */ | 1 /* SingleQuote */);
+});
+test('Strings: f-string with replacement field', () => {
+    const t = new tokenizer_1.Tokenizer();
+    const results = t.tokenize("f'hello { a + b}'");
+    assert_1.default.equal(results.tokens.count, 8 + _implicitTokenCount);
+    assert_1.default.equal(results.tokens.getItemAt(0).type, 24 /* FStringStart */);
+    assert_1.default.equal(results.tokens.getItemAt(1).type, 25 /* FStringMiddle */);
+    assert_1.default.equal(results.tokens.getItemAt(2).type, 17 /* OpenCurlyBrace */);
+    assert_1.default.equal(results.tokens.getItemAt(3).type, 7 /* Identifier */);
+    assert_1.default.equal(results.tokens.getItemAt(4).type, 9 /* Operator */);
+    assert_1.default.equal(results.tokens.getItemAt(5).type, 7 /* Identifier */);
+    assert_1.default.equal(results.tokens.getItemAt(6).type, 18 /* CloseCurlyBrace */);
+    assert_1.default.equal(results.tokens.getItemAt(7).type, 26 /* FStringEnd */);
+});
+test('Strings: f-string with format specifier', () => {
+    const t = new tokenizer_1.Tokenizer();
+    const results = t.tokenize("f'hello { a ! b}'");
+    assert_1.default.equal(results.tokens.count, 8 + _implicitTokenCount);
+    assert_1.default.equal(results.tokens.getItemAt(0).type, 24 /* FStringStart */);
+    assert_1.default.equal(results.tokens.getItemAt(1).type, 25 /* FStringMiddle */);
+    assert_1.default.equal(results.tokens.getItemAt(2).type, 17 /* OpenCurlyBrace */);
+    assert_1.default.equal(results.tokens.getItemAt(3).type, 7 /* Identifier */);
+    assert_1.default.equal(results.tokens.getItemAt(4).type, 23 /* ExclamationMark */);
+    assert_1.default.equal(results.tokens.getItemAt(5).type, 7 /* Identifier */);
+    assert_1.default.equal(results.tokens.getItemAt(6).type, 18 /* CloseCurlyBrace */);
+    assert_1.default.equal(results.tokens.getItemAt(7).type, 26 /* FStringEnd */);
+});
+test('Strings: f-string with debug format specifier', () => {
+    const t = new tokenizer_1.Tokenizer();
+    const results = t.tokenize("f'hello { a =}'");
+    assert_1.default.equal(results.tokens.count, 7 + _implicitTokenCount);
+    assert_1.default.equal(results.tokens.getItemAt(0).type, 24 /* FStringStart */);
+    assert_1.default.equal(results.tokens.getItemAt(1).type, 25 /* FStringMiddle */);
+    assert_1.default.equal(results.tokens.getItemAt(2).type, 17 /* OpenCurlyBrace */);
+    assert_1.default.equal(results.tokens.getItemAt(3).type, 7 /* Identifier */);
+    assert_1.default.equal(results.tokens.getItemAt(4).type, 9 /* Operator */);
+    assert_1.default.equal(results.tokens.getItemAt(5).type, 18 /* CloseCurlyBrace */);
+    assert_1.default.equal(results.tokens.getItemAt(6).type, 26 /* FStringEnd */);
+});
+test('Strings: nested f-string', () => {
+    const t = new tokenizer_1.Tokenizer();
+    const results = t.tokenize("f'{f'{a}'}'");
+    assert_1.default.equal(results.tokens.count, 9 + _implicitTokenCount);
+    assert_1.default.equal(results.tokens.getItemAt(0).type, 24 /* FStringStart */);
+    assert_1.default.equal(results.tokens.getItemAt(1).type, 17 /* OpenCurlyBrace */);
+    assert_1.default.equal(results.tokens.getItemAt(2).type, 24 /* FStringStart */);
+    assert_1.default.equal(results.tokens.getItemAt(3).type, 17 /* OpenCurlyBrace */);
+    assert_1.default.equal(results.tokens.getItemAt(4).type, 7 /* Identifier */);
+    assert_1.default.equal(results.tokens.getItemAt(5).type, 18 /* CloseCurlyBrace */);
+    assert_1.default.equal(results.tokens.getItemAt(6).type, 26 /* FStringEnd */);
+    assert_1.default.equal(results.tokens.getItemAt(7).type, 18 /* CloseCurlyBrace */);
+    assert_1.default.equal(results.tokens.getItemAt(8).type, 26 /* FStringEnd */);
+});
+test('Strings: nested f-string formats 1', () => {
+    const t = new tokenizer_1.Tokenizer();
+    const results = t.tokenize("f'{a:x{{b}+:x{c}+}}'");
+    assert_1.default.equal(results.tokens.count, 19 + _implicitTokenCount);
+    assert_1.default.equal(results.tokens.getItemAt(0).type, 24 /* FStringStart */);
+    assert_1.default.equal(results.tokens.getItemAt(1).type, 17 /* OpenCurlyBrace */);
+    assert_1.default.equal(results.tokens.getItemAt(2).type, 7 /* Identifier */);
+    assert_1.default.equal(results.tokens.getItemAt(3).type, 10 /* Colon */);
+    assert_1.default.equal(results.tokens.getItemAt(4).type, 25 /* FStringMiddle */);
+    assert_1.default.equal(results.tokens.getItemAt(5).type, 17 /* OpenCurlyBrace */);
+    assert_1.default.equal(results.tokens.getItemAt(6).type, 17 /* OpenCurlyBrace */);
+    assert_1.default.equal(results.tokens.getItemAt(7).type, 7 /* Identifier */);
+    assert_1.default.equal(results.tokens.getItemAt(8).type, 18 /* CloseCurlyBrace */);
+    assert_1.default.equal(results.tokens.getItemAt(9).type, 9 /* Operator */);
+    assert_1.default.equal(results.tokens.getItemAt(10).type, 10 /* Colon */);
+    assert_1.default.equal(results.tokens.getItemAt(11).type, 25 /* FStringMiddle */);
+    assert_1.default.equal(results.tokens.getItemAt(12).type, 17 /* OpenCurlyBrace */);
+    assert_1.default.equal(results.tokens.getItemAt(13).type, 7 /* Identifier */);
+    assert_1.default.equal(results.tokens.getItemAt(14).type, 18 /* CloseCurlyBrace */);
+    assert_1.default.equal(results.tokens.getItemAt(15).type, 25 /* FStringMiddle */);
+    assert_1.default.equal(results.tokens.getItemAt(16).type, 18 /* CloseCurlyBrace */);
+    assert_1.default.equal(results.tokens.getItemAt(17).type, 18 /* CloseCurlyBrace */);
+    assert_1.default.equal(results.tokens.getItemAt(18).type, 26 /* FStringEnd */);
+});
+test('Strings: nested f-string formats 2', () => {
+    const t = new tokenizer_1.Tokenizer();
+    const results = t.tokenize("f'hi{'x':*^{8:{'':}}0}'");
+    assert_1.default.equal(results.tokens.count, 17 + _implicitTokenCount);
+    assert_1.default.equal(results.tokens.getItemAt(0).type, 24 /* FStringStart */);
+    assert_1.default.equal(results.tokens.getItemAt(1).type, 25 /* FStringMiddle */);
+    assert_1.default.equal(results.tokens.getItemAt(2).type, 17 /* OpenCurlyBrace */);
+    assert_1.default.equal(results.tokens.getItemAt(3).type, 5 /* String */);
+    assert_1.default.equal(results.tokens.getItemAt(4).type, 10 /* Colon */);
+    assert_1.default.equal(results.tokens.getItemAt(5).type, 25 /* FStringMiddle */);
+    assert_1.default.equal(results.tokens.getItemAt(6).type, 17 /* OpenCurlyBrace */);
+    assert_1.default.equal(results.tokens.getItemAt(7).type, 6 /* Number */);
+    assert_1.default.equal(results.tokens.getItemAt(8).type, 10 /* Colon */);
+    assert_1.default.equal(results.tokens.getItemAt(9).type, 17 /* OpenCurlyBrace */);
+    assert_1.default.equal(results.tokens.getItemAt(10).type, 5 /* String */);
+    assert_1.default.equal(results.tokens.getItemAt(11).type, 10 /* Colon */);
+    assert_1.default.equal(results.tokens.getItemAt(12).type, 18 /* CloseCurlyBrace */);
+    assert_1.default.equal(results.tokens.getItemAt(13).type, 18 /* CloseCurlyBrace */);
+    assert_1.default.equal(results.tokens.getItemAt(14).type, 25 /* FStringMiddle */);
+    assert_1.default.equal(results.tokens.getItemAt(15).type, 18 /* CloseCurlyBrace */);
+    assert_1.default.equal(results.tokens.getItemAt(16).type, 26 /* FStringEnd */);
 });
 test('Strings: escape at the end of single quoted string', () => {
     const t = new tokenizer_1.Tokenizer();
@@ -782,7 +996,7 @@ test('Strings: bad name escapes', () => {
 });
 test('Comments', () => {
     const t = new tokenizer_1.Tokenizer();
-    const results = t.tokenize(' #co"""mment1\n\t\n#comm\'ent2 ');
+    const results = t.tokenize(' #co"""mment1\n\t\n#x\'y2 ');
     assert_1.default.equal(results.tokens.count, 1 + _implicitTokenCountNoImplicitNewLine);
     assert_1.default.equal(results.tokens.getItemAt(0).type, 2 /* NewLine */);
 });
@@ -1226,7 +1440,7 @@ test('TypeIgnoreAll2', () => {
 });
 test('TypeIgnoreAll3', () => {
     const t = new tokenizer_1.Tokenizer();
-    const results = t.tokenize('\n#    type:     ignoressss\n');
+    const results = t.tokenize('\n#    type:     ignoreSsss\n');
     (0, assert_1.default)(!results.typeIgnoreAll);
 });
 test('TypeIgnoreAll3', () => {