bobe 0.0.41 → 0.0.42

package/dist/bobe.compiler.cjs.js

@@ -235,7 +235,7 @@ class Tokenizer {
     if (!this.token) return false;
     return this.token.type & TokenType.Identifier && this.token.value === Tokenizer.EofId;
   }
-  setToken(type, value) {
+  setToken(type, value, dt = 1) {
     this.token = {
       type,
       typeName: TokenType[type],
@@ -247,11 +247,11 @@ class Tokenizer {
           column: this.preCol
         },
         end: {
-          offset: this.i + 1,
+          offset: this.i + dt,
           line: this.line,
-          column: this.column + 1
+          column: this.column + dt
         },
-        source: this.code.slice(this.preI, this.i + 1)
+        source: this.code.slice(this.preI, this.i + dt)
       } : null
     };
     this.isFirstToken = false;
@@ -361,26 +361,22 @@ class Tokenizer {
     }
     let value = '';
     this.token = null;
-    try {
-      if (this.code[this.i] === '\n') {
-        this.setToken(TokenType.Identifier, true);
-        return this.token;
-      }
-      while (this.code[this.i + 1] !== '\n') {
-        value += this.code[this.i];
-        this.next();
+    let char = this.code[this.i];
+    while (char !== '\n') {
+      if (char === '"' || char === "'") {
+        value += char + this.getStr(char);
       }
       value += this.code[this.i];
-      const trimmed = value.replace(/\/\/[\s\S]+/, '').trim();
-      this.setToken(TokenType.Identifier, trimmed ? trimmed : true);
-      return this.token;
-    } finally {
       this.next();
-      this.handledTokens.push(this.token);
-      {
-        this.needLoc = false;
-      }
+      char = this.code[this.i];
+    }
+    const trimmed = value.replace(/\/\/[\s\S]+/, '').trim();
+    this.setToken(TokenType.Identifier, trimmed ? trimmed : true, 0);
+    this.handledTokens.push(this.token);
+    {
+      this.needLoc = false;
     }
+    return this.token;
   }
   isEol(i) {
     return this.code[i] === '\n' || this.code[i] === '/';
@@ -393,28 +389,21 @@ class Tokenizer {
     }
     this.token = null;
     let value = '';
-    try {
-      const char = this.code[this.i];
-      if (char === ';' || char === '\n') {
-        this.setToken(TokenType.Identifier, value);
-        return this.token;
-      }
-      let nextC = this.code[this.i + 1];
-      while (nextC !== ';' && nextC !== '\n') {
-        value += this.code[this.i];
-        this.next();
-        nextC = this.code[this.i + 1];
+    let char = this.code[this.i];
+    while (char !== ';' && char !== '\n') {
+      if (char === '"' || char === "'") {
+        value += char + this.getStr(char);
       }
       value += this.code[this.i];
-      this.setToken(TokenType.Identifier, value);
-      return this.token;
-    } finally {
       this.next();
-      this.handledTokens.push(this.token);
-      {
-        this.needLoc = false;
-      }
+      char = this.code[this.i];
+    }
+    this.setToken(TokenType.Identifier, value, 0);
+    this.handledTokens.push(this.token);
+    {
+      this.needLoc = false;
     }
+    return this.token;
   }
   peekChar() {
     let i = this.i;
@@ -472,10 +461,8 @@ class Tokenizer {
       startLine = this.line,
       startCol = this.preCol;
     let inComment,
-      inString,
       count = 0,
-      value = '',
-      backslashCount = 0;
+      value = '';
     while (1) {
       const char = this.code[this.i];
       if (char === undefined) {
@@ -488,11 +475,6 @@ class Tokenizer {
         inComment = null;
         value += this.code[this.i];
         this.next();
-      } else if (inString) {
-        if (char === inString && backslashCount % 2 === 0) {
-          inString = null;
-        }
-        backslashCount = char === '\\' ? backslashCount + 1 : 0;
       } else {
         if (char === '/' && nextChar === '/') {
           inComment = 'single';
@@ -502,15 +484,15 @@ class Tokenizer {
           inComment = 'multi';
           value += this.code[this.i];
           this.next();
-        } else if (char === "'" || char === '"' || char === '`') {
-          inString = char;
+        } else if (char === "'" || char === '"') {
+          value += char + this.getStr(char);
         } else if (char === '{') {
           count++;
         } else if (char === '}') {
           count--;
         }
       }
-      if (count === 0 && inString == null && inComment == null) {
+      if (count === 0 && inComment == null) {
         return value.slice(1);
       }
       value += this.code[this.i];
@@ -706,7 +688,7 @@ class Tokenizer {
     }
     this.setToken(tokenType, realValue);
   }
-  str(char) {
+  getStr(head, parseEscape = true) {
     const startOffset = this.preI,
       startLine = this.line,
       startCol = this.preCol;
@@ -725,11 +707,18 @@ class Tokenizer {
         continuousBackslashCount = 0;
       }
       this.next();
-      if (nextC === char && memoCount % 2 === 0) {
+      if (nextC === head && memoCount % 2 === 0) {
         break;
       }
-      value += nextC;
+      bobeShared.escapeMap[nextC];
+      {
+        value += nextC;
+      }
     }
+    return value;
+  }
+  str(char) {
+    const value = this.getStr(char, false);
     this.setToken(TokenType.String, value);
   }
   number(char) {
@@ -1169,7 +1158,7 @@ class Compiler {
   }
   parseLoopNode(node) {
     const forLoc = this.tokenizer.token.loc ?? this.tokenizer.emptyLoc();
-    this.tokenizer.nextToken();
+    this.tokenizer.jsExp();
     const collection = this.parseJsExp();
     if (!collection.value && collection.value !== 0) {
       this.addError(ParseErrorCode.MISSING_FOR_COLLECTION, '"for" 缺少集合表达式', forLoc, node);
@@ -1630,7 +1619,7 @@ class Interpreter {
     return _node;
   }
   forDeclaration() {
-    const arrExp = this.tokenizer.nextToken().value;
+    const arrExp = this.tokenizer.jsExp().value;
     this.tokenizer.nextToken();
     const itemToken = this.tokenizer.nextToken();
     const isDestruct = itemToken.type === TokenType.InsertionExp;