exercism-analysis 0.1.1

data/vendor/javascript/node_modules/jshint/src/lex.js

@@ -0,0 +1,1650 @@
+/*
+ * Lexical analysis and token construction.
+ */
+
+"use strict";
+
+var _      = require("underscore");
+var events = require("events");
+var reg    = require("./reg.js");
+var state  = require("./state.js").state;
+
+var unicodeData = require("../data/ascii-identifier-data.js");
+var asciiIdentifierStartTable = unicodeData.asciiIdentifierStartTable;
+var asciiIdentifierPartTable = unicodeData.asciiIdentifierPartTable;
+var nonAsciiIdentifierStartTable = require("../data/non-ascii-identifier-start.js");
+var nonAsciiIdentifierPartTable = require("../data/non-ascii-identifier-part-only.js");
+
+// Some of these token types are from JavaScript Parser API
+// while others are specific to JSHint parser.
+// JS Parser API: https://developer.mozilla.org/en-US/docs/SpiderMonkey/Parser_API
+
+var Token = {
+  Identifier: 1,
+  Punctuator: 2,
+  NumericLiteral: 3,
+  StringLiteral: 4,
+  Comment: 5,
+  Keyword: 6,
+  NullLiteral: 7,
+  BooleanLiteral: 8,
+  RegExp: 9,
+  TemplateLiteral: 10
+};
+
+// Object that handles postponed lexing verifications that checks the parsed
+// environment state.
+
+function asyncTrigger() {
+  var _checks = [];
+
+  return {
+    push: function (fn) {
+      _checks.push(fn);
+    },
+
+    check: function () {
+      for (var check = 0; check < _checks.length; ++check) {
+        _checks[check]();
+      }
+
+      _checks.splice(0, _checks.length);
+    }
+  };
+}
+
+/*
+ * Lexer for JSHint.
+ *
+ * This object does a char-by-char scan of the provided source code
+ * and produces a sequence of tokens.
+ *
+ *   var lex = new Lexer("var i = 0;");
+ *   lex.start();
+ *   lex.token(); // returns the next token
+ *
+ * You have to use the token() method to move the lexer forward
+ * but you don't have to use its return value to get tokens. In addition
+ * to token() method returning the next token, the Lexer object also
+ * emits events.
+ *
+ *   lex.on("Identifier", function (data) {
+ *     if (data.name.indexOf("_") >= 0) {
+ *       // Produce a warning.
+ *     }
+ *   });
+ *
+ * Note that the token() method returns tokens in a JSLint-compatible
+ * format while the event emitter uses a slightly modified version of
+ * Mozilla's JavaScript Parser API. Eventually, we will move away from
+ * JSLint format.
+ */
+function Lexer(source) {
+  var lines = source;
+
+  if (typeof lines === "string") {
+    lines = lines
+      .replace(/\r\n/g, "\n")
+      .replace(/\r/g, "\n")
+      .split("\n");
+  }
+
+  // If the first line is a shebang (#!), make it a blank and move on.
+  // Shebangs are used by Node scripts.
+
+  if (lines[0] && lines[0].substr(0, 2) === "#!") {
+    if (lines[0].indexOf("node") !== -1) {
+      state.option.node = true;
+    }
+    lines[0] = "";
+  }
+
+  this.emitter = new events.EventEmitter();
+  this.source = source;
+  this.setLines(lines);
+  this.prereg = true;
+
+  this.line = 0;
+  this.char = 1;
+  this.from = 1;
+  this.input = "";
+  this.inComment = false;
+
+  for (var i = 0; i < state.option.indent; i += 1) {
+    state.tab += " ";
+  }
+}
+
+Lexer.prototype = {
+  _lines: [],
+
+  getLines: function () {
+    this._lines = state.lines;
+    return this._lines;
+  },
+
+  setLines: function (val) {
+    this._lines = val;
+    state.lines = this._lines;
+  },
+
+  /*
+   * Return the next i character without actually moving the
+   * char pointer.
+   */
+  peek: function (i) {
+    return this.input.charAt(i || 0);
+  },
+
+  /*
+   * Move the char pointer forward i times.
+   */
+  skip: function (i) {
+    i = i || 1;
+    this.char += i;
+    this.input = this.input.slice(i);
+  },
+
+  /*
+   * Subscribe to a token event. The API for this method is similar
+   * Underscore.js i.e. you can subscribe to multiple events with
+   * one call:
+   *
+   *   lex.on("Identifier Number", function (data) {
+   *     // ...
+   *   });
+   */
+  on: function (names, listener) {
+    names.split(" ").forEach(function (name) {
+      this.emitter.on(name, listener);
+    }.bind(this));
+  },
+
+  /*
+   * Trigger a token event. All arguments will be passed to each
+   * listener.
+   */
+  trigger: function () {
+    this.emitter.emit.apply(this.emitter, Array.prototype.slice.call(arguments));
+  },
+
+  /*
+   * Postpone a token event. the checking condition is set as
+   * last parameter, and the trigger function is called in a
+   * stored callback. To be later called using the check() function
+   * by the parser. This avoids parser's peek() to give the lexer
+   * a false context.
+   */
+  triggerAsync: function (type, args, checks, fn) {
+    checks.push(function () {
+      if (fn()) {
+        this.trigger(type, args);
+      }
+    }.bind(this));
+  },
+
+  /*
+   * Extract a punctuator out of the next sequence of characters
+   * or return 'null' if its not possible.
+   *
+   * This method's implementation was heavily influenced by the
+   * scanPunctuator function in the Esprima parser's source code.
+   */
+  scanPunctuator: function () {
+    var ch1 = this.peek();
+    var ch2, ch3, ch4;
+
+    switch (ch1) {
+    // Most common single-character punctuators
+    case ".":
+      if ((/^[0-9]$/).test(this.peek(1))) {
+        return null;
+      }
+      if (this.peek(1) === "." && this.peek(2) === ".") {
+        return {
+          type: Token.Punctuator,
+          value: "..."
+        };
+      }
+      /* falls through */
+    case "(":
+    case ")":
+    case ";":
+    case ",":
+    case "{":
+    case "}":
+    case "[":
+    case "]":
+    case ":":
+    case "~":
+    case "?":
+      return {
+        type: Token.Punctuator,
+        value: ch1
+      };
+
+    // A pound sign (for Node shebangs)
+    case "#":
+      return {
+        type: Token.Punctuator,
+        value: ch1
+      };
+
+    // We're at the end of input
+    case "":
+      return null;
+    }
+
+    // Peek more characters
+
+    ch2 = this.peek(1);
+    ch3 = this.peek(2);
+    ch4 = this.peek(3);
+
+    // 4-character punctuator: >>>=
+
+    if (ch1 === ">" && ch2 === ">" && ch3 === ">" && ch4 === "=") {
+      return {
+        type: Token.Punctuator,
+        value: ">>>="
+      };
+    }
+
+    // 3-character punctuators: === !== >>> <<= >>=
+
+    if (ch1 === "=" && ch2 === "=" && ch3 === "=") {
+      return {
+        type: Token.Punctuator,
+        value: "==="
+      };
+    }
+
+    if (ch1 === "!" && ch2 === "=" && ch3 === "=") {
+      return {
+        type: Token.Punctuator,
+        value: "!=="
+      };
+    }
+
+    if (ch1 === ">" && ch2 === ">" && ch3 === ">") {
+      return {
+        type: Token.Punctuator,
+        value: ">>>"
+      };
+    }
+
+    if (ch1 === "<" && ch2 === "<" && ch3 === "=") {
+      return {
+        type: Token.Punctuator,
+        value: "<<="
+      };
+    }
+
+    if (ch1 === ">" && ch2 === ">" && ch3 === "=") {
+      return {
+        type: Token.Punctuator,
+        value: ">>="
+      };
+    }
+
+    // Fat arrow punctuator
+    if (ch1 === "=" && ch2 === ">") {
+      return {
+        type: Token.Punctuator,
+        value: ch1 + ch2
+      };
+    }
+
+    // 2-character punctuators: <= >= == != ++ -- << >> && ||
+    // += -= *= %= &= |= ^= (but not /=, see below)
+    if (ch1 === ch2 && ("+-<>&|".indexOf(ch1) >= 0)) {
+      return {
+        type: Token.Punctuator,
+        value: ch1 + ch2
+      };
+    }
+
+    if ("<>=!+-*%&|^".indexOf(ch1) >= 0) {
+      if (ch2 === "=") {
+        return {
+          type: Token.Punctuator,
+          value: ch1 + ch2
+        };
+      }
+
+      return {
+        type: Token.Punctuator,
+        value: ch1
+      };
+    }
+
+    // Special case: /=. We need to make sure that this is an
+    // operator and not a regular expression.
+
+    if (ch1 === "/") {
+      if (ch2 === "=" && /\/=(?!(\S*\/[gim]?))/.test(this.input)) {
+        // /= is not a part of a regular expression, return it as a
+        // punctuator.
+        return {
+          type: Token.Punctuator,
+          value: "/="
+        };
+      }
+
+      return {
+        type: Token.Punctuator,
+        value: "/"
+      };
+    }
+
+    return null;
+  },
+
+  /*
+   * Extract a comment out of the next sequence of characters and/or
+   * lines or return 'null' if its not possible. Since comments can
+   * span across multiple lines this method has to move the char
+   * pointer.
+   *
+   * In addition to normal JavaScript comments (// and /*) this method
+   * also recognizes JSHint- and JSLint-specific comments such as
+   * /*jshint, /*jslint, /*globals and so on.
+   */
+  scanComments: function () {
+    var ch1 = this.peek();
+    var ch2 = this.peek(1);
+    var rest = this.input.substr(2);
+    var startLine = this.line;
+    var startChar = this.char;
+
+    // Create a comment token object and make sure it
+    // has all the data JSHint needs to work with special
+    // comments.
+
+    function commentToken(label, body, opt) {
+      var special = ["jshint", "jslint", "members", "member", "globals", "global", "exported"];
+      var isSpecial = false;
+      var value = label + body;
+      var commentType = "plain";
+      opt = opt || {};
+
+      if (opt.isMultiline) {
+        value += "*/";
+      }
+
+      special.forEach(function (str) {
+        if (isSpecial) {
+          return;
+        }
+
+        // Don't recognize any special comments other than jshint for single-line
+        // comments. This introduced many problems with legit comments.
+        if (label === "//" && str !== "jshint") {
+          return;
+        }
+
+        if (body.substr(0, str.length) === str) {
+          isSpecial = true;
+          label = label + str;
+          body = body.substr(str.length);
+        }
+
+        if (!isSpecial && body.charAt(0) === " " && body.substr(1, str.length) === str) {
+          isSpecial = true;
+          label = label + " " + str;
+          body = body.substr(str.length + 1);
+        }
+
+        if (!isSpecial) {
+          return;
+        }
+
+        switch (str) {
+        case "member":
+          commentType = "members";
+          break;
+        case "global":
+          commentType = "globals";
+          break;
+        default:
+          commentType = str;
+        }
+      });
+
+      return {
+        type: Token.Comment,
+        commentType: commentType,
+        value: value,
+        body: body,
+        isSpecial: isSpecial,
+        isMultiline: opt.isMultiline || false,
+        isMalformed: opt.isMalformed || false
+      };
+    }
+
+    // End of unbegun comment. Raise an error and skip that input.
+    if (ch1 === "*" && ch2 === "/") {
+      this.trigger("error", {
+        code: "E018",
+        line: startLine,
+        character: startChar
+      });
+
+      this.skip(2);
+      return null;
+    }
+
+    // Comments must start either with // or /*
+    if (ch1 !== "/" || (ch2 !== "*" && ch2 !== "/")) {
+      return null;
+    }
+
+    // One-line comment
+    if (ch2 === "/") {
+      this.skip(this.input.length); // Skip to the EOL.
+      return commentToken("//", rest);
+    }
+
+    var body = "";
+
+    /* Multi-line comment */
+    if (ch2 === "*") {
+      this.inComment = true;
+      this.skip(2);
+
+      while (this.peek() !== "*" || this.peek(1) !== "/") {
+        if (this.peek() === "") { // End of Line
+          body += "\n";
+
+          // If we hit EOF and our comment is still unclosed,
+          // trigger an error and end the comment implicitly.
+          if (!this.nextLine()) {
+            this.trigger("error", {
+              code: "E017",
+              line: startLine,
+              character: startChar
+            });
+
+            this.inComment = false;
+            return commentToken("/*", body, {
+              isMultiline: true,
+              isMalformed: true
+            });
+          }
+        } else {
+          body += this.peek();
+          this.skip();
+        }
+      }
+
+      this.skip(2);
+      this.inComment = false;
+      return commentToken("/*", body, { isMultiline: true });
+    }
+  },
+
+  /*
+   * Extract a keyword out of the next sequence of characters or
+   * return 'null' if its not possible.
+   */
+  scanKeyword: function () {
+    var result = /^[a-zA-Z_$][a-zA-Z0-9_$]*/.exec(this.input);
+    var keywords = [
+      "if", "in", "do", "var", "for", "new",
+      "try", "let", "this", "else", "case",
+      "void", "with", "enum", "while", "break",
+      "catch", "throw", "const", "yield", "class",
+      "super", "return", "typeof", "delete",
+      "switch", "export", "import", "default",
+      "finally", "extends", "function", "continue",
+      "debugger", "instanceof"
+    ];
+
+    if (result && keywords.indexOf(result[0]) >= 0) {
+      return {
+        type: Token.Keyword,
+        value: result[0]
+      };
+    }
+
+    return null;
+  },
+
+  /*
+   * Extract a JavaScript identifier out of the next sequence of
+   * characters or return 'null' if its not possible. In addition,
+   * to Identifier this method can also produce BooleanLiteral
+   * (true/false) and NullLiteral (null).
+   */
+  scanIdentifier: function () {
+    var id = "";
+    var index = 0;
+    var type, char;
+
+    function isNonAsciiIdentifierStart(code) {
+      return nonAsciiIdentifierStartTable.indexOf(code) > -1;
+    }
+
+    function isNonAsciiIdentifierPart(code) {
+      return isNonAsciiIdentifierStart(code) || nonAsciiIdentifierPartTable.indexOf(code) > -1;
+    }
+
+    function isHexDigit(str) {
+      return (/^[0-9a-fA-F]$/).test(str);
+    }
+
+    var readUnicodeEscapeSequence = function () {
+      /*jshint validthis:true */
+      index += 1;
+
+      if (this.peek(index) !== "u") {
+        return null;
+      }
+
+      var ch1 = this.peek(index + 1);
+      var ch2 = this.peek(index + 2);
+      var ch3 = this.peek(index + 3);
+      var ch4 = this.peek(index + 4);
+      var code;
+
+      if (isHexDigit(ch1) && isHexDigit(ch2) && isHexDigit(ch3) && isHexDigit(ch4)) {
+        code = parseInt(ch1 + ch2 + ch3 + ch4, 16);
+
+        if (asciiIdentifierPartTable[code] || isNonAsciiIdentifierPart(code)) {
+          index += 5;
+          return "\\u" + ch1 + ch2 + ch3 + ch4;
+        }
+
+        return null;
+      }
+
+      return null;
+    }.bind(this);
+
+    var getIdentifierStart = function () {
+      /*jshint validthis:true */
+      var chr = this.peek(index);
+      var code = chr.charCodeAt(0);
+
+      if (code === 92) {
+        return readUnicodeEscapeSequence();
+      }
+
+      if (code < 128) {
+        if (asciiIdentifierStartTable[code]) {
+          index += 1;
+          return chr;
+        }
+
+        return null;
+      }
+
+      if (isNonAsciiIdentifierStart(code)) {
+        index += 1;
+        return chr;
+      }
+
+      return null;
+    }.bind(this);
+
+    var getIdentifierPart = function () {
+      /*jshint validthis:true */
+      var chr = this.peek(index);
+      var code = chr.charCodeAt(0);
+
+      if (code === 92) {
+        return readUnicodeEscapeSequence();
+      }
+
+      if (code < 128) {
+        if (asciiIdentifierPartTable[code]) {
+          index += 1;
+          return chr;
+        }
+
+        return null;
+      }
+
+      if (isNonAsciiIdentifierPart(code)) {
+        index += 1;
+        return chr;
+      }
+
+      return null;
+    }.bind(this);
+
+    char = getIdentifierStart();
+    if (char === null) {
+      return null;
+    }
+
+    id = char;
+    for (;;) {
+      char = getIdentifierPart();
+
+      if (char === null) {
+        break;
+      }
+
+      id += char;
+    }
+
+    switch (id) {
+    case "true":
+    case "false":
+      type = Token.BooleanLiteral;
+      break;
+    case "null":
+      type = Token.NullLiteral;
+      break;
+    default:
+      type = Token.Identifier;
+    }
+
+    return {
+      type: type,
+      value: id
+    };
+  },
+
+  /*
+   * Extract a numeric literal out of the next sequence of
+   * characters or return 'null' if its not possible. This method
+   * supports all numeric literals described in section 7.8.3
+   * of the EcmaScript 5 specification.
+   *
+   * This method's implementation was heavily influenced by the
+   * scanNumericLiteral function in the Esprima parser's source code.
+   */
+  scanNumericLiteral: function () {
+    var index = 0;
+    var value = "";
+    var length = this.input.length;
+    var char = this.peek(index);
+    var bad;
+
+    function isDecimalDigit(str) {
+      return (/^[0-9]$/).test(str);
+    }
+
+    function isOctalDigit(str) {
+      return (/^[0-7]$/).test(str);
+    }
+
+    function isHexDigit(str) {
+      return (/^[0-9a-fA-F]$/).test(str);
+    }
+
+    function isIdentifierStart(ch) {
+      return (ch === "$") || (ch === "_") || (ch === "\\") ||
+        (ch >= "a" && ch <= "z") || (ch >= "A" && ch <= "Z");
+    }
+
+    // Numbers must start either with a decimal digit or a point.
+
+    if (char !== "." && !isDecimalDigit(char)) {
+      return null;
+    }
+
+    if (char !== ".") {
+      value = this.peek(index);
+      index += 1;
+      char = this.peek(index);
+
+      if (value === "0") {
+        // Base-16 numbers.
+        if (char === "x" || char === "X") {
+          index += 1;
+          value += char;
+
+          while (index < length) {
+            char = this.peek(index);
+            if (!isHexDigit(char)) {
+              break;
+            }
+            value += char;
+            index += 1;
+          }
+
+          if (value.length <= 2) { // 0x
+            return {
+              type: Token.NumericLiteral,
+              value: value,
+              isMalformed: true
+            };
+          }
+
+          if (index < length) {
+            char = this.peek(index);
+            if (isIdentifierStart(char)) {
+              return null;
+            }
+          }
+
+          return {
+            type: Token.NumericLiteral,
+            value: value,
+            base: 16,
+            isMalformed: false
+          };
+        }
+
+        // Base-8 numbers.
+        if (isOctalDigit(char)) {
+          index += 1;
+          value += char;
+          bad = false;
+
+          while (index < length) {
+            char = this.peek(index);
+
+            // Numbers like '019' (note the 9) are not valid octals
+            // but we still parse them and mark as malformed.
+
+            if (isDecimalDigit(char)) {
+              bad = true;
+            } else if (!isOctalDigit(char)) {
+              break;
+            }
+            value += char;
+            index += 1;
+          }
+
+          if (index < length) {
+            char = this.peek(index);
+            if (isIdentifierStart(char)) {
+              return null;
+            }
+          }
+
+          return {
+            type: Token.NumericLiteral,
+            value: value,
+            base: 8,
+            isMalformed: false
+          };
+        }
+
+        // Decimal numbers that start with '0' such as '09' are illegal
+        // but we still parse them and return as malformed.
+
+        if (isDecimalDigit(char)) {
+          index += 1;
+          value += char;
+        }
+      }
+
+      while (index < length) {
+        char = this.peek(index);
+        if (!isDecimalDigit(char)) {
+          break;
+        }
+        value += char;
+        index += 1;
+      }
+    }
+
+    // Decimal digits.
+
+    if (char === ".") {
+      value += char;
+      index += 1;
+
+      while (index < length) {
+        char = this.peek(index);
+        if (!isDecimalDigit(char)) {
+          break;
+        }
+        value += char;
+        index += 1;
+      }
+    }
+
+    // Exponent part.
+
+    if (char === "e" || char === "E") {
+      value += char;
+      index += 1;
+      char = this.peek(index);
+
+      if (char === "+" || char === "-") {
+        value += this.peek(index);
+        index += 1;
+      }
+
+      char = this.peek(index);
+      if (isDecimalDigit(char)) {
+        value += char;
+        index += 1;
+
+        while (index < length) {
+          char = this.peek(index);
+          if (!isDecimalDigit(char)) {
+            break;
+          }
+          value += char;
+          index += 1;
+        }
+      } else {
+        return null;
+      }
+    }
+
+    if (index < length) {
+      char = this.peek(index);
+      if (isIdentifierStart(char)) {
+        return null;
+      }
+    }
+
+    return {
+      type: Token.NumericLiteral,
+      value: value,
+      base: 10,
+      isMalformed: !isFinite(value)
+    };
+  },
+
+  /*
+   * Extract a template literal out of the next sequence of characters
+   * and/or lines or return 'null' if its not possible. Since template
+   * literals can span across multiple lines, this method has to move
+   * the char pointer.
+   */
+  scanTemplateLiteral: function () {
+    // String must start with a backtick.
+    if (!state.option.esnext || this.peek() !== "`") {
+      return null;
+    }
+
+    var startLine = this.line;
+    var startChar = this.char;
+    var jump = 1;
+    var value = "";
+
+    // For now, do not perform any linting of the content of the template
+    // string. Just skip until the next backtick is found.
+    this.skip();
+
+    while (this.peek() !== "`") {
+      while (this.peek() === "") {
+        // End of line --- For template literals in ES6, no backslash is
+        // required to precede newlines.
+        if (!this.nextLine()) {
+          this.trigger("error", {
+            code: "E052",
+            line: startLine,
+            character: startChar
+          });
+
+          return {
+            type: Token.TemplateLiteral,
+            value: value,
+            isUnclosed: true
+          };
+        }
+        value += "\n";
+      }
+
+      // TODO: do more interesting linting here, similar to string literal
+      // linting.
+      var char = this.peek();
+      this.skip(jump);
+      value += char;
+    }
+
+    this.skip();
+    return {
+      type: Token.TemplateLiteral,
+      value: value,
+      isUnclosed: false
+    };
+  },
+
+  /*
+   * Extract a string out of the next sequence of characters and/or
+   * lines or return 'null' if its not possible. Since strings can
+   * span across multiple lines this method has to move the char
+   * pointer.
+   *
+   * This method recognizes pseudo-multiline JavaScript strings:
+   *
+   *   var str = "hello\
+   *   world";
+   */
+  scanStringLiteral: function (checks) {
+    /*jshint loopfunc:true */
+    var quote = this.peek();
+
+    // String must start with a quote.
+    if (quote !== "\"" && quote !== "'") {
+      return null;
+    }
+
+    // In JSON strings must always use double quotes.
+    this.triggerAsync("warning", {
+      code: "W108",
+      line: this.line,
+      character: this.char // +1?
+    }, checks, function () { return state.jsonMode && quote !== "\""; });
+
+    var value = "";
+    var startLine = this.line;
+    var startChar = this.char;
+    var allowNewLine = false;
+
+    this.skip();
+
+    while (this.peek() !== quote) {
+      while (this.peek() === "") { // End Of Line
+
+        // If an EOL is not preceded by a backslash, show a warning
+        // and proceed like it was a legit multi-line string where
+        // author simply forgot to escape the newline symbol.
+        //
+        // Another approach is to implicitly close a string on EOL
+        // but it generates too many false positives.
+
+        if (!allowNewLine) {
+          this.trigger("warning", {
+            code: "W112",
+            line: this.line,
+            character: this.char
+          });
+        } else {
+          allowNewLine = false;
+
+          // Otherwise show a warning if multistr option was not set.
+          // For JSON, show warning no matter what.
+
+          this.triggerAsync("warning", {
+            code: "W043",
+            line: this.line,
+            character: this.char
+          }, checks, function () { return !state.option.multistr; });
+
+          this.triggerAsync("warning", {
+            code: "W042",
+            line: this.line,
+            character: this.char
+          }, checks, function () { return state.jsonMode && state.option.multistr; });
+        }
+
+        // If we get an EOF inside of an unclosed string, show an
+        // error and implicitly close it at the EOF point.
+
+        if (!this.nextLine()) {
+          this.trigger("error", {
+            code: "E029",
+            line: startLine,
+            character: startChar
+          });
+
+          return {
+            type: Token.StringLiteral,
+            value: value,
+            isUnclosed: true,
+            quote: quote
+          };
+        }
+      }
+
+      allowNewLine = false;
+      var char = this.peek();
+      var jump = 1; // A length of a jump, after we're done
+                    // parsing this character.
+
+      if (char < " ") {
+        // Warn about a control character in a string.
+        this.trigger("warning", {
+          code: "W113",
+          line: this.line,
+          character: this.char,
+          data: [ "<non-printable>" ]
+        });
+      }
+
+      // Special treatment for some escaped characters.
+
+      if (char === "\\") {
+        this.skip();
+        char = this.peek();
+
+        switch (char) {
+        case "'":
+          this.triggerAsync("warning", {
+            code: "W114",
+            line: this.line,
+            character: this.char,
+            data: [ "\\'" ]
+          }, checks, function () {return state.jsonMode; });
+          break;
+        case "b":
+          char = "\\b";
+          break;
+        case "f":
+          char = "\\f";
+          break;
+        case "n":
+          char = "\\n";
+          break;
+        case "r":
+          char = "\\r";
+          break;
+        case "t":
+          char = "\\t";
+          break;
+        case "0":
+          char = "\\0";
+
+          // Octal literals fail in strict mode.
+          // Check if the number is between 00 and 07.
+          var n = parseInt(this.peek(1), 10);
+          this.triggerAsync("warning", {
+            code: "W115",
+            line: this.line,
+            character: this.char
+          }, checks,
+          function () { return n >= 0 && n <= 7 && state.directive["use strict"]; });
+          break;
+        case "u":
+          char = String.fromCharCode(parseInt(this.input.substr(1, 4), 16));
+          jump = 5;
+          break;
+        case "v":
+          this.triggerAsync("warning", {
+            code: "W114",
+            line: this.line,
+            character: this.char,
+            data: [ "\\v" ]
+          }, checks, function () { return state.jsonMode; });
+
+          char = "\v";
+          break;
+        case "x":
+          var  x = parseInt(this.input.substr(1, 2), 16);
+
+          this.triggerAsync("warning", {
+            code: "W114",
+            line: this.line,
+            character: this.char,
+            data: [ "\\x-" ]
+          }, checks, function () { return state.jsonMode; });
+
+          char = String.fromCharCode(x);
+          jump = 3;
+          break;
+        case "\\":
+          char = "\\\\";
+          break;
+        case "\"":
+          char = "\\\"";
+          break;
+        case "/":
+          break;
+        case "":
+          allowNewLine = true;
+          char = "";
+          break;
+        case "!":
+          if (value.slice(value.length - 2) === "<") {
+            break;
+          }
+
+          /*falls through */
+        default:
+          // Weird escaping.
+          this.trigger("warning", {
+            code: "W044",
+            line: this.line,
+            character: this.char
+          });
+        }
+      }
+
+      value += char;
+      this.skip(jump);
+    }
+
+    this.skip();
+    return {
+      type: Token.StringLiteral,
+      value: value,
+      isUnclosed: false,
+      quote: quote
+    };
+  },
+
+  /*
+   * Extract a regular expression out of the next sequence of
+   * characters and/or lines or return 'null' if its not possible.
+   *
+   * This method is platform dependent: it accepts almost any
+   * regular expression values but then tries to compile and run
+   * them using system's RegExp object. This means that there are
+   * rare edge cases where one JavaScript engine complains about
+   * your regular expression while others don't.
+   */
+  scanRegExp: function () {
+    var index = 0;
+    var length = this.input.length;
+    var char = this.peek();
+    var value = char;
+    var body = "";
+    var flags = [];
+    var malformed = false;
+    var isCharSet = false;
+    var terminated;
+
+    var scanUnexpectedChars = function () {
+      // Unexpected control character
+      if (char < " ") {
+        malformed = true;
+        this.trigger("warning", {
+          code: "W048",
+          line: this.line,
+          character: this.char
+        });
+      }
+
+      // Unexpected escaped character
+      if (char === "<") {
+        malformed = true;
+        this.trigger("warning", {
+          code: "W049",
+          line: this.line,
+          character: this.char,
+          data: [ char ]
+        });
+      }
+    }.bind(this);
+
+    // Regular expressions must start with '/'
+    if (!this.prereg || char !== "/") {
+      return null;
+    }
+
+    index += 1;
+    terminated = false;
+
+    // Try to get everything in between slashes. A couple of
+    // cases aside (see scanUnexpectedChars) we don't really
+    // care whether the resulting expression is valid or not.
+    // We will check that later using the RegExp object.
+
+    while (index < length) {
+      char = this.peek(index);
+      value += char;
+      body += char;
+
+      if (isCharSet) {
+        if (char === "]") {
+          if (this.peek(index - 1) !== "\\" || this.peek(index - 2) === "\\") {
+            isCharSet = false;
+          }
+        }
+
+        if (char === "\\") {
+          index += 1;
+          char = this.peek(index);
+          body += char;
+          value += char;
+
+          scanUnexpectedChars();
+        }
+
+        index += 1;
+        continue;
+      }
+
+      if (char === "\\") {
+        index += 1;
+        char = this.peek(index);
+        body += char;
+        value += char;
+
+        scanUnexpectedChars();
+
+        if (char === "/") {
+          index += 1;
+          continue;
+        }
+
+        if (char === "[") {
+          index += 1;
+          continue;
+        }
+      }
+
+      if (char === "[") {
+        isCharSet = true;
+        index += 1;
+        continue;
+      }
+
+      if (char === "/") {
+        body = body.substr(0, body.length - 1);
+        terminated = true;
+        index += 1;
+        break;
+      }
+
+      index += 1;
+    }
+
+    // A regular expression that was never closed is an
+    // error from which we cannot recover.
+
+    if (!terminated) {
+      this.trigger("error", {
+        code: "E015",
+        line: this.line,
+        character: this.from
+      });
+
+      return void this.trigger("fatal", {
+        line: this.line,
+        from: this.from
+      });
+    }
+
+    // Parse flags (if any).
+
+    while (index < length) {
+      char = this.peek(index);
+      if (!/[gim]/.test(char)) {
+        break;
+      }
+      flags.push(char);
+      value += char;
+      index += 1;
+    }
+
+    // Check regular expression for correctness.
+
+    try {
+      new RegExp(body, flags.join(""));
+    } catch (err) {
+      malformed = true;
+      this.trigger("error", {
+        code: "E016",
+        line: this.line,
+        character: this.char,
+        data: [ err.message ] // Platform dependent!
+      });
+    }
+
+    return {
+      type: Token.RegExp,
+      value: value,
+      flags: flags,
+      isMalformed: malformed
+    };
+  },
+
+  /*
+   * Scan for any occurence of non-breaking spaces. Non-breaking spaces
+   * can be mistakenly typed on OS X with option-space. Non UTF-8 web
+   * pages with non-breaking pages produce syntax errors.
+   */
+  scanNonBreakingSpaces: function () {
+    return state.option.nonbsp ?
+      this.input.search(/(\u00A0)/) : -1;
+  },
+
+  /*
+   * Scan for characters that get silently deleted by one or more browsers.
+   */
+  scanUnsafeChars: function () {
+    return this.input.search(reg.unsafeChars);
+  },
+
+  /*
+   * Produce the next raw token or return 'null' if no tokens can be matched.
+   * This method skips over all space characters.
+   */
+  next: function (checks) {
+    this.from = this.char;
+
+    // Move to the next non-space character.
+    var start;
+    if (/\s/.test(this.peek())) {
+      start = this.char;
+
+      while (/\s/.test(this.peek())) {
+        this.from += 1;
+        this.skip();
+      }
+    }
+
+    // Methods that work with multi-line structures and move the
+    // character pointer.
+
+    var match = this.scanComments() ||
+      this.scanStringLiteral(checks) ||
+      this.scanTemplateLiteral();
+
+    if (match) {
+      return match;
+    }
+
+    // Methods that don't move the character pointer.
+
+    match =
+      this.scanRegExp() ||
+      this.scanPunctuator() ||
+      this.scanKeyword() ||
+      this.scanIdentifier() ||
+      this.scanNumericLiteral();
+
+    if (match) {
+      this.skip(match.value.length);
+      return match;
+    }
+
+    // No token could be matched, give up.
+
+    return null;
+  },
+
+  /*
+   * Switch to the next line and reset all char pointers. Once
+   * switched, this method also checks for other minor warnings.
+   */
+  nextLine: function () {
+    var char;
+
+    if (this.line >= this.getLines().length) {
+      return false;
+    }
+
+    this.input = this.getLines()[this.line];
+    this.line += 1;
+    this.char = 1;
+    this.from = 1;
+
+    var inputTrimmed = this.input.trim();
+
+    var startsWith = function () {
+      return _.some(arguments, function (prefix) {
+        return inputTrimmed.indexOf(prefix) === 0;
+      });
+    };
+
+    var endsWith = function () {
+      return _.some(arguments, function (suffix) {
+        return inputTrimmed.indexOf(suffix, inputTrimmed.length - suffix.length) !== -1;
+      });
+    };
+
+    // If we are ignoring linter errors, replace the input with empty string
+    // if it doesn't already at least start or end a multi-line comment
+    if (state.ignoreLinterErrors === true) {
+      if (!startsWith("/*", "//") && !endsWith("*/")) {
+        this.input = "";
+      }
+    }
+
+    char = this.scanNonBreakingSpaces();
+    if (char >= 0) {
+      this.trigger("warning", { code: "W125", line: this.line, character: char + 1 });
+    }
+
+    this.input = this.input.replace(/\t/g, state.tab);
+    char = this.scanUnsafeChars();
+
+    if (char >= 0) {
+      this.trigger("warning", { code: "W100", line: this.line, character: char });
+    }
+
+    // If there is a limit on line length, warn when lines get too
+    // long.
+
+    if (state.option.maxlen && state.option.maxlen < this.input.length) {
+      var inComment = this.inComment ||
+        startsWith.call(inputTrimmed, "//") ||
+        startsWith.call(inputTrimmed, "/*");
+
+      var shouldTriggerError = !inComment || !reg.maxlenException.test(inputTrimmed);
+
+      if (shouldTriggerError) {
+        this.trigger("warning", { code: "W101", line: this.line, character: this.input.length });
+      }
+    }
+
+    return true;
+  },
+
+  /*
+   * This is simply a synonym for nextLine() method with a friendlier
+   * public name.
+   */
+  start: function () {
+    this.nextLine();
+  },
+
+  /*
+   * Produce the next token. This function is called by advance() to get
+   * the next token. It retuns a token in a JSLint-compatible format.
+   */
+  token: function () {
+    /*jshint loopfunc:true */
+    var checks = asyncTrigger();
+    var token;
+
+
+    function isReserved(token, isProperty) {
+      if (!token.reserved) {
+        return false;
+      }
+      var meta = token.meta;
+
+      if (meta && meta.isFutureReservedWord && state.option.inES5()) {
+        // ES3 FutureReservedWord in an ES5 environment.
+        if (!meta.es5) {
+          return false;
+        }
+
+        // Some ES5 FutureReservedWord identifiers are active only
+        // within a strict mode environment.
+        if (meta.strictOnly) {
+          if (!state.option.strict && !state.directive["use strict"]) {
+            return false;
+          }
+        }
+
+        if (isProperty) {
+          return false;
+        }
+      }
+
+      return true;
+    }
+
+    // Produce a token object.
+    var create = function (type, value, isProperty) {
+      /*jshint validthis:true */
+      var obj;
+
+      if (type !== "(endline)" && type !== "(end)") {
+        this.prereg = false;
+      }
+
+      if (type === "(punctuator)") {
+        switch (value) {
+        case ".":
+        case ")":
+        case "~":
+        case "#":
+        case "]":
+          this.prereg = false;
+          break;
+        default:
+          this.prereg = true;
+        }
+
+        obj = Object.create(state.syntax[value] || state.syntax["(error)"]);
+      }
+
+      if (type === "(identifier)") {
+        if (value === "return" || value === "case" || value === "typeof") {
+          this.prereg = true;
+        }
+
+        if (_.has(state.syntax, value)) {
+          obj = Object.create(state.syntax[value] || state.syntax["(error)"]);
+
+          // If this can't be a reserved keyword, reset the object.
+          if (!isReserved(obj, isProperty && type === "(identifier)")) {
+            obj = null;
+          }
+        }
+      }
+
+      if (!obj) {
+        obj = Object.create(state.syntax[type]);
+      }
+
+      obj.identifier = (type === "(identifier)");
+      obj.type = obj.type || type;
+      obj.value = value;
+      obj.line = this.line;
+      obj.character = this.char;
+      obj.from = this.from;
+
+      if (isProperty && obj.identifier) {
+        obj.isProperty = isProperty;
+      }
+
+      obj.check = checks.check;
+
+      return obj;
+    }.bind(this);
+
+    for (;;) {
+      if (!this.input.length) {
+        return create(this.nextLine() ? "(endline)" : "(end)", "");
+      }
+
+      token = this.next(checks);
+
+      if (!token) {
+        if (this.input.length) {
+          // Unexpected character.
+          this.trigger("error", {
+            code: "E024",
+            line: this.line,
+            character: this.char,
+            data: [ this.peek() ]
+          });
+
+          this.input = "";
+        }
+
+        continue;
+      }
+
+      switch (token.type) {
+      case Token.StringLiteral:
+        this.triggerAsync("String", {
+          line: this.line,
+          char: this.char,
+          from: this.from,
+          value: token.value,
+          quote: token.quote
+        }, checks, function () { return true; });
+
+        return create("(string)", token.value);
+
+      case Token.TemplateLiteral:
+        this.trigger("Template", {
+          line: this.line,
+          char: this.char,
+          from: this.from,
+          value: token.value
+        });
+        return create("(template)", token.value);
+
+      case Token.Identifier:
+        this.trigger("Identifier", {
+          line: this.line,
+          char: this.char,
+          from: this.form,
+          name: token.value,
+          isProperty: state.tokens.curr.id === "."
+        });
+
+        /* falls through */
+      case Token.Keyword:
+      case Token.NullLiteral:
+      case Token.BooleanLiteral:
+        return create("(identifier)", token.value, state.tokens.curr.id === ".");
+
+      case Token.NumericLiteral:
+        if (token.isMalformed) {
+          this.trigger("warning", {
+            code: "W045",
+            line: this.line,
+            character: this.char,
+            data: [ token.value ]
+          });
+        }
+
+        this.triggerAsync("warning", {
+          code: "W114",
+          line: this.line,
+          character: this.char,
+          data: [ "0x-" ]
+        }, checks, function () { return token.base === 16 && state.jsonMode; });
+
+        this.triggerAsync("warning", {
+          code: "W115",
+          line: this.line,
+          character: this.char
+        }, checks, function () {
+          return state.directive["use strict"] && token.base === 8;
+        });
+
+        this.trigger("Number", {
+          line: this.line,
+          char: this.char,
+          from: this.from,
+          value: token.value,
+          base: token.base,
+          isMalformed: token.malformed
+        });
+
+        return create("(number)", token.value);
+
+      case Token.RegExp:
+        return create("(regexp)", token.value);
+
+      case Token.Comment:
+        state.tokens.curr.comment = true;
+
+        if (token.isSpecial) {
+          return {
+            id: '(comment)',
+            value: token.value,
+            body: token.body,
+            type: token.commentType,
+            isSpecial: token.isSpecial,
+            line: this.line,
+            character: this.char,
+            from: this.from
+          };
+        }
+
+        break;
+
+      case "":
+        break;
+
+      default:
+        return create("(punctuator)", token.value);
+      }
+    }
+  }
+};
+
+exports.Lexer = Lexer;