rufus-lua-win 5.1.5

data/vendor/lua/lib/lua/luaidl/lex.lua

@@ -0,0 +1,793 @@
+--
+-- Project:  LuaIDL
+-- Author:   Ricardo Cosme <rcosme@tecgraf.puc-rio.br>
+-- Filename: lex.lua
+--
+
+local type     = type
+local pairs    = pairs
+local tonumber = tonumber
+local error    = error
+local ipairs   = ipairs
+local table    = table
+local string   = require "string"
+
+module 'luaidl.lex'
+
+tab_tokens = {
+  TK_ID = 257, TK_ABSTRACT = 258, TK_ANY = 259, TK_ATTRIBUTE = 260,
+  TK_BOOLEAN = 261, TK_CASE = 262, TK_CHAR = 263, TK_COMPONENT = 264,
+  TK_CONST = 265, TK_CONSUMES = 266, TK_CONTEXT = 267, TK_CUSTOM = 268,
+  TK_DEFAULT = 269, TK_DOUBLE = 270, TK_EXCEPTION = 271, TK_EMITS = 272,
+  TK_ENUM = 273, TK_EVENTTYPE = 274, TK_FACTORY = 275, TK_FALSE = 276,
+  TK_FINDER = 277, TK_FIXED = 278, TK_FLOAT = 279, TK_GETRAISES = 280,
+  TK_HOME = 281, TK_IMPORT = 282, TK_IN = 283, TK_INOUT = 284,
+  TK_INTERFACE = 285, TK_LOCAL = 286, TK_LONG = 287, TK_MODULE = 288,
+  TK_MULTIPLE = 289, TK_NATIVE = 290, TK_OBJECT = 291, TK_OCTET = 292,
+  TK_ONEWAY = 293, TK_OUT = 294, TK_PRIMARYKEY = 295, TK_PRIVATE = 296,
+  TK_PROVIDES = 297, TK_PUBLIC = 298, TK_PUBLISHES = 299, TK_RAISES = 300,
+  TK_READONLY = 301, TK_SETRAISES = 302, TK_SEQUENCE = 303, TK_SHORT = 304,
+  TK_STRING = 305, TK_STRUCT = 306, TK_SUPPORTS = 307, TK_SWITCH = 308,
+  TK_TRUE = 309, TK_TRUNCATABLE = 310, TK_TYPEDEF = 311, TK_TYPEID = 312,
+  TK_TYPEPREFIX = 313, TK_UNSIGNED = 314, TK_UNION = 315, TK_USES = 316,
+  TK_VALUEBASE = 317, TK_VALUETYPE = 318, TK_VOID = 319, TK_WCHAR = 320,
+  TK_WSTRING = 321, TK_INTEGER_LITERAL = 322, TK_FLOAT_LITERAL = 323,
+  TK_CHAR_LITERAL = 324, TK_WCHAR_LITERAL = 325, TK_STRING_LITERAL = 326,
+  TK_WSTRING_LITERAL = 327, TK_FIXED_LITERAL = 328, TK_PRAGMA_PREFIX = 329,
+  TK_PRAGMA_ID = 330, TK_MANAGES = 332,
+}
+
+local tab_keywords = {
+  ['abstract']      = { token = tab_tokens.TK_ABSTRACT },
+  ['any']           = { token = tab_tokens.TK_ANY },
+  ['attribute']     = { token = tab_tokens.TK_ATTRIBUTE },
+  ['boolean']       = { token = tab_tokens.TK_BOOLEAN },
+  ['case']          = { token = tab_tokens.TK_CASE },
+  ['char']          = { token = tab_tokens.TK_CHAR },
+  ['component']     = { token = tab_tokens.TK_COMPONENT },
+  ['const']         = { token = tab_tokens.TK_CONST },
+  ['consumes']      = { token = tab_tokens.TK_CONSUMES },
+  ['context']       = { token = tab_tokens.TK_CONTEXT },
+  ['custom']        = { token = tab_tokens.TK_CUSTOM },
+  ['default']       = { token = tab_tokens.TK_DEFAULT },
+  ['double']        = { token = tab_tokens.TK_DOUBLE },
+  ['exception']     = { token = tab_tokens.TK_EXCEPTION },
+  ['emits']         = { token = tab_tokens.TK_EMITS },
+  ['enum']          = { token = tab_tokens.TK_ENUM },
+  ['eventtype']     = { token = tab_tokens.TK_EVENTTYPE },
+  ['factory']       = { token = tab_tokens.TK_FACTORY },
+  ['FALSE']         = { token = tab_tokens.TK_FALSE },
+  ['finder']        = { token = tab_tokens.TK_FINDER },
+  ['fixed']         = { token = tab_tokens.TK_FIXED },
+  ['float']         = { token = tab_tokens.TK_FLOAT },
+  ['getraises']     = { token = tab_tokens.TK_GETRAISES },
+  ['home']          = { token = tab_tokens.TK_HOME },
+  ['import']        = { token = tab_tokens.TK_IMPORT },
+  ['in']            = { token = tab_tokens.TK_IN },
+  ['inout']         = { token = tab_tokens.TK_INOUT },
+  ['interface']     = { token = tab_tokens.TK_INTERFACE },
+  ['local']         = { token = tab_tokens.TK_LOCAL },
+  ['long']          = { token = tab_tokens.TK_LONG },
+  ['manages']       = { token = tab_tokens.TK_MANAGES },
+  ['module']        = { token = tab_tokens.TK_MODULE },
+  ['multiple']      = { token = tab_tokens.TK_MULTIPLE },
+  ['native']        = { token = tab_tokens.TK_NATIVE },
+  ['Object']        = { token = tab_tokens.TK_OBJECT },
+  ['octet']         = { token = tab_tokens.TK_OCTET },
+  ['oneway']        = { token = tab_tokens.TK_ONEWAY },
+  ['out']           = { token = tab_tokens.TK_OUT },
+  ['primarykey']    = { token = tab_tokens.TK_PRIMARYKEY },
+  ['private']       = { token = tab_tokens.TK_PRIVATE },
+  ['provides']      = { token = tab_tokens.TK_PROVIDES },
+  ['public']        = { token = tab_tokens.TK_PUBLIC },
+  ['publishes']     = { token = tab_tokens.TK_PUBLISHES },
+  ['raises']        = { token = tab_tokens.TK_RAISES },
+  ['readonly']      = { token = tab_tokens.TK_READONLY },
+  ['setraises']     = { token = tab_tokens.TK_SETRAISES },
+  ['sequence']      = { token = tab_tokens.TK_SEQUENCE },
+  ['short']         = { token = tab_tokens.TK_SHORT },
+  ['string']        = { token = tab_tokens.TK_STRING },
+  ['struct']        = { token = tab_tokens.TK_STRUCT },
+  ['supports']      = { token = tab_tokens.TK_SUPPORTS },
+  ['switch']        = { token = tab_tokens.TK_SWITCH },
+  ['TRUE']          = { token = tab_tokens.TK_TRUE },
+  ['truncatable']   = { token = tab_tokens.TK_TRUNCATABLE },
+  ['typedef']       = { token = tab_tokens.TK_TYPEDEF },
+  ['typeid']        = { token = tab_tokens.TK_TYPEID },
+  ['typeprefix']    = { token = tab_tokens.TK_TYPEPREFIX },
+  ['unsigned']      = { token = tab_tokens.TK_UNSIGNED },
+  ['union']         = { token = tab_tokens.TK_UNION },
+  ['uses']          = { token = tab_tokens.TK_USES },
+  ['ValueBase']     = { token = tab_tokens.TK_VALUEBASE },
+  ['valuetype']     = { token = tab_tokens.TK_VALUETYPE },
+  ['void']          = { token = tab_tokens.TK_VOID },
+  ['wchar']         = { token = tab_tokens.TK_WCHAR },
+  ['wstring']       = { token = tab_tokens.TK_WSTRING },
+}
+
+local tab_symbols
+
+PRAGMA_VERSION          = '1.0'
+local ERROR_MSG_TYPE    = '[lexical error]:_LINE:_ERRORMSG.'
+
+local token
+local lookahead
+local i
+local stridllen
+local linemarkDeclared
+
+local function is_blank(char)
+  if (lookahead == ' ' or lookahead == '\f' or lookahead == '\r' or
+       lookahead == '\t' or lookahead == '\v') then
+    return true
+  else
+    return false
+  end
+end
+
+local function is_digit(char)
+  return string.find(char, '%d')
+end
+
+local function is_hex_digit(char)
+  return string.find(char, '%x')
+end
+
+local function is_octal_digit(char)
+  return string.find(char, '[0-7]')
+end
+
+local function is_alpha(char)
+  return string.find(char, '%w')
+end
+
+local function is_near_value(lookahead)
+  if string.find(lookahead, '%w') then
+    return ' near \''..lookahead..'\''
+  else
+    return ''
+  end
+end
+
+local function insert_symbols(lexeme)
+  tab_symbols[lexeme] = {token = tab_tokens.TK_ID, descriptions = {}}
+end
+
+local function validate_symbol(lexeme, tab_symbols)
+  if tab_symbols[lexeme] then
+    return lexeme, tab_symbols[lexeme].token
+  else
+    for key, value in pairs(tab_symbols) do
+      if string.upper(lexeme) == string.upper(key) then
+        return key, 'collide'
+      end
+    end
+  end
+  return nil, nil
+end
+
+local function search_symbol(lexeme, tab_symbols)
+  if tab_symbols[lexeme] then
+    return lexeme, tab_symbols[lexeme].token
+  end
+  return nil, nil
+end
+
+local function error_lex(error_msg)
+  if (type(ERROR_MSG_TYPE) ~= 'string') then
+    error('bad value to \'error_msg_type\' variable (string expected)', 2)
+  end
+  init()
+  local _LINE =  line
+  error(string.gsub(string.gsub(ERROR_MSG_TYPE,  '_LINE',  _LINE), 
+         '_ERRORMSG',  error_msg),  3)
+end
+
+local function get_constructor (symbol_name)
+  if tab_symbols[symbol_name] then
+    return tab_symbols[symbol_name].descriptions
+  else
+    return nil
+  end
+end
+
+local function set_constructor (symbol_name, field_name, value)
+  for k,  v in ipairs(tab_symbols) do
+    if (v.lexeme == symbol_name) then
+      v.descriptions[field_name] = value
+    end
+  end
+  return nil
+end
+
+local function lineCount()
+  line = line + 1
+end
+
+local function getchar(stridl)
+  if (i > stridllen) then
+    return nil
+  else
+    local c = string.sub(stridl, i, i)
+    i = i + 1
+    return c
+  end
+end
+
+function init()
+  tab_symbols           = { }
+  token                 = nil
+  tokenvalue            = '<EOF>'
+  tokenvalue_previous   = '<EOF>'
+  line                  = 1
+  srcfilename           = ''
+  tab_linemarks         = { }
+  lookahead             = ' '
+  i                     = 1
+  stridllen             = nil
+end
+
+function lexer(stridl)
+  if not stridllen then
+    init()
+    stridllen = string.len(stridl)
+  end
+
+  while true do
+    tokenvalue_previous = tokenvalue
+
+    if not lookahead then
+      init()
+      return token
+    elseif (lookahead == '#') then
+      lookahead = getchar(stridl)
+    -- C preprocessor
+    -- # <linenum> "<filename>" <flags>*
+      if (lookahead == ' ') then
+        linemarkDeclared = true
+      -- linenum
+        local linenum = getchar(stridl)
+        lookahead = getchar(stridl)
+        while is_digit(lookahead) do
+          linenum = linenum..lookahead
+          lookahead = getchar(stridl)
+        end
+        line = tonumber(linenum)
+      -- filename
+      -- '"' char
+        local _ = getchar(stridl)
+        local filename = ''
+        lookahead = getchar(stridl)
+        while (lookahead ~= '"') do
+          filename = filename..lookahead
+          lookahead = getchar(stridl)
+        end
+        srcfilename = filename
+        lookahead = getchar(stridl)
+      -- flags
+        local flags = { }
+        while (lookahead == ' ') do
+          lookahead = getchar(stridl)
+          if is_digit(lookahead) then
+            flags[lookahead] = true
+          end
+          lookahead = getchar(stridl)
+        end
+        table.insert(tab_linemarks, flags)
+    -- pragma declarations
+      elseif (lookahead == 'p') then
+        lookahead = getchar(stridl)
+        if (lookahead == 'r') then
+          lookahead = getchar(stridl)
+          if (lookahead == 'a') then
+            lookahead = getchar(stridl)
+            if (lookahead == 'g') then
+              lookahead = getchar(stridl)
+              if (lookahead == 'm') then
+                lookahead = getchar(stridl)
+                if (lookahead == 'a') then
+                  lookahead = getchar(stridl)
+                  if is_blank(lookahead) then
+                    lookahead = getchar(stridl)
+                  end
+                -- pragma prefix
+                  if (lookahead == 'p') then
+                    lookahead = getchar(stridl)
+                    if (lookahead == 'r') then
+                      lookahead = getchar(stridl)
+                      if (lookahead == 'e') then
+                        lookahead = getchar(stridl)
+                        if (lookahead == 'f') then
+                          lookahead = getchar(stridl)
+                          if (lookahead == 'i') then
+                            lookahead = getchar(stridl)
+                            if (lookahead == 'x') then
+                              lookahead = getchar(stridl)
+                              token = tab_tokens.TK_PRAGMA_PREFIX
+                              return token
+                            end
+                          end
+                        end
+                      end
+                    end
+                -- pragma ID
+                  elseif (lookahead == 'I') then
+                    lookahead = getchar(stridl)
+                    if (lookahead == 'D') then
+                      lookahead = getchar(stridl)
+                      token = tab_tokens.TK_PRAGMA_ID
+                      return token
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+  -- new lines
+    elseif (lookahead == '\n') then
+    -- We does not consider linemark declarations to line counts.
+      if not linemarkDeclared then
+        lineCount()
+      else
+        linemarkDeclared = false
+      end
+      lookahead = getchar(stridl)
+  -- blank characters
+    elseif is_blank(lookahead) then
+      lookahead = getchar(stridl)
+  -- comments
+    elseif (lookahead == '/') then
+      lookahead = getchar(stridl)
+      if (lookahead == '/') then
+        while (lookahead ~= '\n') do
+          lookahead = getchar(stridl)
+        end
+        lineCount()
+        lookahead = getchar(stridl)
+      elseif (lookahead == '*') then
+        local first_line = line
+        lookahead = getchar(stridl)
+        while true do
+          if (lookahead == '\n') then
+            lookahead = getchar(stridl)
+            lineCount()
+          elseif not lookahead then
+          -- where begin nonterminated comment ?
+            line = first_line
+            error_lex('nonterminated comment')
+          elseif (lookahead == '*') then
+             lookahead = getchar(stridl)
+             if (lookahead == '/') then
+               break
+             end
+          else
+             lookahead = getchar(stridl)
+          end
+        end
+        lookahead = getchar(stridl)
+      else
+        tokenvalue = '/'
+        token = tokenvalue
+        return token
+      end
+  -- floating-point literals
+  -- fixed-point literals
+    elseif (lookahead == '.') then
+      tokenvalue = '.'
+      lookahead = getchar(stridl)
+      if is_digit(lookahead) then
+        tokenvalue = tokenvalue..lookahead
+        lookahead = getchar(stridl)
+        while is_digit(lookahead) do
+          tokenvalue = tokenvalue..lookahead
+          lookahead  = getchar()
+        end
+        if (lookahead == 'e' or lookahead == 'E') then
+          tokenvalue = tokenvalue..lookahead
+          lookahead = getchar(stridl)
+          if (lookahead == '-') then
+            tokenvalue = tokenvalue..lookahead
+            lookahead = getchar(stridl)
+            if not is_digit(lookahead) then
+              error_lex('malformed number'..is_near_value(tokenvalue))
+            else
+              tokenvalue = tokenvalue..lookahead
+              lookahead = getchar(stridl)
+              while is_digit(lookahead) do
+                tokenvalue = tokenvalue..lookahead
+                lookahead = getchar(stridl)
+              end
+            end
+          elseif is_digit(lookahead) then
+            tokenvalue = tokenvalue..lookahead
+            lookahead = getchar(stridl)
+            while is_digit(lookahead) do
+              tokenvalue = tokenvalue..lookahead
+              lookahead = getchar(stridl)
+            end
+          else
+            error_lex('malformed number'..is_near_value(tokenvalue))
+          end
+          tokenvalue = tonumber(tokenvalue, 10)
+          token = tab_tokens.TK_FLOAT_LITERAL
+          return token
+        elseif (lookahead == 'd' or lookahead == 'D') then
+          tokenvalue = tonumber(tokenvalue, 10)
+          lookahead = getchar(stridl)
+          token = tab_tokens.TK_FIXED_LITERAL
+          return token
+        else
+          tokenvalue = tonumber(tokenvalue, 10)
+          token = tab_tokens.TK_FLOAT_LITERAL
+          return token
+        end
+      else
+        tokenvalue = '.'
+        token = tokenvalue
+        return token
+      end
+  -- integer literal (decimal)
+  -- integer literal (hexa)
+  -- integer literal (octal)
+  -- floating-point literals
+  -- fixed-point literals
+    elseif (lookahead == '0') then
+      tokenvalue = '0'
+      lookahead = getchar(stridl)
+      if (lookahead == 'x' or lookahead == 'X') then
+        tokenvalue = tokenvalue..lookahead
+        lookahead = getchar(stridl)
+        while is_hex_digit(lookahead) do
+          tokenvalue = tokenvalue..lookahead
+          lookahead = getchar(stridl)
+        end
+        tokenvalue = tonumber(tokenvalue, 10)
+        token = tab_tokens.TK_INTEGER_LITERAL
+        return token
+      end
+      while is_digit(lookahead) do
+        tokenvalue = tokenvalue..lookahead
+        lookahead = getchar(stridl)
+      end
+      if (lookahead == '.' or lookahead == 'e' or lookahead == 'E' or
+          lookahead == 'd' or lookahead == 'd')
+      then
+        if (lookahead == '.') then
+          tokenvalue = tokenvalue..'.'
+          lookahead = getchar(stridl)
+          if is_digit(lookahead) then
+            tokenvalue = tokenvalue..lookahead
+            lookahead = getchar(stridl)
+            while is_digit(lookahead) do
+              tokenvalue = tokenvalue..lookahead
+              lookahead = getchar(stridl)
+            end
+          end
+        end
+        if (lookahead == 'e' or lookahead == 'E') then
+          tokenvalue = tokenvalue..lookahead
+          lookahead = getchar(stridl)
+          if (lookahead == '-') then
+            tokenvalue = tokenvalue..lookahead
+            lookahead = getchar(stridl)
+            if not is_digit(lookahead) then
+              error_lex('malformed number near'..is_near_value(tokenvalue))
+            else
+              tokenvalue = tokenvalue..lookahead
+              lookahead = getchar(stridl)
+              while is_digit(lookahead) do
+                tokenvalue = tokenvalue..lookahead
+                lookahead = getchar(stridl)
+              end
+            end
+          elseif is_digit(lookahead) then
+            tokenvalue = tokenvalue..lookahead
+            lookahead = getchar(stridl)
+            while is_digit(lookahead) do
+              tokenvalue = tokenvalue..lookahead
+              lookahead = getchar(stridl)
+            end
+          else
+            error_lex('malformed number near'..is_near_value(tokenvalue))
+          end
+          tokenvalue = tonumber(tokenvalue, 10)
+          token = tab_tokens.TK_FLOAT_LITERAL
+          return token
+        elseif (lookahead == 'd' or lookahead == 'D') then
+          tokenvalue = tonumber(tokenvalue, 10)
+          lookahead = getchar(stridl)
+          token = tab_tokens.TK_FIXED_LITERAL
+          return token
+        else
+          tokenvalue = tonumber(tokenvalue, 10)
+          token = tab_tokens.TK_FLOAT_LITERAL
+          return token
+        end
+      else
+        if not (string.find(tokenvalue, '8') or string.find(tokenvalue, '9')) then
+          tokenvalue = tonumber(tokenvalue, 8)
+        end
+        token = tab_tokens.TK_INTEGER_LITERAL
+        return token
+      end
+  -- integer literal (decimal)
+  -- floating-point literals
+  -- fixed-point literals
+    elseif is_digit(lookahead) then
+      tokenvalue = lookahead
+      lookahead = getchar(stridl)
+      while is_digit(lookahead) do
+        tokenvalue = tokenvalue..lookahead
+        lookahead = getchar(stridl)
+      end
+      if (lookahead == '.' or lookahead == 'e' or lookahead == 'E' or
+          lookahead == 'd' or lookahead == 'd')
+      then
+        if (lookahead == '.') then
+          tokenvalue = tokenvalue..'.'
+          lookahead = getchar(stridl)
+          if is_digit(lookahead) then
+            tokenvalue = tokenvalue..lookahead
+            lookahead = getchar(stridl)
+            while is_digit(lookahead) do
+              tokenvalue = tokenvalue..lookahead
+              lookahead = getchar(stridl)
+            end
+          end
+        end
+        if (lookahead == 'e' or lookahead == 'E') then
+          tokenvalue = tokenvalue..lookahead
+          lookahead = getchar(stridl)
+          if (lookahead == '-') then
+            tokenvalue = tokenvalue..lookahead
+            lookahead = getchar(stridl)
+            if not is_digit(lookahead) then
+              error_lex('malformed number near'..is_near_value(tokenvalue))
+            else
+              tokenvalue = tokenvalue..lookahead
+              lookahead = getchar(stridl)
+              while is_digit(lookahead) do
+                tokenvalue = tokenvalue..lookahead
+                lookahead = getchar(stridl)
+              end
+            end
+          elseif is_digit(lookahead) then
+            tokenvalue = tokenvalue..lookahead
+            lookahead = getchar(stridl)
+            while is_digit(lookahead) do
+              tokenvalue = tokenvalue..lookahead
+              lookahead = getchar(stridl)
+            end
+          else
+            error_lex('malformed number near'..is_near_value(tokenvalue))
+          end
+          tokenvalue = tonumber(tokenvalue, 10)
+          token = tab_tokens.TK_FLOAT_LITERAL
+          return token
+        elseif (lookahead == 'd' or lookahead == 'D') then
+          tokenvalue = tonumber(tokenvalue, 10)
+          lookahead = getchar(stridl)
+          token = tab_tokens.TK_FIXED_LITERAL
+          return token
+        else
+          tokenvalue = tonumber(tokenvalue, 10)
+          token = tab_tokens.TK_FLOAT_LITERAL
+          return token
+        end
+      else
+        token = tab_tokens.TK_INTEGER_LITERAL
+        return token
+      end
+  -- char literal
+  -- "The value of a null is 0" ????
+    elseif (lookahead == '\'') then
+      tokenvalue = ''
+      lookahead = getchar(stridl)
+      if (lookahead == '\\') then
+        lookahead = getchar(stridl)
+        if (lookahead == 'n') then
+          tokenvalue = tokenvalue..'\n'
+          lookahead = getchar(stridl)
+        elseif (lookahead == 't') then
+          tokenvalue = tokenvalue..'\t'
+          lookahead = getchar(stridl)
+        elseif (lookahead == 'v') then
+          tokenvalue = tokenvalue..'\v'
+          lookahead = getchar(stridl)
+        elseif (lookahead == 'b') then
+          tokenvalue = tokenvalue..'\b'
+          lookahead = getchar(stridl)
+        elseif (lookahead == 'r') then
+          tokenvalue = tokenvalue..'\r'
+          lookahead = getchar(stridl)
+        elseif (lookahead == 'f') then
+          tokenvalue = tokenvalue..'\f'
+          lookahead = getchar(stridl)
+        elseif (lookahead == 'a') then
+          tokenvalue = tokenvalue..'\a'
+          lookahead = getchar(stridl)
+        elseif (lookahead == '\\') then
+          tokenvalue = tokenvalue..'\\'
+          lookahead = getchar(stridl)
+        elseif (lookahead == '?') then
+          tokenvalue = tokenvalue..'?'
+          lookahead = getchar(stridl)
+        elseif (lookahead == '\'') then
+          tokenvalue = tokenvalue..'\''
+          lookahead = getchar(stridl)
+        elseif (lookahead == '"') then
+          tokenvalue = tokenvalue..'"'
+          lookahead = getchar(stridl)
+        elseif is_octal_digit(lookahead) then
+          local num_digits = 1
+          local tokenvalue_tmp = lookahead
+          lookahead = getchar(stridl)
+          while is_octal_digit(lookahead) do
+            tokenvalue_tmp = tokenvalue_tmp..lookahead
+            lookahead = getchar(stridl)
+            num_digits = num_digits + 1
+            if (num_digits == 3) then
+              lookahed = getchar(stridl)
+              break
+            end
+          end
+          tokenvalue = tokenvalue..string.char(tonumber(tokenvalue_tmp, 8))
+        elseif (lookahead == 'x') then
+          local tokenvalue_tmp = '0x'
+          lookahead = getchar(stridl)
+          if is_hex_digit(lookahead) then
+            tokenvalue_tmp = tokenvalue_tmp..lookahead
+            lookahead = getchar(stridl)
+            if is_hex_digit(lookahead) then
+              tokenvalue_tmp = tokenvalue_tmp..lookahead
+              lookahead = getchar(stridl)
+            end
+          end
+          tokenvalue = tokenvalue..string.char(tonumber(tokenvalue_tmp, 10))
+        elseif (lookahead == 'u') then
+          error_lex('it doest not permited unicode characters in char type')
+        else
+        -- When occur an unknown escape sequence, then we apply a common
+        -- behavior that is to return a proper character. Ex.: '\e' -> e
+          tokenvalue = tokenvalue..lookahead
+          lookahead = getchar(stridl)
+        end
+      elseif (lookahead ~= '\'') then
+        tokenvalue = lookahead
+        lookahead = getchar(stridl)
+      end
+      if (lookahead ~= '\'') then
+        if not lookahead then
+          error_lex('"\'" expected near \'<eof>\'')
+        else
+          error_lex('"\'" expected'..is_near_value(tokenvalue))
+        end
+      end
+      lookahead = getchar(stridl)
+      token = tab_tokens.TK_CHAR_LITERAL
+      return token
+  -- string literal
+    elseif (lookahead == '"') then
+      lookahead = getchar(stridl)
+      tokenvalue = ''
+      while true do
+        if (lookahead == '\\') then
+          lookahead = getchar(stridl)
+          if (lookahead == 'n') then
+            tokenvalue = tokenvalue..'\n'
+            lookahead = getchar(stridl)
+          elseif (lookahead == 't') then
+            tokenvalue = tokenvalue..'\t'
+            lookahead = getchar(stridl)
+          elseif (lookahead == 'v') then
+            tokenvalue = tokenvalue..'\v'
+            lookahead = getchar(stridl)
+          elseif (lookahead == 'b') then
+            tokenvalue = tokenvalue..'\b'
+            lookahead = getchar(stridl)
+          elseif (lookahead == 'r') then
+            tokenvalue = tokenvalue..'\r'
+            lookahead = getchar(stridl)
+          elseif (lookahead == 'f') then
+            tokenvalue = tokenvalue..'\f'
+            lookahead = getchar(stridl)
+          elseif (lookahead == 'a') then
+            tokenvalue = tokenvalue..'\a'
+            lookahead = getchar(stridl)
+          elseif (lookahead == '\\') then
+            tokenvalue = tokenvalue..'\\'
+            lookahead = getchar(stridl)
+          elseif (lookahead == '?') then
+            tokenvalue = tokenvalue..'?'
+            lookahead = getchar(stridl)
+          elseif (lookahead == '\'') then
+            tokenvalue = tokenvalue..'\''
+            lookahead = getchar(stridl)
+          elseif (lookahead == '"') then
+            tokenvalue = tokenvalue..'"'
+            lookahead = getchar(stridl)
+          elseif is_octal_digit(lookahead) then
+            local num_digits = 1
+            local tokenvalue_tmp = lookahead
+            lookahead = getchar(stridl)
+            while is_octal_digit(lookahead) do
+              tokenvalue_tmp = tokenvalue_tmp..lookahead
+              lookahead = getchar(stridl)
+              num_digits = num_digits + 1
+              if (num_digits == 3) then
+                lookahed = getchar(stridl)
+                break
+              end
+            end
+            tokenvalue = tokenvalue..string.char(tonumber(tokenvalue_tmp, 8))
+          elseif (lookahead == 'x') then
+            local tokenvalue_tmp = '0x'
+            lookahead = getchar(stridl)
+            if is_hex_digit(lookahead) then
+              tokenvalue_tmp = tokenvalue_tmp..lookahead
+              lookahead = getchar(stridl)
+              if is_hex_digit(lookahead) then
+                tokenvalue_tmp = tokenvalue_tmp..lookahead
+                lookahead = getchar(stridl)
+              end
+            end
+            tokenvalue = tokenvalue..string.char(tonumber(tokenvalue_tmp,  10))
+          elseif (lookahead == 'u') then
+            error_lex('it doest not permited unicode characters in char type')
+          else
+          -- When occur an unknown escape sequence, then we apply a common
+          -- behavior that is to return a proper character. Ex.: '\e' -> e
+            tokenvalue = tokenvalue..lookahead
+            lookahead = getchar(stridl)
+          end
+        elseif (lookahead == '"') then
+          break
+        elseif not lookahead then
+          error_lex('nonterminated string')
+        else
+          tokenvalue = tokenvalue..lookahead
+          lookahead = getchar(stridl)
+        end
+      end
+      lookahead = getchar(stridl)
+      token = tab_tokens.TK_STRING_LITERAL
+      return token
+  -- identifiers
+  -- keywords
+    elseif (is_alpha(lookahead) or (lookahead == '_')) then
+      local lexbuf = lookahead
+      lookahead = getchar(stridl)
+      while is_alpha(lookahead) or (lookahead == '_') or is_digit(lookahead) do
+        lexbuf = lexbuf..lookahead
+        lookahead = getchar(stridl)
+      end
+    -- Escaped identifiers - turns off keyword checking.
+      if (string.sub(lexbuf, 1, 1) == '_') then
+        lexbuf = string.sub(lexbuf, 2)
+      else
+        tokenvalue, _token = validate_symbol(lexbuf, tab_keywords)
+        if (_token == "collide") then
+          error_lex("'"..lexbuf.."' collides with keyword '"..tokenvalue.."'")
+        elseif _token then
+          token = _token
+          return token
+        end
+      end
+      tokenvalue, token = search_symbol(lexbuf, tab_symbols)
+      if not token then
+        insert_symbols(lexbuf)
+        tokenvalue = lexbuf
+        token = tab_tokens.TK_ID
+      end
+      return token
+  -- operators and other characters
+    else
+      tokenvalue = lookahead
+      lookahead = getchar(stridl)
+      token = tokenvalue
+      return token
+    end
+  end
+end