lex 0.1.0

data/Rakefile

@@ -0,0 +1,8 @@
+# coding: utf-8
+
+require 'bundler/gem_tasks'
+
+FileList['tasks/**/*.rake'].each(&method(:import))
+
+desc 'Run all specs'
+task ci: %w[ spec ]

data/lex.gemspec

@@ -0,0 +1,22 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'lex/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "lex"
+  spec.version       = Lex::VERSION
+  spec.authors       = ["Piotr Murach"]
+  spec.email         = [""]
+  spec.summary       = %q{Lex is an implementation of complier constuction tool lex in Ruby.}
+  spec.description   = %q{Lex is an implementation of compiler construction tool lex in Ruby. The goal is to stay close to the way the original tool works and combine it with the expressivness or Ruby.}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.6"
+end

data/lib/lex.rb

@@ -0,0 +1,22 @@
+# coding: utf-8
+
+require 'strscan'
+require 'logger'
+require 'forwardable'
+
+require 'lex/logger'
+require 'lex/linter'
+require 'lex/lexeme'
+require 'lex/source_line'
+require 'lex/state'
+require 'lex/token'
+require 'lex/lexer'
+require "lex/version"
+
+module Lex
+  # A base class for all Lexer errors
+  class Error < StandardError; end
+
+  # Raised when lexing fails
+  class LexerError < Error; end
+end # Lex

data/lib/lex/lexeme.rb

@@ -0,0 +1,27 @@
+# coding: utf-8
+
+module Lex
+  # Represents token definition
+  class Lexeme
+    attr_reader :name, :pattern, :action
+
+    def initialize(name, pattern, &action)
+      @name    = name
+      @pattern = pattern
+      @action  = action
+    end
+
+    def match(scanner)
+      match = scanner.check(pattern)
+      if match
+        return Token.new(name, match.to_s, &action)
+      end
+      match
+    end
+
+    # @api public
+    def ==(other)
+      @name == other.name
+    end
+  end # Lexeme
+end # Lex

data/lib/lex/lexer.rb

@@ -0,0 +1,210 @@
+# coding: utf-8
+
+require 'lex/lexer/dsl'
+
+module Lex
+  # An abstract lexer that doesn't provide any lexing rules.
+  #
+  # @api public
+  class Lexer
+    extend Forwardable
+    include DSL
+
+    attr_reader :input,
+                :logger,
+                :debug,
+                :current_state,
+                :current_line
+
+    def_delegators :@dsl,
+                   :lex_tokens,
+                   :state_info,
+                   :state_re,
+                   :state_names,
+                   :state_ignore,
+                   :state_error,
+                   :state_lexemes
+
+    def initialize(options = {}, &block)
+      @current_line     = 1
+      @current_pos      = 1 # Position in input
+      @char_pos_in_line = 0
+      @current_state    = :initial
+      @state_stack      = []
+      @logger           = Lex::Logger.new
+      @linter           = Lex::Linter.new
+      @debug            = options[:debug]
+      @dsl              = self.class.dsl
+
+      @dsl.instance_eval(&block) if block
+      @linter.lint(self)
+    end
+
+    # Tokenizes input and returns all tokens
+    #
+    # @param [String] input
+    #
+    # @return [Enumerator]
+    #   the tokens found
+    #
+    # @api public
+    def lex(input)
+      @input = input
+
+      return enum_for(:lex, input) unless block_given?
+
+      if debug
+        logger.info "lex: tokens   = #{@dsl.lex_tokens}"
+        logger.info "lex: states   = #{@dsl.state_info}"
+        logger.info "lex: ignore   = #{@dsl.state_ignore}"
+        logger.info "lex: error    = #{@dsl.state_error}"
+      end
+
+      stream_tokens(input) do |token|
+        yield token
+      end
+    end
+
+    # Advances through input and streams tokens
+    #
+    # @param [String] input
+    #
+    # @yield [Lex::Token]
+    #
+    # @api public
+    def stream_tokens(input, &block)
+      scanner = StringScanner.new(input)
+      while !scanner.eos?
+        current_char = scanner.peek(1)
+        if @dsl.state_ignore[current_state].include?(current_char)
+          scanner.pos += current_char.size
+          @char_pos_in_line += current_char.size
+          next
+        end
+
+        if debug
+          logger.info "lex: [#{current_state}]: lexemes = #{@dsl.state_lexemes[current_state].map(&:name)}"
+        end
+        # Look for regex match
+        longest_token = nil
+        @dsl.state_lexemes[current_state].each do |lexeme|
+          match = lexeme.match(scanner)
+          next if match.nil?
+          longest_token = match if longest_token.nil?
+          next if longest_token.value.length >= match.value.length
+          longest_token = match
+        end
+
+        if longest_token
+          if longest_token.action
+            new_token = longest_token.action.call(self, longest_token)
+            # No value returned from action move to the next token
+            if new_token.nil? || !new_token.is_a?(Token)
+              chars_to_skip = longest_token.value.to_s.length
+              scanner.pos += chars_to_skip
+              unless longest_token.name == :newline
+                @char_pos_in_line += chars_to_skip
+              end
+              next
+            end
+          end
+          move_by = longest_token.value.to_s.length
+          start_char_pos_in_token = @char_pos_in_line + current_char.size
+          longest_token.update_line(current_line, start_char_pos_in_token)
+          advance_column(move_by)
+          scanner.pos += move_by
+        end
+
+        # No match
+        if longest_token.nil?
+          # Check in errors
+          if @dsl.state_error[current_state]
+            token = Token.new(:error, current_char)
+            start_char_pos_in_token = @char_pos_in_line + current_char.size
+            token.update_line(current_line, start_char_pos_in_token)
+            new_token = @dsl.state_error[current_state].call(self, token)
+            advance_column(current_char.length)
+            scanner.pos += current_char.length
+            if new_token.is_a?(Token) || !new_token.nil?
+              longest_token = new_token
+            else
+              next
+            end
+          end
+
+          if longest_token.nil?
+            complain("Illegal character `#{current_char}`")
+          end
+        end
+
+        logger.info "lex: #{longest_token}" if debug
+        block.call(longest_token)
+      end
+    end
+
+    # Switches the state
+    #
+    # @param [Symbol] state
+    #   the name of the state
+    #
+    # @api public
+    def begin(state)
+      unless @dsl.state_info.key?(state)
+        complain("Undefined state: #{state}")
+      end
+      @current_state = state
+    end
+
+    # Enter new state and save old one on stack
+    #
+    # @param [Symbol] state
+    #   the name of the state
+    #
+    # @api public
+    def push_state(state)
+      @state_stack << @current_state
+      self.begin(state)
+    end
+
+    # Restore previous state
+    #
+    # @param [Symbol] state
+    #   the name of the state
+    #
+    # @api public
+    def pop_state
+      self.begin(@state_stack.pop)
+    end
+
+    # Skip ahead n characters
+    #
+    # @api public
+    def skip(n)
+      @current_pos += n
+    end
+
+    def advance_line(value)
+      @current_line += value
+      @char_pos_in_line = 0
+    end
+
+    def advance_column(value)
+      @char_pos_in_line += value
+    end
+
+    # Reset the internal state of the lexer
+    # @api public
+    def rewind
+      @line = 1
+      @column = 1
+      @stack = []
+    end
+
+    private
+
+    # @api private
+    def complain(*args)
+      raise LexerError, *args
+    end
+  end # Lexer
+end # Lex

data/lib/lex/lexer/dsl.rb

@@ -0,0 +1,49 @@
+# coding: utf-8
+
+require 'lex/lexer/rule_dsl'
+
+module Lex
+  class Lexer
+    # Lexer DSL
+    module DSL
+      # Extend lexer class with DSL methods
+      #
+      # @api private
+      def self.included(klass)
+        klass.extend(ClassMethods)
+      end
+
+      # Class methods for a lexer
+      #
+      # @api private
+      module ClassMethods
+        # Set dsl for lexer
+        #
+        # @api private
+        def inherited(klass)
+          super
+
+          klass.instance_variable_set('@dsl', nil)
+        end
+
+        # Return the rule DSL used by Lexer
+        #
+        # @api private
+        def dsl
+          @dsl ||= RuleDSL.new
+        end
+
+        # Delegate calls to RuleDSL
+        #
+        # @api private
+        def method_missing(name, *args, &block)
+          if dsl.respond_to?(name)
+            dsl.public_send(name, *args, &block)
+          else
+            super
+          end
+        end
+      end # ClassMethods
+    end # DSL
+  end # Lexer
+end # Lex

data/lib/lex/lexer/rule_dsl.rb

@@ -0,0 +1,165 @@
+# coding: utf-8
+
+module Lex
+  class Lexer
+    # Rules DSL used internally by {Lexer}
+    #
+    # @api private
+    class RuleDSL
+      attr_reader :lex_tokens,
+                  :state_info,
+                  :state_re,
+                  :state_names,
+                  :state_ignore,
+                  :state_error,
+                  :state_lexemes
+
+      # @api private
+      def initialize
+        @state_info    = { initial: :inclusive }
+        @state_ignore  = { initial: '' }  # Ignored characters for each state
+        @state_error   = {} # Error conditions for each state
+        @state_re      = Hash.new { |hash, name| hash[name] = {}} # Regexes for each state
+        @state_names   = {} # Symbol names for each state
+        @state_lexemes = Hash.new { |hash, name| hash[name] = State.new(name) }
+        @lex_tokens    = []  # List of valid tokens
+      end
+
+      # Add tokens to lexer
+      #
+      # @api public
+      def tokens(*value)
+        @lex_tokens = value
+      end
+
+      # Add states to lexer
+      #
+      # @api public
+      def states(value)
+        @state_info.merge!(value)
+      end
+
+      # Specify lexing rule
+      #
+      # @param [Symbol] name
+      #   the rule name
+      #
+      # @param [Regex] pattern
+      #   the regex pattern
+      #
+      # @api public
+      def rule(name, pattern, &action)
+        state_names, token_name = *extract_state_token(name)
+        if token_name =~ /^[[:upper:]]*$/ && !@lex_tokens.include?(token_name)
+          complain("Rule '#{name}' defined for" \
+                   " an unspecified token #{token_name}")
+        end
+        state_names.each do |state_name|
+          state = @state_lexemes[state_name]
+          state << Lexeme.new(token_name, pattern, &action)
+        end
+        update_inclusive_states
+        state_names.each do |state_name|
+          if @state_re[state_name].key?(token_name)
+            complain("Rule '#{name}' redefined.")
+          end
+          @state_re[state_name][token_name] = pattern
+        end
+      end
+
+      # Define ignore condition for a state
+      #
+      # @param [Symbol] states
+      #   the optional state names
+      #
+      # @param [String] value
+      #   the characters to ignore
+      #
+      # @api public
+      def ignore(states, value = (not_set = true))
+        if not_set
+          value = states
+          state_names = [:initial]
+        else
+          state_names = states.to_s.split('_').map(&:to_sym)
+        end
+        if !value.is_a?(String)
+          logger.error("Ignore rule '#{value}' has to be defined with a string")
+        end
+        state_names.each do |state_name|
+          @state_ignore[state_name] = value
+        end
+        @state_info.each do |state_name, state_type|
+          if state_name != :initial && state_type == :inclusive
+            if !@state_ignore.key?(state_name)
+              @state_ignore[state_name] = @state_ignore[:initial]
+            end
+          end
+        end
+      end
+
+      # Define error condition for a state
+      #
+      # @api public
+      def error(states = :initial, &action)
+        state_names = states.to_s.split('_').map(&:to_sym)
+        state_names.each do |state_name|
+          @state_error[state_name] = action
+        end
+        @state_info.each do |state_name, state_type|
+          if state_name != :initial && state_type == :inclusive
+            if !@state_error.key?(state_name)
+              @state_error[state_name] = @state_error[:initial]
+            end
+          end
+        end
+      end
+
+      private
+
+      # For inclusive states copy over initial state rules
+      #
+      # @api private
+      def update_inclusive_states
+        @state_info.each do |state_name, state_type|
+          if state_name != :initial && state_type == :inclusive
+            initial_state = @state_lexemes[:initial]
+            @state_lexemes[state_name].update(initial_state.lexemes)
+          end
+        end
+      end
+
+      # Extract tuple of state names and token name
+      #
+      # @param [Symbol] name
+      #   the rule name
+      #
+      # @return [Array[Symbol], Symbol]
+      #   returns tuples [states, token]
+      #
+      # @api private
+      def extract_state_token(name)
+        parts = name.to_s.split('_')
+        state_i = 0
+        parts.each_with_index do |part, i|
+          if !@state_info.keys.include?(part.to_sym)
+            state_i = i
+            break
+          end
+        end
+        states = if state_i > 0
+                   parts[0...state_i].map(&:to_sym)
+                 else
+                   [:initial]
+                 end
+        token_name = parts[state_i..-1].join('_').to_sym
+        [states, token_name]
+      end
+
+      # @api private
+      def complain(*args)
+        raise LexerError, *args
+      end
+    end # RuleDSL
+  end # Lexer
+end # Lex