simply_stored 0.3.9 → 0.5.0

data/test/vendor/dhaka-2.2.1/lib/dhaka/lexer/dfa.rb

@@ -1,121 +0,0 @@
-module Dhaka
-  module LexerSupport
-  
-    # Raised when an invalid regular expression pattern is encountered 
-    # in a LexerSpecification 
-    class InvalidRegexException < StandardError 
-    end
-
-    class CheckpointAction
-      attr_reader :pattern
-      def initialize(pattern)
-        @pattern = pattern
-      end
-      
-      def call(lexer_run)
-        lexer_run.save_checkpoint(pattern)
-      end
-      
-      def compile_to_ruby_source
-        "add_checkpoint(#{pattern.inspect})"
-      end
-    end
-  
-  
-    class DFA < StateMachine #:nodoc:
-      def initialize(regex)
-        @regex = regex
-      
-        tokenize_result = RegexTokenizer.tokenize(@regex)
-        raise InvalidRegexException.new(tokenize_error_message(tokenize_result)) if tokenize_result.has_error?
-        
-        parse_result = RegexParser.parse(tokenize_result)
-        raise InvalidRegexException.new(parse_error_message(parse_result)) if parse_result.has_error?
-      
-        ast = parse_result
-        ast.calculate_follow_sets
-
-        super(ItemSet.new(ast.first))
-      end
-    
-      def tokenize_error_message(tokenize_result)
-        "Invalid character #{@regex[tokenize_result.unexpected_char_index].chr}: #{@regex.dup.insert(tokenize_result.unexpected_char_index, '>>>')}"
-      end
-
-      def parse_error_message(parse_result)
-        unexpected_token = parse_result.unexpected_token
-        if unexpected_token.symbol_name == END_SYMBOL_NAME
-          "Unexpected end of regex."
-        else
-          "Unexpected token #{parse_result.unexpected_token.symbol_name}: #{@regex.dup.insert(parse_result.unexpected_token.input_position, '>>>')}"
-        end
-      end
-    
-      def dest_key_for key, char
-        result = ItemSet.new
-        key.each do |position|
-          result.merge(position.follow_set) if position.character == char
-        end
-        result
-      end
-    
-      def new_state_for_key key
-        accepting = key.detect {|position| position.accepting} 
-        if accepting
-          new_state = State.new(self, accepting.action(@regex))
-        else
-          new_state = State.new(self)
-        end
-        if key.any? {|position| position.checkpoint}
-          new_state.checkpoint_actions << CheckpointAction.new(@regex)
-        end
-        new_state
-      end
-    
-      def transition_characters key
-        result = Set.new
-        key.each do |node|
-          result << node.character unless (node.accepting || node.checkpoint)
-        end
-        result
-      end
-    
-      def match(input)
-        DFARun.new(self, input).match
-      end
-    end
-    
-    class DFARun
-      def initialize(dfa, input)
-        @dfa, @input = dfa, input
-        @matched = ""
-        @not_yet_accepted = ""
-        @curr_state = @dfa.start_state
-      end
-      
-      def match
-        @input.unpack("C*").each do |i|
-          break unless dest_state = @curr_state.transitions[i.chr]
-          @not_yet_accepted << i.chr
-          @curr_state = dest_state
-          @curr_state.process(self)
-        end
-        @matched
-      end
-
-      def save_checkpoint(pattern)
-        @last_saved_checkpoint = @matched + @not_yet_accepted
-      end
-      
-      def accept(pattern)
-        @matched.concat @not_yet_accepted
-        @not_yet_accepted = ""
-      end
-
-      def accept_last_saved_checkpoint(pattern)
-        @matched = @last_saved_checkpoint
-        @not_yet_accepted = ""
-      end
-    end
-  end
-end

data/test/vendor/dhaka-2.2.1/lib/dhaka/lexer/lexeme.rb

@@ -1,32 +0,0 @@
-module Dhaka
-  # Represents a portion of the input string that has been recognized as matching a given lexer pattern.
-  class Lexeme
-    # The pattern matched by this lexeme.
-    attr_accessor :pattern, :characters
-
-    # +input_position+ is the index in the input stream that this lexeme starts at.
-    attr_reader :input_position
-    
-    def initialize(input_position) #:nodoc:
-      @input_position = input_position
-      @characters = []
-    end
-    
-    # The substring of the input stream that this lexeme is comprised of. 
-    def value
-      characters.join
-    end
-
-    def accepted? #:nodoc:
-      pattern
-    end
-  
-    def << char #:nodoc:
-      characters << char
-    end
-  
-    def concat chars #:nodoc:
-      characters.concat chars
-    end
-  end
-end 

data/test/vendor/dhaka-2.2.1/lib/dhaka/lexer/lexer.rb

@@ -1,70 +0,0 @@
-module Dhaka
-  # The lexer generator. To generate a lexer from a lexer specification +MyLexerSpecification+:
-  #   lexer = Dhaka::Lexer.new(MyLexerSpecification)
-  # 
-  # To compile this lexer as +MyLexer+ to a string of Ruby source:
-  #   lexer.compile_to_ruby_source_as(:MyLexer)
-  class Lexer < LexerSupport::StateMachine
-    attr_reader :specification
-
-    # Creates a new lexer from a given specification. 
-    def initialize(specification)
-      dfas           = {}
-      @specification = specification
-      specification.items.each do |pattern, item|
-        dfas[pattern] = LexerSupport::DFA.new(pattern)
-      end
-      super(ItemSet.new(dfas.values.collect{|dfa| dfa.start_state}))
-    end
-  
-    # Compiles the lexer to Ruby code that when executed, reloads all the states and actions of the lexer 
-    # into a class named +lexer_class_name+.
-    def compile_to_ruby_source_as lexer_class_name
-      result  =   "class #{lexer_class_name} < Dhaka::CompiledLexer\n\n"
-      result <<   "  self.specification = #{specification.name}\n\n"
-      result <<   "  start_with #{start_state.object_id}\n\n"
-      @states.each do |key, state|
-        result << "#{state.compile_to_ruby_source}\n\n"
-      end
-      result <<   "end"
-      result
-    end
-  
-    # Returns a LexerRun that tokenizes +input+.
-    def lex input
-      LexerRun.new(self, input)
-    end
-    
-    def action_for_pattern pattern #:nodoc
-      @specification.items[pattern].action
-    end
-    
-    private
-      def new_state_for_key key
-        accepting_states = key.select {|state| state.accepting?}
-        unless accepting_states.empty?
-          highest_precedence_state = accepting_states.min {|a, b| @specification.items[a.action.pattern] <=> @specification.items[b.action.pattern]}
-          new_state = LexerSupport::State.new(self, highest_precedence_state.action)
-        else
-          new_state = LexerSupport::State.new(self)
-        end
-        key.select {|state| !state.checkpoint_actions.empty?}.each do |state|
-          new_state.checkpoint_actions.concat state.checkpoint_actions
-        end
-        new_state
-      end
-  
-      def transition_characters states
-        states.collect{|state| state.transitions.keys}.flatten.uniq
-      end
-
-      def dest_key_for states, char
-        result = ItemSet.new
-        states.each do |state|
-          dest_state = state.transitions[char]
-          result << dest_state if dest_state
-        end  
-        result
-      end
-  end
-end

data/test/vendor/dhaka-2.2.1/lib/dhaka/lexer/lexer_run.rb

@@ -1,78 +0,0 @@
-module Dhaka
-  # Represents a run of a lexer on a given input string.
-  class LexerRun
-    include Enumerable
-    
-    attr_reader :current_lexeme
-    def initialize lexer, input
-      @lexer, @input          = lexer, input
-      @input_position         = 0
-      @not_yet_accepted_chars = []
-      @last_saved_checkpoints = {}
-    end
-
-    # Constructs a token of type +symbol_name+ from the +current_lexeme+.
-    def create_token(symbol_name, value = current_lexeme.characters.join)
-      Token.new(symbol_name, value, current_lexeme.input_position)
-    end
-  
-    # Yields each token as it is recognized. Returns a TokenizerErrorResult if an error occurs during tokenization.
-    def each
-      reset_and_rewind
-      loop do
-        c = curr_char
-        break if (c == "\0" && @not_yet_accepted_chars.empty? && !@current_lexeme.accepted?)
-        dest_state  = @curr_state.transitions[c]
-        unless dest_state
-          return TokenizerErrorResult.new(@input_position) unless @current_lexeme.accepted?
-          token = get_token
-          yield token if token
-          reset_and_rewind
-        else
-          @curr_state = dest_state
-          @not_yet_accepted_chars << c
-          @curr_state.process(self) 
-          advance
-        end
-      end
-      yield Token.new(END_SYMBOL_NAME, nil, nil)
-    end
-    
-    def accept(pattern) #:nodoc:
-      @current_lexeme.pattern = pattern
-      @current_lexeme.concat @not_yet_accepted_chars
-      @not_yet_accepted_chars = []
-    end
-    
-    def save_checkpoint(pattern) #:nodoc:
-      @last_saved_checkpoints[pattern] = (@current_lexeme.characters + @not_yet_accepted_chars)
-    end
-    
-    def accept_last_saved_checkpoint(pattern) #:nodoc:
-      @current_lexeme.pattern = pattern
-      @current_lexeme.concat @not_yet_accepted_chars
-      @not_yet_accepted_chars = @current_lexeme.characters[(@last_saved_checkpoints[pattern].size)..-1]
-      @current_lexeme.characters = @last_saved_checkpoints[pattern].dup
-    end
-    
-    private
-      def reset_and_rewind
-        @input_position -= @not_yet_accepted_chars.size
-        @current_lexeme = Lexeme.new(@input_position)
-        @curr_state     = @lexer.start_state
-        @not_yet_accepted_chars = []
-      end
-
-      def curr_char
-        (@input[@input_position] || 0).chr
-      end
-    
-      def advance
-        @input_position += 1
-      end
-    
-      def get_token 
-        instance_eval(&@lexer.action_for_pattern(@current_lexeme.pattern))
-      end
-  end
-end   

data/test/vendor/dhaka-2.2.1/lib/dhaka/lexer/regex_grammar.rb

@@ -1,392 +0,0 @@
-module Dhaka
-  module LexerSupport #:nodoc:all
-
-    class RegexGrammar < Dhaka::Grammar
-
-      for_symbol(Dhaka::START_SYMBOL_NAME) do
-        regex                         %w| Disjunction |                         do Dhaka::LexerSupport::RootNode.new(child_nodes[0], Dhaka::LexerSupport::AcceptingNode.new) end
-        regex_with_lookahead          %w| Disjunction / Disjunction |           do Dhaka::LexerSupport::RootNode.new(Dhaka::LexerSupport::LookaheadNode.new(child_nodes[0], child_nodes[2]), Dhaka::LexerSupport::LookaheadAcceptingNode.new) end
-      end
-      
-      for_symbol('Disjunction') do                                   
-        disjunction                   %w| Alternative \| Disjunction |          do Dhaka::LexerSupport::OrNode.new(child_nodes[0], child_nodes[2]) end
-        alternative                   %w| Alternative |                         do child_nodes[0] end
-      end                                                   
-                                                          
-      for_symbol('Alternative') do                            
-        concatenation                 %w| Alternative Term |                    do Dhaka::LexerSupport::CatNode.new(child_nodes[0], child_nodes[1]) end
-        term                          %w| Term |                                do child_nodes[0] end
-      end                                                   
-                                                          
-      for_symbol('Term') do                             
-        zero_or_more                  %w| Atom * |                              do Dhaka::LexerSupport::ZeroOrMoreNode.new(child_nodes[0]) end
-        one_or_more                   %w| Atom + |                              do Dhaka::LexerSupport::OneOrMoreNode.new(child_nodes[0]) end
-        zero_or_one                   %w| Atom ? |                              do Dhaka::LexerSupport::ZeroOrOneNode.new(child_nodes[0]) end
-        atom                          %w| Atom |                                do child_nodes[0] end
-      end                                                   
-                                                          
-      for_symbol('Atom') do                        
-        group                         %w| ( Disjunction ) |                     do child_nodes[1] end
-        char                          %w| Character |                           do Dhaka::LexerSupport::LeafNode.new(child_nodes[0]) end 
-        anything                      %w| . |                                   do Dhaka::LexerSupport::OrNode.new(*(Dhaka::LexerSupport::ALL_CHARACTERS - ["\r", "\n"]).collect {|char| Dhaka::LexerSupport::LeafNode.new(char)}) end
-        positive_set                  %w| [ SetContents ] |                     do OrNode.new(*child_nodes[1].collect{|char| Dhaka::LexerSupport::LeafNode.new(char)}) end
-        negative_set                  %w| [ ^ SetContents ] |                   do Dhaka::LexerSupport::OrNode.new(*(Dhaka::LexerSupport::ALL_CHARACTERS - child_nodes[2]).collect {|char| Dhaka::LexerSupport::LeafNode.new(char)}) end
-
-        Dhaka::LexerSupport::CLASSES.each do |char, expansion|
-          send("character_class_#{char}", ['\\', char]) do
-            Dhaka::LexerSupport::OrNode.new(*Dhaka::LexerSupport::CLASSES[char].collect {|c| Dhaka::LexerSupport::LeafNode.new(c)})
-          end 
-        end
-
-        Dhaka::LexerSupport::OPERATOR_CHARACTERS.each do |char, method_name|
-          send(method_name, ['\\', char]) do
-            Dhaka::LexerSupport::LeafNode.new(char)
-          end
-        end
-      end
-      
-      for_symbol('Character') do
-        letter_character              %w| Letter |                              do child_nodes[0] end
-        digit_character               %w| Digit |                               do child_nodes[0] end  
-        white_space_character         %w| Whitespace |                          do child_nodes[0] end
-        symbol_character              %w| Symbol |                              do child_nodes[0] end
-      end
-      
-      
-      for_symbol('SetContents') do
-        single_item                   %w| SetItem |                             do child_nodes[0] end
-        multiple_items                %w| SetContents SetItem |                 do child_nodes[0].concat child_nodes[1] end
-      end
-    
-      for_symbol('SetItem') do
-        single_char_item              %w| SetCharacter |                        do [child_nodes[0]] end
-        lower_case_letter_range       %w| LowercaseLetter - LowercaseLetter |   do (child_nodes[0]..child_nodes[2]).to_a end
-        upper_case_letter_range       %w| UppercaseLetter - UppercaseLetter |   do (child_nodes[0]..child_nodes[2]).to_a end
-        digit_range                   %w| Digit - Digit |                       do (child_nodes[0]..child_nodes[2]).to_a end
-      end
-
-      
-      
-      for_symbol('Letter') do
-        lower_case_letter             %w| LowercaseLetter |                     do child_nodes[0] end
-        upper_case_letter             %w| UppercaseLetter |                     do child_nodes[0] end
-      end
-      
-      for_symbol('LowercaseLetter') do
-        Dhaka::LexerSupport::LOWERCASE_LETTERS.each do |letter|
-          send("lower_char_letter_#{letter}", letter) do
-            letter
-          end
-        end
-      end
-      
-      for_symbol('UppercaseLetter') do
-        Dhaka::LexerSupport::UPPERCASE_LETTERS.each do |letter|
-          send("upper_case_letter_#{letter}", letter) do
-            letter
-          end
-        end
-      end
-
-      for_symbol('Digit') do
-        Dhaka::LexerSupport::DIGITS.each do |digit|
-          send("digit_#{digit}", digit) do
-            digit
-          end
-        end
-      end
-
-      for_symbol('Whitespace') do
-        Dhaka::LexerSupport::WHITESPACE.each do |whitespace_char|
-          send("whitespace_#{to_byte(whitespace_char)}", whitespace_char) do
-            whitespace_char
-          end
-        end
-      end
-      
-      for_symbol('Symbol') do
-        Dhaka::LexerSupport::SYMBOLS.each do |symbol_char|
-          send("symbol_char_#{to_byte(symbol_char)}", symbol_char) do
-            symbol_char
-          end
-        end
-      end
-
-      for_symbol('SetCharacter') do
-        (Dhaka::LexerSupport::ALL_CHARACTERS - Dhaka::LexerSupport::SET_OPERATOR_CHARACTERS).each do |char|
-          send("set_character_#{to_byte(char)}", char) do
-            char
-          end
-        end
-        Dhaka::LexerSupport::SET_OPERATOR_CHARACTERS.each do |char|
-          send("set_operator_character_#{to_byte(char)}", ['\\', char]) do
-            char
-          end
-        end
-      end
-    end
-  
-  
-    class ASTNode
-      def checkpoint
-        false
-      end
-      
-      def accepting
-        false
-      end
-    end
-  
-    class BinaryNode < ASTNode
-      attr_reader :left, :right
-      def initialize left, right
-        @left, @right = left, right
-      end
-
-      def to_dot(graph)
-        graph.node(self, :label => label)
-        graph.edge(self, left)
-        graph.edge(self, right)
-        left.to_dot(graph)
-        right.to_dot(graph)
-      end
-    
-      def calculate_follow_sets
-        left.calculate_follow_sets
-        right.calculate_follow_sets
-      end
-    end
-
-    class OrNode < ASTNode
-      attr_reader :children
-      def initialize(*children)
-        @children = children
-      end
-      def label
-        "|"
-      end
-
-      def nullable
-        children.any? {|child| child.nullable}
-      end
-
-      def first
-        result = Set.new
-        children.each do |child|
-          result.merge child.first
-        end
-        result
-      end
-    
-      def last
-        result = Set.new
-        children.each do |child|
-          result.merge child.last
-        end
-        result
-      end
-    
-      def to_dot(graph)
-        graph.node(self, :label => label)
-        children.each do |child|
-          graph.edge(self, child)
-          child.to_dot(graph)
-        end
-      end
-    
-      def calculate_follow_sets
-        children.each do |child|
-          child.calculate_follow_sets
-        end
-      end
-    end
-  
-    class CatNode < BinaryNode
-      def label
-        "cat"
-      end
-
-      def nullable
-        left.nullable && right.nullable
-      end
-    
-      def first
-        left.nullable ? (left.first | right.first) : left.first
-      end
-    
-      def last
-        right.nullable ? (left.last | right.last) : right.last
-      end
-    
-      def calculate_follow_sets
-        super
-        left.last.each do |leaf_node|
-          leaf_node.follow_set.merge right.first
-        end
-      end
-    end
-
-    class LookaheadNode < CatNode
-      def label
-        "/"
-      end
-      
-      def calculate_follow_sets
-        super
-        left.last.each do |leaf_node|
-          leaf_node.follow_set.merge(Set.new([CheckpointNode.new]))
-        end
-      end
-    end
-  
-    class UnaryNode < ASTNode
-      attr_reader :child
-      def initialize child
-        @child = child
-      end
-
-      def to_dot(graph)
-        graph.node(self, :label => label)
-        graph.edge(self, child)
-        child.to_dot(graph)
-      end
-
-      def nullable
-        child.nullable
-      end
-
-      def first
-        child.first
-      end
-
-      def last
-        child.last
-      end
-    
-      def calculate_follow_sets
-        child.calculate_follow_sets
-      end
-    end
-  
-    class RootNode < CatNode
-      def label
-        "start"
-      end
-
-      def head_node?
-        true
-      end
-    end
-  
-    class ZeroOrMoreNode < UnaryNode
-      def label
-        "*"
-      end
-
-      def nullable
-        true
-      end
-    
-      def calculate_follow_sets
-        super
-        last.each do |leaf_node| 
-          leaf_node.follow_set.merge first
-        end
-      end
-    end
-  
-    class ZeroOrOneNode < UnaryNode
-      def label
-        "?"
-      end
-    
-      def nullable
-        true
-      end
-    end
-  
-    class OneOrMoreNode < UnaryNode
-      def label
-        "+"
-      end
-    
-      def calculate_follow_sets
-        super
-        last.each do |leaf_node| 
-          leaf_node.follow_set.merge first
-        end
-      end
-    end
-  
-    class LeafNode < ASTNode
-      attr_reader :character, :follow_set
-      def initialize character
-        @character = character
-        @follow_set = Set.new
-      end
-
-      def to_dot(graph)
-        graph.node(self, :label => character)
-      end
-
-      def nullable
-        false
-      end
-
-      def first
-        Set.new([self])
-      end
-
-      def last
-        Set.new([self])
-      end
-    
-      def calculate_follow_sets
-      end
-    end
-    
-    class CheckpointNode < ASTNode
-      def to_dot(graph)
-        graph.node(self, :label => "lookahead")
-      end
-      
-      def character
-      end
-      
-      def checkpoint
-        true
-      end
-    end
-  
-    class AcceptingNode < ASTNode
-      def accepting
-        true
-      end
-    
-      def character
-      end
-      
-      def action(pattern)
-        AcceptAction.new(pattern)
-      end
-
-      def first
-        Set.new([self])
-      end
-
-      def calculate_follow_sets
-      end
-    
-      def to_dot(graph)
-        graph.node(self, :label => '#')
-      end
-    end
-    
-    class LookaheadAcceptingNode < AcceptingNode
-      def action(pattern)
-        LookaheadAcceptAction.new(pattern)
-      end
-    end
-    
-  end
-end