simply_stored 0.1.4

data/test/vendor/dhaka-2.2.1/lib/dhaka/parser/parser.rb

@@ -0,0 +1,165 @@
+module Dhaka
+  # The parser generator. To generate a parser from a grammar specification +ArithmeticPrecedenceGrammar+, one would
+  # write:
+  #   parser = Dhaka::Parser.new(ArithmeticPrecedenceGrammar)
+  # 
+  # To compile this parser to Ruby source as +ArithmeticPrecedenceParser+:
+  #   parser.compile_to_ruby_source_as(:ArithmeticPrecedenceParser)
+  # which returns a string of Ruby code.
+  class Parser
+    include ParserMethods
+    attr_reader :grammar
+  
+    # Creates a new parser from the given grammar. Messages are logged by default to STDOUT
+    # and the log level is WARN. Shift-reduce conflicts are reported at WARN and reduce-reduce conflicts
+    # at ERROR. You may pass in your own logger. Logging at DEBUG shows a lot of progress output.
+    def initialize(grammar, logger = nil)
+      @shift_actions  = Hash.new {|hash, state| hash[state] = ShiftAction.new(state)}
+      @reduce_actions = Hash.new {|hash, production| hash[production] = ReduceAction.new(production)}
+      @logger         = logger || default_logger
+      @transitions    = Hash.new {|hash, state| hash[state] = {}}
+      @grammar        = grammar
+      @channels       = Hash.new {|hash, start_item| hash[start_item] = []}
+      @states = Hash.new do |hash, kernel|
+          closure, channels = grammar.closure(kernel)
+          channels.each do |start_item, channel_set|
+            @channels[start_item].concat channel_set.to_a
+          end
+          new_state    = ParserState.new(self, closure)
+          hash[kernel] = new_state
+          @logger.debug("Created #{new_state.unique_name}.")
+          new_state.transition_items.each do |symbol, items|
+            destination_kernel = ItemSet.new(items.collect{|item| item.next_item})
+            destination_state  = hash[destination_kernel]
+            items.each {|item| @channels[item] << grammar.passive_channel(item, destination_state.items[item.next_item])}
+            @transitions[new_state][symbol] = destination_state
+          end
+          new_state
+      end
+      initialize_states
+    end
+
+    # Returns the Ruby source of the generated parser compiled as +parser_class_name+. This can be written out to a file.
+    def compile_to_ruby_source_as parser_class_name
+      result = "class #{parser_class_name} < Dhaka::CompiledParser\n\n"
+      result << "  self.grammar = #{grammar.name}\n\n"
+      result << "  start_with #{start_state.id}\n\n"
+      states.each do |state|
+        result << "#{state.compile_to_ruby_source}\n\n"
+      end
+      result << "end"
+      result
+    end
+
+    # Returns the dot representation of the parser. If <tt>:hide_lookaheads</tt> is set to true in the 
+    # options hash, lookaheads are not written out to the parser states, which is helpful when there are dozens
+    # of lookahead symbols for every item in every state.
+    def to_dot(options = {})
+      Dot::Digraph.new(:fontsize => 10, :shape => :box, :size => 5) do |g|
+        states.each do |state|
+          g.node(state, :label => state.items.values.collect{|item| item.to_s(options)}.join("\n"))
+          @transitions[state].each do |symbol, dest_state|
+            g.edge(state, dest_state, :label => symbol.name)
+          end
+        end
+      end.to_dot
+    end
+
+    def inspect
+      "<Dhaka::Parser grammar : #{grammar}>"
+    end
+  
+    private
+      attr_reader :start_state
+      
+    def states
+      @states.values
+    end
+    
+    def default_logger
+      logger           = Logger.new(STDOUT)
+      logger.level     = Logger::WARN
+      logger.formatter = ParserLogOutputFormatter.new
+      logger
+    end
+
+    def initialize_states
+      start_productions = grammar.productions_for_symbol(grammar.start_symbol)
+      raise NoStartProductionsError.new(grammar) if start_productions.empty?
+      start_items = ItemSet.new(start_productions.collect {|production| Item.new(production, 0)})
+      start_items.each {|start_item| start_item.lookaheadset << grammar.end_symbol}
+      @start_state = @states[start_items]
+      @logger.debug("Pumping #{@channels.keys.size} dirty items...")
+      pump_channels @channels.keys
+      @logger.debug("Generating shift actions...")
+      generate_shift_actions
+      @logger.debug("Generating reduce actions...")
+      generate_reduce_actions
+    end
+
+    def generate_shift_actions
+      @states.values.each do |state|
+        @transitions[state].keys.each do |symbol|
+          state.actions[symbol.name] = @shift_actions[@transitions[state][symbol]]
+        end
+      end
+    end
+
+    def generate_reduce_actions
+      @states.values.each do |state|  
+        state.items.values.select{ |item| !item.next_symbol }.each do |item| 
+          create_reduction_actions_for_item_and_state item, state
+        end
+      end
+    end
+
+    def create_reduction_actions_for_item_and_state item, state
+      item.lookaheadset.each do |lookahead| 
+        new_action = @reduce_actions[item.production]
+        if existing_action = state.actions[lookahead.name]
+          if ReduceAction === existing_action
+            message = ReduceReduceConflict.new(state, lookahead, new_action).resolve
+            @logger.error(message)
+          else
+            message = ShiftReduceConflict.new(state, lookahead, new_action).resolve
+            @logger.warn(message)
+          end
+        else
+          state.actions[lookahead.name] = new_action
+        end
+      end
+    end
+  
+    def pump_channels dirty_items
+      loop do
+        new_dirty_items = Set.new
+        dirty_items.each do |dirty_item| 
+          @channels[dirty_item].each do |channel| 
+            new_dirty_items << channel.end_item if channel.pump
+          end
+        end
+        break if new_dirty_items.empty?
+        @logger.debug("#{new_dirty_items.size} dirty items...")
+        dirty_items = new_dirty_items
+      end
+    end
+  end
+
+  # Raised when trying to create a Parser for a grammar that has no productions for the start symbol
+  class NoStartProductionsError < StandardError
+    def initialize(grammar) #:nodoc:
+      @grammar = grammar
+    end
+    def to_s #:nodoc:
+      "No start productions defined for #{@grammar.name}"
+    end
+  end
+
+  class ParserLogOutputFormatter < Logger::Formatter #:nodoc:
+    def call(severity, time, progname, msg)
+      "\n%s -- %s: %s\n" % [ severity, progname, msg2str(msg)]
+    end
+  end
+
+end
+

data/test/vendor/dhaka-2.2.1/lib/dhaka/parser/parser_methods.rb

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+module Dhaka
+  # This module is included both in Parser and CompiledParser. 
+  module ParserMethods
+    # +token_stream+ is an Enumerable of Token-s. Returns either a ParseSuccessResult or a ParseErrorResult. 
+    def parse token_stream
+      parser_run = ParserRun.new(grammar, start_state, token_stream)
+      parser_run.run
+    end
+  end
+end

data/test/vendor/dhaka-2.2.1/lib/dhaka/parser/parser_run.rb

@@ -0,0 +1,39 @@
+module Dhaka
+  class ParserRun #:nodoc:
+
+    def initialize(grammar, start_state, token_stream)
+      @grammar      = grammar
+      @node_stack   = []
+      @state_stack  = [start_state]
+      @token_stream = token_stream
+      @symbol_queue = []
+    end
+
+    def run
+      tokenize_result = token_stream.each do |token|
+        @current_token = token
+        @symbol_queue << @current_token.symbol_name
+        error = execute_actions
+        return error if error
+        node_stack << ParseTreeLeafNode.new(@current_token)
+        state_stack.last
+      end
+      return tokenize_result if TokenizerErrorResult === tokenize_result
+      ParseSuccessResult.new(node_stack.first) if node_stack.first.head_node?
+    end
+
+    private
+
+    attr_reader :state_stack, :token_stream, :node_stack
+
+    def execute_actions
+      while symbol_name = @symbol_queue.pop
+        action = state_stack.last.actions[symbol_name]
+        return ParseErrorResult.new(@current_token, state_stack.last) unless action
+        instance_eval(&action.action_code)
+      end
+      nil
+    end
+
+  end
+end

data/test/vendor/dhaka-2.2.1/lib/dhaka/parser/parser_state.rb

@@ -0,0 +1,74 @@
+module Dhaka
+  class ParserState #:nodoc:
+    attr_accessor :items, :actions, :id
+  
+    @@state_id = 0
+  
+    def self.next_state_id
+      result      = @@state_id
+      @@state_id += 1
+      result
+    end
+  
+    def initialize(parser, items, id=nil)
+      @parser  = parser
+      @items   = items
+      @actions = {}
+      @id      = id || ParserState.next_state_id
+    end
+
+    def transition_items
+      result = Hash.new {|h, k| h[k] = ItemSet.new()}
+      items.values.each do |item|
+        result[item.next_symbol] << item if item.next_symbol
+      end
+      result
+    end
+
+    def unique_name
+      "State#{id}"
+    end
+
+    def compile_to_ruby_source
+      result = "  at_state(#{id}) {\n"
+
+      symbol_names_by_action = Hash.new {|hash, key| hash[key] = []}
+      actions.each do |symbol_name, action|
+        symbol_names_by_action[action] << symbol_name
+      end
+      
+      symbol_names_by_action.keys.each do |action|
+        symbol_names = symbol_names_by_action[action].collect {|symbol_name| "#{symbol_name.inspect}"}.join(', ')
+        result << "    for_symbols(#{symbol_names}) { #{action.compile_to_ruby_source} }\n"
+      end
+      
+      result << "  }"
+      result
+    end
+    
+    def for_symbols *symbol_names, &blk
+      symbol_names.each do |symbol_name|
+        actions[symbol_name] = @parser.instance_eval(&blk)
+      end
+    end
+  
+    alias :for_symbol :for_symbols
+  
+    def to_s(options = {})
+      items.values.collect{|item| item.to_s(options)}.join("\n")
+    end
+  
+  end
+
+  class ItemSet < Set #:nodoc:
+    def hash
+      result = 5381
+      each { |item| result ^= item.hash }
+      result
+    end
+    
+    def eql? other
+      self == other
+    end
+  end
+end

data/test/vendor/dhaka-2.2.1/lib/dhaka/parser/token.rb

@@ -0,0 +1,22 @@
+module Dhaka
+  # Represents a portion of the input character stream that is mapped by the tokenizer
+  # to a symbol in the grammar. The attribute +input_position+ contains the start index position of the original
+  # string input that this token came from. It can be used to report errors by indicating the specific portion
+  # of the input where the error occurred.
+  class Token
+    attr_accessor :symbol_name, :value, :input_position
+    def initialize(symbol_name, value, input_position)
+      @symbol_name    = symbol_name
+      @value          = value
+      @input_position = input_position
+    end
+    
+    def to_s #:nodoc:
+      value ? "#{symbol_name} : #{value}" : "#{symbol_name}"
+    end
+    
+    def == other
+      symbol_name == other.symbol_name && value == other.value
+    end
+  end
+end

data/test/vendor/dhaka-2.2.1/lib/dhaka/runtime.rb

@@ -0,0 +1,51 @@
+#--
+# Copyright (c) 2006, 2007 Mushfeq Khan
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require 'set'
+require 'logger'
+require 'delegate'
+
+%w[
+grammar/grammar_symbol
+grammar/production
+grammar/grammar
+grammar/precedence
+parser/parse_tree
+parser/parse_result
+parser/parser_methods
+parser/parser_state
+parser/token
+parser/action
+parser/parser_run
+parser/compiled_parser
+tokenizer/tokenizer
+evaluator/evaluator
+lexer/accept_actions
+lexer/alphabet
+lexer/state_machine
+lexer/state
+lexer/specification
+lexer/lexeme
+lexer/lexer_run
+lexer/compiled_lexer
+].each {|path| require File.join(File.dirname(__FILE__), path)}

data/test/vendor/dhaka-2.2.1/lib/dhaka/tokenizer/tokenizer.rb

@@ -0,0 +1,190 @@
+module Dhaka
+  # Reserved constant used to identify the idle state of the tokenizer.
+  TOKENIZER_IDLE_STATE = :idle_state 
+
+  # Returned on successful tokenizing of the input stream. Supports iteration by including Enumerable, so it can
+  # be passed in directly to the parser.
+  class TokenizerSuccessResult
+    include Enumerable
+    
+    def initialize(tokens)
+      @tokens = tokens
+    end
+    
+    # Returns false.
+    def has_error?
+      false
+    end
+    
+    def each(&block)
+      @tokens.each(&block)
+    end
+  end  
+
+  # Returned when tokenizing fails due to an unexpected character in the input stream. 
+  class TokenizerErrorResult
+    # The index of the character that caused the error.
+    attr_reader :unexpected_char_index
+    
+    def initialize(unexpected_char_index)
+      @unexpected_char_index = unexpected_char_index
+    end
+    
+    # Returns true.
+    def has_error?
+      true
+    end
+  end
+
+  # A tokenizer state encapsulates actions that should be performed upon
+  # encountering each permissible character for that state.
+  class TokenizerState
+    attr_reader :actions, :default_action
+
+    def initialize
+      @actions = {}
+    end
+
+    # Define the action (+blk+) to be performed when encountering any of +characters+ in the token stream.
+    def for_characters(characters, &blk)
+      characters.each do |character|
+        actions[character] = blk
+      end
+    end
+
+    alias for_character for_characters
+
+    # define the action (+blk+) to be performed for any +characters+ that don't have an action to perform.
+    def for_default(&blk)
+      @default_action = blk
+    end
+  
+    def to_s #:nodoc:
+      actions.inspect
+    end
+
+  end
+
+  # This abstract class contains a DSL for hand-coding tokenizers. Subclass it to implement tokenizers for specific grammars. 
+  #
+  # Tokenizers are state machines. Each state of a tokenizer is identified
+  # by a Ruby symbol. The constant Dhaka::TOKENIZER_IDLE_STATE is reserved for the idle state of the tokenizer (the one
+  # that it starts in).
+  # 
+  # The following is a tokenizer for arithmetic expressions with integer terms. The tokenizer starts in the idle state
+  # creating single-character tokens for all characters excepts digits and whitespace. It shifts to
+  # <tt>:get_integer_literal</tt> when it encounters a digit character and creates a token on the stack on which it
+  # accumulates the value of the literal. When it again encounters a non-digit character, it shifts back to idle.
+  # Whitespace is treated as a delimiter, but not shifted as a token.
+  #
+  #  class ArithmeticPrecedenceTokenizer < Dhaka::Tokenizer
+  #
+  #    digits = ('0'..'9').to_a
+  #    parenths = ['(', ')']
+  #    operators = ['-', '+', '/', '*', '^']
+  #    functions = ['h', 'l']
+  #    arg_separator = [',']
+  #    whitespace = [' ']
+  #
+  #    all_characters = digits + parenths + operators + functions + arg_separator + whitespace
+  #
+  #    for_state Dhaka::TOKENIZER_IDLE_STATE do
+  #      for_characters(all_characters - (digits + whitespace)) do
+  #        create_token(curr_char, nil)
+  #        advance
+  #      end
+  #      for_characters digits do
+  #        create_token('n', '')
+  #        switch_to :get_integer_literal
+  #      end
+  #      for_character whitespace do
+  #        advance
+  #      end
+  #    end
+  #
+  #    for_state :get_integer_literal do
+  #      for_characters all_characters - digits do
+  #        switch_to Dhaka::TOKENIZER_IDLE_STATE
+  #      end
+  #      for_characters digits do
+  #        curr_token.value << curr_char
+  #        advance
+  #      end
+  #    end
+  #
+  #  end
+  #
+  # For languages where the lexical structure is very complicated, it may be too tedious to implement a Tokenizer by hand. 
+  # In such cases, it's a lot easier to write a LexerSpecification using regular expressions and create a Lexer from that.
+  class Tokenizer
+    class << self
+      # Define the action for the state named +state_name+.
+      def for_state(state_name, &blk)
+        states[state_name].instance_eval(&blk)
+      end
+    
+      # Tokenizes a string +input+ and returns a TokenizerErrorResult on failure or a TokenizerSuccessResult on sucess.
+      def tokenize(input)
+        new(input).run
+      end
+      
+      private
+        def inherited(tokenizer)
+          class << tokenizer
+            attr_accessor :states, :grammar
+          end
+          tokenizer.states = Hash.new {|hash, key| hash[key] = TokenizerState.new}
+        end
+    end
+    
+    # The tokens shifted so far.
+    attr_reader :tokens
+    
+    def initialize(input) #:nodoc:
+      @input           = input
+      @current_state   = self.class.states[TOKENIZER_IDLE_STATE]
+      @curr_char_index = 0
+      @tokens          = []
+    end
+  
+    # The character currently being processed.
+    def curr_char
+      @input[@curr_char_index] and @input[@curr_char_index].chr 
+    end
+    
+    # Advance to the next character.
+    def advance
+      @curr_char_index += 1
+    end
+
+    def inspect
+      "<Dhaka::Tokenizer grammar : #{grammar}>"
+    end
+    
+    # The token currently on top of the stack. 
+    def curr_token
+      tokens.last
+    end
+    
+    # Push a new token on to the stack with symbol corresponding to +symbol_name+ and a value of +value+.
+    def create_token(symbol_name, value)
+      new_token = Dhaka::Token.new(symbol_name, value, @curr_char_index)
+      tokens << new_token
+    end
+    
+    # Change the active state of the tokenizer to the state identified by the symbol +state_name+.
+    def switch_to state_name
+      @current_state = self.class.states[state_name]
+    end
+
+    def run #:nodoc:
+      while curr_char
+        blk = @current_state.actions[curr_char] || @current_state.default_action
+        return TokenizerErrorResult.new(@curr_char_index) unless blk
+        instance_eval(&blk)
+      end
+      tokens << Dhaka::Token.new(Dhaka::END_SYMBOL_NAME, nil, nil)
+      TokenizerSuccessResult.new(tokens)
+    end
+  end
+end

data/test/vendor/dhaka-2.2.1/lib/dhaka.rb

@@ -0,0 +1,62 @@
+#--
+# Copyright (c) 2006, 2007 Mushfeq Khan
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require 'set'
+require 'logger'
+require 'delegate'
+
+%w[
+dot/dot
+grammar/grammar_symbol
+grammar/production
+grammar/closure_hash
+grammar/grammar
+grammar/precedence
+parser/parse_tree
+parser/parse_result
+parser/item
+parser/channel
+parser/parser_methods
+parser/parser_state
+parser/conflict
+parser/token
+parser/action
+parser/parser_run
+parser/parser
+parser/compiled_parser
+tokenizer/tokenizer
+evaluator/evaluator
+lexer/accept_actions
+lexer/alphabet
+lexer/regex_grammar
+lexer/regex_tokenizer
+lexer/regex_parser
+lexer/state_machine
+lexer/dfa
+lexer/state
+lexer/specification
+lexer/lexeme
+lexer/lexer_run
+lexer/lexer
+lexer/compiled_lexer
+].each {|path| require File.join(File.dirname(__FILE__), 'dhaka/' + path)}

data/test/vendor/dhaka-2.2.1/test/all_tests.rb

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+Dir['**/*test.rb'].each do |test_file|
+  puts test_file
+  require File.join(File.dirname(__FILE__), test_file)
+end

data/test/vendor/dhaka-2.2.1/test/arithmetic/arithmetic_evaluator.rb

@@ -0,0 +1,64 @@
+require File.dirname(__FILE__) + '/arithmetic_grammar'
+
+class ArithmeticEvaluator < Dhaka::Evaluator
+  
+  self.grammar = ArithmeticGrammar
+  
+  define_evaluation_rules do
+    
+    for_subtraction do 
+      evaluate(child_nodes[0]) - evaluate(child_nodes[2])
+    end
+
+    for_addition do
+      evaluate(child_nodes[0]) + evaluate(child_nodes[2])
+    end
+
+    for_division do
+      evaluate(child_nodes[0]).to_f/evaluate(child_nodes[2]) 
+    end
+
+    for_multiplication do
+      evaluate(child_nodes[0]) * evaluate(child_nodes[2])
+    end
+
+    for_getting_literals do
+      child_nodes[0].token.value
+    end
+
+    for_unpacking_parenthetized_expression do
+      evaluate(child_nodes[1])
+    end
+    
+    for_empty_args do
+      []
+    end
+
+    for_evaluating_function do
+      evaluate(child_nodes[0]).call evaluate(child_nodes[2])
+    end
+  
+    for_concatenating_args do
+      [evaluate(child_nodes[0])]+evaluate(child_nodes[2])
+    end
+  
+    for_single_args do
+      [evaluate(child_nodes[0])]
+    end
+  
+    for_min_function do
+      @min_function
+    end
+  
+    for_max_function do
+      @max_function
+    end
+
+  end
+  
+  def initialize(min_function, max_function)
+    @min_function = min_function
+    @max_function = max_function
+  end
+
+end