rpdf2txt 0.8.2

data/lib/rpdf2txt-rockit/graphdrawing.rb

@@ -0,0 +1,107 @@
+
+def stack_to_node(stack, includeShape = false)
+  str = "#{stack.state} (#{stack.lexer.position.char_position})"
+  if includeShape
+    str = str.inspect + " [shape=box]"
+  end
+  str
+end
+
+def parsestack_as_dot_digraph(stack, nodes = Hash.new, links = Hash.new)
+  nodes[stack] = stack_to_node(stack, true)
+  stack.links.each do |link| 
+    nodes, links = parsestack_as_dot_digraph(link.stack, nodes, links)
+    links[link] = "#{stack_to_node(stack).inspect} -> #{stack_to_node(link.stack).inspect} [label=#{link.tree.inspect_compact.inspect}]"
+  end
+  return nodes, links
+end
+
+def parsestacks_as_dot_digraph(stacks)
+  nodes, links = {}, {}
+  stacks.each do |stack|
+    nodes, links = parsestack_as_dot_digraph(stack, nodes, links)
+  end
+  "digraph G {\nsize=\"8,11\"\n" +
+    nodes.values.uniq.join("\n") + "\n" +
+    links.values.uniq.join("\n") + "\n" + 
+    "}"
+end
+
+class DotGraphPrinter
+  def initialize(size = "11,9", orientation = "landscape")
+    @size, @orientation = size, orientation
+  end
+
+  def to_graph(ast)
+    @nodes, @links = Hash.new, Hash.new
+    eval_to_dot(ast, nil)
+    to_graph_from_nodes_and_links(@nodes, @links)
+  end
+
+  protected
+
+  def to_graph_from_nodes_and_links(nodes, links)
+    "digraph G {\n" +
+      "size = #{@size.inspect}\n" +
+      "orientation = #{@orientation}\n" +
+      @nodes.values.uniq.join("\n") + "\n" +
+      @links.values.uniq.join("\n") + "\n" + 
+      "}"
+  end
+end
+
+class SyntaxTreeAsDotGraph < DotGraphPrinter
+  def add_parent_link(parent, child, label = nil, weight = nil)
+    if parent
+      @links[[parent, child]] = 
+	"#{parent.id} -> #{child.id}"
+      if label or weight
+	@links[[parent, child]] += " [" +
+	  (label ? "label=#{label.inspect}" : "") +
+	  ((label and weight) ? "," : "") +
+	  (weight ? "weight=#{weight.inspect}" : "") +
+	  "]"
+      end
+    end
+  end
+
+  def eval_to_dot(ast, parent = nil, linkname = nil, weight = nil)
+    if ast
+      if ast.kind_of?(SyntaxTree)
+	case ast.name
+	when "_ArrayNode"
+	  add_parent_link(parent, ast, linkname, weight)
+	  @nodes[ast] = "#{ast.id} [label=" + '"[]"]'
+	  ast.each_with_index {|c,i| eval_to_dot(c, ast, i.inspect)}
+	else
+	  if parent
+	  end
+	  # Special handling of Token nodes since we only want to print 
+	  # the lexeme
+	  if ast.children_names.sort == ["lexeme", "value"].sort
+	    @nodes[ast] = "#{ast.id} [shape=box,label=#{ast.lexeme.inspect}]"
+	    add_parent_link(parent, ast, linkname, weight)
+	  else
+	    add_parent_link(parent, ast, linkname, weight)
+	    @nodes[ast] = "#{ast.id} [label=#{ast.name.inspect}]" 
+	    ast.childrens.each_with_index {|c,i| 
+	      eval_to_dot(c, ast, ast.children_names[i])
+	    }
+	  end
+	end
+      elsif ast.class == Array
+	# Or nodes return array but they should return ArrayNodes...
+	add_parent_link(parent, ast, linkname, weight)
+	@nodes[ast] = "#{ast.id} [label=\"[]\"]"
+	ast.each_with_index {|c,i| eval_to_dot(c, ast, i)}
+      end
+    end
+  end 
+end
+
+def syntaxtree_as_dot_digraph(syntaxtree)
+  SyntaxTreeAsDotGraph.new.to_graph(syntaxtree)
+end
+
+
+

data/lib/rpdf2txt-rockit/graphviz_dot.rb

@@ -0,0 +1,63 @@
+class DotGraphFormatter
+  @@default_node_shaper = proc{|n| "box"}
+  @@default_node_labeler = proc{|n| n.inspect}
+  @@default_link_labeler = proc{|info| info ? info.inspect : nil}
+
+  def initialize(nodeShaper = nil, nodeLabeler = nil, linkLabeler = nil,
+		 size = "11,9", orientation = "landscape")
+    @node_shaper = nodeShaper || @@default_node_shaper
+    @node_labeler = nodeLabeler || @@default_node_labeler
+    @link_labeler = linkLabeler || @@default_link_labeler
+    @size, @orientation = size, orientation
+  end
+
+  # nodes is array of node objects
+  # links is either array of 
+  #                   arrays [fromNode, toNode [, infoOnLink]], or
+  #                   objects with attributes :from, :to, :info
+  def format(nodes, links)
+    DotGraph.new("digraph G {\n" +
+		   "size = #{@size.inspect}\n" +
+		   "orientation = #{@orientation}\n" +
+		   nodes.uniq.map {|n| format_node(n)}.join("\n") + "\n" +
+		   links.uniq.map {|l| format_link(l)}.join("\n") + "\n" +
+		 "}"
+		 )
+  end
+
+  protected
+
+  def format_node(node)
+    node.id.inspect + " [" + 
+      "shape=" + @node_shaper.call(node).inspect + ", " +
+      "label=" + @node_labeler.call(node).inspect + "]" 
+  end
+
+  def get_link_data(link)
+    begin
+      return link.from, link.to, link.info
+    rescue Exception
+      return link[0], link[1], link[2]
+    end
+  end
+  
+  def format_link(link)
+    from, to, info = get_link_data(link)
+    label = @link_labeler.call(info)
+    from.id.inspect + " -> " + to.id.inspect +
+      (label ? " [label=" + label.inspect + "]" : "")
+  end
+end
+
+DotGraph = Struct.new("DotGraph", :description)
+class DotGraph
+  def write_to_file(filename)
+    tmpfile = filename + rand(100000).inspect
+    while test(?f, tmpfile)
+      tmpfile = filename + rand(100000).inspect
+    end
+    File.open(tmpfile, "w") {|f| f.write description}
+    system "dot -Tps -o #{filename} #{tmpfile}"
+    File.delete(tmpfile)
+  end
+end

data/lib/rpdf2txt-rockit/indexable.rb

@@ -0,0 +1,53 @@
+module Indexable
+  attr_accessor :index_number  # attr_reader_once_write instead?
+  attr_accessor :factory
+end
+
+class IndexableFactory
+  attr_reader :instances, :start_index, :next_index
+  
+  def initialize(klass, startIndex = 0)
+    unless klass.ancestors.include?(Indexable)
+      raise ArgumentError, "#{klass.inspect} is not Indexable" 
+    end
+    @klass, @start_index, @next_index = klass, startIndex, startIndex
+    @instance_map, @instances = Hash.new, Array.new
+  end
+
+  def make(*args)
+    obj = @instance_map[args]
+    unless obj
+      @instance_map[args] = obj = make_new_obj(args)
+      @instances.push obj
+    end
+    obj
+  end
+
+  def make_unless_exists(*args)
+    new_instance = @instance_map[args] == nil
+    return make(*args), new_instance
+  end
+
+  def get_instance(*args)
+    @instance_map[args]
+  end
+
+  def instance_with_args(*args)
+    @instance_map[args] || (@instance_map[args] = make(*args))
+  end
+
+  protected
+
+  def make_new_obj(arguments)
+    obj = @klass.new(*arguments)
+    obj.index_number = advance_index_number
+    obj.factory = self
+    obj
+  end
+  
+  def advance_index_number
+    i = @next_index
+    @next_index += 1
+    i
+  end
+end

data/lib/rpdf2txt-rockit/lalr_parsetable_generator.rb

@@ -0,0 +1,144 @@
+require 'rpdf2txt-rockit/grammar'
+require 'rpdf2txt-rockit/parse_table'
+require 'rpdf2txt-rockit/parsetable_generation'
+require 'rpdf2txt-rockit/reduce_actions_generator'
+
+require 'rpdf2txt-rockit/profiler'
+
+module Parse
+  class StateGraph < BackLinkedDirectedGraph
+    attr_reader :start_state
+    attr_reader :consistent_reduce_states, :inconsistent_reduce_states
+
+    def initialize(startState, *args)
+      super(*args)
+      @start_state = startState
+      @consistent_reduce_states = Array.new
+      @inconsistent_reduce_states = Array.new
+    end
+
+    def add_node(state)
+      super(state)
+      if state.reduce_state?
+	if state.consistent?
+	  a = @consistent_reduce_states
+	else
+	  a = @inconsistent_reduce_states
+	end
+	a.push state unless a.include?(state)
+      end
+    end
+
+    @@node_labeler_with_kernels = proc{|state|
+      "S" + state.index_number.inspect + ": " + state.kernel_items.inspect
+    }
+
+    @@node_labeler = proc{|state|
+      "S" + state.index_number.inspect
+    }
+
+    def to_postscript_file(filename, withKernelItems = true)
+      super(filename, nil, 
+	    withKernelItems ? @@node_labeler_with_kernels : @@node_labeler )
+    end
+  end
+
+  class LaLr1ParseTableGenerator
+    def initialize(grammar, 
+		   lookaheadCalculatorKlass = 
+		     ReduceActionsGenerator)
+      @grammar, @lookahead_calculator_klass = grammar, lookaheadCalculatorKlass
+    end
+
+    def generate_parse_table(parseTableKlass = ParseTable)
+      @grammar.augment
+      #puts "\n  NoN = #{@grammar.productions.map{|p| p.nonterminal}.uniq.length}"
+      #puts "  NoP = #{@grammar.productions.length}"
+      time_and_puts("\n  Normalizing grammar") {
+	@grammar.normalize
+      }
+      #puts "  NoPN = #{@grammar.productions.length}"
+      state_graph = nil
+      time_and_puts("  Calculating states") {
+	@item_factory = IndexableFactory.new(Item, 0)
+	@state_factory = IndexableFactory.new(LrState, 0)
+	precalc_nonkernel_items_for_nonterminals
+	@parse_table = parseTableKlass.new_from_grammar(@grammar)
+	state_graph = calculate_state_graph # also adds gotos and shift actions
+      }
+      time_and_puts("  Calculating lalr1_lookaheads") {      
+	lookahead_alg = 
+	  @lookahead_calculator_klass.new(state_graph, @grammar, @parse_table, 
+					  @item_factory.instances)
+	lookahead_alg.add_reduce_actions
+      }
+      @parse_table
+    end
+
+    def test_nonterminal_uniqueness
+      nts = DefaultInitHash.new {|k| Array.new}
+      @grammar.productions.each do |production|
+	nts[production.nonterminal.name].push(production.nonterminal.id)
+	nts[production.nonterminal.name].uniq!
+	production.elements.each do |e| 
+	  if e.nonterminal?
+	    nts[e.name].push(e.id)
+	    nts[e.name].uniq!
+	  end
+	end
+      end
+      puts nts.inspect
+    end
+
+    protected
+
+    def precalc_nonkernel_items_for_nonterminals
+      @grammar.nonterminals.each do |nt| 
+	nt.calc_nonkernel_items(@grammar, @item_factory)
+      end
+    end
+
+    def add_state_unless_exists(kernel_items)
+      @state_factory.make_unless_exists(kernel_items)
+    end
+
+    # Calculate the state graph by constructing the sets-of-lr0-items 
+    # collection.
+    # See page 224 (basic algorithm) and 240 (representing the states by
+    # their kernel items) in the Dragon book
+    def calculate_state_graph
+      Profiler.__enter__(:calculate_state_graph) if $PROFILE
+      start_item = @item_factory.make(@grammar.productions[0], 0)
+      state_graph = StateGraph.new(add_state_unless_exists([start_item]).first)
+      states, current = [state_graph.start_state], 0
+      dest_sets, next_item = DefaultInitHash.new {|k| Array.new}, nil
+      while current < states.length
+	state = states[current]
+	dest_sets.clear
+	state.closure.each do |item|
+	  symbol = item.symbol
+	  if symbol
+	    next_item = item.next_item
+	    dest_sets[symbol].push(next_item) if next_item
+	  end
+	end
+	dest_sets.each do |symbol, kernel_item_set|
+	  kernel_item_set.uniq! # Needed?
+	  dest_state, new_state = add_state_unless_exists(kernel_item_set)
+	  states.push(dest_state) if new_state
+	  state_graph.add_link(state, dest_state, symbol)
+	  if symbol.nonterminal?
+	    @parse_table.add_goto(state.index_number, symbol, 
+				  dest_state.index_number)
+	  else
+	    @parse_table.add_action(state.index_number, symbol, 
+				    [:SHIFT, dest_state.index_number])
+	  end
+	end
+	current += 1
+      end
+      Profiler.__leave__(:calculate_state_graph, state_graph) if $PROFILE
+      state_graph
+    end
+  end
+end

data/lib/rpdf2txt-rockit/parse_table.rb

@@ -0,0 +1,273 @@
+require 'rpdf2txt-rockit/sourcecode_dumpable'
+require 'rpdf2txt-rockit/grammar'
+require 'rpdf2txt-rockit/base_extensions'
+
+require 'rpdf2txt-rockit/profiler'
+
+class ParseTable
+  include SourceCodeDumpable
+  attr_reader :start_state, :tokens, :priorities
+  attr_accessor :language
+
+  def ParseTable.new_from_grammar(aGrammar)
+    pt = self.new(aGrammar.productions, aGrammar.tokens, aGrammar.priorities)
+    pt.language = aGrammar.name || "UNNAMED_LANGUAGE"
+    pt
+  end
+
+  # We save the actions in a compact numerical way to save space and time:
+  #   The action table is an array of arrays. Each state number is an index
+  #   into the array and its array contains an even number of integers.
+  #   Each pair of integers represent one unique action. The first of the 
+  #   integers is the action and the second is the number representing the
+  #   terminals for which it apply. The least significant 'action_bits' bits
+  #   of the action number determines the type of action by giving an index
+  #   into the 'action_map'. Its default value is:
+  #     [:REDUCE, :SHIFT, :ACCEPT]
+  #   so that 
+  @@default_action_map = [:REDUCE, :SHIFT, :ACCEPT]
+
+  def initialize(productions, tokens, priorities = nil,
+		 actionTable = ArrayOfArrays.new, gotoHash = Hash.new,
+		 actionBits = 2, actionMap = @@default_action_map)
+    @productions, @start_state, @language = productions, 0, "UNNAMED_LANGUAGE"
+    @priorities = priorities
+    @tokens, @nonterminals = tokens, nonterminals(productions)
+    @action_table, @goto_hash = actionTable, gotoHash
+    @action_cache = ArrayOfHashes.new
+    @mask = Array.new
+    @action_map, @action_bits, @action_mask = actionMap, actionBits, 0
+    while actionBits > 0
+      @action_mask = (@action_mask << 1) | 1
+      actionBits -= 1
+    end
+    init_productionnum_to_nonterminal_number_hash
+    init_tokentype_to_token_number_hash
+  end
+
+  def num_states
+    @action_table.length
+  end
+
+  def ==(other)
+    other.class == self.class and
+      other.productions == @productions and
+      other.tokens == @tokens and
+      other.action_table == @action_table and
+      other.goto_hash == @goto_hash and
+      other.start_state == @start_state
+  end
+
+  def add_action(state, aTokenType, action)
+    Profiler.__enter__(:ParseTable_add_action, state, aTokenType, action) if $PROFILE
+    @action_cache[state].clear
+    actionnum = action_to_actionnum(action)
+    @action_table[state] << actionnum << token_to_terminalset(aTokenType)
+    Profiler.__leave__(:ParseTable_add_action) if $PROFILE
+  end
+
+  def add_action_for_terminalset(state, action, terminalSet)
+    Profiler.__enter__(:ParseTable_add_action_for_terminalset, state, action, terminalSet) if $PROFILE
+    @action_table[state] << action_to_actionnum(action) << terminalSet.to_i
+    Profiler.__leave__(:ParseTable_add_action_for_terminalset) if $PROFILE
+  end
+
+  # Unify terminal sets for identical actions
+  def compact!
+    Profiler.__enter__(:ParseTable_compact!) if $PROFILE
+    actions, i, new_index = Hash.new, 0, 0
+    @action_table.map! do |actionnums|
+      actions.clear; 
+      i, new_actionnums, new_index = 0, Array.new, 0
+      while i < actionnums.length
+	if (index = actions[actionnums[i]])
+	  new_actionnums[index+1] |= actionnums[i+1]
+	else
+	  actions[actionnums[i]] = new_index
+	  new_index += 2
+	  new_actionnums << actionnums[i] << actionnums[i+1]
+	end
+	i += 2
+      end
+      new_actionnums
+    end
+    Profiler.__leave__(:ParseTable_compact!) if $PROFILE
+  end
+
+  def token_to_terminalset(aTokenType)
+    mask(@token_to_number[aTokenType])
+  end
+
+  def mask(index)
+    @mask[index] || (@mask[index] = (1 << index))
+  end
+
+  def add_goto(state, aNonTerminal, newState)
+    begin
+      @goto_hash[state][@nonterminals.index(aNonTerminal)] = newState
+    rescue NameError
+      @goto_hash[state] = Hash.new
+      retry
+    end
+  end
+
+  def actions(state, tokenType)
+    actions = @action_cache[state][tokenType]
+    unless actions
+      actions = Array.new
+      actionnums = @action_table[state]
+      token_mask = mask(@token_to_number[tokenType])
+      i = 0
+      while i < actionnums.length
+	if(actionnums[i+1] & token_mask > 0)
+	  actions.push actionnum_to_action(actionnums[i])
+	end
+	i += 2
+      end
+      @action_cache[state][tokenType] = actions
+    end
+    actions
+  end
+
+  def valid_tokens(state)
+    terminal_set = 0
+    each_terminalset(state) {|ts| terminal_set |= ts}
+    terminalset_to_terminals(terminal_set)
+  end
+
+  def each_terminalset(state)
+    @action_table[state].each_with_index {|e,i| yield(e) if i % 2 == 1}
+  end
+
+  def terminalset_to_terminals(terminalSet)
+    @tokens.select {|t| terminalSet & mask(@token_to_number[t]) > 0}
+  end
+
+  def actionnum_to_action(actionNumber)
+    [@action_map[actionNumber & @action_mask], actionNumber >> @action_bits]
+  end
+
+  def action_to_actionnum(action)
+    Profiler.__enter__(:ParseTable_action_to_actionnum, action) if $PROFILE
+    res = @action_map.index(action[0]) + (action[1] << @action_bits)
+    Profiler.__leave__(:ParseTable_action_to_actionnum) if $PROFILE
+    res
+  end
+
+  def goto(state, productionNumber)
+    begin
+      @goto_hash[state][@productionnum_to_nonterminal_num[productionNumber]]
+    rescue Exception
+      nil
+    end
+  end
+
+  def production(number)
+    @productions[number]
+  end
+
+  def to_src(name = "parse_table", nameHash = {})
+    names = name_hash(@tokens) {|t| "t"}
+    str = @tokens.to_src("tokens", names) + "\n"
+    names.update(name_hash(@productions) {|p| "p"})
+    str << @productions.to_src("productions", names) + "\n"
+    str << @priorities.to_src("priorities", names) + "\n"
+    #str << "r = :REDUCE\n"
+    #str << "s = :SHIFT\n"
+    str << @action_table.to_compact_src("action_table") + "\n"
+    str << @goto_hash.to_compact_src("goto_hash") + "\n"
+    str << assign_to(name, 
+		     new_of_my_type(as_code("productions"), 
+				    as_code("tokens"),
+				    as_code("priorities"),
+				    as_code("action_table"), 
+				    as_code("goto_hash"),
+				    @action_bits,
+				    @action_map))
+    str
+  end
+
+  def inspect
+    str = "ParseTable\n"
+    str += "Tokens: #{@tokens.inspect}\n"
+    str += "NonTerminals: #{@nonterminals.inspect}\n"
+    str += "Productions:\n#{productions_inspect}\n"
+    str += "Actions: \n"
+    max_state = @action_table.length-1
+    (max_state+1).times do |state|
+      str += "#{state}:\t"
+      @tokens.each do |t|
+	str += inspect_actions(actions(state, t)) + ","
+      end
+      str += "| "
+      @nonterminals.each do |nt|
+	i = @productions.index(@productions.detect {|p| p.nonterminal == nt})
+	str += ((ns=goto(state, i)) ? "#{ns}" : " ") + ","
+      end
+      str += "\n"
+    end
+    str
+  end
+
+  protected
+
+  def productions_inspect
+    str = ""
+    @productions.each_with_index do |production, i|
+      str += "  #{i}: #{production.inspect}\n"
+    end
+    str
+  end
+
+  def inspect_actions(actions)
+    unless actions
+      "   "
+    else
+      return "   " if actions.length == 0
+      if actions.length > 1
+	"[" + actions.map {|a| inspect_actions([a])}.join(',') + "]"
+      else
+	case actions[0][0]
+	when :ACCEPT
+	  " a "
+	when :SHIFT
+	  "s#{actions[0][1]} "
+	when :REDUCE
+	  "r#{actions[0][1]} "
+	end
+      end
+    end
+  end
+
+  attr_reader :productions, :action_table, :goto_hash
+
+  def nonterminals(anArrayOfProductions)
+    anArrayOfProductions.map {|p| p.nonterminal}.equality_uniq
+  end
+
+  def init_nonterminal_index(nonterminals, productions)
+    @nonterminal_index = Hash.new
+    productions.each_with_index do |prod, i|
+      @nonterminal_index[i] = nonterminals.index(prod.nonterminal)
+    end
+    @nonterminal_index
+  end
+
+  def init_token_index(tokens)
+    @token_index = Hash.new
+    tokens.each_with_index {|t,i| @token_index[t] = i}
+    @token_index
+  end
+
+  def init_productionnum_to_nonterminal_number_hash
+    @productionnum_to_nonterminal_num = Hash.new
+    @productions.each_with_index do |p, n| 
+      @productionnum_to_nonterminal_num[n] = @nonterminals.index(p.nonterminal)
+    end
+  end
+
+  def init_tokentype_to_token_number_hash
+    @token_to_number = Hash.new
+    @tokens.each_with_index {|t,i| @token_to_number[t] = i}
+  end
+end