andyjeffries-journey 1.0.0

data/lib/journey/gtg/transition_table.rb

@@ -0,0 +1,152 @@
+require 'journey/nfa/dot'
+
+module Journey
+  module GTG
+    class TransitionTable
+      include Journey::NFA::Dot
+
+      attr_reader :memos
+
+      def initialize
+        @regexp_states = Hash.new { |h,k| h[k] = {} }
+        @string_states = Hash.new { |h,k| h[k] = {} }
+        @accepting  = {}
+        @memos      = Hash.new { |h,k| h[k] = [] }
+      end
+
+      def add_accepting state
+        @accepting[state] = true
+      end
+
+      def accepting_states
+        @accepting.keys
+      end
+
+      def accepting? state
+        @accepting[state]
+      end
+
+      def add_memo idx, memo
+        @memos[idx] << memo
+      end
+
+      def memo idx
+        @memos[idx]
+      end
+
+      def eclosure t
+        Array(t)
+      end
+
+      def move t, a
+        t = Array(t)
+        move_string(t, a) + move_regexp(t, a)
+      end
+
+      def to_json
+        require 'json'
+
+        simple_regexp = Hash.new { |h,k| h[k] = {} }
+
+        @regexp_states.each do |from, hash|
+          hash.each do |re, to|
+            simple_regexp[from][re.source] = to
+          end
+        end
+
+        JSON.dump({
+          :regexp_states => simple_regexp,
+          :string_states => @string_states,
+          :accepting     => @accepting
+        })
+      end
+
+      def to_svg
+        svg = IO.popen("dot -Tsvg", 'w+') { |f|
+          f.write to_dot
+          f.close_write
+          f.readlines
+        }
+        3.times { svg.shift }
+        svg.join.sub(/width="[^"]*"/, '').sub(/height="[^"]*"/, '')
+      end
+
+      def visualizer paths, title = 'FSM'
+        viz_dir   = File.join File.dirname(__FILE__), '..', 'visualizer'
+        fsm_js    = File.read File.join(viz_dir, 'fsm.js')
+        d3_js     = File.read File.join(viz_dir, 'd3.min.js')
+        reset_css = File.read File.join(viz_dir, 'reset.css')
+        fsm_css   = File.read File.join(viz_dir, 'fsm.css')
+        erb       = File.read File.join(viz_dir, 'index.html.erb')
+        states    = "function tt() { return #{to_json}; }"
+
+        fun_routes = paths.shuffle.first(3).map do |ast|
+          ast.map { |n|
+            case n
+            when Nodes::Symbol
+              case n.left
+              when ':id' then rand(100).to_s
+              when ':format' then %w{ xml json }.shuffle.first
+              else
+                'omg'
+              end
+            when Nodes::Terminal then n.symbol
+            else
+              nil
+            end
+          }.compact.join
+        end
+
+        stylesheets = [reset_css, fsm_css]
+        svg         = to_svg
+        javascripts = [d3_js, states, fsm_js]
+
+        # Annoying hack for 1.9 warnings
+        fun_routes  = fun_routes
+        stylesheets = stylesheets
+        svg         = svg
+        javascripts = javascripts
+
+        require 'erb'
+        template = ERB.new erb
+        template.result(binding)
+      end
+
+      def []= from, to, sym
+        case sym
+        when String
+          @string_states[from][sym] = to
+        when Regexp
+          @regexp_states[from][sym] = to
+        else
+          raise ArgumentError, 'unknown symbol: %s' % sym.class
+        end
+      end
+
+      def states
+        ss = @string_states.keys + @string_states.values.map(&:values).flatten
+        rs = @regexp_states.keys + @regexp_states.values.map(&:values).flatten
+        (ss + rs).uniq
+      end
+
+      def transitions
+        @string_states.map { |from, hash|
+          hash.map { |s, to| [from, s, to] }
+        }.flatten(1) + @regexp_states.map { |from, hash|
+          hash.map { |s, to| [from, s, to] }
+        }.flatten(1)
+      end
+
+      private
+      def move_regexp t, a
+        t.map { |s|
+          @regexp_states[s].find_all { |re,_| re === a }.map(&:last)
+        }.flatten.uniq
+      end
+
+      def move_string t, a
+        t.map { |s| @string_states[s][a] }.compact
+      end
+    end
+  end
+end

data/lib/journey/nfa/builder.rb

@@ -0,0 +1,74 @@
+require 'journey/nfa/transition_table'
+require 'journey/gtg/transition_table'
+
+module Journey
+  module NFA
+    class Visitor < Visitors::Visitor
+      def initialize tt
+        @tt = tt
+        @i  = -1
+      end
+
+      def visit_CAT node
+        left  = visit node.left
+        right = visit node.right
+
+        @tt.merge left.last, right.first
+
+        [left.first, right.last]
+      end
+
+      def visit_GROUP node
+        from  = @i += 1
+        left  = visit node.left
+        to    = @i += 1
+
+        @tt.accepting = to
+
+        @tt[from, left.first] = nil
+        @tt[left.last, to] = nil
+        @tt[from, to] = nil
+
+        [from, to]
+      end
+
+      def visit_OR node
+        from  = @i += 1
+        children = node.children.map { |c| visit c }
+        to    = @i += 1
+
+        children.each do |child|
+          @tt[from, child.first]  = nil
+          @tt[child.last, to]     = nil
+        end
+
+        @tt.accepting = to
+
+        [from, to]
+      end
+
+      def terminal node
+        from_i = @i += 1 # new state
+        to_i   = @i += 1 # new state
+
+        @tt[from_i, to_i] = node
+        @tt.accepting = to_i
+        @tt.add_memo to_i, node.memo
+
+        [from_i, to_i]
+      end
+    end
+
+    class Builder
+      def initialize ast
+        @ast = ast
+      end
+
+      def transition_table
+        tt = TransitionTable.new
+        Visitor.new(tt).accept @ast
+        tt
+      end
+    end
+  end
+end

data/lib/journey/nfa/dot.rb

@@ -0,0 +1,34 @@
+# encoding: utf-8
+
+module Journey
+  module NFA
+    module Dot
+      def to_dot
+        edges = transitions.map { |from, sym, to|
+          "  #{from} -> #{to} [label=\"#{sym || 'ε'}\"];"
+        }
+
+        #memo_nodes = memos.values.flatten.map { |n|
+        #  label = n
+        #  if Journey::Route === n
+        #    label = "#{n.verb.source} #{n.path.spec}"
+        #  end
+        #  "  #{n.object_id} [label=\"#{label}\", shape=box];"
+        #}
+        #memo_edges = memos.map { |k, memos|
+        #  (memos || []).map { |v| "  #{k} -> #{v.object_id};" }
+        #}.flatten.uniq
+
+        <<-eodot
+digraph nfa {
+  rankdir=LR;
+  node [shape = doublecircle];
+  #{accepting_states.join ' '};
+  node [shape = circle];
+#{edges.join "\n"}
+}
+        eodot
+      end
+    end
+  end
+end

data/lib/journey/nfa/simulator.rb

@@ -0,0 +1,45 @@
+require 'strscan'
+
+module Journey
+  module NFA
+    class MatchData
+      attr_reader :memos
+
+      def initialize memos
+        @memos = memos
+      end
+    end
+
+    class Simulator
+      attr_reader :tt
+
+      def initialize transition_table
+        @tt = transition_table
+      end
+
+      def simulate string
+        input = StringScanner.new string
+        state = tt.eclosure 0
+        until input.eos?
+          sym   = input.scan(/[\/\.\?]|[^\/\.\?]+/)
+
+          # FIXME: tt.eclosure is not needed for the GTG
+          state = tt.eclosure tt.move(state, sym)
+        end
+
+        acceptance_states = state.find_all { |s|
+          tt.accepting? tt.eclosure(s).sort.last
+        }
+
+        return if acceptance_states.empty?
+
+        memos = acceptance_states.map { |x| tt.memo x }.flatten.compact
+
+        MatchData.new memos
+      end
+
+      alias :=~    :simulate
+      alias :match :simulate
+    end
+  end
+end

data/lib/journey/nfa/transition_table.rb

@@ -0,0 +1,164 @@
+require 'journey/nfa/dot'
+
+module Journey
+  module NFA
+    class TransitionTable
+      include Journey::NFA::Dot
+
+      attr_accessor :accepting
+      attr_reader :memos
+
+      def initialize
+        @table     = Hash.new { |h,f| h[f] = {} }
+        @memos     = {}
+        @accepting = nil
+        @inverted  = nil
+      end
+
+      def accepting? state
+        accepting == state
+      end
+
+      def accepting_states
+        [accepting]
+      end
+
+      def add_memo idx, memo
+        @memos[idx] = memo
+      end
+
+      def memo idx
+        @memos[idx]
+      end
+
+      def []= i, f, s
+        @table[f][i] = s
+      end
+
+      def merge left, right
+        @memos[right] = @memos.delete left
+        @table[right] = @table.delete(left)
+      end
+
+      def states
+        (@table.keys + @table.values.map(&:keys).flatten).uniq
+      end
+
+      ###
+      # Returns a generalized transition graph with reduced states.  The states
+      # are reduced like a DFA, but the table must be simulated like an NFA.
+      #
+      # Edges of the GTG are regular expressions
+      def generalized_table
+        gt       = GTG::TransitionTable.new
+        marked   = {}
+        state_id = Hash.new { |h,k| h[k] = h.length }
+        alphabet = self.alphabet
+
+        stack = [eclosure(0)]
+
+        until stack.empty?
+          state = stack.pop
+          next if marked[state] || state.empty?
+
+          marked[state] = true
+
+          alphabet.each do |alpha|
+            next_state = eclosure(following_states(state, alpha))
+            next if next_state.empty?
+
+            gt[state_id[state], state_id[next_state]] = alpha
+            stack << next_state
+          end
+        end
+
+        final_groups = state_id.keys.find_all { |s|
+          s.sort.last == accepting
+        }
+
+        final_groups.each do |states|
+          id = state_id[states]
+
+          gt.add_accepting id
+          save = states.find { |s|
+            @memos.key?(s) && eclosure(s).sort.last == accepting
+          }
+
+          gt.add_memo id, memo(save)
+        end
+
+        gt
+      end
+
+      ###
+      # Returns set of NFA states to which there is a transition on ast symbol
+      # +a+ from some state +s+ in +t+.
+      def following_states t, a
+        Array(t).map { |s| inverted[s][a] }.flatten.uniq
+      end
+
+      ###
+      # Returns set of NFA states to which there is a transition on ast symbol
+      # +a+ from some state +s+ in +t+.
+      def move t, a
+        Array(t).map { |s|
+          inverted[s].keys.compact.find_all { |sym|
+            sym === a
+          }.map { |sym| inverted[s][sym] }
+        }.flatten.uniq
+      end
+
+      def alphabet
+        inverted.values.map(&:keys).flatten.compact.uniq
+      end
+
+      ###
+      # Returns a set of NFA states reachable from some NFA state +s+ in set
+      # +t+ on nil-transitions alone.
+      def eclosure t
+        stack = Array(t)
+        seen  = {}
+        children = []
+
+        until stack.empty?
+          s = stack.pop
+          next if seen[s]
+
+          seen[s] = true
+          children << s
+
+          stack.concat inverted[s][nil]
+        end
+
+        children.uniq
+      end
+
+      def transitions
+        @table.map { |to, hash|
+          hash.map { |from, sym| [from, sym, to] }
+        }.flatten(1)
+      end
+
+      private
+      def inverted
+        return @inverted if @inverted
+
+        @inverted = Hash.new { |h,from|
+          h[from] = Hash.new { |j,s| j[s] = [] }
+        }
+
+        @table.each { |to, hash|
+          hash.each { |from, sym|
+            if sym
+              sym = Nodes::Symbol === sym ? sym.regexp : sym.left
+            end
+
+            @inverted[from][sym] << to
+          }
+        }
+
+        @inverted
+      end
+    end
+  end
+end

data/lib/journey/nodes/node.rb

@@ -0,0 +1,104 @@
+require 'journey/visitors'
+
+module Journey
+  module Nodes
+    class Node # :nodoc:
+      include Enumerable
+
+      attr_accessor :left, :memo
+
+      def initialize left
+        @left = left
+        @memo = nil
+      end
+
+      def each(&block)
+        Visitors::Each.new(block).accept(self)
+      end
+
+      def to_s
+        Visitors::String.new.accept(self)
+      end
+
+      def to_dot
+        Visitors::Dot.new.accept(self)
+      end
+
+      def to_sym
+        name.to_sym
+      end
+
+      def name
+        left.tr ':', ''
+      end
+
+      def type
+        raise NotImplementedError
+      end
+    end
+
+    class Terminal < Node
+      alias :symbol :left
+    end
+
+    %w{ Symbol Slash Literal Dot }.each do |t|
+      class_eval %{
+        class #{t} < Terminal
+          def type; :#{t.upcase}; end
+        end
+      }
+    end
+
+    class Symbol < Terminal
+      attr_accessor :regexp
+      alias :symbol :regexp
+
+      DEFAULT_EXP = /[^\.\/\?]+/
+      def initialize left
+        super
+        @regexp = DEFAULT_EXP
+      end
+
+      def default_regexp?
+        regexp == DEFAULT_EXP
+      end
+    end
+
+    class Unary < Node
+      def children; [value] end
+    end
+
+    class Group < Unary
+      def type; :GROUP; end
+    end
+
+    class Star < Unary
+      def type; :STAR; end
+    end
+
+    class Binary < Node
+      attr_accessor :right
+
+      def initialize left, right
+        super(left)
+        @right = right
+      end
+
+      def children; [left, right] end
+    end
+
+    class Cat < Binary
+      def type; :CAT; end
+    end
+
+    class Or < Node
+      attr_reader :children
+
+      def initialize children
+        @children = children
+      end
+
+      def type; :OR; end
+    end
+  end
+end