andyjeffries-journey 1.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (51) hide show
  1. data/.autotest +8 -0
  2. data/.gemtest +0 -0
  3. data/CHANGELOG.rdoc +6 -0
  4. data/Gemfile +11 -0
  5. data/Manifest.txt +49 -0
  6. data/README.rdoc +48 -0
  7. data/Rakefile +31 -0
  8. data/journey.gemspec +42 -0
  9. data/lib/journey/backwards.rb +5 -0
  10. data/lib/journey/core-ext/hash.rb +11 -0
  11. data/lib/journey/formatter.rb +129 -0
  12. data/lib/journey/gtg/builder.rb +159 -0
  13. data/lib/journey/gtg/simulator.rb +44 -0
  14. data/lib/journey/gtg/transition_table.rb +152 -0
  15. data/lib/journey/nfa/builder.rb +74 -0
  16. data/lib/journey/nfa/dot.rb +34 -0
  17. data/lib/journey/nfa/simulator.rb +45 -0
  18. data/lib/journey/nfa/transition_table.rb +164 -0
  19. data/lib/journey/nodes/node.rb +104 -0
  20. data/lib/journey/parser.rb +204 -0
  21. data/lib/journey/parser.y +47 -0
  22. data/lib/journey/parser_extras.rb +21 -0
  23. data/lib/journey/path/pattern.rb +190 -0
  24. data/lib/journey/route.rb +92 -0
  25. data/lib/journey/router/strexp.rb +22 -0
  26. data/lib/journey/router/utils.rb +57 -0
  27. data/lib/journey/router.rb +138 -0
  28. data/lib/journey/routes.rb +74 -0
  29. data/lib/journey/scanner.rb +58 -0
  30. data/lib/journey/visitors.rb +186 -0
  31. data/lib/journey/visualizer/d3.min.js +2 -0
  32. data/lib/journey/visualizer/fsm.css +34 -0
  33. data/lib/journey/visualizer/fsm.js +134 -0
  34. data/lib/journey/visualizer/index.html.erb +50 -0
  35. data/lib/journey/visualizer/reset.css +48 -0
  36. data/lib/journey.rb +5 -0
  37. data/test/gtg/test_builder.rb +77 -0
  38. data/test/gtg/test_transition_table.rb +113 -0
  39. data/test/helper.rb +4 -0
  40. data/test/nfa/test_simulator.rb +96 -0
  41. data/test/nfa/test_transition_table.rb +70 -0
  42. data/test/nodes/test_symbol.rb +15 -0
  43. data/test/path/test_pattern.rb +260 -0
  44. data/test/route/definition/test_parser.rb +108 -0
  45. data/test/route/definition/test_scanner.rb +52 -0
  46. data/test/router/test_strexp.rb +30 -0
  47. data/test/router/test_utils.rb +19 -0
  48. data/test/test_route.rb +95 -0
  49. data/test/test_router.rb +464 -0
  50. data/test/test_routes.rb +38 -0
  51. metadata +171 -0
@@ -0,0 +1,152 @@
1
+ require 'journey/nfa/dot'
2
+
3
+ module Journey
4
+ module GTG
5
+ class TransitionTable
6
+ include Journey::NFA::Dot
7
+
8
+ attr_reader :memos
9
+
10
+ def initialize
11
+ @regexp_states = Hash.new { |h,k| h[k] = {} }
12
+ @string_states = Hash.new { |h,k| h[k] = {} }
13
+ @accepting = {}
14
+ @memos = Hash.new { |h,k| h[k] = [] }
15
+ end
16
+
17
+ def add_accepting state
18
+ @accepting[state] = true
19
+ end
20
+
21
+ def accepting_states
22
+ @accepting.keys
23
+ end
24
+
25
+ def accepting? state
26
+ @accepting[state]
27
+ end
28
+
29
+ def add_memo idx, memo
30
+ @memos[idx] << memo
31
+ end
32
+
33
+ def memo idx
34
+ @memos[idx]
35
+ end
36
+
37
+ def eclosure t
38
+ Array(t)
39
+ end
40
+
41
+ def move t, a
42
+ t = Array(t)
43
+ move_string(t, a) + move_regexp(t, a)
44
+ end
45
+
46
+ def to_json
47
+ require 'json'
48
+
49
+ simple_regexp = Hash.new { |h,k| h[k] = {} }
50
+
51
+ @regexp_states.each do |from, hash|
52
+ hash.each do |re, to|
53
+ simple_regexp[from][re.source] = to
54
+ end
55
+ end
56
+
57
+ JSON.dump({
58
+ :regexp_states => simple_regexp,
59
+ :string_states => @string_states,
60
+ :accepting => @accepting
61
+ })
62
+ end
63
+
64
+ def to_svg
65
+ svg = IO.popen("dot -Tsvg", 'w+') { |f|
66
+ f.write to_dot
67
+ f.close_write
68
+ f.readlines
69
+ }
70
+ 3.times { svg.shift }
71
+ svg.join.sub(/width="[^"]*"/, '').sub(/height="[^"]*"/, '')
72
+ end
73
+
74
+ def visualizer paths, title = 'FSM'
75
+ viz_dir = File.join File.dirname(__FILE__), '..', 'visualizer'
76
+ fsm_js = File.read File.join(viz_dir, 'fsm.js')
77
+ d3_js = File.read File.join(viz_dir, 'd3.min.js')
78
+ reset_css = File.read File.join(viz_dir, 'reset.css')
79
+ fsm_css = File.read File.join(viz_dir, 'fsm.css')
80
+ erb = File.read File.join(viz_dir, 'index.html.erb')
81
+ states = "function tt() { return #{to_json}; }"
82
+
83
+ fun_routes = paths.shuffle.first(3).map do |ast|
84
+ ast.map { |n|
85
+ case n
86
+ when Nodes::Symbol
87
+ case n.left
88
+ when ':id' then rand(100).to_s
89
+ when ':format' then %w{ xml json }.shuffle.first
90
+ else
91
+ 'omg'
92
+ end
93
+ when Nodes::Terminal then n.symbol
94
+ else
95
+ nil
96
+ end
97
+ }.compact.join
98
+ end
99
+
100
+ stylesheets = [reset_css, fsm_css]
101
+ svg = to_svg
102
+ javascripts = [d3_js, states, fsm_js]
103
+
104
+ # Annoying hack for 1.9 warnings
105
+ fun_routes = fun_routes
106
+ stylesheets = stylesheets
107
+ svg = svg
108
+ javascripts = javascripts
109
+
110
+ require 'erb'
111
+ template = ERB.new erb
112
+ template.result(binding)
113
+ end
114
+
115
+ def []= from, to, sym
116
+ case sym
117
+ when String
118
+ @string_states[from][sym] = to
119
+ when Regexp
120
+ @regexp_states[from][sym] = to
121
+ else
122
+ raise ArgumentError, 'unknown symbol: %s' % sym.class
123
+ end
124
+ end
125
+
126
+ def states
127
+ ss = @string_states.keys + @string_states.values.map(&:values).flatten
128
+ rs = @regexp_states.keys + @regexp_states.values.map(&:values).flatten
129
+ (ss + rs).uniq
130
+ end
131
+
132
+ def transitions
133
+ @string_states.map { |from, hash|
134
+ hash.map { |s, to| [from, s, to] }
135
+ }.flatten(1) + @regexp_states.map { |from, hash|
136
+ hash.map { |s, to| [from, s, to] }
137
+ }.flatten(1)
138
+ end
139
+
140
+ private
141
+ def move_regexp t, a
142
+ t.map { |s|
143
+ @regexp_states[s].find_all { |re,_| re === a }.map(&:last)
144
+ }.flatten.uniq
145
+ end
146
+
147
+ def move_string t, a
148
+ t.map { |s| @string_states[s][a] }.compact
149
+ end
150
+ end
151
+ end
152
+ end
@@ -0,0 +1,74 @@
1
+ require 'journey/nfa/transition_table'
2
+ require 'journey/gtg/transition_table'
3
+
4
+ module Journey
5
+ module NFA
6
+ class Visitor < Visitors::Visitor
7
+ def initialize tt
8
+ @tt = tt
9
+ @i = -1
10
+ end
11
+
12
+ def visit_CAT node
13
+ left = visit node.left
14
+ right = visit node.right
15
+
16
+ @tt.merge left.last, right.first
17
+
18
+ [left.first, right.last]
19
+ end
20
+
21
+ def visit_GROUP node
22
+ from = @i += 1
23
+ left = visit node.left
24
+ to = @i += 1
25
+
26
+ @tt.accepting = to
27
+
28
+ @tt[from, left.first] = nil
29
+ @tt[left.last, to] = nil
30
+ @tt[from, to] = nil
31
+
32
+ [from, to]
33
+ end
34
+
35
+ def visit_OR node
36
+ from = @i += 1
37
+ children = node.children.map { |c| visit c }
38
+ to = @i += 1
39
+
40
+ children.each do |child|
41
+ @tt[from, child.first] = nil
42
+ @tt[child.last, to] = nil
43
+ end
44
+
45
+ @tt.accepting = to
46
+
47
+ [from, to]
48
+ end
49
+
50
+ def terminal node
51
+ from_i = @i += 1 # new state
52
+ to_i = @i += 1 # new state
53
+
54
+ @tt[from_i, to_i] = node
55
+ @tt.accepting = to_i
56
+ @tt.add_memo to_i, node.memo
57
+
58
+ [from_i, to_i]
59
+ end
60
+ end
61
+
62
+ class Builder
63
+ def initialize ast
64
+ @ast = ast
65
+ end
66
+
67
+ def transition_table
68
+ tt = TransitionTable.new
69
+ Visitor.new(tt).accept @ast
70
+ tt
71
+ end
72
+ end
73
+ end
74
+ end
@@ -0,0 +1,34 @@
1
+ # encoding: utf-8
2
+
3
+ module Journey
4
+ module NFA
5
+ module Dot
6
+ def to_dot
7
+ edges = transitions.map { |from, sym, to|
8
+ " #{from} -> #{to} [label=\"#{sym || 'ε'}\"];"
9
+ }
10
+
11
+ #memo_nodes = memos.values.flatten.map { |n|
12
+ # label = n
13
+ # if Journey::Route === n
14
+ # label = "#{n.verb.source} #{n.path.spec}"
15
+ # end
16
+ # " #{n.object_id} [label=\"#{label}\", shape=box];"
17
+ #}
18
+ #memo_edges = memos.map { |k, memos|
19
+ # (memos || []).map { |v| " #{k} -> #{v.object_id};" }
20
+ #}.flatten.uniq
21
+
22
+ <<-eodot
23
+ digraph nfa {
24
+ rankdir=LR;
25
+ node [shape = doublecircle];
26
+ #{accepting_states.join ' '};
27
+ node [shape = circle];
28
+ #{edges.join "\n"}
29
+ }
30
+ eodot
31
+ end
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,45 @@
1
+ require 'strscan'
2
+
3
+ module Journey
4
+ module NFA
5
+ class MatchData
6
+ attr_reader :memos
7
+
8
+ def initialize memos
9
+ @memos = memos
10
+ end
11
+ end
12
+
13
+ class Simulator
14
+ attr_reader :tt
15
+
16
+ def initialize transition_table
17
+ @tt = transition_table
18
+ end
19
+
20
+ def simulate string
21
+ input = StringScanner.new string
22
+ state = tt.eclosure 0
23
+ until input.eos?
24
+ sym = input.scan(/[\/\.\?]|[^\/\.\?]+/)
25
+
26
+ # FIXME: tt.eclosure is not needed for the GTG
27
+ state = tt.eclosure tt.move(state, sym)
28
+ end
29
+
30
+ acceptance_states = state.find_all { |s|
31
+ tt.accepting? tt.eclosure(s).sort.last
32
+ }
33
+
34
+ return if acceptance_states.empty?
35
+
36
+ memos = acceptance_states.map { |x| tt.memo x }.flatten.compact
37
+
38
+ MatchData.new memos
39
+ end
40
+
41
+ alias :=~ :simulate
42
+ alias :match :simulate
43
+ end
44
+ end
45
+ end
@@ -0,0 +1,164 @@
1
+ require 'journey/nfa/dot'
2
+
3
+ module Journey
4
+ module NFA
5
+ class TransitionTable
6
+ include Journey::NFA::Dot
7
+
8
+ attr_accessor :accepting
9
+ attr_reader :memos
10
+
11
+ def initialize
12
+ @table = Hash.new { |h,f| h[f] = {} }
13
+ @memos = {}
14
+ @accepting = nil
15
+ @inverted = nil
16
+ end
17
+
18
+ def accepting? state
19
+ accepting == state
20
+ end
21
+
22
+ def accepting_states
23
+ [accepting]
24
+ end
25
+
26
+ def add_memo idx, memo
27
+ @memos[idx] = memo
28
+ end
29
+
30
+ def memo idx
31
+ @memos[idx]
32
+ end
33
+
34
+ def []= i, f, s
35
+ @table[f][i] = s
36
+ end
37
+
38
+ def merge left, right
39
+ @memos[right] = @memos.delete left
40
+ @table[right] = @table.delete(left)
41
+ end
42
+
43
+ def states
44
+ (@table.keys + @table.values.map(&:keys).flatten).uniq
45
+ end
46
+
47
+ ###
48
+ # Returns a generalized transition graph with reduced states. The states
49
+ # are reduced like a DFA, but the table must be simulated like an NFA.
50
+ #
51
+ # Edges of the GTG are regular expressions
52
+ def generalized_table
53
+ gt = GTG::TransitionTable.new
54
+ marked = {}
55
+ state_id = Hash.new { |h,k| h[k] = h.length }
56
+ alphabet = self.alphabet
57
+
58
+ stack = [eclosure(0)]
59
+
60
+ until stack.empty?
61
+ state = stack.pop
62
+ next if marked[state] || state.empty?
63
+
64
+ marked[state] = true
65
+
66
+ alphabet.each do |alpha|
67
+ next_state = eclosure(following_states(state, alpha))
68
+ next if next_state.empty?
69
+
70
+ gt[state_id[state], state_id[next_state]] = alpha
71
+ stack << next_state
72
+ end
73
+ end
74
+
75
+ final_groups = state_id.keys.find_all { |s|
76
+ s.sort.last == accepting
77
+ }
78
+
79
+ final_groups.each do |states|
80
+ id = state_id[states]
81
+
82
+ gt.add_accepting id
83
+ save = states.find { |s|
84
+ @memos.key?(s) && eclosure(s).sort.last == accepting
85
+ }
86
+
87
+ gt.add_memo id, memo(save)
88
+ end
89
+
90
+ gt
91
+ end
92
+
93
+ ###
94
+ # Returns set of NFA states to which there is a transition on ast symbol
95
+ # +a+ from some state +s+ in +t+.
96
+ def following_states t, a
97
+ Array(t).map { |s| inverted[s][a] }.flatten.uniq
98
+ end
99
+
100
+ ###
101
+ # Returns set of NFA states to which there is a transition on ast symbol
102
+ # +a+ from some state +s+ in +t+.
103
+ def move t, a
104
+ Array(t).map { |s|
105
+ inverted[s].keys.compact.find_all { |sym|
106
+ sym === a
107
+ }.map { |sym| inverted[s][sym] }
108
+ }.flatten.uniq
109
+ end
110
+
111
+ def alphabet
112
+ inverted.values.map(&:keys).flatten.compact.uniq
113
+ end
114
+
115
+ ###
116
+ # Returns a set of NFA states reachable from some NFA state +s+ in set
117
+ # +t+ on nil-transitions alone.
118
+ def eclosure t
119
+ stack = Array(t)
120
+ seen = {}
121
+ children = []
122
+
123
+ until stack.empty?
124
+ s = stack.pop
125
+ next if seen[s]
126
+
127
+ seen[s] = true
128
+ children << s
129
+
130
+ stack.concat inverted[s][nil]
131
+ end
132
+
133
+ children.uniq
134
+ end
135
+
136
+ def transitions
137
+ @table.map { |to, hash|
138
+ hash.map { |from, sym| [from, sym, to] }
139
+ }.flatten(1)
140
+ end
141
+
142
+ private
143
+ def inverted
144
+ return @inverted if @inverted
145
+
146
+ @inverted = Hash.new { |h,from|
147
+ h[from] = Hash.new { |j,s| j[s] = [] }
148
+ }
149
+
150
+ @table.each { |to, hash|
151
+ hash.each { |from, sym|
152
+ if sym
153
+ sym = Nodes::Symbol === sym ? sym.regexp : sym.left
154
+ end
155
+
156
+ @inverted[from][sym] << to
157
+ }
158
+ }
159
+
160
+ @inverted
161
+ end
162
+ end
163
+ end
164
+ end
@@ -0,0 +1,104 @@
1
+ require 'journey/visitors'
2
+
3
+ module Journey
4
+ module Nodes
5
+ class Node # :nodoc:
6
+ include Enumerable
7
+
8
+ attr_accessor :left, :memo
9
+
10
+ def initialize left
11
+ @left = left
12
+ @memo = nil
13
+ end
14
+
15
+ def each(&block)
16
+ Visitors::Each.new(block).accept(self)
17
+ end
18
+
19
+ def to_s
20
+ Visitors::String.new.accept(self)
21
+ end
22
+
23
+ def to_dot
24
+ Visitors::Dot.new.accept(self)
25
+ end
26
+
27
+ def to_sym
28
+ name.to_sym
29
+ end
30
+
31
+ def name
32
+ left.tr ':', ''
33
+ end
34
+
35
+ def type
36
+ raise NotImplementedError
37
+ end
38
+ end
39
+
40
+ class Terminal < Node
41
+ alias :symbol :left
42
+ end
43
+
44
+ %w{ Symbol Slash Literal Dot }.each do |t|
45
+ class_eval %{
46
+ class #{t} < Terminal
47
+ def type; :#{t.upcase}; end
48
+ end
49
+ }
50
+ end
51
+
52
+ class Symbol < Terminal
53
+ attr_accessor :regexp
54
+ alias :symbol :regexp
55
+
56
+ DEFAULT_EXP = /[^\.\/\?]+/
57
+ def initialize left
58
+ super
59
+ @regexp = DEFAULT_EXP
60
+ end
61
+
62
+ def default_regexp?
63
+ regexp == DEFAULT_EXP
64
+ end
65
+ end
66
+
67
+ class Unary < Node
68
+ def children; [value] end
69
+ end
70
+
71
+ class Group < Unary
72
+ def type; :GROUP; end
73
+ end
74
+
75
+ class Star < Unary
76
+ def type; :STAR; end
77
+ end
78
+
79
+ class Binary < Node
80
+ attr_accessor :right
81
+
82
+ def initialize left, right
83
+ super(left)
84
+ @right = right
85
+ end
86
+
87
+ def children; [left, right] end
88
+ end
89
+
90
+ class Cat < Binary
91
+ def type; :CAT; end
92
+ end
93
+
94
+ class Or < Node
95
+ attr_reader :children
96
+
97
+ def initialize children
98
+ @children = children
99
+ end
100
+
101
+ def type; :OR; end
102
+ end
103
+ end
104
+ end