rley 0.4.02 → 0.4.03
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +16 -14
- data/README.md +12 -3
- data/examples/data_formats/JSON/JSON_demo.rb +19 -4
- data/examples/data_formats/JSON/JSON_grammar.rb +26 -20
- data/examples/data_formats/JSON/JSON_lexer.rb +15 -17
- data/lib/rley.rb +2 -1
- data/lib/rley/constants.rb +1 -1
- data/lib/rley/formatter/bracket_notation.rb +68 -0
- data/lib/rley/parse_tree_visitor.rb +7 -7
- data/lib/rley/parser/dotted_item.rb +0 -5
- data/lib/rley/parser/parse_tree_builder.rb +0 -14
- data/lib/rley/ptree/parse_tree_node.rb +1 -0
- data/spec/rley/formatter/bracket_notation_spec.rb +96 -0
- data/spec/rley/gfg/item_vertex_spec.rb +6 -0
- data/spec/rley/parse_tree_visitor_spec.rb +58 -0
- data/spec/rley/parser/state_set_spec.rb +13 -0
- data/spec/rley/ptree/parse_tree_node_spec.rb +11 -0
- data/spec/rley/sppf/non_terminal_node_spec.rb +4 -0
- metadata +5 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 2d13db61565b20ef542592a59bcd056d4dd2192e
|
|
4
|
+
data.tar.gz: a8d8c12a600d9621d6be682570cb165d526641ef
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 679e3f7d6747264092f317521e1f8f9e241ad912de71280c0c6d68454110bd3ba75fdc6f1f4a9c76ac61622ed87fba86f2662e3f15b5cd9a0e2d571b340e2988
|
|
7
|
+
data.tar.gz: edb83ffb5fb673f27abef6c7ef10014fb8c6c312de093251f6ce509e2893c20cf5114cb77abf1b97c1406d9d22d474b0ab577829acb9d59f58117e43e96eaef2
|
data/CHANGELOG.md
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
|
-
### 0.4.
|
|
2
|
-
* [
|
|
3
|
-
* [
|
|
1
|
+
### 0.4.03 / 2017-04-17
|
|
2
|
+
* [FIX] File `rley.rb` reference to obsolete `EarleyParser` class removed.
|
|
3
|
+
* [NEW] `BracketNotation` formatter class. Allows parse tree output in Labelled Bracket Notation.
|
|
4
|
+
* [CHANGE] Code refactoring in directory `examples\data_formats\JSON`. The demo command-line tool parses JSON and converts it into LBN (Labelled Bracket Notation). There are two diagrams (in SVG format) generated from the LBN output.
|
|
5
|
+
|
|
4
6
|
|
|
5
7
|
### 0.4.01 / 2016-12-21
|
|
6
8
|
* [NEW] File `appveyor.yml`. Add AppVeyor CI to Github commits. AppVeyor complements Travis by running builds under Windows OS.
|
|
@@ -48,7 +50,7 @@
|
|
|
48
50
|
### 0.3.06 / 2016-11-06
|
|
49
51
|
* [FIX] There were missing links to shared parse forest nodes for ambiguous parses.
|
|
50
52
|
* [NEW] RSpec file `ambiguous_parse_spec.rb` added in order to test the parse forest building for an ambiguous parse.
|
|
51
|
-
* [CHANGE] Attribute `ParseWalkerContext#nterm2start`: previous implementation assumed -wrongly- that for each non terminal there was only one start entry.
|
|
53
|
+
* [CHANGE] Attribute `ParseWalkerContext#nterm2start`: previous implementation assumed -wrongly- that for each non terminal there was only one start entry.
|
|
52
54
|
Now this attribute uses nested hashes as data structure in order to disambiguate the mapping.
|
|
53
55
|
* [CHANGE] Method `ParseWalkerFactory#visit_entry` updated to reflect change in the `ParseWalkerContext#nterm2start` attribute.
|
|
54
56
|
* [CHANGE] Method `ParseWalkerFactory#visit_entry` now emits an event if an item entry is re-visited (previously, no such event were generated)
|
|
@@ -73,10 +75,10 @@
|
|
|
73
75
|
### 0.3.01 / 2016-10-23
|
|
74
76
|
* [CHANGE] Method `ParseWalkerFactory#build_walker`. Signature change in order prevent direct dependency on `GFGParsing` class.
|
|
75
77
|
* [CHANGE] Class `ParseForestBuilder`. Removal of `parsing` attribute, no direct dependency on `GFGParsing` class.
|
|
76
|
-
* [CHANGE] Internal changed to `ParseForestFactory` class.
|
|
78
|
+
* [CHANGE] Internal changed to `ParseForestFactory` class.
|
|
77
79
|
|
|
78
80
|
### 0.3.00 / 2016-10-23
|
|
79
|
-
* [CHANGE] Many new classes. The gem bundles a second parser that copes with ambiguous grammars.
|
|
81
|
+
* [CHANGE] Many new classes. The gem bundles a second parser that copes with ambiguous grammars.
|
|
80
82
|
|
|
81
83
|
|
|
82
84
|
### 0.2.15 / 2016-02-14
|
|
@@ -90,11 +92,11 @@
|
|
|
90
92
|
* [CHANGED] method `Parsing#success?`. New implementation that relies on start symbol derivation.
|
|
91
93
|
* [NEW] New method `Chart#start_symbol` added. Returns the start symbol of the grammar.
|
|
92
94
|
* [NEW] New method `StateSet#ambiguities` added. Returns the parse sets that are ambiguous (= distinct derivation for same input tokens).
|
|
93
|
-
* [FIX] In special cases the parsing didn't work correctly when there more than one
|
|
95
|
+
* [FIX] In special cases the parsing didn't work correctly when there more than one
|
|
94
96
|
production rule for the start symbol of a grammar.
|
|
95
97
|
|
|
96
98
|
### 0.2.12 / 2015-11-20
|
|
97
|
-
* [FIX] In special cases the parsing didn't work correctly when there more than one
|
|
99
|
+
* [FIX] In special cases the parsing didn't work correctly when there more than one
|
|
98
100
|
production rule for the start symbol of a grammar.
|
|
99
101
|
|
|
100
102
|
### 0.2.11 / 2015-09-05
|
|
@@ -150,7 +152,7 @@ Version number bump: major re-design of the parse tree generation.
|
|
|
150
152
|
* [FIX] Method `Parsing#parse_tree` now handles situations where there are multiple complete parse states for a non-terminal.
|
|
151
153
|
|
|
152
154
|
### 0.1.12 / 2014-12-22
|
|
153
|
-
* [FIX] Fixed `Parsing#parse_tree`: code couldn't cope with parse state set containing more
|
|
155
|
+
* [FIX] Fixed `Parsing#parse_tree`: code couldn't cope with parse state set containing more
|
|
154
156
|
than one parse state that expected the same symbol.
|
|
155
157
|
* [NEW] Added one more parser example (for very basic arithmetic expression)
|
|
156
158
|
|
|
@@ -239,19 +241,19 @@ in method argument names between source code and documentation.
|
|
|
239
241
|
|
|
240
242
|
|
|
241
243
|
### 0.0.11 / 2014-11-16
|
|
242
|
-
* [CHANGE] Usage of `GrammarBuilder`simplified: the call to method `GrammarBuilder#add_non_terminal` isn't necessary. Method is removed
|
|
244
|
+
* [CHANGE] Usage of `GrammarBuilder`simplified: the call to method `GrammarBuilder#add_non_terminal` isn't necessary. Method is removed
|
|
243
245
|
* [CHANGE] Updated the `examples` folder accordingly.
|
|
244
246
|
|
|
245
247
|
### 0.0.10 / 2014-11-15
|
|
246
|
-
* [NEW] New folder `examples` added with two examples of grammar creation
|
|
248
|
+
* [NEW] New folder `examples` added with two examples of grammar creation
|
|
247
249
|
|
|
248
250
|
### 0.0.09 / 2014-11-15
|
|
249
|
-
* [NEW] New class `GrammarBuilder` added and tested, its purpose is
|
|
251
|
+
* [NEW] New class `GrammarBuilder` added and tested, its purpose is
|
|
250
252
|
to simplify the construction of grammars.
|
|
251
253
|
|
|
252
254
|
### 0.0.08 / 2014-11-14
|
|
253
255
|
* [CHANGE] `EarleyParser#parse` method: Initial API documentation.
|
|
254
|
-
* [INFO] This version was committed to force Travis CI to execute a complete build
|
|
256
|
+
* [INFO] This version was committed to force Travis CI to execute a complete build
|
|
255
257
|
failed because Travis couldn't connect to GitHub)
|
|
256
258
|
|
|
257
259
|
### 0.0.07 / 2014-11-14
|
|
@@ -280,4 +282,4 @@ failed because Travis couldn't connect to GitHub)
|
|
|
280
282
|
|
|
281
283
|
|
|
282
284
|
### 0.0.00 / 2014-11-07
|
|
283
|
-
* [FEATURE] Initial public working version
|
|
285
|
+
* [FEATURE] Initial public working version
|
data/README.md
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
|
|
1
|
+
[Rley](https://github.com/famished-tiger/Rley)
|
|
2
2
|
|
|
3
3
|
[](https://travis-ci.org/famished-tiger/Rley)
|
|
4
4
|
[](https://ci.appveyor.com/project/famished-tiger/rley)
|
|
@@ -28,7 +28,7 @@ encode efficiently all the possible parse trees that result from an ambiguous
|
|
|
28
28
|
grammar.
|
|
29
29
|
|
|
30
30
|
As another distinctive mark, __Rley__ is also the first Ruby implementation of a
|
|
31
|
-
parsing library based on the new _Grammar Flow Graph_ approach
|
|
31
|
+
parsing library based on the new _Grammar Flow Graph_ approach [References on GFG](#references-on-gfg).
|
|
32
32
|
|
|
33
33
|
### What it can do?
|
|
34
34
|
Maybe parsing algorithms and internal implementation details are of lesser
|
|
@@ -217,7 +217,7 @@ ask it to generate a message.
|
|
|
217
217
|
puts result.failure_reason.message unless result.success?
|
|
218
218
|
```
|
|
219
219
|
|
|
220
|
-
Re-running the example with the error,
|
|
220
|
+
Re-running the example with the error, results in the error message:
|
|
221
221
|
```
|
|
222
222
|
Syntax error at or near token 2 >>>Mary<<<
|
|
223
223
|
Expected one 'Verb', found a 'Proper-Noun' instead.
|
|
@@ -281,6 +281,15 @@ Here are a few other ones:
|
|
|
281
281
|
## Thanks to:
|
|
282
282
|
* Professor Keshav Pingali, one of the creators of the Grammar Flow Graph parsing approach for his encouraging e-mail exchanges.
|
|
283
283
|
|
|
284
|
+
## References on GFG
|
|
285
|
+
Since the __G__rammar __F__low __G__raph parsing approach is quite new, it has not yet taken a place in
|
|
286
|
+
standard parser textbooks. Here are a few references (and links) of papers on GFG:
|
|
287
|
+
- K. Pingali, G. Bilardi. [Parsing with Pictures](http://apps.cs.utexas.edu/tech_reports/reports/tr/TR-2102.pdf)
|
|
288
|
+
- K. Pingali, G. Bilardi. [A Graphical Model for Context-Free Grammar Parsing.](https://link.springer.com/chapter/10.1007/978-3-662-46663-6_1)
|
|
289
|
+
In : International Conference on Compiler Construction. Springer Berlin Heidelberg, 2015. p. 3-27.
|
|
290
|
+
- M. Fulbright. [An Evaluation of Two Approaches to Parsing](http://apps.cs.utexas.edu/tech_reports/reports/tr/TR-2199.pdf)
|
|
291
|
+
|
|
292
|
+
|
|
284
293
|
Copyright
|
|
285
294
|
---------
|
|
286
295
|
Copyright (c) 2014-2017, Dimitri Geshef.
|
|
@@ -6,7 +6,12 @@ parser = JSONParser.new
|
|
|
6
6
|
# Parse the input file with name given in command-line
|
|
7
7
|
if ARGV.empty?
|
|
8
8
|
msg = <<-END_MSG
|
|
9
|
-
|
|
9
|
+
A demo utility that converts a JSON file into labelled square notation (LBN).
|
|
10
|
+
Use online tools (e.g. http://yohasebe.com/rsyntaxtree/) to visualize
|
|
11
|
+
parse trees from LBN output.
|
|
12
|
+
|
|
13
|
+
Command-line syntax:
|
|
14
|
+
|
|
10
15
|
ruby #{__FILE__} filename
|
|
11
16
|
where:
|
|
12
17
|
filename is the name of a JSON file
|
|
@@ -27,7 +32,17 @@ unless result.success?
|
|
|
27
32
|
exit(1)
|
|
28
33
|
end
|
|
29
34
|
|
|
30
|
-
# Generate a parse
|
|
31
|
-
|
|
32
|
-
|
|
35
|
+
# Generate a parse tree from the parse result
|
|
36
|
+
ptree = result.parse_tree
|
|
37
|
+
require 'yaml'
|
|
38
|
+
File.open('json1.yml', 'w') {|f| YAML.dump(ptree, f)}
|
|
39
|
+
|
|
40
|
+
# Let's create a parse tree visitor
|
|
41
|
+
visitor = Rley::ParseTreeVisitor.new(ptree)
|
|
42
|
+
|
|
43
|
+
# Output the labelled bracket notation of the tree
|
|
44
|
+
use_notation = Rley::Formatter::BracketNotation.new($stdout)
|
|
45
|
+
use_notation.render(visitor)
|
|
46
|
+
|
|
47
|
+
|
|
33
48
|
# End of file
|
|
@@ -4,27 +4,33 @@ require 'rley' # Load the gem
|
|
|
4
4
|
|
|
5
5
|
########################################
|
|
6
6
|
# Define a grammar for JSON
|
|
7
|
+
# Original JSON grammar is available http://www.json.org/fatfree.html
|
|
8
|
+
# Official JSON grammar: http://rfc7159.net/rfc7159#rfc.section.2
|
|
9
|
+
# Names of grammar elements are based on the RFC 7159 documentation
|
|
7
10
|
builder = Rley::Syntax::GrammarBuilder.new do
|
|
8
|
-
add_terminals('
|
|
9
|
-
add_terminals('
|
|
10
|
-
add_terminals('
|
|
11
|
-
add_terminals('
|
|
12
|
-
add_terminals('
|
|
13
|
-
rule '
|
|
14
|
-
rule '
|
|
15
|
-
rule '
|
|
16
|
-
rule '
|
|
17
|
-
rule '
|
|
18
|
-
rule '
|
|
19
|
-
rule '
|
|
20
|
-
rule '
|
|
21
|
-
rule '
|
|
22
|
-
rule '
|
|
23
|
-
rule
|
|
24
|
-
rule '
|
|
25
|
-
rule '
|
|
26
|
-
rule '
|
|
27
|
-
rule '
|
|
11
|
+
add_terminals('false', 'null', 'true') # Literal names
|
|
12
|
+
add_terminals('string', 'number')
|
|
13
|
+
add_terminals('begin-object', 'end-object') # For '{', '}' delimiters
|
|
14
|
+
add_terminals('begin-array', 'end-array') # For '[', ']' delimiters
|
|
15
|
+
add_terminals('name-separator', 'value-separator') # For ':', ',' separators
|
|
16
|
+
rule 'JSON-text' => 'value'
|
|
17
|
+
rule 'value' => 'false'
|
|
18
|
+
rule 'value' => 'null'
|
|
19
|
+
rule 'value' => 'true'
|
|
20
|
+
rule 'value' => 'object'
|
|
21
|
+
rule 'value' => 'array'
|
|
22
|
+
rule 'value' => 'number'
|
|
23
|
+
rule 'value' => 'string'
|
|
24
|
+
rule 'object' => %w(begin-object member-list end-object)
|
|
25
|
+
rule 'object' => %w(begin-object end-object)
|
|
26
|
+
# Next rule is an example of a left recursive rule
|
|
27
|
+
rule 'member-list' => %w(member-list value-separator member)
|
|
28
|
+
rule 'member-list' => 'member'
|
|
29
|
+
rule 'member' => %w(string name-separator value)
|
|
30
|
+
rule 'array' => %w(begin-array array_items end-array)
|
|
31
|
+
rule 'array' => %w(begin-array end-array)
|
|
32
|
+
rule 'array_items' => %w(array_items value-separator value)
|
|
33
|
+
rule 'array_items' => %w(value)
|
|
28
34
|
end
|
|
29
35
|
|
|
30
36
|
# And now build the grammar...
|
|
@@ -11,12 +11,12 @@ class JSONLexer
|
|
|
11
11
|
attr_reader(:name2symbol)
|
|
12
12
|
|
|
13
13
|
@@lexeme2name = {
|
|
14
|
-
'{' => '
|
|
15
|
-
'}' => '
|
|
16
|
-
'[' => '
|
|
17
|
-
']' => '
|
|
18
|
-
',' => '
|
|
19
|
-
':' => '
|
|
14
|
+
'{' => 'begin-object',
|
|
15
|
+
'}' => 'end-object',
|
|
16
|
+
'[' => 'begin-array',
|
|
17
|
+
']' => 'end-array',
|
|
18
|
+
',' => 'value-separator',
|
|
19
|
+
':' => 'name-separator'
|
|
20
20
|
}
|
|
21
21
|
|
|
22
22
|
class ScanError < StandardError ; end
|
|
@@ -53,14 +53,6 @@ private
|
|
|
53
53
|
token_type = name2symbol[type_name]
|
|
54
54
|
token = Rley::Tokens::Token.new(curr_ch, token_type)
|
|
55
55
|
|
|
56
|
-
# LITERALS
|
|
57
|
-
when '"' # Start string delimiter found
|
|
58
|
-
value = scanner.scan(/([^"\\]|\\.)*/)
|
|
59
|
-
end_delimiter = scanner.getch()
|
|
60
|
-
raise ScanError.new('No closing quotes (") found') if end_delimiter.nil?
|
|
61
|
-
token_type = name2symbol['JSON_STRING']
|
|
62
|
-
token = Rley::Tokens::Token.new(value, token_type)
|
|
63
|
-
|
|
64
56
|
when /[ftn]/ # First letter of keywords
|
|
65
57
|
@scanner.pos = scanner.pos - 1 # Simulate putback
|
|
66
58
|
keyw = scanner.scan(/false|true|null/)
|
|
@@ -68,18 +60,24 @@ private
|
|
|
68
60
|
invalid_keyw = scanner.scan(/\w+/)
|
|
69
61
|
raise ScanError.new("Invalid keyword: #{invalid_keyw}")
|
|
70
62
|
else
|
|
71
|
-
token_type = name2symbol[
|
|
63
|
+
token_type = name2symbol[keyw]
|
|
72
64
|
token = Rley::Tokens::Token.new(keyw, token_type)
|
|
73
65
|
end
|
|
74
66
|
|
|
67
|
+
# LITERALS
|
|
68
|
+
when '"' # Start string delimiter found
|
|
69
|
+
value = scanner.scan(/([^"\\]|\\.)*/)
|
|
70
|
+
end_delimiter = scanner.getch()
|
|
71
|
+
raise ScanError.new('No closing quotes (") found') if end_delimiter.nil?
|
|
72
|
+
token_type = name2symbol['string']
|
|
73
|
+
token = Rley::Tokens::Token.new(value, token_type)
|
|
75
74
|
|
|
76
75
|
when /[-0-9]/ # Start character of number literal found
|
|
77
76
|
@scanner.pos = scanner.pos - 1 # Simulate putback
|
|
78
77
|
value = scanner.scan(/-?[0-9]+(\.[0-9]+)?([eE][-+]?[0-9])?/)
|
|
79
|
-
token_type = name2symbol['
|
|
78
|
+
token_type = name2symbol['number']
|
|
80
79
|
token = Rley::Tokens::Token.new(value, token_type)
|
|
81
80
|
|
|
82
|
-
|
|
83
81
|
else # Unknown token
|
|
84
82
|
erroneous = curr_ch.nil? ? '' : curr_ch
|
|
85
83
|
sequel = scanner.scan(/.{1,20}/)
|
data/lib/rley.rb
CHANGED
|
@@ -5,10 +5,11 @@
|
|
|
5
5
|
require_relative './rley/constants'
|
|
6
6
|
require_relative './rley/syntax/grammar_builder'
|
|
7
7
|
require_relative './rley/tokens/token'
|
|
8
|
-
require_relative './rley/parser/earley_parser'
|
|
9
8
|
require_relative './rley/parser/gfg_earley_parser'
|
|
10
9
|
require_relative './rley/parse_tree_visitor'
|
|
11
10
|
require_relative './rley/formatter/debug'
|
|
12
11
|
require_relative './rley/formatter/json'
|
|
12
|
+
require_relative './rley/formatter/debug'
|
|
13
|
+
require_relative './rley/formatter/bracket_notation'
|
|
13
14
|
|
|
14
15
|
# End of file
|
data/lib/rley/constants.rb
CHANGED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
require_relative 'base_formatter'
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
module Rley # This module is used as a namespace
|
|
5
|
+
# Namespace dedicated to parse tree formatters.
|
|
6
|
+
module Formatter
|
|
7
|
+
# A formatter class that generates the labelled bracket notation (LBN)
|
|
8
|
+
# representation of a parse tree.
|
|
9
|
+
# The output can be then fed to an application or library that is
|
|
10
|
+
# capable of displaying parse tree diagrams.
|
|
11
|
+
# For Ruby developers, there is RSyntaxTree by Yoichiro Hasebe.
|
|
12
|
+
# (accessible via: http://yohasebe.com/rsyntaxtree/)
|
|
13
|
+
class BracketNotation < BaseFormatter
|
|
14
|
+
|
|
15
|
+
# Constructor.
|
|
16
|
+
# @param anIO [IO] The output stream to which the rendered grammar
|
|
17
|
+
# is written.
|
|
18
|
+
def initialize(anIO)
|
|
19
|
+
super(anIO)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
# Method called by a ParseTreeVisitor to which the formatter subscribed.
|
|
24
|
+
# Notification of a visit event: the visitor is about to visit
|
|
25
|
+
# a non-terminal node
|
|
26
|
+
# @param nonterm [NonTerminalNode]
|
|
27
|
+
def before_non_terminal(aNonTerm)
|
|
28
|
+
write("[#{aNonTerm.symbol.name} ")
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
# Method called by a ParseTreeVisitor to which the formatter subscribed.
|
|
33
|
+
# Notification of a visit event: the visitor is about to visit
|
|
34
|
+
# a terminal node
|
|
35
|
+
# @param _term [TerminalNode]
|
|
36
|
+
def before_terminal(aTerm)
|
|
37
|
+
write("[#{aTerm.symbol.name} ")
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Method called by a ParseTreeVisitor to which the formatter subscribed.
|
|
41
|
+
# Notification of a visit event: the visitor completed the visit of
|
|
42
|
+
# a terminal node.
|
|
43
|
+
# @param _term [TerminalNode]
|
|
44
|
+
def after_terminal(aTerm)
|
|
45
|
+
# Escape all opening and closing square brackets
|
|
46
|
+
escape_lbrackets = aTerm.token.lexeme.gsub(/\[/, "\\[")
|
|
47
|
+
escaped = escape_lbrackets.gsub(/\]/, "\\]")
|
|
48
|
+
write(escaped + ']')
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# Method called by a ParseTreeVisitor to which the formatter subscribed.
|
|
52
|
+
# Notification of a visit event: the visitor completed the visit of
|
|
53
|
+
# a non-terminal node
|
|
54
|
+
# @param _nonterm [NonTerminalNode]
|
|
55
|
+
def after_non_terminal(_nonterm)
|
|
56
|
+
write(']')
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
private
|
|
60
|
+
|
|
61
|
+
def write(aText)
|
|
62
|
+
output.write(aText)
|
|
63
|
+
end
|
|
64
|
+
end # class
|
|
65
|
+
end # module
|
|
66
|
+
end # module
|
|
67
|
+
|
|
68
|
+
# End of file
|
|
@@ -7,7 +7,7 @@ module Rley # This module is used as a namespace
|
|
|
7
7
|
|
|
8
8
|
# List of objects that subscribed to the visit event notification.
|
|
9
9
|
attr_reader(:subscribers)
|
|
10
|
-
|
|
10
|
+
|
|
11
11
|
# Indicates the kind of tree traversal to perform: :post_order, :pre-order
|
|
12
12
|
attr_reader(:traversal)
|
|
13
13
|
|
|
@@ -52,7 +52,7 @@ module Rley # This module is used as a namespace
|
|
|
52
52
|
broadcast(:before_non_terminal, aNonTerminalNode)
|
|
53
53
|
traverse_subnodes(aNonTerminalNode)
|
|
54
54
|
else
|
|
55
|
-
traverse_subnodes(aNonTerminalNode)
|
|
55
|
+
traverse_subnodes(aNonTerminalNode)
|
|
56
56
|
broadcast(:before_non_terminal, aNonTerminalNode)
|
|
57
57
|
end
|
|
58
58
|
broadcast(:after_non_terminal, aNonTerminalNode)
|
|
@@ -67,7 +67,7 @@ module Rley # This module is used as a namespace
|
|
|
67
67
|
end
|
|
68
68
|
|
|
69
69
|
|
|
70
|
-
# Visit event. The visitor has completed its visit of the given
|
|
70
|
+
# Visit event. The visitor has completed its visit of the given
|
|
71
71
|
# non-terminal node.
|
|
72
72
|
# @param aNonTerminalNode [NonTerminalNode] the node to visit.
|
|
73
73
|
def end_visit_nonterminal(aNonTerminalNode)
|
|
@@ -81,17 +81,17 @@ module Rley # This module is used as a namespace
|
|
|
81
81
|
end
|
|
82
82
|
|
|
83
83
|
private
|
|
84
|
-
|
|
85
|
-
# Visit event. The visitor is about to visit the subnodes of a non
|
|
84
|
+
|
|
85
|
+
# Visit event. The visitor is about to visit the subnodes of a non
|
|
86
86
|
# terminal node.
|
|
87
87
|
# @param aParentNode [NonTeminalNode] the (non-terminal) parent node.
|
|
88
88
|
def traverse_subnodes(aParentNode)
|
|
89
89
|
subnodes = aParentNode.subnodes
|
|
90
90
|
broadcast(:before_subnodes, aParentNode, subnodes)
|
|
91
|
-
|
|
91
|
+
|
|
92
92
|
# Let's proceed with the visit of subnodes
|
|
93
93
|
subnodes.each { |a_node| a_node.accept(self) }
|
|
94
|
-
|
|
94
|
+
|
|
95
95
|
broadcast(:after_subnodes, aParentNode, subnodes)
|
|
96
96
|
end
|
|
97
97
|
|
|
@@ -98,11 +98,6 @@ module Rley # This module is used as a namespace
|
|
|
98
98
|
return result
|
|
99
99
|
end
|
|
100
100
|
|
|
101
|
-
# An item with the dot in front of a terminal is called a shift item
|
|
102
|
-
def shift_item?()
|
|
103
|
-
return position.zero?
|
|
104
|
-
end
|
|
105
|
-
|
|
106
101
|
# Return true if this dotted item has a dot one place
|
|
107
102
|
# to the right compared to the dotted item argument.
|
|
108
103
|
def successor_of?(another)
|
|
@@ -114,11 +114,6 @@ module Rley # This module is used as a namespace
|
|
|
114
114
|
create_token_node(anEntry, anIndex)
|
|
115
115
|
|
|
116
116
|
when NilClass # Dot at the beginning of production
|
|
117
|
-
if anEntry.vertex.dotted_item.production.empty?
|
|
118
|
-
# Empty rhs => create an epsilon node ...
|
|
119
|
-
# ... without changing current path
|
|
120
|
-
create_epsilon_node(anEntry, anIndex)
|
|
121
|
-
end
|
|
122
117
|
curr_path.pop if curr_parent.kind_of?(SPPF::AlternativeNode)
|
|
123
118
|
end
|
|
124
119
|
|
|
@@ -195,15 +190,6 @@ module Rley # This module is used as a namespace
|
|
|
195
190
|
return candidate
|
|
196
191
|
end
|
|
197
192
|
|
|
198
|
-
|
|
199
|
-
def create_epsilon_node(anEntry, anIndex)
|
|
200
|
-
new_node = PTree::EpsilonNode.new(anIndex)
|
|
201
|
-
candidate = add_node_to_tree(new_node)
|
|
202
|
-
entry2node[anEntry] = candidate
|
|
203
|
-
|
|
204
|
-
return candidate
|
|
205
|
-
end
|
|
206
|
-
|
|
207
193
|
# Add the given node if not yet present in parse tree
|
|
208
194
|
def add_node_to_tree(aNode)
|
|
209
195
|
new_node = aNode
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
require_relative '../../spec_helper'
|
|
2
|
+
require 'stringio'
|
|
3
|
+
|
|
4
|
+
require_relative '../support/grammar_abc_helper'
|
|
5
|
+
require_relative '../../../lib/rley/tokens/token'
|
|
6
|
+
require_relative '../../../lib/rley/parser/gfg_earley_parser'
|
|
7
|
+
require_relative '../../../lib/rley/ptree/parse_tree'
|
|
8
|
+
require_relative '../../../lib/rley/parse_tree_visitor'
|
|
9
|
+
# Load the class under test
|
|
10
|
+
require_relative '../../../lib/rley/formatter/bracket_notation'
|
|
11
|
+
|
|
12
|
+
module Rley # Re-open the module to get rid of qualified names
|
|
13
|
+
module Formatter
|
|
14
|
+
describe BracketNotation do
|
|
15
|
+
include GrammarABCHelper # Mix-in module for grammar abc
|
|
16
|
+
|
|
17
|
+
# Factory method. Build a production with the given sequence
|
|
18
|
+
# of symbols as its rhs.
|
|
19
|
+
let(:grammar_abc) do
|
|
20
|
+
builder = grammar_abc_builder
|
|
21
|
+
builder.grammar
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# Variables for the terminal symbols
|
|
25
|
+
let(:a_) { grammar_abc.name2symbol['a'] }
|
|
26
|
+
let(:b_) { grammar_abc.name2symbol['b'] }
|
|
27
|
+
let(:c_) { grammar_abc.name2symbol['c'] }
|
|
28
|
+
|
|
29
|
+
# Helper method that mimicks the output of a tokenizer
|
|
30
|
+
# for the language specified by gramma_abc
|
|
31
|
+
let(:grm_abc_tokens1) do
|
|
32
|
+
[
|
|
33
|
+
Tokens::Token.new('a', a_),
|
|
34
|
+
Tokens::Token.new('a', a_),
|
|
35
|
+
Tokens::Token.new('b', b_),
|
|
36
|
+
Tokens::Token.new('c', c_),
|
|
37
|
+
Tokens::Token.new('c', c_)
|
|
38
|
+
]
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# Factory method that builds a sample parse tree.
|
|
42
|
+
# Generated tree has the following structure:
|
|
43
|
+
# S[0,5]
|
|
44
|
+
# +- A[0,5]
|
|
45
|
+
# +- a[0,0]
|
|
46
|
+
# +- A[1,4]
|
|
47
|
+
# | +- a[1,1]
|
|
48
|
+
# | +- A[2,3]
|
|
49
|
+
# | | +- b[2,3]
|
|
50
|
+
# | +- c[3,4]
|
|
51
|
+
# +- c[4,5]
|
|
52
|
+
# Capital letters represent non-terminal nodes
|
|
53
|
+
let(:grm_abc_ptree1) do
|
|
54
|
+
parser = Parser::GFGEarleyParser.new(grammar_abc)
|
|
55
|
+
parse_result = parser.parse(grm_abc_tokens1)
|
|
56
|
+
parse_result.parse_tree
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
let(:destination) { StringIO.new('', 'w') }
|
|
60
|
+
subject { BracketNotation.new(destination) }
|
|
61
|
+
|
|
62
|
+
context 'Standard creation & initialization:' do
|
|
63
|
+
it 'should be initialized with an IO argument' do
|
|
64
|
+
expect { BracketNotation.new(StringIO.new('', 'w')) }.not_to raise_error
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
it 'should know its output destination' do
|
|
68
|
+
expect(subject.output).to eq(destination)
|
|
69
|
+
end
|
|
70
|
+
end # context
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
context 'Formatting events:' do
|
|
74
|
+
it 'should support visit events of a parse tree' do
|
|
75
|
+
visitor = Rley::ParseTreeVisitor.new(grm_abc_ptree1)
|
|
76
|
+
subject.render(visitor)
|
|
77
|
+
expectations = '[S [A [a a][A [a a][A [b b]][c c]][c c]]]'
|
|
78
|
+
expect(destination.string).to eq(expectations)
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
it 'should escape square brackets' do
|
|
82
|
+
f_node = double('fake-node')
|
|
83
|
+
f_token = double('fake-token')
|
|
84
|
+
expect(f_node).to receive(:token).and_return(f_token)
|
|
85
|
+
expect(f_token).to receive(:lexeme).and_return('[][]')
|
|
86
|
+
|
|
87
|
+
subject.after_terminal(f_node)
|
|
88
|
+
expectations = '\[\]\[\]]'
|
|
89
|
+
expect(destination.string).to eq(expectations)
|
|
90
|
+
end
|
|
91
|
+
end # context
|
|
92
|
+
end # describe
|
|
93
|
+
end # module
|
|
94
|
+
end # module
|
|
95
|
+
|
|
96
|
+
# End of file
|
|
@@ -110,6 +110,12 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
|
110
110
|
shortcut = ShortcutEdge.new(subject, next_vertex)
|
|
111
111
|
expect(subject.shortcut).to eq(shortcut)
|
|
112
112
|
end
|
|
113
|
+
|
|
114
|
+
it 'should reject an invalid shortcut edge' do
|
|
115
|
+
err = StandardError
|
|
116
|
+
err_msg = 'Invalid shortcut argument'
|
|
117
|
+
expect { subject.shortcut = 'invalid'}.to raise_error(err, err_msg)
|
|
118
|
+
end
|
|
113
119
|
end # context
|
|
114
120
|
end # describe
|
|
115
121
|
end # module
|
|
@@ -212,6 +212,64 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
|
212
212
|
# Here we go...
|
|
213
213
|
subject.start
|
|
214
214
|
end
|
|
215
|
+
|
|
216
|
+
it 'should also visit in pre-order' do
|
|
217
|
+
# Reminder: parse tree structure is
|
|
218
|
+
# S[0,5]
|
|
219
|
+
# +- A[0,5]
|
|
220
|
+
# +- a[0,0]
|
|
221
|
+
# +- A[1,4]
|
|
222
|
+
# | +- a[1,1]
|
|
223
|
+
# | +- A[2,3]
|
|
224
|
+
# | | +- b[2,3]
|
|
225
|
+
# | +- c[3,4]
|
|
226
|
+
# +- c[4,5]
|
|
227
|
+
root = grm_abc_ptree1.root
|
|
228
|
+
# Here we defeat encapsulation for the good cause
|
|
229
|
+
subject.instance_variable_set(:@traversal, :pre_order)
|
|
230
|
+
|
|
231
|
+
children = root.subnodes
|
|
232
|
+
big_a_1 = children[0]
|
|
233
|
+
big_a_1_children = big_a_1.subnodes
|
|
234
|
+
big_a_2 = big_a_1_children[1]
|
|
235
|
+
big_a_2_children = big_a_2.subnodes
|
|
236
|
+
big_a_3 = big_a_2_children[1]
|
|
237
|
+
big_a_3_children = big_a_3.subnodes
|
|
238
|
+
expectations = [
|
|
239
|
+
[:before_ptree, [grm_abc_ptree1]],
|
|
240
|
+
# TODO: fix this test
|
|
241
|
+
#[:before_subnodes, [root, children]],
|
|
242
|
+
#[:before_non_terminal, [root]],
|
|
243
|
+
|
|
244
|
+
# [:before_non_terminal, [big_a_1]],
|
|
245
|
+
# [:before_subnodes, [big_a_1, big_a_1_children]],
|
|
246
|
+
# [:before_terminal, [big_a_1_children[0]]],
|
|
247
|
+
# [:after_terminal, [big_a_1_children[0]]],
|
|
248
|
+
# [:before_non_terminal, [big_a_2]],
|
|
249
|
+
# [:before_subnodes, [big_a_2, big_a_2_children]],
|
|
250
|
+
# [:before_terminal, [big_a_2_children[0]]],
|
|
251
|
+
# [:after_terminal, [big_a_2_children[0]]],
|
|
252
|
+
# [:before_non_terminal, [big_a_3]],
|
|
253
|
+
# [:before_subnodes, [big_a_3, big_a_3_children]],
|
|
254
|
+
# [:before_terminal, [big_a_3_children[0]]],
|
|
255
|
+
# [:after_terminal, [big_a_3_children[0]]],
|
|
256
|
+
# [:after_subnodes, [big_a_3, big_a_3_children]],
|
|
257
|
+
# [:before_terminal, [big_a_2_children[2]]],
|
|
258
|
+
# [:after_terminal, [big_a_2_children[2]]],
|
|
259
|
+
# [:after_subnodes, [big_a_2, big_a_2_children]],
|
|
260
|
+
# [:before_terminal, [big_a_1_children[2]]],
|
|
261
|
+
# [:after_terminal, [big_a_1_children[2]]],
|
|
262
|
+
# [:after_subnodes, [big_a_1, big_a_1_children]],
|
|
263
|
+
# [:after_subnodes, [root, children]],
|
|
264
|
+
# [:after_ptree, [grm_abc_ptree1]]
|
|
265
|
+
]
|
|
266
|
+
expectations.each do |(msg, args)|
|
|
267
|
+
expect(listener1).to receive(msg).with(*args).ordered
|
|
268
|
+
end
|
|
269
|
+
|
|
270
|
+
# Here we go...
|
|
271
|
+
subject.start
|
|
272
|
+
end
|
|
215
273
|
end # context
|
|
216
274
|
end # describe
|
|
217
275
|
end # module
|
|
@@ -31,6 +31,19 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
|
31
31
|
subject.push_state(state2)
|
|
32
32
|
expect(subject.states).to eq([state1, state2])
|
|
33
33
|
end
|
|
34
|
+
|
|
35
|
+
it 'should ignore a second push of a state' do
|
|
36
|
+
expect(subject.states).to be_empty
|
|
37
|
+
subject.push_state(state1)
|
|
38
|
+
subject.push_state(state2)
|
|
39
|
+
expect(subject.states).to eq([state1, state2])
|
|
40
|
+
|
|
41
|
+
# One tries to push an already pushed state
|
|
42
|
+
expect(subject.push_state(state1)).to be_falsy
|
|
43
|
+
|
|
44
|
+
# ...It is not added
|
|
45
|
+
expect(subject.states).to eq([state1, state2])
|
|
46
|
+
end
|
|
34
47
|
|
|
35
48
|
it 'should list the states expecting a given terminal' do
|
|
36
49
|
# Case of no state
|
|
@@ -25,6 +25,17 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
|
25
25
|
expect(subject.range).to eq(sample_range)
|
|
26
26
|
end
|
|
27
27
|
end # context
|
|
28
|
+
|
|
29
|
+
context 'Initialization:' do
|
|
30
|
+
it 'should assign undefined range bounds' do
|
|
31
|
+
partial_range = { low: 0 } # High bound left undefined
|
|
32
|
+
instance = ParseTreeNode.new(sample_symbol, partial_range)
|
|
33
|
+
|
|
34
|
+
another = { low: 1, high: 4 } # High bound is specified
|
|
35
|
+
instance.range = another
|
|
36
|
+
expect(instance.range).to eq({low: 0, high: 4})
|
|
37
|
+
end
|
|
38
|
+
end # context
|
|
28
39
|
end # describe
|
|
29
40
|
end # module
|
|
30
41
|
end # module
|
|
@@ -55,6 +55,10 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
|
55
55
|
it 'should have a string representation' do
|
|
56
56
|
expect(subject.to_string(0)).to eq('VP[0, 3]')
|
|
57
57
|
end
|
|
58
|
+
|
|
59
|
+
it 'should return a key value of itself' do
|
|
60
|
+
expect(subject.key).to eq('VP[0, 3]')
|
|
61
|
+
end
|
|
58
62
|
end # context
|
|
59
63
|
end # describe
|
|
60
64
|
end # module
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: rley
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.4.
|
|
4
|
+
version: 0.4.03
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Dimitri Geshef
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2017-04-
|
|
11
|
+
date: 2017-04-17 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: rake
|
|
@@ -142,6 +142,7 @@ files:
|
|
|
142
142
|
- lib/rley.rb
|
|
143
143
|
- lib/rley/constants.rb
|
|
144
144
|
- lib/rley/formatter/base_formatter.rb
|
|
145
|
+
- lib/rley/formatter/bracket_notation.rb
|
|
145
146
|
- lib/rley/formatter/debug.rb
|
|
146
147
|
- lib/rley/formatter/json.rb
|
|
147
148
|
- lib/rley/gfg/call_edge.rb
|
|
@@ -201,6 +202,7 @@ files:
|
|
|
201
202
|
- lib/rley/syntax/verbatim_symbol.rb
|
|
202
203
|
- lib/rley/tokens/token.rb
|
|
203
204
|
- lib/rley/tokens/token_range.rb
|
|
205
|
+
- spec/rley/formatter/bracket_notation_spec.rb
|
|
204
206
|
- spec/rley/formatter/debug_spec.rb
|
|
205
207
|
- spec/rley/formatter/json_spec.rb
|
|
206
208
|
- spec/rley/gfg/call_edge_spec.rb
|
|
@@ -291,6 +293,7 @@ signing_key:
|
|
|
291
293
|
specification_version: 4
|
|
292
294
|
summary: Ruby implementation of the Earley's parsing algorithm
|
|
293
295
|
test_files:
|
|
296
|
+
- spec/rley/formatter/bracket_notation_spec.rb
|
|
294
297
|
- spec/rley/formatter/debug_spec.rb
|
|
295
298
|
- spec/rley/formatter/json_spec.rb
|
|
296
299
|
- spec/rley/gfg/call_edge_spec.rb
|