rley 0.8.05 → 0.8.06
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -0
- data/examples/general/calc_iter2/sample_result.txt +60 -0
- data/examples/tokenizer/README.md +6 -1
- data/examples/tokenizer/loxxy_raw_scanner.rex +13 -2
- data/examples/tokenizer/loxxy_raw_scanner.rex.rb +2 -0
- data/examples/tokenizer/loxxy_tokenizer.rb +20 -1
- data/examples/tokenizer/run_tokenizer.rb +6 -2
- data/examples/tokenizer/tokens.yaml +249 -0
- data/lib/rley/constants.rb +1 -1
- metadata +4 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: e15e0d34742a0b6c95f3cc7c7d84020d6a486a1e47c2b3fb38bf33c42ff71ea9
|
|
4
|
+
data.tar.gz: 2a842d65101d8d833478f70023a4c5263b5dbb24b36799329ef82dffef70be37
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 78ee8ff8d729f52f710cf6c52eccb880c78a851b45e6a4df146aa1138f77275557c87ebc5b591896695d3ab8c9072d5b6a7aa32bf600fb14d09658fd834779fe
|
|
7
|
+
data.tar.gz: 2c0cfc070c05d0049f65cb938e03162a305fdab926a8e47d9825cf7a36841262a076407a9c6ac86685954a5b3b174137b24bfc7c976fee443d791d82346b1fe4
|
data/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,8 @@
|
|
|
1
|
+
### 0.8.06 / 2021-10-06
|
|
2
|
+
- Added more comment in `/examples/tokenizer` files
|
|
3
|
+
|
|
4
|
+
* [CHANGE] File `rley.gemspec` some files weren't put the gem files.
|
|
5
|
+
|
|
1
6
|
### 0.8.05 / 2021-10-04
|
|
2
7
|
- Added example of generated scanner. Unused class removal, code re-styling to please `Rubocop`
|
|
3
8
|
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
2 * 3 + (1 + 3 ** 2)
|
|
2
|
+
Concrete Syntax Tree (CST)
|
|
3
|
+
==========================
|
|
4
|
+
expression
|
|
5
|
+
+-- simple_expression
|
|
6
|
+
+-- simple_expression
|
|
7
|
+
| +-- term
|
|
8
|
+
| +-- term
|
|
9
|
+
| | +-- factor
|
|
10
|
+
| | +-- simple_factor
|
|
11
|
+
| | +-- sign
|
|
12
|
+
| | +-- NUMBER: '2'
|
|
13
|
+
| +-- mul_operator
|
|
14
|
+
| | +-- STAR: '*'
|
|
15
|
+
| +-- factor
|
|
16
|
+
| +-- simple_factor
|
|
17
|
+
| +-- sign
|
|
18
|
+
| +-- NUMBER: '3'
|
|
19
|
+
+-- add_operator
|
|
20
|
+
| +-- PLUS: '+'
|
|
21
|
+
+-- term
|
|
22
|
+
+-- factor
|
|
23
|
+
+-- simple_factor
|
|
24
|
+
+-- LPAREN: '('
|
|
25
|
+
+-- expression
|
|
26
|
+
| +-- simple_expression
|
|
27
|
+
| +-- simple_expression
|
|
28
|
+
| | +-- term
|
|
29
|
+
| | +-- factor
|
|
30
|
+
| | +-- simple_factor
|
|
31
|
+
| | +-- sign
|
|
32
|
+
| | +-- NUMBER: '1'
|
|
33
|
+
| +-- add_operator
|
|
34
|
+
| | +-- PLUS: '+'
|
|
35
|
+
| +-- term
|
|
36
|
+
| +-- factor
|
|
37
|
+
| +-- simple_factor
|
|
38
|
+
| | +-- sign
|
|
39
|
+
| | +-- NUMBER: '3'
|
|
40
|
+
| +-- POWER: '**'
|
|
41
|
+
| +-- simple_factor
|
|
42
|
+
| +-- sign
|
|
43
|
+
| +-- NUMBER: '2'
|
|
44
|
+
+-- RPAREN: ')'
|
|
45
|
+
|
|
46
|
+
Abstract Syntax Tree (AST)
|
|
47
|
+
==========================
|
|
48
|
+
PLUS
|
|
49
|
+
+-- STAR
|
|
50
|
+
| +-- NUMBER: '2'
|
|
51
|
+
| +-- NUMBER: '3'
|
|
52
|
+
+-- PLUS
|
|
53
|
+
+-- NUMBER: '1'
|
|
54
|
+
+-- POWER
|
|
55
|
+
+-- NUMBER: '3'
|
|
56
|
+
+-- NUMBER: '2'
|
|
57
|
+
|
|
58
|
+
Result:
|
|
59
|
+
=======
|
|
60
|
+
16
|
|
@@ -43,4 +43,9 @@ There were two reasons:
|
|
|
43
43
|
Without the flexbility of Ruby, handling keywords directly in the `rex` can become cumbersone.
|
|
44
44
|
- Second, the `LoxxyTokenizer` class acts as an Adapter between the parser-neutral generated scanner and the expectations of a Rley parser.
|
|
45
45
|
For instance, Rley expects the tokenizer to deliver a sequence of `Rley::Lexical::Token` instances.
|
|
46
|
-
In addition, that class performs some convertion methods that are better implemented directly in Ruby.
|
|
46
|
+
In addition, that class performs some convertion methods that are better implemented directly in Ruby.
|
|
47
|
+
|
|
48
|
+
## Can I see the tokenizer in action?
|
|
49
|
+
Sure, there is a little script `run_tokenizer.rb` that contains a Lox snippet.
|
|
50
|
+
The script creates a tokenizer, launches the lexical analysis of the snippet,
|
|
51
|
+
then saves the result in a YAML file.
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# As Rubocop shouts about "offences" in the generated code,
|
|
2
|
+
# we disable the detection of most of them...
|
|
1
3
|
# rubocop: disable Style/MutableConstant
|
|
2
4
|
# rubocop: disable Layout/SpaceBeforeSemicolon
|
|
3
5
|
# rubocop: disable Style/Alias
|
|
@@ -19,16 +21,20 @@
|
|
|
19
21
|
# rubocop: disable Layout/EmptyLineBetweenDefs
|
|
20
22
|
# rubocop: disable Layout/IndentationConsistency
|
|
21
23
|
|
|
24
|
+
# The scanner for the Lox programming language to generate.
|
|
22
25
|
class LoxxyRawScanner
|
|
23
26
|
option
|
|
24
|
-
lineno
|
|
25
|
-
column
|
|
27
|
+
lineno # Option to generate line number handling
|
|
28
|
+
column # Option to generate column number handling
|
|
26
29
|
|
|
30
|
+
# Macros in `oedipus_lex` define name regexps that can be reused
|
|
31
|
+
# via interpolation in other macros of rule patterns
|
|
27
32
|
macro
|
|
28
33
|
DIGIT /\d/
|
|
29
34
|
ALPHA /[a-zA-Z_]/
|
|
30
35
|
|
|
31
36
|
rule
|
|
37
|
+
# Rule syntax: state? regex (block|method)?
|
|
32
38
|
# Delimiters, punctuators, operators
|
|
33
39
|
/[ \t]+/
|
|
34
40
|
/\/\/[^\r\n]*/
|
|
@@ -50,6 +56,8 @@ rule
|
|
|
50
56
|
|
|
51
57
|
inner
|
|
52
58
|
|
|
59
|
+
# Method called in `parse` method.
|
|
60
|
+
# @return [Array<Array>]
|
|
53
61
|
def do_parse
|
|
54
62
|
tokens = []
|
|
55
63
|
while (tok = next_token) do
|
|
@@ -65,6 +73,9 @@ inner
|
|
|
65
73
|
tokens
|
|
66
74
|
end
|
|
67
75
|
|
|
76
|
+
# Increment the line number in case the \r\n? occurs.
|
|
77
|
+
# Generated code works correctly with Linux end-of-line only.
|
|
78
|
+
# @param txt [String]
|
|
68
79
|
def newline(txt)
|
|
69
80
|
if txt == '\r'
|
|
70
81
|
ss.skip(/\n/) # CR LF sequence
|
|
@@ -3,13 +3,22 @@
|
|
|
3
3
|
require 'rley'
|
|
4
4
|
require_relative 'loxxy_raw_scanner.rex'
|
|
5
5
|
|
|
6
|
+
# Tokenizer for the Lox language that is compatible with a Rley parser.
|
|
7
|
+
# It works as an adapter between the parser and the scanner generated
|
|
8
|
+
# with `oedipus_lex` scanner generator.
|
|
9
|
+
# The purpose is to transform the output of a `LoxxyRawScanner` instance
|
|
10
|
+
# into o sequence of tokens in the format expected by Rley.
|
|
6
11
|
class LoxxyTokenizer
|
|
7
12
|
# @return [LoxxyRawScanner] Scanner generated by `oedipus_lex`gem.
|
|
8
13
|
attr_reader :scanner
|
|
9
14
|
|
|
10
|
-
|
|
15
|
+
# @return [String] Lox input text to tokenize
|
|
11
16
|
attr_reader :input
|
|
12
17
|
|
|
18
|
+
# A Hash that maps a Lox keyword to its uppercase version
|
|
19
|
+
# (a convention used in the grammar).
|
|
20
|
+
# If a search is unsuccessful, it returns the string 'IDENTIFIER'
|
|
21
|
+
# @return [{String => String}]
|
|
13
22
|
Keyword2name = begin
|
|
14
23
|
lookup = %w[
|
|
15
24
|
and class else false fun for if nil or
|
|
@@ -19,6 +28,8 @@ class LoxxyTokenizer
|
|
|
19
28
|
lookup.freeze
|
|
20
29
|
end
|
|
21
30
|
|
|
31
|
+
# A Hash that maps a special character of Lox into a name
|
|
32
|
+
# @return [{String => String}]
|
|
22
33
|
Special2name = {
|
|
23
34
|
'(' => 'LEFT_PAREN',
|
|
24
35
|
')' => 'RIGHT_PAREN',
|
|
@@ -41,15 +52,22 @@ class LoxxyTokenizer
|
|
|
41
52
|
'<=' => 'LESS_EQUAL'
|
|
42
53
|
}.freeze
|
|
43
54
|
|
|
55
|
+
# Constructor
|
|
56
|
+
# @param source [String, NilClass] Optional. The text to tokenize
|
|
44
57
|
def initialize(source = nil)
|
|
45
58
|
@scanner = LoxxyRawScanner.new
|
|
46
59
|
start_with(source)
|
|
47
60
|
end
|
|
48
61
|
|
|
62
|
+
# Set the text to tokenize.
|
|
63
|
+
# @param source [String] The text to tokenize
|
|
49
64
|
def start_with(source)
|
|
50
65
|
@input = source
|
|
51
66
|
end
|
|
52
67
|
|
|
68
|
+
# Launch the scanning/tokenization of input text.
|
|
69
|
+
# Returns an array of Rley token objects.
|
|
70
|
+
# @return [Array<Rley::Lexical::Token>] array of token objects.
|
|
53
71
|
def tokens
|
|
54
72
|
raw_tokens = scanner.parse(input)
|
|
55
73
|
cooked = raw_tokens.map do |(raw_type, raw_text, line, col)|
|
|
@@ -65,6 +83,7 @@ class LoxxyTokenizer
|
|
|
65
83
|
|
|
66
84
|
private
|
|
67
85
|
|
|
86
|
+
# Convert the raw tokens from generated scanner into something edible by Rley.
|
|
68
87
|
def convert(token_kind, token_text, pos)
|
|
69
88
|
result = case token_kind
|
|
70
89
|
when :SPECIAL
|
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
# Demo script that illustrates how to scan a Lox code snippet
|
|
4
|
+
# into a stream of Rley token objects & serialize them into a YAML file.
|
|
5
|
+
|
|
3
6
|
require 'yaml'
|
|
4
7
|
require_relative 'loxxy_tokenizer'
|
|
5
8
|
|
|
9
|
+
# Here is a Lox code snippet
|
|
6
10
|
lox_source = <<LOX_END
|
|
7
11
|
class Base {
|
|
8
12
|
foo() {
|
|
@@ -22,8 +26,8 @@ Derived().foo();
|
|
|
22
26
|
// expect: Base.foo()
|
|
23
27
|
LOX_END
|
|
24
28
|
|
|
25
|
-
|
|
26
|
-
loxxy_tokenizer.
|
|
29
|
+
|
|
30
|
+
loxxy_tokenizer = LoxxyTokenizer.new(lox_source)
|
|
27
31
|
tokens = loxxy_tokenizer.tokens
|
|
28
32
|
File::open('tokens.yaml', 'w') { |f| YAML.dump(tokens, f) }
|
|
29
33
|
puts 'Done: tokenizer results saved in YAML.'
|
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
---
|
|
2
|
+
- !ruby/object:Rley::Lexical::Token
|
|
3
|
+
lexeme: class
|
|
4
|
+
terminal: CLASS
|
|
5
|
+
position: !ruby/struct:Rley::Lexical::Position
|
|
6
|
+
line: 1
|
|
7
|
+
column: 1
|
|
8
|
+
- !ruby/object:Rley::Lexical::Token
|
|
9
|
+
lexeme: Base
|
|
10
|
+
terminal: IDENTIFIER
|
|
11
|
+
position: !ruby/struct:Rley::Lexical::Position
|
|
12
|
+
line: 1
|
|
13
|
+
column: 7
|
|
14
|
+
- !ruby/object:Rley::Lexical::Token
|
|
15
|
+
lexeme: "{"
|
|
16
|
+
terminal: LEFT_BRACE
|
|
17
|
+
position: !ruby/struct:Rley::Lexical::Position
|
|
18
|
+
line: 1
|
|
19
|
+
column: 12
|
|
20
|
+
- !ruby/object:Rley::Lexical::Token
|
|
21
|
+
lexeme: foo
|
|
22
|
+
terminal: IDENTIFIER
|
|
23
|
+
position: !ruby/struct:Rley::Lexical::Position
|
|
24
|
+
line: 2
|
|
25
|
+
column: 3
|
|
26
|
+
- !ruby/object:Rley::Lexical::Token
|
|
27
|
+
lexeme: "("
|
|
28
|
+
terminal: LEFT_PAREN
|
|
29
|
+
position: !ruby/struct:Rley::Lexical::Position
|
|
30
|
+
line: 2
|
|
31
|
+
column: 6
|
|
32
|
+
- !ruby/object:Rley::Lexical::Token
|
|
33
|
+
lexeme: ")"
|
|
34
|
+
terminal: RIGHT_PAREN
|
|
35
|
+
position: !ruby/struct:Rley::Lexical::Position
|
|
36
|
+
line: 2
|
|
37
|
+
column: 7
|
|
38
|
+
- !ruby/object:Rley::Lexical::Token
|
|
39
|
+
lexeme: "{"
|
|
40
|
+
terminal: LEFT_BRACE
|
|
41
|
+
position: !ruby/struct:Rley::Lexical::Position
|
|
42
|
+
line: 2
|
|
43
|
+
column: 9
|
|
44
|
+
- !ruby/object:Rley::Lexical::Token
|
|
45
|
+
lexeme: print
|
|
46
|
+
terminal: PRINT
|
|
47
|
+
position: !ruby/struct:Rley::Lexical::Position
|
|
48
|
+
line: 3
|
|
49
|
+
column: 5
|
|
50
|
+
- !ruby/object:Rley::Lexical::Literal
|
|
51
|
+
lexeme: '"Base.foo()"'
|
|
52
|
+
terminal: STRING
|
|
53
|
+
position: !ruby/struct:Rley::Lexical::Position
|
|
54
|
+
line: 3
|
|
55
|
+
column: 11
|
|
56
|
+
value: Base.foo()
|
|
57
|
+
- !ruby/object:Rley::Lexical::Token
|
|
58
|
+
lexeme: ";"
|
|
59
|
+
terminal: SEMICOLON
|
|
60
|
+
position: !ruby/struct:Rley::Lexical::Position
|
|
61
|
+
line: 3
|
|
62
|
+
column: 23
|
|
63
|
+
- !ruby/object:Rley::Lexical::Token
|
|
64
|
+
lexeme: "}"
|
|
65
|
+
terminal: RIGHT_BRACE
|
|
66
|
+
position: !ruby/struct:Rley::Lexical::Position
|
|
67
|
+
line: 4
|
|
68
|
+
column: 3
|
|
69
|
+
- !ruby/object:Rley::Lexical::Token
|
|
70
|
+
lexeme: "}"
|
|
71
|
+
terminal: RIGHT_BRACE
|
|
72
|
+
position: !ruby/struct:Rley::Lexical::Position
|
|
73
|
+
line: 5
|
|
74
|
+
column: 1
|
|
75
|
+
- !ruby/object:Rley::Lexical::Token
|
|
76
|
+
lexeme: class
|
|
77
|
+
terminal: CLASS
|
|
78
|
+
position: !ruby/struct:Rley::Lexical::Position
|
|
79
|
+
line: 7
|
|
80
|
+
column: 1
|
|
81
|
+
- !ruby/object:Rley::Lexical::Token
|
|
82
|
+
lexeme: Derived
|
|
83
|
+
terminal: IDENTIFIER
|
|
84
|
+
position: !ruby/struct:Rley::Lexical::Position
|
|
85
|
+
line: 7
|
|
86
|
+
column: 7
|
|
87
|
+
- !ruby/object:Rley::Lexical::Token
|
|
88
|
+
lexeme: "<"
|
|
89
|
+
terminal: LESS
|
|
90
|
+
position: !ruby/struct:Rley::Lexical::Position
|
|
91
|
+
line: 7
|
|
92
|
+
column: 15
|
|
93
|
+
- !ruby/object:Rley::Lexical::Token
|
|
94
|
+
lexeme: Base
|
|
95
|
+
terminal: IDENTIFIER
|
|
96
|
+
position: !ruby/struct:Rley::Lexical::Position
|
|
97
|
+
line: 7
|
|
98
|
+
column: 17
|
|
99
|
+
- !ruby/object:Rley::Lexical::Token
|
|
100
|
+
lexeme: "{"
|
|
101
|
+
terminal: LEFT_BRACE
|
|
102
|
+
position: !ruby/struct:Rley::Lexical::Position
|
|
103
|
+
line: 7
|
|
104
|
+
column: 22
|
|
105
|
+
- !ruby/object:Rley::Lexical::Token
|
|
106
|
+
lexeme: foo
|
|
107
|
+
terminal: IDENTIFIER
|
|
108
|
+
position: !ruby/struct:Rley::Lexical::Position
|
|
109
|
+
line: 8
|
|
110
|
+
column: 3
|
|
111
|
+
- !ruby/object:Rley::Lexical::Token
|
|
112
|
+
lexeme: "("
|
|
113
|
+
terminal: LEFT_PAREN
|
|
114
|
+
position: !ruby/struct:Rley::Lexical::Position
|
|
115
|
+
line: 8
|
|
116
|
+
column: 6
|
|
117
|
+
- !ruby/object:Rley::Lexical::Token
|
|
118
|
+
lexeme: ")"
|
|
119
|
+
terminal: RIGHT_PAREN
|
|
120
|
+
position: !ruby/struct:Rley::Lexical::Position
|
|
121
|
+
line: 8
|
|
122
|
+
column: 7
|
|
123
|
+
- !ruby/object:Rley::Lexical::Token
|
|
124
|
+
lexeme: "{"
|
|
125
|
+
terminal: LEFT_BRACE
|
|
126
|
+
position: !ruby/struct:Rley::Lexical::Position
|
|
127
|
+
line: 8
|
|
128
|
+
column: 9
|
|
129
|
+
- !ruby/object:Rley::Lexical::Token
|
|
130
|
+
lexeme: print
|
|
131
|
+
terminal: PRINT
|
|
132
|
+
position: !ruby/struct:Rley::Lexical::Position
|
|
133
|
+
line: 9
|
|
134
|
+
column: 5
|
|
135
|
+
- !ruby/object:Rley::Lexical::Literal
|
|
136
|
+
lexeme: '"Derived.foo()"'
|
|
137
|
+
terminal: STRING
|
|
138
|
+
position: !ruby/struct:Rley::Lexical::Position
|
|
139
|
+
line: 9
|
|
140
|
+
column: 11
|
|
141
|
+
value: Derived.foo()
|
|
142
|
+
- !ruby/object:Rley::Lexical::Token
|
|
143
|
+
lexeme: ";"
|
|
144
|
+
terminal: SEMICOLON
|
|
145
|
+
position: !ruby/struct:Rley::Lexical::Position
|
|
146
|
+
line: 9
|
|
147
|
+
column: 26
|
|
148
|
+
- !ruby/object:Rley::Lexical::Token
|
|
149
|
+
lexeme: super
|
|
150
|
+
terminal: SUPER
|
|
151
|
+
position: !ruby/struct:Rley::Lexical::Position
|
|
152
|
+
line: 10
|
|
153
|
+
column: 5
|
|
154
|
+
- !ruby/object:Rley::Lexical::Token
|
|
155
|
+
lexeme: "."
|
|
156
|
+
terminal: DOT
|
|
157
|
+
position: !ruby/struct:Rley::Lexical::Position
|
|
158
|
+
line: 10
|
|
159
|
+
column: 10
|
|
160
|
+
- !ruby/object:Rley::Lexical::Token
|
|
161
|
+
lexeme: foo
|
|
162
|
+
terminal: IDENTIFIER
|
|
163
|
+
position: !ruby/struct:Rley::Lexical::Position
|
|
164
|
+
line: 10
|
|
165
|
+
column: 11
|
|
166
|
+
- !ruby/object:Rley::Lexical::Token
|
|
167
|
+
lexeme: "("
|
|
168
|
+
terminal: LEFT_PAREN
|
|
169
|
+
position: !ruby/struct:Rley::Lexical::Position
|
|
170
|
+
line: 10
|
|
171
|
+
column: 14
|
|
172
|
+
- !ruby/object:Rley::Lexical::Token
|
|
173
|
+
lexeme: ")"
|
|
174
|
+
terminal: RIGHT_PAREN
|
|
175
|
+
position: !ruby/struct:Rley::Lexical::Position
|
|
176
|
+
line: 10
|
|
177
|
+
column: 15
|
|
178
|
+
- !ruby/object:Rley::Lexical::Token
|
|
179
|
+
lexeme: ";"
|
|
180
|
+
terminal: SEMICOLON
|
|
181
|
+
position: !ruby/struct:Rley::Lexical::Position
|
|
182
|
+
line: 10
|
|
183
|
+
column: 16
|
|
184
|
+
- !ruby/object:Rley::Lexical::Token
|
|
185
|
+
lexeme: "}"
|
|
186
|
+
terminal: RIGHT_BRACE
|
|
187
|
+
position: !ruby/struct:Rley::Lexical::Position
|
|
188
|
+
line: 11
|
|
189
|
+
column: 3
|
|
190
|
+
- !ruby/object:Rley::Lexical::Token
|
|
191
|
+
lexeme: "}"
|
|
192
|
+
terminal: RIGHT_BRACE
|
|
193
|
+
position: !ruby/struct:Rley::Lexical::Position
|
|
194
|
+
line: 12
|
|
195
|
+
column: 1
|
|
196
|
+
- !ruby/object:Rley::Lexical::Token
|
|
197
|
+
lexeme: Derived
|
|
198
|
+
terminal: IDENTIFIER
|
|
199
|
+
position: !ruby/struct:Rley::Lexical::Position
|
|
200
|
+
line: 14
|
|
201
|
+
column: 1
|
|
202
|
+
- !ruby/object:Rley::Lexical::Token
|
|
203
|
+
lexeme: "("
|
|
204
|
+
terminal: LEFT_PAREN
|
|
205
|
+
position: !ruby/struct:Rley::Lexical::Position
|
|
206
|
+
line: 14
|
|
207
|
+
column: 8
|
|
208
|
+
- !ruby/object:Rley::Lexical::Token
|
|
209
|
+
lexeme: ")"
|
|
210
|
+
terminal: RIGHT_PAREN
|
|
211
|
+
position: !ruby/struct:Rley::Lexical::Position
|
|
212
|
+
line: 14
|
|
213
|
+
column: 9
|
|
214
|
+
- !ruby/object:Rley::Lexical::Token
|
|
215
|
+
lexeme: "."
|
|
216
|
+
terminal: DOT
|
|
217
|
+
position: !ruby/struct:Rley::Lexical::Position
|
|
218
|
+
line: 14
|
|
219
|
+
column: 10
|
|
220
|
+
- !ruby/object:Rley::Lexical::Token
|
|
221
|
+
lexeme: foo
|
|
222
|
+
terminal: IDENTIFIER
|
|
223
|
+
position: !ruby/struct:Rley::Lexical::Position
|
|
224
|
+
line: 14
|
|
225
|
+
column: 11
|
|
226
|
+
- !ruby/object:Rley::Lexical::Token
|
|
227
|
+
lexeme: "("
|
|
228
|
+
terminal: LEFT_PAREN
|
|
229
|
+
position: !ruby/struct:Rley::Lexical::Position
|
|
230
|
+
line: 14
|
|
231
|
+
column: 14
|
|
232
|
+
- !ruby/object:Rley::Lexical::Token
|
|
233
|
+
lexeme: ")"
|
|
234
|
+
terminal: RIGHT_PAREN
|
|
235
|
+
position: !ruby/struct:Rley::Lexical::Position
|
|
236
|
+
line: 14
|
|
237
|
+
column: 15
|
|
238
|
+
- !ruby/object:Rley::Lexical::Token
|
|
239
|
+
lexeme: ";"
|
|
240
|
+
terminal: SEMICOLON
|
|
241
|
+
position: !ruby/struct:Rley::Lexical::Position
|
|
242
|
+
line: 14
|
|
243
|
+
column: 16
|
|
244
|
+
- !ruby/object:Rley::Lexical::Token
|
|
245
|
+
lexeme:
|
|
246
|
+
terminal: EOF
|
|
247
|
+
position: !ruby/struct:Rley::Lexical::Position
|
|
248
|
+
line: 14
|
|
249
|
+
column: 17
|
data/lib/rley/constants.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: rley
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.8.
|
|
4
|
+
version: 0.8.06
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Dimitri Geshef
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2021-10-
|
|
11
|
+
date: 2021-10-06 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: rake
|
|
@@ -118,6 +118,7 @@ files:
|
|
|
118
118
|
- examples/general/calc_iter2/calc_demo.rb
|
|
119
119
|
- examples/general/calc_iter2/calc_grammar.rb
|
|
120
120
|
- examples/general/calc_iter2/calc_lexer.rb
|
|
121
|
+
- examples/general/calc_iter2/sample_result.txt
|
|
121
122
|
- examples/general/calc_iter2/spec/calculator_spec.rb
|
|
122
123
|
- examples/general/general_examples.md
|
|
123
124
|
- examples/general/left.rb
|
|
@@ -127,6 +128,7 @@ files:
|
|
|
127
128
|
- examples/tokenizer/loxxy_raw_scanner.rex.rb
|
|
128
129
|
- examples/tokenizer/loxxy_tokenizer.rb
|
|
129
130
|
- examples/tokenizer/run_tokenizer.rb
|
|
131
|
+
- examples/tokenizer/tokens.yaml
|
|
130
132
|
- lib/rley.rb
|
|
131
133
|
- lib/rley/base/base_parser.rb
|
|
132
134
|
- lib/rley/base/dotted_item.rb
|