rley 0.8.05 → 0.8.06

Sign up to get free protection for your applications and to get access to all the features.
Files changed (11) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +5 -0
  3. data/examples/general/calc_iter2/sample_result.txt +60 -0
  4. data/examples/tokenizer/README.md +6 -1
  5. data/examples/tokenizer/loxxy_raw_scanner.rex +13 -2
  6. data/examples/tokenizer/loxxy_raw_scanner.rex.rb +2 -0
  7. data/examples/tokenizer/loxxy_tokenizer.rb +20 -1
  8. data/examples/tokenizer/run_tokenizer.rb +6 -2
  9. data/examples/tokenizer/tokens.yaml +249 -0
  10. data/lib/rley/constants.rb +1 -1
  11. metadata +4 -2
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: a94c23265bf2996856da4d4b1fc6f2749873626951d30bacec9f1ca7619d4d4f
4
- data.tar.gz: fc6a6ae19388b532dc711915a9c33ffc455a602ea9e6a1ae13e53522bc0616d0
3
+ metadata.gz: e15e0d34742a0b6c95f3cc7c7d84020d6a486a1e47c2b3fb38bf33c42ff71ea9
4
+ data.tar.gz: 2a842d65101d8d833478f70023a4c5263b5dbb24b36799329ef82dffef70be37
5
5
  SHA512:
6
- metadata.gz: 9b25f6a34e4e5603d4b88f5cf3220e998d9e0c38ffef204ebbf4a8eacaf57853c2bb9bc2bb5a3765440aaed41ee5a4883edadc533c6df0dc907b58457b085469
7
- data.tar.gz: 57c3021f24eb5764fef2be6bdd58a9cda1c9b9343737c357d6c1e1862beceaf041ed9eee9fb625cf9acc9492bac0fcb6a6ac8cd9dc464dcf2096fccd5b51e6e9
6
+ metadata.gz: 78ee8ff8d729f52f710cf6c52eccb880c78a851b45e6a4df146aa1138f77275557c87ebc5b591896695d3ab8c9072d5b6a7aa32bf600fb14d09658fd834779fe
7
+ data.tar.gz: 2c0cfc070c05d0049f65cb938e03162a305fdab926a8e47d9825cf7a36841262a076407a9c6ac86685954a5b3b174137b24bfc7c976fee443d791d82346b1fe4
data/CHANGELOG.md CHANGED
@@ -1,3 +1,8 @@
1
+ ### 0.8.06 / 2021-10-06
2
+ - Added more comment in `/examples/tokenizer` files
3
+
4
+ * [CHANGE] File `rley.gemspec` some files weren't put the gem files.
5
+
1
6
  ### 0.8.05 / 2021-10-04
2
7
  - Added example of generated scanner. Unused class removal, code re-styling to please `Rubocop`
3
8
 
data/examples/general/calc_iter2/sample_result.txt ADDED
@@ -0,0 +1,60 @@
1
+ 2 * 3 + (1 + 3 ** 2)
2
+ Concrete Syntax Tree (CST)
3
+ ==========================
4
+ expression
5
+ +-- simple_expression
6
+ +-- simple_expression
7
+ | +-- term
8
+ | +-- term
9
+ | | +-- factor
10
+ | | +-- simple_factor
11
+ | | +-- sign
12
+ | | +-- NUMBER: '2'
13
+ | +-- mul_operator
14
+ | | +-- STAR: '*'
15
+ | +-- factor
16
+ | +-- simple_factor
17
+ | +-- sign
18
+ | +-- NUMBER: '3'
19
+ +-- add_operator
20
+ | +-- PLUS: '+'
21
+ +-- term
22
+ +-- factor
23
+ +-- simple_factor
24
+ +-- LPAREN: '('
25
+ +-- expression
26
+ | +-- simple_expression
27
+ | +-- simple_expression
28
+ | | +-- term
29
+ | | +-- factor
30
+ | | +-- simple_factor
31
+ | | +-- sign
32
+ | | +-- NUMBER: '1'
33
+ | +-- add_operator
34
+ | | +-- PLUS: '+'
35
+ | +-- term
36
+ | +-- factor
37
+ | +-- simple_factor
38
+ | | +-- sign
39
+ | | +-- NUMBER: '3'
40
+ | +-- POWER: '**'
41
+ | +-- simple_factor
42
+ | +-- sign
43
+ | +-- NUMBER: '2'
44
+ +-- RPAREN: ')'
45
+
46
+ Abstract Syntax Tree (AST)
47
+ ==========================
48
+ PLUS
49
+ +-- STAR
50
+ | +-- NUMBER: '2'
51
+ | +-- NUMBER: '3'
52
+ +-- PLUS
53
+ +-- NUMBER: '1'
54
+ +-- POWER
55
+ +-- NUMBER: '3'
56
+ +-- NUMBER: '2'
57
+
58
+ Result:
59
+ =======
60
+ 16
data/examples/tokenizer/README.md CHANGED
@@ -43,4 +43,9 @@ There were two reasons:
43
43
  Without the flexbility of Ruby, handling keywords directly in the `rex` can become cumbersone.
44
44
  - Second, the `LoxxyTokenizer` class acts as an Adapter between the parser-neutral generated scanner and the expectations of a Rley parser.
45
45
  For instance, Rley expects the tokenizer to deliver a sequence of `Rley::Lexical::Token` instances.
46
- In addition, that class performs some convertion methods that are better implemented directly in Ruby.
46
+ In addition, that class performs some convertion methods that are better implemented directly in Ruby.
47
+
48
+ ## Can I see the tokenizer in action?
49
+ Sure, there is a little script `run_tokenizer.rb` that contains a Lox snippet.
50
+ The script creates a tokenizer, launches the lexical analysis of the snippet,
51
+ then saves the result in a YAML file.
data/examples/tokenizer/loxxy_raw_scanner.rex CHANGED
@@ -1,3 +1,5 @@
1
+ # As Rubocop shouts about "offences" in the generated code,
2
+ # we disable the detection of most of them...
1
3
  # rubocop: disable Style/MutableConstant
2
4
  # rubocop: disable Layout/SpaceBeforeSemicolon
3
5
  # rubocop: disable Style/Alias
@@ -19,16 +21,20 @@
19
21
  # rubocop: disable Layout/EmptyLineBetweenDefs
20
22
  # rubocop: disable Layout/IndentationConsistency
21
23
 
24
+ # The scanner for the Lox programming language to generate.
22
25
  class LoxxyRawScanner
23
26
  option
24
- lineno
25
- column
27
+ lineno # Option to generate line number handling
28
+ column # Option to generate column number handling
26
29
 
30
+ # Macros in `oedipus_lex` define name regexps that can be reused
31
+ # via interpolation in other macros of rule patterns
27
32
  macro
28
33
  DIGIT /\d/
29
34
  ALPHA /[a-zA-Z_]/
30
35
 
31
36
  rule
37
+ # Rule syntax: state? regex (block|method)?
32
38
  # Delimiters, punctuators, operators
33
39
  /[ \t]+/
34
40
  /\/\/[^\r\n]*/
@@ -50,6 +56,8 @@ rule
50
56
 
51
57
  inner
52
58
 
59
+ # Method called in `parse` method.
60
+ # @return [Array<Array>]
53
61
  def do_parse
54
62
  tokens = []
55
63
  while (tok = next_token) do
@@ -65,6 +73,9 @@ inner
65
73
  tokens
66
74
  end
67
75
 
76
+ # Increment the line number in case the \r\n? occurs.
77
+ # Generated code works correctly with Linux end-of-line only.
78
+ # @param txt [String]
68
79
  def newline(txt)
69
80
  if txt == '\r'
70
81
  ss.skip(/\n/) # CR LF sequence
data/examples/tokenizer/loxxy_raw_scanner.rex.rb CHANGED
@@ -28,6 +28,8 @@
28
28
  # rubocop: disable Layout/EmptyLineBetweenDefs
29
29
  # rubocop: disable Layout/IndentationConsistency
30
30
 
31
+ # The scanner for the Lox programming language to generate.
32
+
31
33
 
32
34
  ##
33
35
  # The generated lexer LoxxyRawScanner
data/examples/tokenizer/loxxy_tokenizer.rb CHANGED
@@ -3,13 +3,22 @@
3
3
  require 'rley'
4
4
  require_relative 'loxxy_raw_scanner.rex'
5
5
 
6
+ # Tokenizer for the Lox language that is compatible with a Rley parser.
7
+ # It works as an adapter between the parser and the scanner generated
8
+ # with `oedipus_lex` scanner generator.
9
+ # The purpose is to transform the output of a `LoxxyRawScanner` instance
10
+ # into o sequence of tokens in the format expected by Rley.
6
11
  class LoxxyTokenizer
7
12
  # @return [LoxxyRawScanner] Scanner generated by `oedipus_lex`gem.
8
13
  attr_reader :scanner
9
14
 
10
- # @return [String] Input text to tokenize
15
+ # @return [String] Lox input text to tokenize
11
16
  attr_reader :input
12
17
 
18
+ # A Hash that maps a Lox keyword to its uppercase version
19
+ # (a convention used in the grammar).
20
+ # If a search is unsuccessful, it returns the string 'IDENTIFIER'
21
+ # @return [{String => String}]
13
22
  Keyword2name = begin
14
23
  lookup = %w[
15
24
  and class else false fun for if nil or
@@ -19,6 +28,8 @@ class LoxxyTokenizer
19
28
  lookup.freeze
20
29
  end
21
30
 
31
+ # A Hash that maps a special character of Lox into a name
32
+ # @return [{String => String}]
22
33
  Special2name = {
23
34
  '(' => 'LEFT_PAREN',
24
35
  ')' => 'RIGHT_PAREN',
@@ -41,15 +52,22 @@ class LoxxyTokenizer
41
52
  '<=' => 'LESS_EQUAL'
42
53
  }.freeze
43
54
 
55
+ # Constructor
56
+ # @param source [String, NilClass] Optional. The text to tokenize
44
57
  def initialize(source = nil)
45
58
  @scanner = LoxxyRawScanner.new
46
59
  start_with(source)
47
60
  end
48
61
 
62
+ # Set the text to tokenize.
63
+ # @param source [String] The text to tokenize
49
64
  def start_with(source)
50
65
  @input = source
51
66
  end
52
67
 
68
+ # Launch the scanning/tokenization of input text.
69
+ # Returns an array of Rley token objects.
70
+ # @return [Array<Rley::Lexical::Token>] array of token objects.
53
71
  def tokens
54
72
  raw_tokens = scanner.parse(input)
55
73
  cooked = raw_tokens.map do |(raw_type, raw_text, line, col)|
@@ -65,6 +83,7 @@ class LoxxyTokenizer
65
83
 
66
84
  private
67
85
 
86
+ # Convert the raw tokens from generated scanner into something edible by Rley.
68
87
  def convert(token_kind, token_text, pos)
69
88
  result = case token_kind
70
89
  when :SPECIAL
data/examples/tokenizer/run_tokenizer.rb CHANGED
@@ -1,8 +1,12 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # Demo script that illustrates how to scan a Lox code snippet
4
+ # into a stream of Rley token objects & serialize them into a YAML file.
5
+
3
6
  require 'yaml'
4
7
  require_relative 'loxxy_tokenizer'
5
8
 
9
+ # Here is a Lox code snippet
6
10
  lox_source = <<LOX_END
7
11
  class Base {
8
12
  foo() {
@@ -22,8 +26,8 @@ Derived().foo();
22
26
  // expect: Base.foo()
23
27
  LOX_END
24
28
 
25
- loxxy_tokenizer = LoxxyTokenizer.new
26
- loxxy_tokenizer.start_with(lox_source)
29
+
30
+ loxxy_tokenizer = LoxxyTokenizer.new(lox_source)
27
31
  tokens = loxxy_tokenizer.tokens
28
32
  File::open('tokens.yaml', 'w') { |f| YAML.dump(tokens, f) }
29
33
  puts 'Done: tokenizer results saved in YAML.'
data/examples/tokenizer/tokens.yaml ADDED
@@ -0,0 +1,249 @@
1
+ ---
2
+ - !ruby/object:Rley::Lexical::Token
3
+ lexeme: class
4
+ terminal: CLASS
5
+ position: !ruby/struct:Rley::Lexical::Position
6
+ line: 1
7
+ column: 1
8
+ - !ruby/object:Rley::Lexical::Token
9
+ lexeme: Base
10
+ terminal: IDENTIFIER
11
+ position: !ruby/struct:Rley::Lexical::Position
12
+ line: 1
13
+ column: 7
14
+ - !ruby/object:Rley::Lexical::Token
15
+ lexeme: "{"
16
+ terminal: LEFT_BRACE
17
+ position: !ruby/struct:Rley::Lexical::Position
18
+ line: 1
19
+ column: 12
20
+ - !ruby/object:Rley::Lexical::Token
21
+ lexeme: foo
22
+ terminal: IDENTIFIER
23
+ position: !ruby/struct:Rley::Lexical::Position
24
+ line: 2
25
+ column: 3
26
+ - !ruby/object:Rley::Lexical::Token
27
+ lexeme: "("
28
+ terminal: LEFT_PAREN
29
+ position: !ruby/struct:Rley::Lexical::Position
30
+ line: 2
31
+ column: 6
32
+ - !ruby/object:Rley::Lexical::Token
33
+ lexeme: ")"
34
+ terminal: RIGHT_PAREN
35
+ position: !ruby/struct:Rley::Lexical::Position
36
+ line: 2
37
+ column: 7
38
+ - !ruby/object:Rley::Lexical::Token
39
+ lexeme: "{"
40
+ terminal: LEFT_BRACE
41
+ position: !ruby/struct:Rley::Lexical::Position
42
+ line: 2
43
+ column: 9
44
+ - !ruby/object:Rley::Lexical::Token
45
+ lexeme: print
46
+ terminal: PRINT
47
+ position: !ruby/struct:Rley::Lexical::Position
48
+ line: 3
49
+ column: 5
50
+ - !ruby/object:Rley::Lexical::Literal
51
+ lexeme: '"Base.foo()"'
52
+ terminal: STRING
53
+ position: !ruby/struct:Rley::Lexical::Position
54
+ line: 3
55
+ column: 11
56
+ value: Base.foo()
57
+ - !ruby/object:Rley::Lexical::Token
58
+ lexeme: ";"
59
+ terminal: SEMICOLON
60
+ position: !ruby/struct:Rley::Lexical::Position
61
+ line: 3
62
+ column: 23
63
+ - !ruby/object:Rley::Lexical::Token
64
+ lexeme: "}"
65
+ terminal: RIGHT_BRACE
66
+ position: !ruby/struct:Rley::Lexical::Position
67
+ line: 4
68
+ column: 3
69
+ - !ruby/object:Rley::Lexical::Token
70
+ lexeme: "}"
71
+ terminal: RIGHT_BRACE
72
+ position: !ruby/struct:Rley::Lexical::Position
73
+ line: 5
74
+ column: 1
75
+ - !ruby/object:Rley::Lexical::Token
76
+ lexeme: class
77
+ terminal: CLASS
78
+ position: !ruby/struct:Rley::Lexical::Position
79
+ line: 7
80
+ column: 1
81
+ - !ruby/object:Rley::Lexical::Token
82
+ lexeme: Derived
83
+ terminal: IDENTIFIER
84
+ position: !ruby/struct:Rley::Lexical::Position
85
+ line: 7
86
+ column: 7
87
+ - !ruby/object:Rley::Lexical::Token
88
+ lexeme: "<"
89
+ terminal: LESS
90
+ position: !ruby/struct:Rley::Lexical::Position
91
+ line: 7
92
+ column: 15
93
+ - !ruby/object:Rley::Lexical::Token
94
+ lexeme: Base
95
+ terminal: IDENTIFIER
96
+ position: !ruby/struct:Rley::Lexical::Position
97
+ line: 7
98
+ column: 17
99
+ - !ruby/object:Rley::Lexical::Token
100
+ lexeme: "{"
101
+ terminal: LEFT_BRACE
102
+ position: !ruby/struct:Rley::Lexical::Position
103
+ line: 7
104
+ column: 22
105
+ - !ruby/object:Rley::Lexical::Token
106
+ lexeme: foo
107
+ terminal: IDENTIFIER
108
+ position: !ruby/struct:Rley::Lexical::Position
109
+ line: 8
110
+ column: 3
111
+ - !ruby/object:Rley::Lexical::Token
112
+ lexeme: "("
113
+ terminal: LEFT_PAREN
114
+ position: !ruby/struct:Rley::Lexical::Position
115
+ line: 8
116
+ column: 6
117
+ - !ruby/object:Rley::Lexical::Token
118
+ lexeme: ")"
119
+ terminal: RIGHT_PAREN
120
+ position: !ruby/struct:Rley::Lexical::Position
121
+ line: 8
122
+ column: 7
123
+ - !ruby/object:Rley::Lexical::Token
124
+ lexeme: "{"
125
+ terminal: LEFT_BRACE
126
+ position: !ruby/struct:Rley::Lexical::Position
127
+ line: 8
128
+ column: 9
129
+ - !ruby/object:Rley::Lexical::Token
130
+ lexeme: print
131
+ terminal: PRINT
132
+ position: !ruby/struct:Rley::Lexical::Position
133
+ line: 9
134
+ column: 5
135
+ - !ruby/object:Rley::Lexical::Literal
136
+ lexeme: '"Derived.foo()"'
137
+ terminal: STRING
138
+ position: !ruby/struct:Rley::Lexical::Position
139
+ line: 9
140
+ column: 11
141
+ value: Derived.foo()
142
+ - !ruby/object:Rley::Lexical::Token
143
+ lexeme: ";"
144
+ terminal: SEMICOLON
145
+ position: !ruby/struct:Rley::Lexical::Position
146
+ line: 9
147
+ column: 26
148
+ - !ruby/object:Rley::Lexical::Token
149
+ lexeme: super
150
+ terminal: SUPER
151
+ position: !ruby/struct:Rley::Lexical::Position
152
+ line: 10
153
+ column: 5
154
+ - !ruby/object:Rley::Lexical::Token
155
+ lexeme: "."
156
+ terminal: DOT
157
+ position: !ruby/struct:Rley::Lexical::Position
158
+ line: 10
159
+ column: 10
160
+ - !ruby/object:Rley::Lexical::Token
161
+ lexeme: foo
162
+ terminal: IDENTIFIER
163
+ position: !ruby/struct:Rley::Lexical::Position
164
+ line: 10
165
+ column: 11
166
+ - !ruby/object:Rley::Lexical::Token
167
+ lexeme: "("
168
+ terminal: LEFT_PAREN
169
+ position: !ruby/struct:Rley::Lexical::Position
170
+ line: 10
171
+ column: 14
172
+ - !ruby/object:Rley::Lexical::Token
173
+ lexeme: ")"
174
+ terminal: RIGHT_PAREN
175
+ position: !ruby/struct:Rley::Lexical::Position
176
+ line: 10
177
+ column: 15
178
+ - !ruby/object:Rley::Lexical::Token
179
+ lexeme: ";"
180
+ terminal: SEMICOLON
181
+ position: !ruby/struct:Rley::Lexical::Position
182
+ line: 10
183
+ column: 16
184
+ - !ruby/object:Rley::Lexical::Token
185
+ lexeme: "}"
186
+ terminal: RIGHT_BRACE
187
+ position: !ruby/struct:Rley::Lexical::Position
188
+ line: 11
189
+ column: 3
190
+ - !ruby/object:Rley::Lexical::Token
191
+ lexeme: "}"
192
+ terminal: RIGHT_BRACE
193
+ position: !ruby/struct:Rley::Lexical::Position
194
+ line: 12
195
+ column: 1
196
+ - !ruby/object:Rley::Lexical::Token
197
+ lexeme: Derived
198
+ terminal: IDENTIFIER
199
+ position: !ruby/struct:Rley::Lexical::Position
200
+ line: 14
201
+ column: 1
202
+ - !ruby/object:Rley::Lexical::Token
203
+ lexeme: "("
204
+ terminal: LEFT_PAREN
205
+ position: !ruby/struct:Rley::Lexical::Position
206
+ line: 14
207
+ column: 8
208
+ - !ruby/object:Rley::Lexical::Token
209
+ lexeme: ")"
210
+ terminal: RIGHT_PAREN
211
+ position: !ruby/struct:Rley::Lexical::Position
212
+ line: 14
213
+ column: 9
214
+ - !ruby/object:Rley::Lexical::Token
215
+ lexeme: "."
216
+ terminal: DOT
217
+ position: !ruby/struct:Rley::Lexical::Position
218
+ line: 14
219
+ column: 10
220
+ - !ruby/object:Rley::Lexical::Token
221
+ lexeme: foo
222
+ terminal: IDENTIFIER
223
+ position: !ruby/struct:Rley::Lexical::Position
224
+ line: 14
225
+ column: 11
226
+ - !ruby/object:Rley::Lexical::Token
227
+ lexeme: "("
228
+ terminal: LEFT_PAREN
229
+ position: !ruby/struct:Rley::Lexical::Position
230
+ line: 14
231
+ column: 14
232
+ - !ruby/object:Rley::Lexical::Token
233
+ lexeme: ")"
234
+ terminal: RIGHT_PAREN
235
+ position: !ruby/struct:Rley::Lexical::Position
236
+ line: 14
237
+ column: 15
238
+ - !ruby/object:Rley::Lexical::Token
239
+ lexeme: ";"
240
+ terminal: SEMICOLON
241
+ position: !ruby/struct:Rley::Lexical::Position
242
+ line: 14
243
+ column: 16
244
+ - !ruby/object:Rley::Lexical::Token
245
+ lexeme:
246
+ terminal: EOF
247
+ position: !ruby/struct:Rley::Lexical::Position
248
+ line: 14
249
+ column: 17
data/lib/rley/constants.rb CHANGED
@@ -5,7 +5,7 @@
5
5
 
6
6
  module Rley # Module used as a namespace
7
7
  # The version number of the gem.
8
- Version = '0.8.05'
8
+ Version = '0.8.06'
9
9
 
10
10
  # Brief description of the gem.
11
11
  Description = "Ruby implementation of the Earley's parsing algorithm"
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rley
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.8.05
4
+ version: 0.8.06
5
5
  platform: ruby
6
6
  authors:
7
7
  - Dimitri Geshef
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-10-04 00:00:00.000000000 Z
11
+ date: 2021-10-06 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake
@@ -118,6 +118,7 @@ files:
118
118
  - examples/general/calc_iter2/calc_demo.rb
119
119
  - examples/general/calc_iter2/calc_grammar.rb
120
120
  - examples/general/calc_iter2/calc_lexer.rb
121
+ - examples/general/calc_iter2/sample_result.txt
121
122
  - examples/general/calc_iter2/spec/calculator_spec.rb
122
123
  - examples/general/general_examples.md
123
124
  - examples/general/left.rb
@@ -127,6 +128,7 @@ files:
127
128
  - examples/tokenizer/loxxy_raw_scanner.rex.rb
128
129
  - examples/tokenizer/loxxy_tokenizer.rb
129
130
  - examples/tokenizer/run_tokenizer.rb
131
+ - examples/tokenizer/tokens.yaml
130
132
  - lib/rley.rb
131
133
  - lib/rley/base/base_parser.rb
132
134
  - lib/rley/base/dotted_item.rb