dendroid 0.0.9 → 0.0.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +4 -1
- data/CHANGELOG.md +16 -0
- data/dendroid.gemspec +2 -2
- data/lib/dendroid/grm_analysis/alternative_item.rb +1 -2
- data/lib/dendroid/grm_analysis/choice_items.rb +2 -1
- data/lib/dendroid/grm_analysis/dotted_item.rb +2 -0
- data/lib/dendroid/grm_analysis/grm_analyzer.rb +26 -9
- data/lib/dendroid/grm_analysis/production_items.rb +2 -1
- data/lib/dendroid/lexical/token.rb +1 -1
- data/lib/dendroid/lexical/token_position.rb +9 -1
- data/lib/dendroid/recognizer/chart.rb +53 -0
- data/lib/dendroid/recognizer/e_item.rb +48 -0
- data/lib/dendroid/recognizer/item_set.rb +37 -0
- data/lib/dendroid/recognizer/recognizer.rb +282 -0
- data/lib/dendroid/syntax/grm_symbol.rb +1 -1
- data/lib/dendroid/utils/base_tokenizer.rb +54 -15
- data/spec/dendroid/grm_analysis/grm_analyzer_spec.rb +7 -77
- data/spec/dendroid/lexical/token_position_spec.rb +1 -1
- data/spec/dendroid/recognizer/chart_spec.rb +2 -0
- data/spec/dendroid/recognizer/e_item_spec.rb +55 -0
- data/spec/dendroid/recognizer/item_set_spec.rb +63 -0
- data/spec/dendroid/recognizer/recognizer_spec.rb +186 -0
- data/spec/dendroid/support/sample_grammars.rb +76 -0
- data/spec/dendroid/utils/base_tokenizer_spec.rb +4 -4
- data/version.txt +1 -1
- metadata +13 -4
@@ -0,0 +1,76 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative '../../../lib/dendroid/grm_dsl/base_grm_builder'
|
4
|
+
require_relative '../../../lib/dendroid/utils/base_tokenizer'
|
5
|
+
|
6
|
+
module SampleGrammars
|
7
|
+
def grammar_l1
|
8
|
+
builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
|
9
|
+
# Grammar inspired from Wikipedia entry on Earley parsing
|
10
|
+
declare_terminals('PLUS', 'STAR', 'INTEGER')
|
11
|
+
rule('p' => 's')
|
12
|
+
rule('s' => ['s PLUS m', 'm'])
|
13
|
+
# rule('s' => 'm')
|
14
|
+
rule('m' => ['m STAR t', 't'])
|
15
|
+
# rule('m' => 't')
|
16
|
+
rule('t' => 'INTEGER')
|
17
|
+
end
|
18
|
+
|
19
|
+
builder.grammar
|
20
|
+
end
|
21
|
+
|
22
|
+
def tokenizer_l1
|
23
|
+
Dendroid::Utils::BaseTokenizer.new do
|
24
|
+
map_verbatim2terminal({ '+' => :PLUS, '*' => :STAR })
|
25
|
+
|
26
|
+
scan_verbatim(['+', '*'])
|
27
|
+
scan_value(/\d+/, :INTEGER, ->(txt) { txt.to_i })
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
|
32
|
+
def grammar_l2
|
33
|
+
builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
|
34
|
+
# Grammar inspired from Loup Vaillant's example
|
35
|
+
# https://loup-vaillant.fr/tutorials/earley-parsing/recogniser
|
36
|
+
declare_terminals('PLUS', 'MINUS', 'STAR', 'SLASH')
|
37
|
+
declare_terminals('LPAREN', 'RPAREN', 'NUMBER')
|
38
|
+
|
39
|
+
rule('p' => 'sum')
|
40
|
+
rule('sum' => ['sum PLUS product', 'sum MINUS product', 'product'])
|
41
|
+
rule('product' => ['product STAR factor', 'product SLASH factor', 'factor'])
|
42
|
+
rule('factor' => ['LPAREN sum RPAREN', 'NUMBER'])
|
43
|
+
end
|
44
|
+
|
45
|
+
builder.grammar
|
46
|
+
end
|
47
|
+
|
48
|
+
def tokenizer_l2
|
49
|
+
Dendroid::Utils::BaseTokenizer.new do
|
50
|
+
map_verbatim2terminal({
|
51
|
+
'+' => :PLUS,
|
52
|
+
'-' => :MINUS,
|
53
|
+
'*' => :STAR,
|
54
|
+
'/' => :SLASH,
|
55
|
+
'(' => :LPAREN,
|
56
|
+
')' => :RPAREN })
|
57
|
+
|
58
|
+
scan_verbatim(['+', '-', '*', '/', '(', ')'])
|
59
|
+
scan_value(/\d+/, :NUMBER, ->(txt) { txt.to_i })
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def grammar_l3
|
64
|
+
builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
|
65
|
+
# Grammar inspired from Andrew Appel's example
|
66
|
+
# Modern Compiler Implementation in Java
|
67
|
+
declare_terminals('a', 'c', 'd')
|
68
|
+
|
69
|
+
rule('Z' => ['d', 'X Y Z'])
|
70
|
+
rule('Y' => ['', 'c'])
|
71
|
+
rule('X' => ['Y', 'a'])
|
72
|
+
end
|
73
|
+
|
74
|
+
builder.grammar
|
75
|
+
end
|
76
|
+
end # module
|
@@ -9,7 +9,7 @@ describe Dendroid::Utils::BaseTokenizer do
|
|
9
9
|
|
10
10
|
context 'Initialization:' do
|
11
11
|
it 'is initialized with an optional block' do
|
12
|
-
expect {described_class.new }.not_to raise_error
|
12
|
+
expect { described_class.new }.not_to raise_error
|
13
13
|
end
|
14
14
|
|
15
15
|
it 'has a scanner at start' do
|
@@ -24,11 +24,11 @@ describe Dendroid::Utils::BaseTokenizer do
|
|
24
24
|
|
25
25
|
context 'Tokenizing:' do
|
26
26
|
subject do
|
27
|
-
described_class.new
|
27
|
+
described_class.new do
|
28
28
|
scan_verbatim(['+', '*'])
|
29
29
|
scan_value(/\d+/, :INTEGER, ->(txt) { txt.to_i })
|
30
30
|
map_verbatim2terminal({ '+' => :PLUS, '*' => :STAR })
|
31
|
-
|
31
|
+
end
|
32
32
|
end
|
33
33
|
|
34
34
|
it 'generates a sequence of tokens from a simple input' do
|
@@ -43,7 +43,7 @@ describe Dendroid::Utils::BaseTokenizer do
|
|
43
43
|
]
|
44
44
|
expectations.each do |tuple|
|
45
45
|
tok = subject.next_token
|
46
|
-
[
|
46
|
+
%i[pos_to_s source terminal value].each_with_index do |message, index|
|
47
47
|
expect(tok.send(message)).to eq(tuple[index]) unless tuple[index].nil?
|
48
48
|
end
|
49
49
|
end
|
data/version.txt
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.0.
|
1
|
+
0.0.11
|
metadata
CHANGED
@@ -1,16 +1,16 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: dendroid
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.11
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Dimitri Geshef
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-11-
|
11
|
+
date: 2023-11-02 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
|
-
description: WIP. A Ruby implementation of
|
13
|
+
description: WIP. A Ruby implementation of an Earley parser
|
14
14
|
email: famished.tiger@yahoo.com
|
15
15
|
executables: []
|
16
16
|
extensions: []
|
@@ -33,6 +33,10 @@ files:
|
|
33
33
|
- lib/dendroid/lexical/literal.rb
|
34
34
|
- lib/dendroid/lexical/token.rb
|
35
35
|
- lib/dendroid/lexical/token_position.rb
|
36
|
+
- lib/dendroid/recognizer/chart.rb
|
37
|
+
- lib/dendroid/recognizer/e_item.rb
|
38
|
+
- lib/dendroid/recognizer/item_set.rb
|
39
|
+
- lib/dendroid/recognizer/recognizer.rb
|
36
40
|
- lib/dendroid/syntax/choice.rb
|
37
41
|
- lib/dendroid/syntax/grammar.rb
|
38
42
|
- lib/dendroid/syntax/grm_symbol.rb
|
@@ -51,6 +55,11 @@ files:
|
|
51
55
|
- spec/dendroid/lexical/literal_spec.rb
|
52
56
|
- spec/dendroid/lexical/token_position_spec.rb
|
53
57
|
- spec/dendroid/lexical/token_spec.rb
|
58
|
+
- spec/dendroid/recognizer/chart_spec.rb
|
59
|
+
- spec/dendroid/recognizer/e_item_spec.rb
|
60
|
+
- spec/dendroid/recognizer/item_set_spec.rb
|
61
|
+
- spec/dendroid/recognizer/recognizer_spec.rb
|
62
|
+
- spec/dendroid/support/sample_grammars.rb
|
54
63
|
- spec/dendroid/syntax/choice_spec.rb
|
55
64
|
- spec/dendroid/syntax/grammar_spec.rb
|
56
65
|
- spec/dendroid/syntax/grm_symbol_spec.rb
|
@@ -84,5 +93,5 @@ requirements: []
|
|
84
93
|
rubygems_version: 3.3.7
|
85
94
|
signing_key:
|
86
95
|
specification_version: 4
|
87
|
-
summary: WIP. A Ruby implementation of
|
96
|
+
summary: WIP. A Ruby implementation of an Earley parser
|
88
97
|
test_files: []
|