rakko 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.yardopts +1 -0
- data/lib/rakko/scanner.rb +165 -0
- data/lib/rakko/token.rb +35 -0
- data/lib/rakko/version.rb +1 -1
- data/rakko.gemspec +15 -12
- metadata +22 -4
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: d3211f0c674ab78099d9cec919b29f26a69cd1d9
|
|
4
|
+
data.tar.gz: 045de341aec2ff82bc6cc5b0793b4d89ebab31cd
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 40f17518bf37127f109c24799cf8fe04407be5c06ff9977d4f042014531d63405a8488b55e76849b0fabc804f3d6fb4f8f83c4dcec203b791ce49121c96536fd
|
|
7
|
+
data.tar.gz: d6024fb9ecedcd2cbd7e4a653e90219121d0e263de027456a9ad7c6629d5daa1d37f1b36d5d8d91b25a0ebd1a6a4779730b6edf3d23880c9318e2240ed188fc0
|
data/.yardopts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
-m markdown
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
require 'rakko/token'
|
|
2
|
+
|
|
3
|
+
module Rakko
|
|
4
|
+
|
|
5
|
+
# Hand-coded scanner which transforms a stream of
|
|
6
|
+
# characters into a stream of tokens in the Rakko language.
|
|
7
|
+
#
|
|
8
|
+
# ### Syntax
|
|
9
|
+
#
|
|
10
|
+
# Comment :: #.*\n
|
|
11
|
+
# Integer :: 0|[1-9][0-9]*
|
|
12
|
+
# Identifier :: [a-zA-Z_][0-9a-zA-Z_]*
|
|
13
|
+
# String :: "([^"]|\")*"
|
|
14
|
+
#
|
|
15
|
+
class Scanner
|
|
16
|
+
include Enumerable
|
|
17
|
+
|
|
18
|
+
attr_reader :lineno, :columnno
|
|
19
|
+
|
|
20
|
+
# @!attribute [r] lineno
|
|
21
|
+
# @return [Integer] The current line position pointed by the scanner (`0` base)
|
|
22
|
+
#
|
|
23
|
+
# @!attribute [r] columnno
|
|
24
|
+
# @return [Integer] The current column position pointed by the scanner (`0` base)
|
|
25
|
+
|
|
26
|
+
# Creates a new Scanner instance from `input` stream.
|
|
27
|
+
#
|
|
28
|
+
# The `input` parameter must be an instance of `String` or I/O object can respond to
|
|
29
|
+
# `#getc` method. If the `input` is an instance of `String`, it will be converted to
|
|
30
|
+
# an instance of `StringIO` internally.
|
|
31
|
+
#
|
|
32
|
+
# @param input [String, #getc] A stream of characters.
|
|
33
|
+
def initialize(input)
|
|
34
|
+
@nextc = nil
|
|
35
|
+
|
|
36
|
+
@columnno = 0
|
|
37
|
+
@lineno = 0
|
|
38
|
+
|
|
39
|
+
@input =
|
|
40
|
+
case input
|
|
41
|
+
when String
|
|
42
|
+
StringIO.new(input)
|
|
43
|
+
else
|
|
44
|
+
unless input.respond_to?(:getc)
|
|
45
|
+
raise ArgumentError, '`input` must be String or IO like object.'
|
|
46
|
+
else
|
|
47
|
+
input
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# Reads the next token from the input stream.
|
|
53
|
+
#
|
|
54
|
+
# @return [Token] The token read, or `nil` if called at end of the input stream.
|
|
55
|
+
def next_token
|
|
56
|
+
# Read the first character if needed.
|
|
57
|
+
advance unless @nextc
|
|
58
|
+
|
|
59
|
+
loop do
|
|
60
|
+
line = self.lineno
|
|
61
|
+
column = self.columnno
|
|
62
|
+
|
|
63
|
+
return case @nextc
|
|
64
|
+
when nil # EOS
|
|
65
|
+
nil
|
|
66
|
+
when "\t", "\f", "\n", "\r", " " # Skip white spaces
|
|
67
|
+
advance
|
|
68
|
+
next
|
|
69
|
+
when '#' # Skip line comment
|
|
70
|
+
advance
|
|
71
|
+
until @nextc == "\n" || @nextc == nil
|
|
72
|
+
advance
|
|
73
|
+
end
|
|
74
|
+
redo
|
|
75
|
+
when "0"
|
|
76
|
+
# TODO octadecimal integer
|
|
77
|
+
# TODO hexadecimal integer
|
|
78
|
+
advance
|
|
79
|
+
Token.new(0, lineno: line, columnno: column)
|
|
80
|
+
when "1".."9" # decimal integer
|
|
81
|
+
n = @nextc.ord - "0".ord
|
|
82
|
+
loop do
|
|
83
|
+
advance
|
|
84
|
+
case @nextc
|
|
85
|
+
when "0".."9"
|
|
86
|
+
n = n * 10 + (@nextc.ord - "0".ord)
|
|
87
|
+
else
|
|
88
|
+
break Token.new(n, lineno: line, columnno: column)
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
when "a".."z", "A".."Z", "_" # identifier
|
|
92
|
+
name = @nextc
|
|
93
|
+
loop do
|
|
94
|
+
advance
|
|
95
|
+
case @nextc
|
|
96
|
+
when "0".."9", "a".."z", "A".."Z", "_"
|
|
97
|
+
name << @nextc
|
|
98
|
+
else
|
|
99
|
+
break Token.new(name.to_sym, lineno: line, columnno: column)
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
when "+", "-", "*", "/", "%", "=", ".", "!",
|
|
103
|
+
"(", ")", "[", "]", "{", "}" # operator
|
|
104
|
+
op = @nextc
|
|
105
|
+
advance
|
|
106
|
+
Token.new(op.to_sym, lineno: line, columnno: column)
|
|
107
|
+
when '"' # string
|
|
108
|
+
str = ''
|
|
109
|
+
loop do
|
|
110
|
+
advance
|
|
111
|
+
case @nextc
|
|
112
|
+
when nil
|
|
113
|
+
raise "Premature end of quoted string"
|
|
114
|
+
when '\\' # Escape sequence
|
|
115
|
+
advance
|
|
116
|
+
case @nextc
|
|
117
|
+
when 't'; str << "\t"
|
|
118
|
+
when 'n'; str << "\n"
|
|
119
|
+
when 'r'; str << "\r"
|
|
120
|
+
when 'f'; str << "\f"
|
|
121
|
+
when '\\'; str << "\\"
|
|
122
|
+
when '"'; str << "\""
|
|
123
|
+
when nil
|
|
124
|
+
raise "Premature end of escape sequence"
|
|
125
|
+
else
|
|
126
|
+
raise %Q{Unrecognized escape sequence: "\\#{@nextc}"}
|
|
127
|
+
end
|
|
128
|
+
when '"'
|
|
129
|
+
advance
|
|
130
|
+
break Token.new(str, lineno: line, columnno: column)
|
|
131
|
+
else
|
|
132
|
+
str << @nextc
|
|
133
|
+
end
|
|
134
|
+
end
|
|
135
|
+
else
|
|
136
|
+
raise "Unrecognized token"
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
def each(&block)
|
|
142
|
+
while t = next_token
|
|
143
|
+
block.yield(t)
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
private
|
|
148
|
+
|
|
149
|
+
def advance
|
|
150
|
+
# Update lineno and columnno of _next character_
|
|
151
|
+
if @nextc
|
|
152
|
+
if @nextc == $/
|
|
153
|
+
@lineno += 1
|
|
154
|
+
@columnno = 0
|
|
155
|
+
else
|
|
156
|
+
@columnno += 1
|
|
157
|
+
end
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
@nextc = @input.getc
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
end # Scanner
|
|
164
|
+
|
|
165
|
+
end
|
data/lib/rakko/token.rb
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
module Rakko
|
|
2
|
+
|
|
3
|
+
# `Token` is a terminal symbol in grammer context.
|
|
4
|
+
#
|
|
5
|
+
# Its `value` attribute represents token's value and its syntactic category.
|
|
6
|
+
# If `Token#value` is an instance of `Symbol`, its syntactic category is `Ident`.
|
|
7
|
+
class Token
|
|
8
|
+
attr_reader :lineno, :columnno, :value
|
|
9
|
+
|
|
10
|
+
# @!attribute [r] lineno
|
|
11
|
+
# @return [Integer] The line position of the token (`0` base).
|
|
12
|
+
#
|
|
13
|
+
# @!attribute [r] columnno
|
|
14
|
+
# @return [Integer] The column position of the token (`0` base).
|
|
15
|
+
|
|
16
|
+
# Creates a new Token instance.
|
|
17
|
+
#
|
|
18
|
+
# @param value token's value. If the value is `String`, it will be duplicated and frozen.
|
|
19
|
+
# @param columnno [Integer] The column number of token position. number is `0` based.
|
|
20
|
+
# @param lineno [Integer] The line number of token position. number is `0` based.
|
|
21
|
+
def initialize(value, columnno: 0, lineno: 0)
|
|
22
|
+
@columnno = columnno
|
|
23
|
+
@lineno = lineno
|
|
24
|
+
@value =
|
|
25
|
+
case value
|
|
26
|
+
when String
|
|
27
|
+
value.dup.freeze
|
|
28
|
+
else
|
|
29
|
+
value
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
end # Token
|
|
34
|
+
|
|
35
|
+
end
|
data/lib/rakko/version.rb
CHANGED
data/rakko.gemspec
CHANGED
|
@@ -4,25 +4,28 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
|
4
4
|
require 'rakko/version'
|
|
5
5
|
|
|
6
6
|
Gem::Specification.new do |spec|
|
|
7
|
-
spec.name =
|
|
7
|
+
spec.name = 'rakko'
|
|
8
8
|
spec.version = Rakko::VERSION
|
|
9
|
-
spec.authors = [
|
|
10
|
-
spec.email = [
|
|
9
|
+
spec.authors = ['Takanori Ishikawa']
|
|
10
|
+
spec.email = ['takanori.ishikawa@gmail.com']
|
|
11
11
|
|
|
12
12
|
spec.summary = %q{A toy language.}
|
|
13
|
-
spec.description = %q{Writing my own homebrew toy programming language to learn compiler
|
|
14
|
-
spec.homepage =
|
|
15
|
-
spec.license =
|
|
13
|
+
spec.description = %q{Writing my own homebrew toy programming language to learn about compiler construction.}
|
|
14
|
+
spec.homepage = 'https://github.com/ishikawa/rakko'
|
|
15
|
+
spec.license = 'MIT'
|
|
16
16
|
|
|
17
17
|
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
|
18
18
|
f.match(%r{^(test|spec|features)/})
|
|
19
19
|
end
|
|
20
|
-
spec.bindir =
|
|
20
|
+
spec.bindir = 'exe'
|
|
21
21
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
|
22
|
-
spec.require_paths = [
|
|
22
|
+
spec.require_paths = ['lib']
|
|
23
23
|
|
|
24
|
-
spec.
|
|
25
|
-
|
|
26
|
-
spec.add_development_dependency
|
|
27
|
-
spec.add_development_dependency
|
|
24
|
+
spec.required_ruby_version = '>= 2.4.0'
|
|
25
|
+
|
|
26
|
+
spec.add_development_dependency 'bundler', '~> 1.13'
|
|
27
|
+
spec.add_development_dependency 'rake', '~> 10.0'
|
|
28
|
+
spec.add_development_dependency 'minitest', '~> 5.0'
|
|
29
|
+
spec.add_development_dependency 'minitest-spec-context', '~> 0.0.3'
|
|
30
|
+
spec.add_development_dependency 'yard', '~> 0.9.5'
|
|
28
31
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: rakko
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.2.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Takanori Ishikawa
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2017-01-
|
|
11
|
+
date: 2017-01-09 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: bundler
|
|
@@ -52,6 +52,20 @@ dependencies:
|
|
|
52
52
|
- - "~>"
|
|
53
53
|
- !ruby/object:Gem::Version
|
|
54
54
|
version: '5.0'
|
|
55
|
+
- !ruby/object:Gem::Dependency
|
|
56
|
+
name: minitest-spec-context
|
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
|
58
|
+
requirements:
|
|
59
|
+
- - "~>"
|
|
60
|
+
- !ruby/object:Gem::Version
|
|
61
|
+
version: 0.0.3
|
|
62
|
+
type: :development
|
|
63
|
+
prerelease: false
|
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
65
|
+
requirements:
|
|
66
|
+
- - "~>"
|
|
67
|
+
- !ruby/object:Gem::Version
|
|
68
|
+
version: 0.0.3
|
|
55
69
|
- !ruby/object:Gem::Dependency
|
|
56
70
|
name: yard
|
|
57
71
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -66,7 +80,8 @@ dependencies:
|
|
|
66
80
|
- - "~>"
|
|
67
81
|
- !ruby/object:Gem::Version
|
|
68
82
|
version: 0.9.5
|
|
69
|
-
description: Writing my own homebrew toy programming language to learn compiler
|
|
83
|
+
description: Writing my own homebrew toy programming language to learn about compiler
|
|
84
|
+
construction.
|
|
70
85
|
email:
|
|
71
86
|
- takanori.ishikawa@gmail.com
|
|
72
87
|
executables:
|
|
@@ -77,6 +92,7 @@ files:
|
|
|
77
92
|
- ".gitignore"
|
|
78
93
|
- ".ruby-version"
|
|
79
94
|
- ".travis.yml"
|
|
95
|
+
- ".yardopts"
|
|
80
96
|
- Gemfile
|
|
81
97
|
- LICENSE.txt
|
|
82
98
|
- README.md
|
|
@@ -85,6 +101,8 @@ files:
|
|
|
85
101
|
- bin/setup
|
|
86
102
|
- exe/rakko
|
|
87
103
|
- lib/rakko.rb
|
|
104
|
+
- lib/rakko/scanner.rb
|
|
105
|
+
- lib/rakko/token.rb
|
|
88
106
|
- lib/rakko/version.rb
|
|
89
107
|
- rakko.gemspec
|
|
90
108
|
homepage: https://github.com/ishikawa/rakko
|
|
@@ -99,7 +117,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
99
117
|
requirements:
|
|
100
118
|
- - ">="
|
|
101
119
|
- !ruby/object:Gem::Version
|
|
102
|
-
version:
|
|
120
|
+
version: 2.4.0
|
|
103
121
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
104
122
|
requirements:
|
|
105
123
|
- - ">="
|