halunke 0.7.0 → 0.8.0

data/docs/number.md

@@ -52,6 +52,16 @@ This multiplies two numbers.
 (0.6 * 0.3) /* => 0.18 */
 ```
 
+## `**`
+
+This performs exponation.
+
+**Example:**
+
+```
+(0.6 ** 0.3) /* => 0.8579172004440949 */
+```
+
 ## `<`
 
 This compares two numbers. It is true, if the first number is smaller than the
@@ -85,14 +95,24 @@ it is false.
 (0.6 = 0.3) /* => false */
 ```
 
-## `to_s`
+## `to_boolean`
+
+This returns true.
+
+**Example:**
+
+```
+(0.6 to_boolean) /* => true */
+```
+
+## `to_string`
 
 This returns a string to represent the number in output.
 
 **Example:**
 
 ```
-(0.6 to_s) /* => "0.6" */
+(0.6 to_string) /* => "0.6" */
 ```
 
 ## `inspect`

data/docs/regexp.md

@@ -8,5 +8,6 @@ Regexp is a regular expression. It can be created like this:
 (Regexp from "a+")
 ```
 
-Currently it doesn't answer to any messages except `inspect` and `to_s`. To use
-a Regexp, see the messages you can send to [String](/string).
+Currently it doesn't answer to any messages except `inspect`, `to_string` and
+`to_boolean`. To use a Regexp, see the messages you can send to
+[String](/string).

data/docs/stdio.md

@@ -9,7 +9,7 @@ isolated.](https://github.com/moonglum/halunke/issues/4)*
 ## `puts`
 
 Prints the object to stdout. To determine how to represent the object, it will
-send the `to_s` message to the object, expecting a string.
+send the `to_string` message to the object, expecting a string.
 
 **Example:**
 
@@ -35,7 +35,7 @@ true
 false
 ```
 
-## `p`
+## `examine`
 
 Prints the object to stdout for debugging. To determine how to represent the
 object, it will send the `inspect` message to the object, expecting a string.
@@ -43,12 +43,12 @@ object, it will send the `inspect` message to the object, expecting a string.
 **Example:**
 
 ```
-(stdio p "Hello World")
-(stdio p 5.2)
-(stdio p @["a" 2 "b" 3])
-(stdio p ["a" "b"])
-(stdio p true)
-(stdio p false)
+(stdio examine "Hello World")
+(stdio examine 5.2)
+(stdio examine @["a" 2 "b" 3])
+(stdio examine ["a" "b"])
+(stdio examine true)
+(stdio examine false)
 ```
 
 This will output:

data/docs/string.md

@@ -38,17 +38,6 @@ by `\1` etc,
 /* => "<a>b<a>b<a>b" */
 ```
 
-## `scan`
-
-Find all occurrences of the given Regex. Returns an Array. If it doesn't
-include any groups, it contains Strings. If it contains groups, then it
-contains arrays of strings.
-
-```
-('result = ("aaabaac" scan (Regexp from "a+")))
-(result @ 0 else "NOT FOUND") /* => "aaa" */
-```
-
 ## `match`
 
 This matches a Regexp over a String an collects the results in a Dictionary.
@@ -67,6 +56,17 @@ referenced by their name and their position.
 (match @ "bar" else "NOT FOUND") /* => "cc" */
 ```
 
+## `scan`
+
+Find all occurrences of the given Regex. Returns an Array. If it doesn't
+include any groups, it contains Strings. If it contains groups, then it
+contains arrays of strings.
+
+```
+('result = ("aaabaac" scan (Regexp from "a+")))
+(result @ 0 else "NOT FOUND") /* => "aaa" */
+```
+
 ## `=`
 
 This compares two strings. It is true, if the two strings are equal. Otherwise,
@@ -88,14 +88,24 @@ Concatenate two strings.
 ("aaa" + "bbb") /* => "aaabbb" */
 ```
 
-## `to_s`
+## `to_boolean`
+
+This returns true.
+
+**Example:**
+
+```
+("aaa" to_boolean) /* => true */
+```
+
+## `to_string`
 
 This returns the string itself.
 
 **Example:**
 
 ```
-("aaa" to_s) /* => "aaa" */
+("aaa" to_string) /* => "aaa" */
 ```
 
 ## `inspect`

data/docs/true-false.md

@@ -45,15 +45,26 @@ false, it will return the second branch.
 (false then { "yes" } else { "no" }) /* => "no" */
 ```
 
-## `to_s`
+## `to_boolean`
+
+This returns the object itself
+
+**Example:**
+
+```
+(true to_boolean) /* => true */
+(false to_boolean) /* => false */
+```
+
+## `to_string`
 
 This returns a string to represent true and false in output.
 
 **Example:**
 
 ```
-(true to_s) /* => "true" */
-(false to_s) /* => "false" */
+(true to_string) /* => "true" */
+(false to_string) /* => "false" */
 ```
 
 ## `inspect`

data/halunke.gemspec

@@ -23,8 +23,8 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "rack", "~> 2.0.4"
 
-  spec.add_development_dependency "bundler", "~> 1.16.1"
-  spec.add_development_dependency "rake", "~> 12.3.0"
+  spec.add_development_dependency "bundler", "~> 1.17.2"
+  spec.add_development_dependency "rake", "~> 12.3.2"
   spec.add_development_dependency "minitest", "~> 5.11.3"
-  spec.add_development_dependency "racc", "~> 1.4.14"
+  spec.add_development_dependency "racc", "~> 1.4.15"
 end

data/lib/halunke/grammar.y

@@ -43,13 +43,13 @@ end
 
 ---- header
 
-require "halunke/lexer"
+require "halunke/tokenizer"
 require "halunke/nodes"
 
 ---- inner
 
 def parse(code)
-  @tokens = Lexer.new.tokenize(code)
+  @tokens = Tokenizer.new.tokenize(code)
   do_parse
 end
 

data/lib/halunke/nodes.rb

@@ -81,7 +81,12 @@ module Halunke
   MessageNode = Struct.new(:nodes) do
     def eval(context)
       if nodes.length == 1
-        [nodes[0].value, []]
+        if nodes[0].is_a? NumberNode
+          # hack to allow expressions like (1+5)
+          ["+", [nodes[0].eval(context)]]
+        else
+          [nodes[0].value, []]
+        end
       elsif nodes.length.even?
         name = []
         message = []

data/lib/halunke/parser.rb

@@ -1,13 +1,13 @@
 #
 # DO NOT MODIFY!!!!
-# This file is automatically generated by Racc 1.4.14
+# This file is automatically generated by Racc 1.4.15
 # from Racc grammer file "".
 #
 
 require 'racc/parser.rb'
 
 
-require "halunke/lexer"
+require "halunke/tokenizer"
 require "halunke/nodes"
 
 module Halunke
@@ -16,7 +16,7 @@ module Halunke
 module_eval(<<'...end grammar.y/module_eval...', 'grammar.y', 50)
 
 def parse(code)
-  @tokens = Lexer.new.tokenize(code)
+  @tokens = Tokenizer.new.tokenize(code)
   do_parse
 end
 

data/lib/halunke/runtime/false.hal

@@ -9,10 +9,10 @@
     "then else" { |'self 'true_branch 'false_branch|
       (false_branch call [])
     }
-    "to_b" { |'self|
+    "to_boolean" { |'self|
       self
     }
-    "to_s" { |'self 'other|
+    "to_string" { |'self 'other|
       "false"
     }
     "inspect" { |'self 'other|

data/lib/halunke/runtime/harray.rb

@@ -4,6 +4,9 @@ module Halunke
       "Array",
       [],
       {
+        "@ else" => HFunction.new([:self, :index, :fallback], lambda { |context|
+          context["self"].ruby_value[context["index"].ruby_value] || context["fallback"]
+        }),
         "=" => HFunction.new([:self, :other], lambda { |context|
           return context["false"] if context["self"].ruby_value.length != context["other"].ruby_value.length
 
@@ -13,9 +16,6 @@ module Halunke
             memo.receive_message(context, "and", [value])
           end
         }),
-        "@ else" => HFunction.new([:self, :index, :fallback], lambda { |context|
-          context["self"].ruby_value[context["index"].ruby_value] || context["fallback"]
-        }),
         "map" => HFunction.new([:self, :fn], lambda { |context|
           return HArray.create_instance(context["self"].ruby_value.map do |x|
             context["fn"].receive_message(context, "call", [HArray.create_instance([x])])
@@ -31,12 +31,12 @@ module Halunke
             context["search_fn"].receive_message(context, "call", [HArray.create_instance([element])]) == context["true"]
           end
         }),
-        "to_b" => HFunction.new([:self], lambda { |context|
+        "to_boolean" => HFunction.new([:self], lambda { |context|
           context["true"]
         }),
-        "to_s" => HFunction.new([:self], lambda { |context|
+        "to_string" => HFunction.new([:self], lambda { |context|
           inspected_members = context["self"].ruby_value.map do |member|
-            member.receive_message(context, "to_s", []).ruby_value
+            member.receive_message(context, "to_string", []).ruby_value
           end
           HString.create_instance("#{inspected_members.join("\n")}")
         }),

data/lib/halunke/runtime/hdictionary.rb

@@ -13,14 +13,14 @@ module Halunke
         "merge" => HFunction.new([:self, :other], lambda { |context|
           HDictionary.create_instance(context["self"].ruby_value.merge(context["other"].ruby_value))
         }),
-        "to_b" => HFunction.new([:self], lambda { |context|
+        "to_boolean" => HFunction.new([:self], lambda { |context|
           context["true"]
         }),
-        "to_s" => HFunction.new([:self], lambda { |context|
+        "to_string" => HFunction.new([:self], lambda { |context|
           x = []
           context["self"].ruby_value.each_pair do |key, value|
-            key_s = key.receive_message(context, "to_s", [])
-            value_s = value.receive_message(context, "to_s", [])
+            key_s = key.receive_message(context, "to_string", [])
+            value_s = value.receive_message(context, "to_string", [])
             x.push("#{key_s.ruby_value} #{value_s.ruby_value}")
           end
           HString.create_instance(x.join("\n"))

data/lib/halunke/runtime/hnumber.rb

@@ -16,6 +16,9 @@ module Halunke
         "*" => HFunction.new([:self, :other], lambda { |context|
           HNumber.create_instance(context["self"].ruby_value * context["other"].ruby_value)
         }),
+        "**" => HFunction.new([:self, :other], lambda { |context|
+          HNumber.create_instance(context["self"].ruby_value ** context["other"].ruby_value)
+        }),
         "<" => HFunction.new([:self, :other], lambda { |context|
           if context["self"].ruby_value < context["other"].ruby_value
             context["true"]
@@ -37,16 +40,16 @@ module Halunke
             context["false"]
           end
         }),
-        "to_b" => HFunction.new([:self], lambda { |context|
+        "to_boolean" => HFunction.new([:self], lambda { |context|
           context["true"]
         }),
-        "to_s" => HFunction.new([:self], lambda { |context|
+        "to_string" => HFunction.new([:self], lambda { |context|
           float_value = context["self"].ruby_value.to_f
           float_value = float_value.to_i if float_value.to_i == float_value
           HString.create_instance(float_value.to_s)
         }),
         "inspect" => HFunction.new([:self], lambda { |context|
-          context["self"].receive_message(context, "to_s", [])
+          context["self"].receive_message(context, "to_string", [])
         })
       },
       {},

data/lib/halunke/runtime/hregexp.rb

@@ -4,10 +4,10 @@ module Halunke
       "Regexp",
       [],
       {
-        "to_b" => HFunction.new([:self], lambda { |context|
+        "to_boolean" => HFunction.new([:self], lambda { |context|
           context["true"]
         }),
-        "to_s" => HFunction.new([:self], lambda { |context|
+        "to_string" => HFunction.new([:self], lambda { |context|
           HString.create_instance(context["self"].ruby_value.inspect)
         }),
         "inspect" => HFunction.new([:self], lambda { |context|

data/lib/halunke/runtime/hstdio.rb

@@ -5,11 +5,11 @@ module Halunke
       [],
       {
         "puts" => HFunction.new([:self, :obj], lambda { |context|
-          str = context["obj"].receive_message(context, "to_s", [])
+          str = context["obj"].receive_message(context, "to_string", [])
           puts str.ruby_value
           str
         }),
-        "p" => HFunction.new([:self, :obj], lambda { |context|
+        "examine" => HFunction.new([:self, :obj], lambda { |context|
           str = context["obj"].receive_message(context, "inspect", [])
           puts str.ruby_value
           context["obj"]

data/lib/halunke/runtime/hstring.rb

@@ -48,10 +48,10 @@ module Halunke
         "+" => HFunction.new([:self, :other], lambda { |context|
           HString.create_instance(context["self"].ruby_value + context["other"].ruby_value)
         }),
-        "to_b" => HFunction.new([:self], lambda { |context|
+        "to_boolean" => HFunction.new([:self], lambda { |context|
           context["true"]
         }),
-        "to_s" => HFunction.new([:self], lambda { |context|
+        "to_string" => HFunction.new([:self], lambda { |context|
           context["self"]
         }),
         "inspect" => HFunction.new([:self], lambda { |context|

data/lib/halunke/runtime/true.hal

@@ -9,10 +9,10 @@
     "then else" { |'self 'true_branch 'false_branch|
       (true_branch call [])
     }
-    "to_b" { |'self|
+    "to_boolean" { |'self|
       self
     }
-    "to_s" { |'self 'other|
+    "to_string" { |'self 'other|
       "true"
     }
     "inspect" { |'self 'other|

data/lib/halunke/{lexer.rb → tokenizer.rb}

@@ -1,21 +1,21 @@
 
-# line 1 "lib/halunke/lexer.rl"
+# line 1 "lib/halunke/tokenizer.rl"
 =begin
 
-# line 40 "lib/halunke/lexer.rl"
+# line 40 "lib/halunke/tokenizer.rl"
 
 =end
 
 module Halunke
-  class Lexer
+  class Tokenizer
     def initialize
       
-# line 14 "lib/halunke/lexer.rb"
+# line 14 "lib/halunke/tokenizer.rb"
 class << self
-	attr_accessor :_lexer_actions
-	private :_lexer_actions, :_lexer_actions=
+	attr_accessor :_tokenizer_actions
+	private :_tokenizer_actions, :_tokenizer_actions=
 end
-self._lexer_actions = [
+self._tokenizer_actions = [
 	0, 1, 0, 1, 1, 1, 2, 1, 
 	5, 1, 6, 1, 7, 1, 8, 1, 
 	9, 1, 10, 1, 11, 1, 12, 1, 
@@ -26,132 +26,132 @@ self._lexer_actions = [
 ]
 
 class << self
-	attr_accessor :_lexer_key_offsets
-	private :_lexer_key_offsets, :_lexer_key_offsets=
+	attr_accessor :_tokenizer_key_offsets
+	private :_tokenizer_key_offsets, :_tokenizer_key_offsets=
 end
-self._lexer_key_offsets = [
-	0, 1, 3, 29, 30, 35, 36, 38, 
-	41, 43, 44, 45
+self._tokenizer_key_offsets = [
+	0, 1, 3, 29, 30, 35, 37, 39, 
+	42, 44, 45, 46
 ]
 
 class << self
-	attr_accessor :_lexer_trans_keys
-	private :_lexer_trans_keys, :_lexer_trans_keys=
+	attr_accessor :_tokenizer_trans_keys
+	private :_tokenizer_trans_keys, :_tokenizer_trans_keys=
 end
-self._lexer_trans_keys = [
+self._tokenizer_trans_keys = [
 	34, 48, 57, 32, 34, 39, 40, 41, 
 	42, 43, 45, 47, 64, 91, 93, 95, 
 	123, 124, 125, 9, 13, 48, 57, 60, 
 	62, 65, 90, 97, 122, 34, 95, 65, 
-	90, 97, 122, 47, 48, 57, 46, 48, 
-	57, 48, 57, 42, 91, 95, 65, 90, 
-	97, 122, 0
+	90, 97, 122, 42, 47, 48, 57, 46, 
+	48, 57, 48, 57, 42, 91, 95, 65, 
+	90, 97, 122, 0
 ]
 
 class << self
-	attr_accessor :_lexer_single_lengths
-	private :_lexer_single_lengths, :_lexer_single_lengths=
+	attr_accessor :_tokenizer_single_lengths
+	private :_tokenizer_single_lengths, :_tokenizer_single_lengths=
 end
-self._lexer_single_lengths = [
-	1, 0, 16, 1, 1, 1, 0, 1, 
+self._tokenizer_single_lengths = [
+	1, 0, 16, 1, 1, 2, 0, 1, 
 	0, 1, 1, 1
 ]
 
 class << self
-	attr_accessor :_lexer_range_lengths
-	private :_lexer_range_lengths, :_lexer_range_lengths=
+	attr_accessor :_tokenizer_range_lengths
+	private :_tokenizer_range_lengths, :_tokenizer_range_lengths=
 end
-self._lexer_range_lengths = [
+self._tokenizer_range_lengths = [
 	0, 1, 5, 0, 2, 0, 1, 1, 
 	1, 0, 0, 2
 ]
 
 class << self
-	attr_accessor :_lexer_index_offsets
-	private :_lexer_index_offsets, :_lexer_index_offsets=
+	attr_accessor :_tokenizer_index_offsets
+	private :_tokenizer_index_offsets, :_tokenizer_index_offsets=
 end
-self._lexer_index_offsets = [
-	0, 2, 4, 26, 28, 32, 34, 36, 
-	39, 41, 43, 45
+self._tokenizer_index_offsets = [
+	0, 2, 4, 26, 28, 32, 35, 37, 
+	40, 42, 44, 46
 ]
 
 class << self
-	attr_accessor :_lexer_trans_targs
-	private :_lexer_trans_targs, :_lexer_trans_targs=
+	attr_accessor :_tokenizer_trans_targs
+	private :_tokenizer_trans_targs, :_tokenizer_trans_targs=
 end
-self._lexer_trans_targs = [
+self._tokenizer_trans_targs = [
 	2, 0, 8, 2, 2, 3, 4, 2, 
 	2, 5, 6, 6, 9, 10, 2, 2, 
 	11, 2, 2, 2, 2, 7, 2, 11, 
 	11, 2, 2, 0, 4, 4, 4, 2, 
-	2, 2, 7, 2, 1, 7, 2, 8, 
-	2, 2, 2, 2, 2, 11, 11, 11, 
-	2, 2, 2, 2, 2, 2, 2, 2, 
-	2, 2, 2, 2, 0
+	2, 2, 2, 7, 2, 1, 7, 2, 
+	8, 2, 2, 2, 2, 2, 11, 11, 
+	11, 2, 2, 2, 2, 2, 2, 2, 
+	2, 2, 2, 2, 2, 0
 ]
 
 class << self
-	attr_accessor :_lexer_trans_actions
-	private :_lexer_trans_actions, :_lexer_trans_actions=
+	attr_accessor :_tokenizer_trans_actions
+	private :_tokenizer_trans_actions, :_tokenizer_trans_actions=
 end
-self._lexer_trans_actions = [
+self._tokenizer_trans_actions = [
 	7, 0, 0, 41, 31, 5, 50, 11, 
 	13, 0, 0, 0, 0, 0, 19, 21, 
 	0, 15, 29, 17, 31, 5, 9, 0, 
 	0, 33, 7, 0, 47, 47, 47, 45, 
-	27, 37, 5, 37, 0, 5, 35, 0, 
-	35, 25, 37, 23, 37, 0, 0, 0, 
-	37, 43, 41, 39, 45, 37, 37, 35, 
-	35, 37, 37, 37, 0
+	9, 27, 37, 5, 37, 0, 5, 35, 
+	0, 35, 25, 37, 23, 37, 0, 0, 
+	0, 37, 43, 41, 39, 45, 37, 37, 
+	35, 35, 37, 37, 37, 0
 ]
 
 class << self
-	attr_accessor :_lexer_to_state_actions
-	private :_lexer_to_state_actions, :_lexer_to_state_actions=
+	attr_accessor :_tokenizer_to_state_actions
+	private :_tokenizer_to_state_actions, :_tokenizer_to_state_actions=
 end
-self._lexer_to_state_actions = [
+self._tokenizer_to_state_actions = [
 	0, 0, 1, 0, 0, 0, 0, 0, 
 	0, 0, 0, 0
 ]
 
 class << self
-	attr_accessor :_lexer_from_state_actions
-	private :_lexer_from_state_actions, :_lexer_from_state_actions=
+	attr_accessor :_tokenizer_from_state_actions
+	private :_tokenizer_from_state_actions, :_tokenizer_from_state_actions=
 end
-self._lexer_from_state_actions = [
+self._tokenizer_from_state_actions = [
 	0, 0, 3, 0, 0, 0, 0, 0, 
 	0, 0, 0, 0
 ]
 
 class << self
-	attr_accessor :_lexer_eof_trans
-	private :_lexer_eof_trans, :_lexer_eof_trans=
+	attr_accessor :_tokenizer_eof_trans
+	private :_tokenizer_eof_trans, :_tokenizer_eof_trans=
 end
-self._lexer_eof_trans = [
-	50, 51, 0, 52, 53, 60, 60, 57, 
-	57, 60, 60, 60
+self._tokenizer_eof_trans = [
+	51, 52, 0, 53, 54, 61, 61, 58, 
+	58, 61, 61, 61
 ]
 
 class << self
-	attr_accessor :lexer_start
+	attr_accessor :tokenizer_start
 end
-self.lexer_start = 2;
+self.tokenizer_start = 2;
 class << self
-	attr_accessor :lexer_first_final
+	attr_accessor :tokenizer_first_final
 end
-self.lexer_first_final = 2;
+self.tokenizer_first_final = 2;
 class << self
-	attr_accessor :lexer_error
+	attr_accessor :tokenizer_error
 end
-self.lexer_error = -1;
+self.tokenizer_error = -1;
 
 class << self
-	attr_accessor :lexer_en_main
+	attr_accessor :tokenizer_en_main
 end
-self.lexer_en_main = 2;
+self.tokenizer_en_main = 2;
 
 
-# line 47 "lib/halunke/lexer.rl"
+# line 47 "lib/halunke/tokenizer.rl"
       @tokens = []
     end
 
@@ -160,19 +160,19 @@ self.lexer_en_main = 2;
       eof = data.length
 
       
-# line 164 "lib/halunke/lexer.rb"
+# line 164 "lib/halunke/tokenizer.rb"
 begin
 	p ||= 0
 	pe ||= data.length
-	cs = lexer_start
+	cs = tokenizer_start
 	ts = nil
 	te = nil
 	act = 0
 end
 
-# line 55 "lib/halunke/lexer.rl"
+# line 55 "lib/halunke/tokenizer.rl"
       
-# line 176 "lib/halunke/lexer.rb"
+# line 176 "lib/halunke/tokenizer.rb"
 begin
 	_klen, _trans, _keys, _acts, _nacts = nil
 	_goto_level = 0
@@ -190,27 +190,27 @@ begin
 	end
 	end
 	if _goto_level <= _resume
-	_acts = _lexer_from_state_actions[cs]
-	_nacts = _lexer_actions[_acts]
+	_acts = _tokenizer_from_state_actions[cs]
+	_nacts = _tokenizer_actions[_acts]
 	_acts += 1
 	while _nacts > 0
 		_nacts -= 1
 		_acts += 1
-		case _lexer_actions[_acts - 1]
+		case _tokenizer_actions[_acts - 1]
 			when 1 then
 # line 1 "NONE"
 		begin
 ts = p
 		end
-# line 206 "lib/halunke/lexer.rb"
+# line 206 "lib/halunke/tokenizer.rb"
 		end # from state action switch
 	end
 	if _trigger_goto
 		next
 	end
-	_keys = _lexer_key_offsets[cs]
-	_trans = _lexer_index_offsets[cs]
-	_klen = _lexer_single_lengths[cs]
+	_keys = _tokenizer_key_offsets[cs]
+	_trans = _tokenizer_index_offsets[cs]
+	_klen = _tokenizer_single_lengths[cs]
 	_break_match = false
 	
 	begin
@@ -222,9 +222,9 @@ ts = p
 	        break if _upper < _lower
 	        _mid = _lower + ( (_upper - _lower) >> 1 )
 
-	        if data[p].ord < _lexer_trans_keys[_mid]
+	        if data[p].ord < _tokenizer_trans_keys[_mid]
 	           _upper = _mid - 1
-	        elsif data[p].ord > _lexer_trans_keys[_mid]
+	        elsif data[p].ord > _tokenizer_trans_keys[_mid]
 	           _lower = _mid + 1
 	        else
 	           _trans += (_mid - _keys)
@@ -236,16 +236,16 @@ ts = p
 	     _keys += _klen
 	     _trans += _klen
 	  end
-	  _klen = _lexer_range_lengths[cs]
+	  _klen = _tokenizer_range_lengths[cs]
 	  if _klen > 0
 	     _lower = _keys
 	     _upper = _keys + (_klen << 1) - 2
 	     loop do
 	        break if _upper < _lower
 	        _mid = _lower + (((_upper-_lower) >> 1) & ~1)
-	        if data[p].ord < _lexer_trans_keys[_mid]
+	        if data[p].ord < _tokenizer_trans_keys[_mid]
 	          _upper = _mid - 2
-	        elsif data[p].ord > _lexer_trans_keys[_mid+1]
+	        elsif data[p].ord > _tokenizer_trans_keys[_mid+1]
 	          _lower = _mid + 2
 	        else
 	          _trans += ((_mid - _keys) >> 1)
@@ -259,137 +259,137 @@ ts = p
 	end while false
 	end
 	if _goto_level <= _eof_trans
-	cs = _lexer_trans_targs[_trans]
-	if _lexer_trans_actions[_trans] != 0
-		_acts = _lexer_trans_actions[_trans]
-		_nacts = _lexer_actions[_acts]
+	cs = _tokenizer_trans_targs[_trans]
+	if _tokenizer_trans_actions[_trans] != 0
+		_acts = _tokenizer_trans_actions[_trans]
+		_nacts = _tokenizer_actions[_acts]
 		_acts += 1
 		while _nacts > 0
 			_nacts -= 1
 			_acts += 1
-			case _lexer_actions[_acts - 1]
+			case _tokenizer_actions[_acts - 1]
 when 2 then
 # line 1 "NONE"
 		begin
 te = p+1
 		end
 when 3 then
-# line 24 "lib/halunke/lexer.rl"
+# line 24 "lib/halunke/tokenizer.rl"
 		begin
 act = 3;		end
 when 4 then
-# line 37 "lib/halunke/lexer.rl"
+# line 37 "lib/halunke/tokenizer.rl"
 		begin
 act = 16;		end
 when 5 then
-# line 23 "lib/halunke/lexer.rl"
+# line 23 "lib/halunke/tokenizer.rl"
 		begin
 te = p+1
  begin  emit(:STRING, data[ts+1...te-1])  end
 		end
 when 6 then
-# line 25 "lib/halunke/lexer.rl"
+# line 25 "lib/halunke/tokenizer.rl"
 		begin
 te = p+1
  begin  emit(:BAREWORD, data[ts...te])  end
 		end
 when 7 then
-# line 26 "lib/halunke/lexer.rl"
+# line 26 "lib/halunke/tokenizer.rl"
 		begin
 te = p+1
  begin  emit(:OPEN_PAREN, data[ts...te])  end
 		end
 when 8 then
-# line 27 "lib/halunke/lexer.rl"
+# line 27 "lib/halunke/tokenizer.rl"
 		begin
 te = p+1
  begin  emit(:CLOSE_PAREN, data[ts...te])  end
 		end
 when 9 then
-# line 28 "lib/halunke/lexer.rl"
+# line 28 "lib/halunke/tokenizer.rl"
 		begin
 te = p+1
  begin  emit(:OPEN_CURLY, data[ts...te])  end
 		end
 when 10 then
-# line 29 "lib/halunke/lexer.rl"
+# line 29 "lib/halunke/tokenizer.rl"
 		begin
 te = p+1
  begin  emit(:CLOSE_CURLY, data[ts...te])  end
 		end
 when 11 then
-# line 30 "lib/halunke/lexer.rl"
+# line 30 "lib/halunke/tokenizer.rl"
 		begin
 te = p+1
  begin  emit(:OPEN_BRACKET, data[ts...te])  end
 		end
 when 12 then
-# line 31 "lib/halunke/lexer.rl"
+# line 31 "lib/halunke/tokenizer.rl"
 		begin
 te = p+1
  begin  emit(:CLOSE_BRACKET, data[ts...te])  end
 		end
 when 13 then
-# line 32 "lib/halunke/lexer.rl"
+# line 32 "lib/halunke/tokenizer.rl"
 		begin
 te = p+1
  begin  emit(:OPEN_DICT_BRACKET, data[ts...te])  end
 		end
 when 14 then
-# line 33 "lib/halunke/lexer.rl"
+# line 33 "lib/halunke/tokenizer.rl"
 		begin
 te = p+1
  begin  emit(:START_COMMENT, data[ts...te])  end
 		end
 when 15 then
-# line 34 "lib/halunke/lexer.rl"
+# line 34 "lib/halunke/tokenizer.rl"
 		begin
 te = p+1
  begin  emit(:END_COMMENT, data[ts...te])  end
 		end
 when 16 then
-# line 35 "lib/halunke/lexer.rl"
+# line 35 "lib/halunke/tokenizer.rl"
 		begin
 te = p+1
  begin  emit(:BAR, data[ts...te])  end
 		end
 when 17 then
-# line 36 "lib/halunke/lexer.rl"
+# line 36 "lib/halunke/tokenizer.rl"
 		begin
 te = p+1
 		end
 when 18 then
-# line 37 "lib/halunke/lexer.rl"
+# line 37 "lib/halunke/tokenizer.rl"
 		begin
 te = p+1
  begin  raise "Could not lex '#{ data[ts...te] }'"  end
 		end
 when 19 then
-# line 22 "lib/halunke/lexer.rl"
+# line 22 "lib/halunke/tokenizer.rl"
 		begin
 te = p
 p = p - 1; begin  emit(:NUMBER, data[ts...te].to_r)  end
 		end
 when 20 then
-# line 25 "lib/halunke/lexer.rl"
+# line 25 "lib/halunke/tokenizer.rl"
 		begin
 te = p
 p = p - 1; begin  emit(:BAREWORD, data[ts...te])  end
 		end
 when 21 then
-# line 37 "lib/halunke/lexer.rl"
+# line 37 "lib/halunke/tokenizer.rl"
 		begin
 te = p
 p = p - 1; begin  raise "Could not lex '#{ data[ts...te] }'"  end
 		end
 when 22 then
-# line 22 "lib/halunke/lexer.rl"
+# line 22 "lib/halunke/tokenizer.rl"
 		begin
  begin p = ((te))-1; end
  begin  emit(:NUMBER, data[ts...te].to_r)  end
 		end
 when 23 then
-# line 37 "lib/halunke/lexer.rl"
+# line 37 "lib/halunke/tokenizer.rl"
 		begin
  begin p = ((te))-1; end
  begin  raise "Could not lex '#{ data[ts...te] }'"  end
@@ -406,7 +406,7 @@ when 24 then
  raise "Could not lex '#{ data[ts...te] }'" end
 end 
 			end
-# line 410 "lib/halunke/lexer.rb"
+# line 410 "lib/halunke/tokenizer.rb"
 			end # action switch
 		end
 	end
@@ -415,18 +415,18 @@ end
 	end
 	end
 	if _goto_level <= _again
-	_acts = _lexer_to_state_actions[cs]
-	_nacts = _lexer_actions[_acts]
+	_acts = _tokenizer_to_state_actions[cs]
+	_nacts = _tokenizer_actions[_acts]
 	_acts += 1
 	while _nacts > 0
 		_nacts -= 1
 		_acts += 1
-		case _lexer_actions[_acts - 1]
+		case _tokenizer_actions[_acts - 1]
 when 0 then
 # line 1 "NONE"
 		begin
 ts = nil;		end
-# line 430 "lib/halunke/lexer.rb"
+# line 430 "lib/halunke/tokenizer.rb"
 		end # to state action switch
 	end
 	if _trigger_goto
@@ -440,8 +440,8 @@ ts = nil;		end
 	end
 	if _goto_level <= _test_eof
 	if p == eof
-	if _lexer_eof_trans[cs] > 0
-		_trans = _lexer_eof_trans[cs] - 1;
+	if _tokenizer_eof_trans[cs] > 0
+		_trans = _tokenizer_eof_trans[cs] - 1;
 		_goto_level = _eof_trans
 		next;
 	end
@@ -453,7 +453,7 @@ end
 	end
 	end
 
-# line 56 "lib/halunke/lexer.rl"
+# line 56 "lib/halunke/tokenizer.rl"
 
       @tokens
     end