keisan 0.8.0 → 0.8.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f2d556d5ef3e67015596fee5856e2fe0b27295b227f7d4edf8cb41c037aed98b
-  data.tar.gz: c8c20ac6ba6a9ad27654931238a2dbafcbbbc509b58fd2a74161b7a38b3edaab
+  metadata.gz: a2a155be79ae75b3c67a097791c2af2bf67196cae16ef5372049b4eb54884117
+  data.tar.gz: b5d38c6111d1803a14c0229b0f54dc8733d581409254068ac9970af1321d9fca
 SHA512:
-  metadata.gz: 9ccc5172adc0b5ebafabe3cd5846fb455ada6e1351c94deab4da5bfd38617c0e0692258e9384d760d25b88a899b46de1899b1688850bfac8ddcdd432091ddf50
-  data.tar.gz: cb2ca195319a744f3ee9b56ef7df14d8dfcf36dbcafcbe1e675a858f232457d39cb3fe96d7f056a26746630d4a7f1c47a8fe6fc75a880683cd2120428a448ff6
+  metadata.gz: 550d595b64739c49e6fdcd12b149fa0f5706d3032f11c61f6f9b9bd3ac6ff7465398a5b04f58868dca803c9cef1b9fb14f9df029bc03c550d8616760751f3886
+  data.tar.gz: d2c25f61cdd3400c198cb6962cd31c0023ef5aa0081d3a0ea2699b7e7098620d8d39dc8af26bee5fe10197f84fb91218763fdc6ba830d2c97b30122a1b30fd3d

data/.travis.yml

@@ -1,7 +1,9 @@
 sudo: false
 language: ruby
+before_install:
+  - gem install bundler
 rvm:
-  - 2.3.6
-  - 2.4.3
-  - 2.5.0
-before_install: gem install bundler -v 1.16.1
+  - 2.4.9
+  - 2.5.7
+  - 2.6.5
+  - 2.7.0

data/README.md

@@ -178,6 +178,22 @@ calculator.evaluate("x = 11; {let x = 12}; x")
 #=> 11
 ```
 
+##### Comments
+
+When working with multi-line blocks of code, sometimes comments are useful to include.
+Comments are parts of a string from the `#` character to the end of a line (indicated by a newline character `"\n"`).
+
+```
+calculator = Keisan::Calculator.new
+calculator.evaluate("""
+  # This is a comment
+  x = 'foo'
+  x += '#bar' # Notice that `#` inside strings is not part of the comment
+  x # Should print 'foo#bar'
+""")
+#=> "foo#bar"
+```
+
 ##### Lists
 
 Just like in Ruby, lists can be defined using square brackets, and indexed using square brackets
@@ -256,6 +272,7 @@ calculator.evaluate("range(1, 6).map(x, [x, x**2]).to_h")
 Keisan supports date and time objects like in Ruby.
 You create a date object using either the method `date` (either a string to be parsed, or year, month, day numerical arguments) or `today`.
 They support methods `year`, `month`, `day`, `weekday`, `strftime`, and `to_time` to convert to a time object.
+`epoch_days` computes the number of days since Unix epoch (Jan 1, 1970).
 
 ```ruby
 calculator = Keisan::Calculator.new
@@ -266,10 +283,13 @@ calculator.evaluate("today() > date(2018, 11, 1)")
 #=> true
 calculator.evaluate("date('1999-12-31').to_time + 10")
 #=> Time.new(1999, 12, 31, 0, 0, 10)
+calculator.evaluate("date(1970, 1, 15).epoch_days")
+#=> 14
 ```
 
 Time objects are created using `time` (either a string to be parsed, or year, month, day, hour, minute, second arguments) or `now`.
 They support methods `year`, `month`, `day`, `hour`, `minute`, `second`, `weekday`, `strftime`, and `to_date` to convert to a date object.
+`epoch_seconds` computes the number of seconds since Unix epoch (00:00:00 on Jan 1, 1970).
 
 ```ruby
 calculator = Keisan::Calculator.new
@@ -279,6 +299,8 @@ calculator.evaluate("time('2000-4-15 12:34:56').minute")
 #=> 34
 calculator.evaluate("time('5000-10-10 20:30:40').strftime('%b %d, %Y')")
 #=> "Oct 10, 5000"
+calculator.evaluate("time(1970, 1, 1, 2, 3, 4).epoch_seconds")
+#=> 7384
 ```
 
 ##### Functional programming methods
@@ -340,7 +362,7 @@ calculator.evaluate("[2, 3, 7, 11].has_element(11)")
 
 ##### Bitwise operations
 
-The basic bitwise operations, NOT `~`, OR `|`, XOR `^`, and AND `&` are also available for use
+The basic bitwise operations, NOT `~`, OR `|`, XOR `^`, AND `&`, and left/right bitwise shifts (`<<` and `>>`) are also available for use
 
 ```ruby
 calculator = Keisan::Calculator.new
@@ -485,6 +507,7 @@ calculator.evaluate("puts x**2") # prints "25\n" to STDOUT
 
 #### Bitwise operators
 - `&`, `|`, `^`: bitwise **and**, **or**, **xor** operators
+- `<<`, `>>` bitwise shift operators
 - `~`: unary bitwise not
 
 #### Indexing of arrays/hashes

data/keisan.gemspec

@@ -21,10 +21,12 @@ Gem::Specification.new do |spec|
 
   spec.required_ruby_version = ">= 2.3.0"
 
-  spec.add_development_dependency "coveralls"
-  spec.add_development_dependency "bundler", "~> 1.14"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_dependency "cmath", "~> 1.0"
+
+  spec.add_development_dependency "bundler", "~> 2.0"
+  spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "pry"
   spec.add_development_dependency "pry-stack_explorer"
+  spec.add_development_dependency "simplecov"
 end

data/lib/keisan.rb

@@ -37,6 +37,8 @@ require "keisan/ast/bitwise_operator"
 require "keisan/ast/bitwise_and"
 require "keisan/ast/bitwise_or"
 require "keisan/ast/bitwise_xor"
+require "keisan/ast/bitwise_left_shift"
+require "keisan/ast/bitwise_right_shift"
 require "keisan/ast/logical_operator"
 require "keisan/ast/logical_and"
 require "keisan/ast/logical_or"
@@ -78,10 +80,12 @@ require "keisan/tokens/assignment"
 require "keisan/tokens/arithmetic_operator"
 require "keisan/tokens/logical_operator"
 require "keisan/tokens/bitwise_operator"
+require "keisan/tokens/bitwise_shift"
 require "keisan/tokens/word"
 require "keisan/tokens/line_separator"
 require "keisan/tokens/unknown"
 
+require "keisan/string_and_group_parser"
 require "keisan/tokenizer"
 
 require "keisan/parsing/component"
@@ -128,6 +132,8 @@ require "keisan/parsing/bitwise_or"
 require "keisan/parsing/bitwise_xor"
 require "keisan/parsing/bitwise_not"
 require "keisan/parsing/bitwise_not_not"
+require "keisan/parsing/bitwise_left_shift"
+require "keisan/parsing/bitwise_right_shift"
 require "keisan/parsing/logical_operator"
 require "keisan/parsing/logical_less_than"
 require "keisan/parsing/logical_greater_than"

data/lib/keisan/ast/bitwise_left_shift.rb

@@ -0,0 +1,17 @@
+module Keisan
+  module AST
+    class BitwiseLeftShift < BitwiseOperator
+      def self.symbol
+        :<<
+      end
+
+      def blank_value
+        0
+      end
+
+      def evaluate(context = nil)
+        children[1..-1].inject(children.first.evaluate(context)) {|total, child| total << child.evaluate(context)}
+      end
+    end
+  end
+end

data/lib/keisan/ast/bitwise_right_shift.rb

@@ -0,0 +1,17 @@
+module Keisan
+  module AST
+    class BitwiseRightShift < BitwiseOperator
+      def self.symbol
+        :>>
+      end
+
+      def blank_value
+        0
+      end
+
+      def evaluate(context = nil)
+        children[1..-1].inject(children.first.evaluate(context)) {|total, child| total >> child.evaluate(context)}
+      end
+    end
+  end
+end

data/lib/keisan/ast/block.rb

@@ -17,6 +17,10 @@ module Keisan
         child.unbound_functions(local)
       end
 
+      def contains_a?(klass)
+        super || child.contains_a?(klass)
+      end
+
       def deep_dup
         dupped = dup
         dupped.instance_variable_set(

data/lib/keisan/ast/cell.rb

@@ -15,6 +15,10 @@ module Keisan
         node.unbound_functions(context)
       end
 
+      def contains_a?(klass)
+        super || node.contains_a?(klass)
+      end
+
       def deep_dup
         dupped = dup
         dupped.instance_variable_set(

data/lib/keisan/ast/function_assignment.rb

@@ -29,8 +29,7 @@ module Keisan
 
       def evaluate
         # Blocks might have local variable/function definitions, so skip check
-        verify_rhs_of_function_assignment_is_valid! unless rhs.is_a?(Block)
-
+        verify_rhs_of_function_assignment_is_valid!
         context.register_function!(lhs.name, expression_function, local: local)
         rhs
       end
@@ -38,15 +37,26 @@ module Keisan
       private
 
       def verify_rhs_of_function_assignment_is_valid!
-        # Only variables that can appear are those that are arguments to the function
-        unless rhs.unbound_variables(context) <= Set.new(argument_names)
-          raise Exceptions::InvalidExpression.new("Unbound variables found in function definition")
-        end
+        verify_unbound_functions!
+        verify_unbound_variables!
+      end
+
+      def verify_unbound_functions!
         # Cannot have undefined functions unless allowed by context
         unless context.allow_recursive || rhs.unbound_functions(context).empty?
           raise Exceptions::InvalidExpression.new("Unbound function definitions are not allowed by current context")
         end
       end
+
+      def verify_unbound_variables!
+        # We allow unbound variables inside block statements, as they could be temporary
+        # variables assigned locally
+        return if rhs.is_a?(Block)
+        # Only variables that can appear are those that are arguments to the function
+        unless rhs.unbound_variables(context) <= Set.new(argument_names)
+          raise Exceptions::InvalidExpression.new("Unbound variables found in function definition")
+        end
+      end
     end
   end
 end

data/lib/keisan/ast/hash.rb

@@ -21,6 +21,10 @@ module Keisan
         end
       end
 
+      def contains_a?(klass)
+        super || @hash.any? {|k, v| k.to_node.contains_a?(klass) || v.contains_a?(klass) }
+      end
+
       def evaluate(context = nil)
         context ||= Context.new
 

data/lib/keisan/ast/node.rb

@@ -37,6 +37,15 @@ module Keisan
         value(context)
       end
 
+      def contains_a?(klass)
+        case klass
+        when Array
+          klass.any? {|k| is_a?(k) }
+        else
+          is_a?(klass)
+        end
+      end
+
       def evaluate_assignments(context = nil)
         self
       end
@@ -45,10 +54,18 @@ module Keisan
         raise Exceptions::NonDifferentiableError.new
       end
 
+      def differentiated(variable, context = nil)
+        deep_dup.differentiate(variable, context)
+      end
+
       def replace(variable, replacement)
         self
       end
 
+      def replaced(variable, replacement)
+        deep_dup.replace(variable, replacement)
+      end
+
       def coerce(other)
         [other.to_node, self]
       end
@@ -132,6 +149,14 @@ module Keisan
         BitwiseOr.new([self, other.to_node])
       end
 
+      def <<(other)
+        BitwiseLeftShift.new([self, other.to_node])
+      end
+
+      def >>(other)
+        BitwiseRightShift.new([self, other.to_node])
+      end
+
       def >(other)
         LogicalGreaterThan.new([self, other.to_node])
       end

data/lib/keisan/ast/number.rb

@@ -111,6 +111,26 @@ module Keisan
         end
       end
 
+      def <<(other)
+        other = other.to_node
+        case other
+        when Number
+          Number.new(value << other.value)
+        else
+          super
+        end
+      end
+
+      def >>(other)
+        other = other.to_node
+        case other
+        when Number
+          Number.new(value >> other.value)
+        else
+          super
+        end
+      end
+
       def >(other)
         other = other.to_node
         case other

data/lib/keisan/ast/operator.rb

@@ -13,6 +13,8 @@ module Keisan
         "%":  [2,  85, :left],   # Modulo
         "+":  [2,  80, :left],   # Plus
         # "-":  [2,  80, :left], # Minus
+        "<<": [2,  75, :left],   # Bitwise left shift
+        ">>": [2,  75, :left],   # Bitwise right shift
         "&":  [2,  70, :left],   # Bitwise and
         "^":  [2,  65, :left],   # Bitwise xor
         "|":  [2,  65, :left],   # Bitwise or

data/lib/keisan/ast/parent.rb

@@ -25,6 +25,10 @@ module Keisan
         end
       end
 
+      def contains_a?(klass)
+        super || children.any? {|child| child.contains_a?(klass) }
+      end
+
       def freeze
         children.each(&:freeze)
         super

data/lib/keisan/ast/unary_inverse.rb

@@ -19,7 +19,7 @@ module Keisan
         @children = [child.simplify(context)]
         case child
         when Number
-          Number.new(Rational(1,child.value(context))).simplify(context)
+          Number.new(child.value**-1)
         else
           (child ** -1).simplify(context)
         end

data/lib/keisan/ast/variable.rb

@@ -13,7 +13,13 @@ module Keisan
 
       def value(context = nil)
         context ||= Context.new
-        variable_node_from_context(context).value(context)
+        node = variable_node_from_context(context)
+        case node
+        when Variable
+          node
+        else
+          node.value(context)
+        end
       end
 
       def unbound_variables(context = nil)
@@ -78,9 +84,7 @@ module Keisan
 
       def variable_node_from_context(context)
         variable = context.variable(name)
-        if variable.is_a?(Cell)
-          variable = variable.node
-        end
+        variable = variable.node if variable.is_a?(Cell)
         variable
       end
     end

data/lib/keisan/calculator.rb

@@ -2,8 +2,14 @@ module Keisan
   class Calculator
     attr_reader :context
 
-    def initialize(context: nil, allow_recursive: false)
-      @context = context || Context.new(allow_recursive: allow_recursive)
+    # Note, allow_recursive would be more appropriately named:
+    # allow_unbound_functions_in_function_definitions, but it is too late for that.
+    def initialize(context: nil, allow_recursive: false, allow_blocks: true, allow_multiline: true)
+      @context = context || Context.new(
+        allow_recursive: allow_recursive,
+        allow_blocks: allow_blocks,
+        allow_multiline: allow_multiline
+      )
     end
 
     def allow_recursive
@@ -14,6 +20,14 @@ module Keisan
       context.allow_recursive!
     end
 
+    def allow_blocks
+      context.allow_blocks
+    end
+
+    def allow_multiline
+      context.allow_multiline
+    end
+
     def evaluate(expression, definitions = {})
       Evaluator.new(self).evaluate(expression, definitions)
     end
@@ -23,7 +37,7 @@ module Keisan
     end
 
     def ast(expression)
-      Evaluator.new(self).ast(expression)
+      Evaluator.new(self).parse_ast(expression)
     end
 
     def define_variable!(name, value)

data/lib/keisan/context.rb

@@ -1,13 +1,24 @@
 module Keisan
   class Context
-    attr_reader :function_registry, :variable_registry, :allow_recursive
-
-    def initialize(parent: nil, random: nil, allow_recursive: false, shadowed: [])
+    attr_reader :function_registry,
+      :variable_registry,
+      :allow_recursive,
+      :allow_multiline,
+      :allow_blocks
+
+    def initialize(parent: nil,
+                   random: nil,
+                   allow_recursive: false,
+                   allow_multiline: true,
+                   allow_blocks: true,
+                   shadowed: [])
       @parent = parent
       @function_registry = Functions::Registry.new(parent: @parent&.function_registry)
       @variable_registry = Variables::Registry.new(parent: @parent&.variable_registry, shadowed: shadowed)
       @random            = random
       @allow_recursive   = allow_recursive
+      @allow_multiline   = allow_multiline
+      @allow_blocks      = allow_blocks
     end
 
     def allow_recursive!
@@ -110,7 +121,13 @@ module Keisan
     end
 
     def pure_child(shadowed: [])
-      self.class.new(parent: self, shadowed: shadowed, allow_recursive: allow_recursive)
+      self.class.new(
+        parent: self,
+        shadowed: shadowed,
+        allow_recursive: allow_recursive,
+        allow_multiline: allow_multiline,
+        allow_blocks: allow_blocks
+      )
     end
   end
 end

data/lib/keisan/evaluator.rb

@@ -8,7 +8,7 @@ module Keisan
 
     def evaluate(expression, definitions = {})
       context = calculator.context.spawn_child(definitions: definitions, transient: true)
-      ast = ast(expression)
+      ast = parse_ast(expression)
       last_line = last_line(ast)
 
       evaluation = ast.evaluated(context)
@@ -24,16 +24,28 @@ module Keisan
 
     def simplify(expression, definitions = {})
       context = calculator.context.spawn_child(definitions: definitions, transient: true)
-      ast = AST.parse(expression)
+      ast = parse_ast(expression)
       ast.simplify(context)
     end
 
-    def ast(expression)
-      AST.parse(expression)
+    def parse_ast(expression)
+      AST.parse(expression).tap do |ast|
+        disallowed = disallowed_nodes
+        if !disallowed.empty? && ast.contains_a?(disallowed)
+          raise Keisan::Exceptions::InvalidExpression.new("Context does not permit expressions with #{disallowed}")
+        end
+      end
     end
 
     private
 
+    def disallowed_nodes
+      disallowed = []
+      disallowed << Keisan::AST::Block unless calculator.allow_blocks
+      disallowed << Keisan::AST::MultiLine unless calculator.allow_multiline
+      disallowed
+    end
+
     def last_line(ast)
       ast.is_a?(AST::MultiLine) ? ast.children.last : ast
     end

data/lib/keisan/functions/cmath_function.rb

@@ -4,7 +4,9 @@ module Keisan
   module Functions
     class CMathFunction < MathFunction
       def initialize(name, proc_function = nil)
-        super(name, proc_function || Proc.new {|arg| CMath.send(name, arg)})
+        super(name, proc_function || Proc.new {|arg|
+          CMath.send(name, arg)
+        })
       end
     end
   end

data/lib/keisan/functions/default_registry.rb

@@ -147,6 +147,9 @@ module Keisan
 
         registry.register!(:to_time, Proc.new {|d| d.to_time }, force: true)
         registry.register!(:to_date, Proc.new {|t| t.to_date }, force: true)
+
+        registry.register!(:epoch_seconds, Proc.new {|d| d.to_time - Time.new(1970, 1, 1, 0, 0, 0) }, force: true)
+        registry.register!(:epoch_days, Proc.new {|t| t.to_date  - Date.new(1970, 1, 1) }, force: true)
       end
 
       def self.register_date_time!(registry)

data/lib/keisan/functions/expression_function.rb

@@ -5,7 +5,11 @@ module Keisan
 
       def initialize(name, arguments, expression, transient_definitions)
         super(name, arguments.count)
-        @expression = expression.deep_dup
+        if expression.is_a?(::String)
+          @expression = AST::parse(expression)
+        else
+          @expression = expression.deep_dup
+        end
         @arguments = arguments
         @transient_definitions = transient_definitions
       end
@@ -70,17 +74,20 @@ module Keisan
 
         local = local_context_for(context)
 
-        # expression.differentiate(variable, context)
-
-        argument_values = ast_function.children.map {|child| child.evaluate(local)}
+        argument_values = ast_function.children.map {|child| child.evaluated(local)}
 
         argument_derivatives = ast_function.children.map do |child|
-          child.differentiate(variable, context)
+          child.differentiated(variable, context)
         end
 
+        partial_derivatives = calculate_partial_derivatives(context)
+
         AST::Plus.new(
           argument_derivatives.map.with_index {|argument_derivative, i|
-            partial_derivative = partial_derivatives[i].replace(argument_variables[i], argument_values[i])
+            partial_derivative = partial_derivatives[i]
+            argument_variables.each.with_index {|argument_variable, j|
+              partial_derivative = partial_derivative.replaced(argument_variable, argument_values[j])
+            }
             AST::Times.new([argument_derivative, partial_derivative])
           }
         )
@@ -92,9 +99,9 @@ module Keisan
         @argument_variables ||= arguments.map {|argument| AST::Variable.new(argument)}
       end
 
-      def partial_derivatives
-        @partial_derivatives ||= argument_variables.map.with_index do |variable, i|
-          partial_derivative = expression.differentiate(variable)
+      def calculate_partial_derivatives(context)
+        argument_variables.map.with_index do |variable, i|
+          partial_derivative = expression.differentiated(variable, context)
         end
       end
 

data/lib/keisan/functions/proc_function.rb

@@ -28,7 +28,7 @@ module Keisan
 
         ast_function.instance_variable_set(
           :@children,
-          ast_function.children.map {|child| child.evaluate(context).to_node}
+          ast_function.children.map {|child| child.simplify(context).to_node}
         )
 
         if ast_function.children.all? {|child| child.well_defined?(context)}
@@ -44,7 +44,7 @@ module Keisan
 
         ast_function.instance_variable_set(
           :@children,
-          ast_function.children.map {|child| child.evaluate(context)}
+          ast_function.children.map {|child| child.simplify(context)}
         )
 
         if ast_function.children.all? {|child| child.is_a?(AST::ConstantLiteral)}

data/lib/keisan/functions/replace.rb

@@ -10,19 +10,19 @@ module Keisan
         evaluate(ast_function, context).value(context)
       end
 
-      def evaluate(ast_function, context = nil)
+      def simplify(ast_function, context = nil)
         context ||= Context.new
         expression, variable, replacement = expression_variable_replacement(ast_function)
 
-        expression = expression.evaluate(context)
-        replacement = replacement.evaluate(context)
+        expression = expression.simplify(context)
+        replacement = replacement.simplify(context)
 
-        expression.replace(variable, replacement).evaluate(context)
+        expression.replace(variable, replacement).simplify(context)
       end
 
-      def simplify(ast_function, context = nil)
+      def evaluate(ast_function, context = nil)
         context ||= Context.new
-        evaluate(ast_function, context).simplify(context)
+        simplify(ast_function, context).evaluate(context)
       end
 
       private

data/lib/keisan/parser.rb

@@ -241,6 +241,8 @@ module Keisan
       :"&"  => Parsing::BitwiseAnd,
       :"|"  => Parsing::BitwiseOr,
       :"^"  => Parsing::BitwiseXor,
+      :<<   => Parsing::BitwiseLeftShift,
+      :>>   => Parsing::BitwiseRightShift,
       :"==" => Parsing::LogicalEqual,
       :"!=" => Parsing::LogicalNotEqual,
       :"&&" => Parsing::LogicalAnd,

data/lib/keisan/parsing/bitwise_left_shift.rb

@@ -0,0 +1,9 @@
+module Keisan
+  module Parsing
+    class BitwiseLeftShift < BitwiseOperator
+      def node_class
+        AST::BitwiseLeftShift
+      end
+    end
+  end
+end

data/lib/keisan/parsing/bitwise_right_shift.rb

@@ -0,0 +1,9 @@
+module Keisan
+  module Parsing
+    class BitwiseRightShift < BitwiseOperator
+      def node_class
+        AST::BitwiseRightShift
+      end
+    end
+  end
+end

data/lib/keisan/string_and_group_parser.rb

@@ -0,0 +1,235 @@
+module Keisan
+  class StringAndGroupParser
+    class Portion
+      attr_reader :start_index, :end_index
+
+      def initialize(start_index)
+        @start_index = start_index
+      end
+    end
+
+    class StringPortion < Portion
+      attr_reader :string, :escaped_string
+
+      def initialize(expression, start_index)
+        super(start_index)
+
+        @string = expression[start_index]
+        @escaped_string = expression[start_index]
+        @end_index = start_index + 1
+
+        while @end_index < expression.size
+          if expression[@end_index] == quote_type
+            @string << quote_type
+            @escaped_string << quote_type
+            @end_index += 1
+            # Successfully parsed the string
+            return
+          end
+
+          n, c = get_potentially_escaped_next_character(expression, @end_index)
+          @escaped_string << c
+          @end_index += n
+        end
+
+        raise Keisan::Exceptions::TokenizingError.new("Tokenizing error, no closing quote #{quote_type}")
+      end
+
+      def size
+        string.size
+      end
+
+      def to_s
+        string
+      end
+
+      private
+
+      # Returns number of processed input characters, and the output character
+      # If a sequence like '\"' is encountered, the first backslash escapes the
+      # second double-quote, and the two characters will act as a one double-quote
+      # character.
+      def get_potentially_escaped_next_character(expression, index)
+        @string << expression[index]
+        if expression[index] == "\\" && index + 1 < expression.size
+          @string << expression[index + 1]
+          return [2, escaped_character(expression[index + 1])]
+        else
+          return [1, expression[index]]
+        end
+      end
+
+      def quote_type
+        @string[0]
+      end
+
+      def escaped_character(character)
+        case character
+        when "\\", '"', "'"
+          character
+        when "a"
+          "\a"
+        when "b"
+          "\b"
+        when "r"
+          "\r"
+        when "n"
+          "\n"
+        when "s"
+          "\s"
+        when "t"
+          "\t"
+        else
+          raise Keisan::Exceptions::TokenizingError.new("Tokenizing error, unknown escape character: \"\\#{character}\"")
+        end
+      end
+    end
+
+    class GroupPortion < Portion
+      attr_reader :opening_brace, :closing_brace ,:portions, :size
+
+      OPENING_TO_CLOSING_BRACE = {
+        "(" => ")",
+        "{" => "}",
+        "[" => "]",
+      }
+
+      def initialize(expression, start_index)
+        super(start_index)
+
+        case expression[start_index]
+        when OPEN_GROUP_REGEX
+          @opening_brace = expression[start_index]
+        else
+          raise Keisan::Exceptions::TokenizingError.new("Internal error, GroupPortion did not start with brace")
+        end
+
+        @closing_brace = OPENING_TO_CLOSING_BRACE[opening_brace]
+
+        parser = StringAndGroupParser.new(expression, start_index: start_index + 1, ending_character: closing_brace)
+        @portions = parser.portions
+        @size = parser.size + 2
+
+        if start_index + size > expression.size || expression[start_index + size - 1] != closing_brace
+          raise Keisan::Exceptions::TokenizingError.new("Tokenizing error, group with opening brace #{opening_brace} did not have closing brace")
+        end
+      end
+
+      def to_s
+        opening_brace + portions.map(&:to_s).join + closing_brace
+      end
+    end
+
+    class OtherPortion < Portion
+      attr_reader :string
+
+      def initialize(expression, start_index)
+        super(start_index)
+
+        case expression[start_index]
+        when STRING_CHARACTER_REGEX, OPEN_GROUP_REGEX, CLOSED_GROUP_REGEX
+          raise Keisan::Exceptions::TokenizingError.new("Internal error, OtherPortion should not have string/braces at start")
+        else
+          index = start_index + 1
+        end
+
+        while index < expression.size
+          case expression[index]
+          when STRING_CHARACTER_REGEX, OPEN_GROUP_REGEX, CLOSED_GROUP_REGEX, COMMENT_CHARACTER_REGEX
+            break
+          else
+            index += 1
+          end
+        end
+
+        @end_index = index
+        @string = expression[start_index...end_index]
+      end
+
+      def size
+        string.size
+      end
+
+      def to_s
+        string
+      end
+    end
+
+    class CommentPortion < Portion
+      attr_reader :string
+
+      def initialize(expression, start_index)
+        super(start_index)
+
+        if expression[start_index] != '#'
+          raise Keisan::Exceptions::TokenizingError.new("Comment should start with '#'")
+        else
+          index = start_index + 1
+        end
+
+        while index < expression.size
+          break if expression[index] == "\n"
+          index += 1
+        end
+
+        @end_index = index
+        @string = expression[start_index...end_index]
+      end
+
+      def size
+        string.size
+      end
+
+      def to_s
+        string
+      end
+    end
+
+    # An ordered array of "portions", which
+    attr_reader :portions, :size
+
+    COMMENT_CHARACTER_REGEX = /[#]/
+    STRING_CHARACTER_REGEX = /["']/
+    OPEN_GROUP_REGEX = /[\(\{\[]/
+    CLOSED_GROUP_REGEX = /[\)\}\]]/
+
+    # Ending character is used as a second ending condition besides expression size
+    def initialize(expression, start_index: 0, ending_character: nil)
+      index = start_index
+      @portions = []
+
+      while index < expression.size && (ending_character.nil? || expression[index] != ending_character)
+        case expression[index]
+        when STRING_CHARACTER_REGEX
+          portion = StringPortion.new(expression, index)
+          index = portion.end_index
+          @portions << portion
+
+        when OPEN_GROUP_REGEX
+          portion = GroupPortion.new(expression, index)
+          index += portion.size
+          @portions << portion
+
+        when CLOSED_GROUP_REGEX
+          raise Keisan::Exceptions::TokenizingError.new("Tokenizing error, unexpected closing brace #{expression[start_index]}")
+
+        when COMMENT_CHARACTER_REGEX
+          portion = CommentPortion.new(expression, index)
+          index += portion.size
+          @portions << portion
+
+        else
+          portion = OtherPortion.new(expression, index)
+          index += portion.size
+          @portions << portion
+        end
+      end
+
+      @size = index - start_index
+    end
+
+    def to_s
+      portions.map(&:to_s).join
+    end
+  end
+end

data/lib/keisan/tokenizer.rb

@@ -1,13 +1,12 @@
 module Keisan
   class Tokenizer
     TOKEN_CLASSES = [
-      Tokens::Group,
-      Tokens::String,
       Tokens::Null,
       Tokens::Boolean,
       Tokens::Word,
       Tokens::Number,
       Tokens::Assignment,
+      Tokens::BitwiseShift,
       Tokens::LogicalOperator,
       Tokens::ArithmeticOperator,
       Tokens::BitwiseOperator,
@@ -25,28 +24,31 @@ module Keisan
     attr_reader :expression, :tokens
 
     def initialize(expression)
-      @expression = self.class.normalize_expression(expression)
-      @scan = @expression.scan(TOKEN_REGEX)
-      @tokens = tokenize!
-    end
+      @expression = expression
 
-    def self.normalize_expression(expression)
-      expression = normalize_line_delimiters(expression)
-      expression = remove_comments(expression)
-    end
+      portions = StringAndGroupParser.new(expression).portions.reject do |portion|
+        portion.is_a? StringAndGroupParser::CommentPortion
+      end
 
-    private
+      @tokens = portions.inject([]) do |tokens, portion|
+        case portion
+        when StringAndGroupParser::StringPortion
+          tokens << Tokens::String.new(portion.escaped_string)
+        when StringAndGroupParser::GroupPortion
+          tokens << Tokens::Group.new(portion.to_s)
+        when StringAndGroupParser::OtherPortion
+          scan = portion.to_s.scan(TOKEN_REGEX)
+          tokens += tokenize!(scan)
+        end
 
-    def self.normalize_line_delimiters(expression)
-      expression.gsub(/\n/, ";")
+        tokens
+      end
     end
 
-    def self.remove_comments(expression)
-      expression.gsub(/#[^;]*/, "")
-    end
+    private
 
-    def tokenize!
-      @scan.map do |scan_result|
+    def tokenize!(scan)
+      scan.map do |scan_result|
         i = scan_result.find_index {|token| !token.nil?}
         token_string = scan_result[i]
         token = TOKEN_CLASSES[i].new(token_string)

data/lib/keisan/tokens/assignment.rb

@@ -13,7 +13,9 @@ module Keisan
          \% |
          \& |
          \| |
-         \^
+         \^ |
+         \<\< |
+         \>\>
         )?
         \=
         (?!\=) # negative lookahead to prevent matching ==

data/lib/keisan/tokens/bitwise_shift.rb

@@ -0,0 +1,23 @@
+module Keisan
+  module Tokens
+    class BitwiseShift < Operator
+      LEFT_SHIFT = /(?:<<)/
+      RIGHT_SHIFT = /(?:>>)/
+
+      REGEX = /(#{LEFT_SHIFT}|#{RIGHT_SHIFT})/
+
+      def self.regex
+        REGEX
+      end
+
+      def operator_type
+        case string
+        when LEFT_SHIFT
+          :<<
+        when RIGHT_SHIFT
+          :>>
+        end
+      end
+    end
+  end
+end

data/lib/keisan/tokens/group.rb

@@ -1,19 +1,13 @@
 module Keisan
   module Tokens
     class Group < Token
-      REGEX = /(\((?:[^\[\]\(\)\{\}]*+\g<1>*+)*+\)|\[(?:[^\[\]\(\)\{\}]*+\g<1>*+)*+\]|\{(?:[^\[\]\(\)\{\}]*+\g<1>*+)*+\})/
-
       attr_reader :sub_tokens
 
       def initialize(string)
-        super
+        @string = string
         @sub_tokens = Tokenizer.new(string[1...-1]).tokens
       end
 
-      def self.regex
-        REGEX
-      end
-
       # Either :round, :square
       def group_type
         case string[0]

data/lib/keisan/tokens/string.rb

@@ -1,10 +1,8 @@
 module Keisan
   module Tokens
     class String < Token
-      REGEX = /(\"[^\"]*\"|\'[^\']*\')/
-
-      def self.regex
-        REGEX
+      def initialize(string)
+        @string = string
       end
 
       def value

data/lib/keisan/version.rb

@@ -1,3 +1,3 @@
 module Keisan
-  VERSION = "0.8.0"
+  VERSION = "0.8.5"
 end

metadata

@@ -1,57 +1,57 @@
 --- !ruby/object:Gem::Specification
 name: keisan
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.8.5
 platform: ruby
 authors:
 - Christopher Locke
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-23 00:00:00.000000000 Z
+date: 2020-09-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: coveralls
+  name: cmath
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
+        version: '1.0'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.14'
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.14'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -94,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: A library for parsing equations into an abstract syntax tree for evaluation
 email:
 - project.eutopia@gmail.com
@@ -117,8 +131,10 @@ files:
 - lib/keisan/ast/arithmetic_operator.rb
 - lib/keisan/ast/assignment.rb
 - lib/keisan/ast/bitwise_and.rb
+- lib/keisan/ast/bitwise_left_shift.rb
 - lib/keisan/ast/bitwise_operator.rb
 - lib/keisan/ast/bitwise_or.rb
+- lib/keisan/ast/bitwise_right_shift.rb
 - lib/keisan/ast/bitwise_xor.rb
 - lib/keisan/ast/block.rb
 - lib/keisan/ast/boolean.rb
@@ -219,10 +235,12 @@ files:
 - lib/keisan/parsing/arithmetic_operator.rb
 - lib/keisan/parsing/assignment.rb
 - lib/keisan/parsing/bitwise_and.rb
+- lib/keisan/parsing/bitwise_left_shift.rb
 - lib/keisan/parsing/bitwise_not.rb
 - lib/keisan/parsing/bitwise_not_not.rb
 - lib/keisan/parsing/bitwise_operator.rb
 - lib/keisan/parsing/bitwise_or.rb
+- lib/keisan/parsing/bitwise_right_shift.rb
 - lib/keisan/parsing/bitwise_xor.rb
 - lib/keisan/parsing/boolean.rb
 - lib/keisan/parsing/component.rb
@@ -266,11 +284,13 @@ files:
 - lib/keisan/parsing/unary_plus.rb
 - lib/keisan/parsing/variable.rb
 - lib/keisan/repl.rb
+- lib/keisan/string_and_group_parser.rb
 - lib/keisan/token.rb
 - lib/keisan/tokenizer.rb
 - lib/keisan/tokens/arithmetic_operator.rb
 - lib/keisan/tokens/assignment.rb
 - lib/keisan/tokens/bitwise_operator.rb
+- lib/keisan/tokens/bitwise_shift.rb
 - lib/keisan/tokens/boolean.rb
 - lib/keisan/tokens/colon.rb
 - lib/keisan/tokens/comma.rb
@@ -308,8 +328,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: An equation parser and evaluator