jaina 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 850df9cc5e4a4962b435569da878367493a801ac0f842760157083e8f5fba578
-  data.tar.gz: 79cb4a3a71d81205b4d6647a04de688b084b6108373d9cc274a11ca390fffadd
+  metadata.gz: 6a90305fd0da0eae831f491b174247e4e057eec761c841615c81b61eb2950ad2
+  data.tar.gz: 9ffe0dd0616419c9dab9d626f5047955d23d9aaa9db3cdd4054d63d0d5acaa10
 SHA512:
-  metadata.gz: 5253d6c057ef220178518f8793cbb7e40e091556a8efec5ee48d443c582408e2122b27cc098747f1b0d620d470604b041e1c8cf66ddd8eb7ffa4486fbd63f08a
-  data.tar.gz: 54d7867f5ee7ec104e2a46d8311f69c6cf3cab25b5d635a64458ee18ce1f94778cd3794425474346e371d576f4a6183f62a4363fb181ebe27507c9b36c44f231
+  metadata.gz: 6c8d4c1c643e519fee75dc68698e6a74e33ef79476a78e5b831cd80c1bc77d8465eddfdc61f451257b04d3eb851f0e40607d2d1d6fff66bf237478ad4a2c9034
+  data.tar.gz: c47ebf4c452f44729936a3210614999debe1e1859fa9fe0f5a775650d217d71ce189f1692332579079c99ff0b412593fd735905084322183ece79d0c4bb69ce2

data/CHANGELOG.md

@@ -1,7 +1,12 @@
 # Changelog
 All notable changes to this project will be documented in this file.
 
+## [0.4.0] - 2019-06-24
+### Added
+- Support for operand attributes and operator attributes;
+
 ## [0.3.0] - 2019-06-21
+### Added
 - Support for initial context: `Jaina.evaluate(program, **initial_context)`
 
 ## [0.2.0] - 2019-06-21

data/README.md

@@ -104,8 +104,13 @@ context.get(:b) # => Jaina::Parser::AST::Contex::UndefinedContextKeyError
 ### Parse your code (build AST)
 
 ```ruby
+# NOTE: without arguments
 Jaina.parse('A AND B AND (C OR D) OR A AND (C OR E)')
 # => #<Jaina::Parser::AST:0x00007fd6f424a2e8>
+
+# NOTE: with arguments
+Jaina.parse('A[1,2] AND B[3,4]')
+# => #<Jaina::Parser::AST:0x00007fd6f424a2e9>
 ```
 
 ---
@@ -113,14 +118,43 @@ Jaina.parse('A AND B AND (C OR D) OR A AND (C OR E)')
 ### Evaluate your code
 
 ```ruby
-ast = Jaina.parse('A AND B AND (C OR D) OR A AND (C OR E)')
+ast = Jaina.parse('A AND B[5,test] AND (C OR D) OR A AND (C OR E)')
 ast.evaluate
 
 # --- or ---
-Jaina.evaluate('A AND B AND (C OR D) OR A AND (C OR E)')
+Jaina.evaluate('A AND B[5,test] AND (C OR D) OR A AND (C OR E)')
 
 # --- you can set initial context of your program ---
-Jaina.evaluate('A AND B', login: 'admin', logged_in: true)
+Jaina.evaluate('A AND B[5,test]', login: 'admin', logged_in: true)
+```
+
+---
+
+### Custom operator/operand arguments
+
+```ruby
+# NOTE: use []
+Jaina.parse('A[1,true] AND B[false,"false"]')
+
+# NOTE:
+#   all your arguments will be typecasted to
+#   the concrete type inferred from the argument literal
+
+Jaina.parse('A[1,true,false,"false"]') # 1, true, false "false"
+
+# NOTE: access to the argument list
+class A < Jaina::TerminalExpr
+  token 'A'
+
+  def evaluate(context)
+    # A[1,true,false,"false"]
+
+    arguments[0] # => 1
+    arguments[1] # => true
+    arguments[2] # => false
+    arguments[3] # => "false"
+  end
+end
 ```
 
 ---
@@ -151,7 +185,7 @@ Jaina.fetch_expression("KEK")
 ### Full example
 
 ```ruby
-# step 1: register new operand
+# step 1: create new operand
 class AddNumber < Jaina::TerminalExpr
   token 'ADD'
 
@@ -160,7 +194,7 @@ class AddNumber < Jaina::TerminalExpr
   end
 end
 
-# step 2: register another new operand
+# step 2: create another new operand
 class CheckNumber < Jaina::TerminalExpr
   token 'CHECK'
 
@@ -169,28 +203,34 @@ class CheckNumber < Jaina::TerminalExpr
   end
 end
 
+# step 4: and another new :)
 class InitState < Jaina::TerminalExpr
   token 'INIT'
 
   def evaluate(context)
-    context.set(:current_value, 0)
+    initial_value = arguments[0] || 0
+
+    context.set(:current_value, initial_value)
   end
 end
 
-# step 3: register new oeprands
+# step 5: register new oeprands
 Jaina.register_expression(AddNumber)
 Jaina.register_expression(CheckNumber)
 Jaina.register_expression(InitState)
 
-# step 4: run your program
+# step 6: run your program
 
 # NOTE: with initial context
-Jaina.evaluate('CHECK AND ADD', current_value: -1) # => false
-Jaina.evaluate('CHECK AND ADD', current_value: 2) # => 12
+Jaina.evaluate('CHECK AND ADD', current_value: -1) # => 9
+Jaina.evaluate('CHECK AND ADD', current_value: 2) # => false
 
 # NOTE: without initial context
 Jaina.evaluate('INIT AND ADD') # => 10
-Jaina.evaluate('INIT AND (CHECK OR ADD)') # => 12
+Jaina.evaluate('INIT AND (CHECK OR ADD)') # => 10
+
+# NOTE: with arguments
+Jaina.evaluate('INIT[100] AND ADD') => # 112
 ```
 
 ---

data/lib/jaina/parser/ast/tree_builder.rb

@@ -64,13 +64,13 @@ class Jaina::Parser::AST::TreeBuilder
   # @since 0.1.0
   attr_reader :prefix_form
 
-  # @return [Array<String>]
+  # @return [Array<Jaina::Parser::Tokenizer::Token>]
   #
   # @api private
   # @since 0.1.0
   attr_reader :tokens
 
-  # @param token_series [Array<String>]
+  # @param token_series [Array<Jaina::Parser::Tokenizer::Token>]
   # @return [Array<Jaina::Parser::Expression::Operator::Abstract>]
   #
   # @api private
@@ -80,14 +80,14 @@ class Jaina::Parser::AST::TreeBuilder
     return if current_token.nil?
 
     case
-    when terminal?(current_token)
+    when terminal?(current_token.token)
       build_terminal_expression(current_token)
-    when non_terminal?(current_token)
+    when non_terminal?(current_token.token)
       build_non_terminal_expression(current_token, token_series)
     end
   end
 
-  # @param token_series [Array<String>]
+  # @param token_series [Array<Jaina::Parser::Tokenizer::Token>]
   # @return [String, NilClass]
   #
   # @api private
@@ -96,7 +96,7 @@ class Jaina::Parser::AST::TreeBuilder
     token_series.shift
   end
 
-  # @param current_token [String]
+  # @param current_token [Jaina::Parser::Tokenizer::Token]
   # @return [Jaina::Parser::Expression::Operator::Abstract]
   #
   # @api private
@@ -105,20 +105,20 @@ class Jaina::Parser::AST::TreeBuilder
     Jaina::Parser::Expression.build(current_token)
   end
 
-  # @param current_token [String]
-  # @param token_series [Array<String>]
+  # @param current_token [Jaina::Parser::Tokenizer::Token]
+  # @param token_series [Array<Jaina::Parser::Tokenizer::Token>]
   # @return [Jaina::Parser::Expression::Operator::Abstract]
   #
   # @api private
   # @since 0.1.0
   def build_non_terminal_expression(current_token, token_series)
     case
-    when acts_as_unary_term?(current_token)
+    when acts_as_unary_term?(current_token.token)
       Jaina::Parser::Expression.build(
         current_token,
         build_expression_tree(token_series)
       )
-    when acts_as_binary_term?(current_token)
+    when acts_as_binary_term?(current_token.token)
       Jaina::Parser::Expression.build(
         current_token,
         build_expression_tree(token_series),

data/lib/jaina/parser/code_converter/to_postfix_form.rb

@@ -51,7 +51,7 @@ class Jaina::Parser::CodeConverter::ToPostfixForm
   # @since 0.1.0
   attr_reader :program
 
-  # @return [Array<String>]
+  # @return [Array<Jaina::Parser::Tokenizer::Token>]
   #
   # @api private
   # @since 0.1.0
@@ -61,7 +61,7 @@ class Jaina::Parser::CodeConverter::ToPostfixForm
   #
   # @api private
   # @since 0.1.0
-  def to_postfix_form
+  def to_postfix_form # rubocop:disable Metrics/AbcSize
     final_expression = []
     structure_operators = []
     token_series = tokens.map(&:dup)
@@ -70,13 +70,13 @@ class Jaina::Parser::CodeConverter::ToPostfixForm
       current_token = token_series.shift
 
       case
-      when non_terminal?(current_token)
+      when non_terminal?(current_token.token)
         process_non_terminal_token(final_expression, structure_operators, current_token)
-      when group_opener?(current_token)
+      when group_opener?(current_token.token)
         process_group_opening_token(final_expression, structure_operators, current_token)
-      when group_closener?(current_token)
+      when group_closener?(current_token.token)
         process_group_closing_token(final_expression, structure_operators, current_token)
-      when terminal?(current_token)
+      when terminal?(current_token.token)
         process_terminal_token(final_expression, structure_operators, current_token)
       end
     end
@@ -89,7 +89,7 @@ class Jaina::Parser::CodeConverter::ToPostfixForm
 
   # @param final_expression [Array<String>]
   # @param structure_operators [Array<String>]
-  # @param current_token [String]
+  # @param current_token [Jaina::Parser::Tokenizer::Token]
   # @return [void]
   #
   # @api private
@@ -98,12 +98,14 @@ class Jaina::Parser::CodeConverter::ToPostfixForm
     if structure_operators.any? # NOTE: check assocaitivity with potential next token
       potential_second_token = structure_operators.last
 
-      if non_terminal?(potential_second_token)
-        current_expression = Jaina::Parser::Expression.fetch(current_token)
-        next_expression    = Jaina::Parser::Expression.fetch(potential_second_token)
+      if non_terminal?(potential_second_token.token)
+        current_expression = Jaina::Parser::Expression.fetch(current_token.token)
+        next_expression    = Jaina::Parser::Expression.fetch(potential_second_token.token)
 
         # NOTE: form infix to postfix form switching
-        final_expression.push(structure_operators.pop) if current_expression.lower?(next_expression)
+        if current_expression.lower?(next_expression)
+          final_expression.push(structure_operators.pop)
+        end
       end
     end
 
@@ -112,7 +114,7 @@ class Jaina::Parser::CodeConverter::ToPostfixForm
 
   # @param final_expression [Array<String>]
   # @param structure_operators [Array<String>]
-  # @param current_token [String]
+  # @param current_token [Jaina::Parser::Tokenizer::Token]
   # @return [void]
   #
   # @api private
@@ -124,13 +126,13 @@ class Jaina::Parser::CodeConverter::ToPostfixForm
 
   # @param final_expression [Array<String>]
   # @param structure_operators [Array<String>]
-  # @param current_token [String]
+  # @param current_token [Jaina::Parser::Tokenizer::Token]
   # @return [void]
   #
   # @api private
   # @since 0.1.0
   def process_group_closing_token(final_expression, structure_operators, current_token)
-    until group_opener?(structure_operators.last)
+    until group_opener?(structure_operators.last.token)
       # NOTE: push all tokens to the final expression
       final_expression.push(structure_operators.pop)
     end
@@ -141,7 +143,7 @@ class Jaina::Parser::CodeConverter::ToPostfixForm
 
   # @param final_expression [Array<String>]
   # @param structure_operators [Array<String>]
-  # @param current_token [String]
+  # @param current_token [Jaina::Parser::Tokenizer::Token]
   # @return [void]
   #
   # @api private
@@ -153,7 +155,7 @@ class Jaina::Parser::CodeConverter::ToPostfixForm
 
   # @param final_expression [Array<String>]
   # @param structure_operators [Array<String>]
-  # @param current_token [String]
+  # @param current_token [Jaina::Parser::Tokenizer::Token]
   # @return [void]
   #
   # @api private

data/lib/jaina/parser/code_converter/to_prefix_form.rb

@@ -53,7 +53,7 @@ class Jaina::Parser::CodeConverter::ToPrefixForm
   # @since 0.1.0
   attr_reader :postfix_form
 
-  # @return [Array<String>]
+  # @return [Array<Jaina::Parser::Tokenizer::Token>]
   #
   # @api private
   # @since 0.1.0
@@ -71,15 +71,15 @@ class Jaina::Parser::CodeConverter::ToPrefixForm
       current_token = token_series.shift
 
       case
-      when terminal?(current_token)
+      when terminal?(current_token.token)
         process_terminal_token(current_token, expression_stack)
-      when non_terminal?(current_token)
+      when non_terminal?(current_token.token)
         process_non_terminal_token(current_token, expression_stack)
       end
     end
 
     # NOTE: build prefixed program string
-    Jaina::Parser::Tokenizer.join(expression_stack)
+    Jaina::Parser::Tokenizer.raw_join(expression_stack)
   end
 
   # @param current_token [String]
@@ -89,7 +89,7 @@ class Jaina::Parser::CodeConverter::ToPrefixForm
   # @api private
   # @since 0.1.0
   def process_terminal_token(current_token, expression_stack)
-    expression_stack.push(current_token)
+    expression_stack.push(current_token.raw_token)
   end
 
   # @param current_token [String]
@@ -99,22 +99,22 @@ class Jaina::Parser::CodeConverter::ToPrefixForm
   # @api private
   # @since 0.1.0
   def process_non_terminal_token(current_token, expression_stack)
-    expression = Jaina::Parser::Expression.fetch(current_token)
+    expression = Jaina::Parser::Expression.fetch(current_token.token)
 
     case # TODO: dry
     when expression.acts_as_binary_term?
       first_operand  = expression_stack.pop
       second_operand = expression_stack.pop
 
-      prefixed_expression_parts = [expression.token, second_operand, first_operand]
-      prefixed_expression = Jaina::Parser::Tokenizer.join(prefixed_expression_parts)
+      prefixed_expression_parts = [current_token.raw_token, second_operand, first_operand]
+      prefixed_expression = Jaina::Parser::Tokenizer.raw_join(prefixed_expression_parts)
 
       expression_stack.push(prefixed_expression)
     when expression.acts_as_unary_term?
       operand = expression_stack.pop
 
-      prefixed_expression_parts = [expression.token, operand]
-      prefixed_expression = Jaina::Parser::Tokenizer.join(prefixed_expression_parts)
+      prefixed_expression_parts = [current_token.raw_token, operand]
+      prefixed_expression = Jaina::Parser::Tokenizer.raw_join(prefixed_expression_parts)
 
       expression_stack.push(prefixed_expression)
     end

data/lib/jaina/parser/expression/operator/abstract.rb

@@ -34,18 +34,26 @@ class Jaina::Parser::Expression::Operator::Abstract
     end
   end
 
+  # @return [Array<Any>]
+  #
+  # @api private
+  # @since 0.4.0
+  attr_reader :arguments
+
   # @return [Array<Jaina::Parser::Expressions::Operator::Abstract>]
   #
   # @api private
   # @since 0.1.0
   attr_reader :expressions
 
-  # @param expressions [Array<Jaina::Parser::Expression::Operator::Abstract>]
+  # @option arguments [Array<Any>]
+  # @option expressions [Array<Jaina::Parser::Expression::Operator::Abstract>]
   # @return [void]
   #
   # @api private
   # @since 0.1.0
-  def initialize(*expressions)
+  def initialize(arguments: [], expressions: [])
+    @arguments   = arguments
     @expressions = expressions
   end
 

data/lib/jaina/parser/expression.rb

@@ -21,15 +21,15 @@ module Jaina::Parser::Expression
   register(Operator::RightCorner)
 
   class << self
-    # @param expression_token [String]
+    # @param expression_token [Jaina::Parser::Tokenizer::Token]
     # @param arguments [Array<Any>]
     # @return [Jaina::Parser::Expression::Operator::Abstract]
     #
     # @api private
     # @since 0.1.0
-    def build(expression_token, *arguments)
-      expression = fetch(expression_token)
-      arguments.any? ? expression.new(*arguments) : expression.new
+    def build(expression_token, *expression_arguments)
+      expression = fetch(expression_token.token)
+      expression.new(arguments: expression_token.arguments, expressions: expression_arguments)
     end
 
     # @param expression_token [String]

data/lib/jaina/parser/tokenizer/token.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+# @api private
+# @since 0.4.0
+class Jaina::Parser::Tokenizer::Token
+  # @return [String]
+  #
+  # @api private
+  # @since 0.4.0
+  attr_reader :raw_token
+
+  # @return [String]
+  #
+  # @api private
+  # @since 0.4.0
+  attr_reader :token
+
+  # @return [Array<String>]
+  #
+  # @api private
+  # @since 0.4.0
+  attr_reader :arguments
+
+  # @param raw_token [String]
+  # @param token [String]
+  # @param arguments [Array<Any>]
+  # @return [void]
+  #
+  # @api private
+  # @since 0.4.0
+  def initialize(raw_token, token, *arguments)
+    @raw_token = raw_token
+    @token = token
+    @arguments = arguments
+  end
+
+  # @return [Jaina::Parser::Tokenizer::Token]
+  #
+  # @api private
+  # @since 0.4.0
+  def dup
+    self.class.new(raw_token.dup, token.dup, *arguments)
+  end
+
+  # @return [String]
+  #
+  # @api private
+  # @since 0.4.0
+  def to_s
+    raw_token
+  end
+end

data/lib/jaina/parser/tokenizer/token_builder/argument_type_caster.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+# @api private
+# @since 0.4.0
+module Jaina::Parser::Tokenizer::TokenBuilder::ArgumentTypeCaster
+  # @return [Regexp]
+  #
+  # @api private
+  # @since 0.4.0
+  INTEGER_PATTERN = /\A\d+\z/.freeze
+
+  # @return [Regexp]
+  #
+  # @api private
+  # @since 0.4.0
+  FLOAT_PATTERN = /\A\d+\.\d+\z/.freeze
+
+  # @return [Regexp]
+  #
+  # @api private
+  # @since 0.4.0
+  TRUE_PATTERN = /\A(t|true)\z/i.freeze
+
+  # @return [Regexp]
+  #
+  # @api private
+  # @since 0.4.0
+  FALSE_PATTERN = /\A(f|false)\z/i.freeze
+
+  # @return [Regexp]
+  #
+  # @api private
+  # @since 0.4.0
+  ARRAY_PATTERN = /\A[^'"].*\s*,\s*.*[^'"]\z/.freeze
+
+  # @return [Regexp]
+  #
+  # @api private
+  # @since 0.4.0
+  QUOTED_STRING_PATTERN = /\A['"].*['"]\z/.freeze
+
+  class << self
+    # @param arguments [Array<String>]
+    # @return [Array<Any>]
+    #
+    # @api private
+    # @since 0.4.0
+    def cast(*arguments)
+      arguments.map { |argument| cast_argument_type(argument) }
+    end
+
+    private
+
+    # @param value [String]
+    # @return [Object]
+    #
+    # @api private
+    # @since 0.4.0
+    def cast_argument_type(value)
+      case value
+      when INTEGER_PATTERN
+        Integer(value)
+      when FLOAT_PATTERN
+        Float(value)
+      when TRUE_PATTERN
+        true
+      when FALSE_PATTERN
+        false
+      when ARRAY_PATTERN
+        value.split(/\s*,\s*/).map(&method(:convert_argument))
+      when QUOTED_STRING_PATTERN
+        value.gsub(/(\A['"]|['"]\z)/, '')
+      else
+        value
+      end
+    end
+  end
+end

data/lib/jaina/parser/tokenizer/token_builder.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+# @api private
+# @since 0.4.0
+class Jaina::Parser::Tokenizer::TokenBuilder
+  require_relative './token_builder/argument_type_caster'
+
+  # @return [Regexp]
+  #
+  # @api private
+  # @since 0.4.0
+  PARTS_SPLITTER = /[^,\[\]]+|\[|\]+/.freeze
+
+  # @return [String]
+  #
+  # @api private
+  # @since 0.4.0
+  OPENING_ATTRIBUTE_GROUP_SYMBOL = '['
+
+  # @return [String]
+  #
+  # @api private
+  # @since 0.4.0
+  CLOSING_ATTRIBUTE_GROUP_SYMBOL = ']'
+
+  # @since 0.4.0
+  Error = Class.new(StandardError)
+  # @since 0.4.0
+  IncorrectTokenDefinitionError = Class.new(Error)
+
+  class << self
+    # @param raw_token [String]
+    # @return [Jaina::Parser::Tokenizer::Token]
+    #
+    # @api private
+    # @since 0.4.0
+    def build(raw_token)
+      new(raw_token).build
+    end
+  end
+
+  # @param raw_token [String]
+  # @return [void]
+  #
+  # @api private
+  # @since 0.4.0
+  def initialize(raw_token)
+    @raw_token = raw_token
+    @parts = raw_token.scan(PARTS_SPLITTER)
+  end
+
+  # @return [Jaina::Parser::Tokenizer::Token]
+  #
+  # @api private
+  # @since 0.4.0
+  def build
+    check_for_correctness!
+
+    # NOTE:
+    #   format: token [ attr1, attr2, attr3 ]
+    #     TOKEN => parts[0]
+    #     OPENING BRAKET => parts[1]
+    #     CLOSING BRAKET => parts[-1]
+    #     ARGUMENTS => parts[2..-2]
+    token = parts[0]
+    arguments = parts.count.zero? ? [] : parts[2..-2]
+    arguments = ArgumentTypeCaster.cast(*arguments)
+
+    Jaina::Parser::Tokenizer::Token.new(raw_token, token, *arguments)
+  end
+
+  private
+
+  # @return [String]
+  #
+  # @api private
+  # @since 0.4.0
+  attr_reader :raw_token
+
+  # @return [Array<String>]
+  #
+  # @api private
+  # @since 0.4.0
+  attr_reader :parts
+
+  # @return [void]
+  #
+  # @api private
+  # @since 0.4.0
+  def check_for_correctness! # rubocop:disable Metrics/AbcSize
+    # NOTE: parts array contains a token only
+    return if parts.count == 1
+
+    # NOTE: prats should contain at least: "__token__", "[", and "]"
+    raise(
+      IncorrectTokenDefinitionError,
+      "Incorrect token definition `#{raw_token}`: " \
+      "token should contain `[` and `]` if `[` is provided)"
+    ) if parts.count < 3
+
+    # NOTE:
+    #   format: token [ attr1 attr2 attr3 ]
+    #     TOKEN => parts[0]
+    #     OPENING BRAKET => parts[1]
+    #     CLOSING BRAKET => parts[-1]
+    #     ARGUMENTS => parts[2..-2]
+
+    opening_corner  = parts[1]
+    closing_corner  = parts[-1]
+    arguments       = parts[2..-2]
+
+    # rubocop:disable Metrics/LineLength
+    if arguments.include?(OPENING_ATTRIBUTE_GROUP_SYMBOL) || opening_corner != OPENING_ATTRIBUTE_GROUP_SYMBOL
+      raise(
+        IncorrectTokenDefinitionError,
+        "Incorrect token definition `#{raw_token}`: `[` should be the first arguments opening symbol."
+      )
+    end
+
+    if arguments.include?(CLOSING_ATTRIBUTE_GROUP_SYMBOL) || closing_corner != CLOSING_ATTRIBUTE_GROUP_SYMBOL
+      raise(
+        IncorrectTokenDefinitionError,
+        "Incorrect token definition `#{raw_token}`: `]` should be the last arguments closing symbol."
+      )
+    end
+    # rubocop:enable Metrics/LineLength
+  end
+end

data/lib/jaina/parser/tokenizer.rb

@@ -3,11 +3,14 @@
 # @api private
 # @since 0.1.0
 module Jaina::Parser::Tokenizer
+  require_relative './tokenizer/token'
+  require_relative './tokenizer/token_builder'
+
   # @return [Regexp]
   #
   # @api private
   # @since 0.1.0
-  TOKEN_SCAN_PATTERN = /\(|\)|[\w\.\*]+/.freeze
+  TOKEN_SCAN_PATTERN = /\(|\)|[\"\'\.\,\w\[\]]+/.freeze
 
   # @return [String]
   #
@@ -17,21 +20,32 @@ module Jaina::Parser::Tokenizer
 
   class << self
     # @param program [String]
-    # @return [Array<String>]
+    # @return [Array<Jaina::Parser::Tokenizer::Token>]
     #
     # @api private
     # @since 0.1.0
     def tokenize(program)
-      program.scan(TOKEN_SCAN_PATTERN)
+      program.scan(TOKEN_SCAN_PATTERN).map do |raw_token|
+        TokenBuilder.build(raw_token)
+      end
     end
 
-    # @param tokens [Array<String>]
+    # @param tokens [Array<Jaina::Parser::Tokenizer::Token>]
     # @return [String]
     #
     # @api private
     # @since 0.1.0
     def join(tokens)
-      tokens.join(TOKEN_SPLITTER)
+      raw_join(tokens.map(&:raw_token))
+    end
+
+    # @param raw_tokens [Array<String>]
+    # @return [String]
+    #
+    # @api private
+    # @since 0.4.0
+    def raw_join(raw_tokens)
+      raw_tokens.join(TOKEN_SPLITTER)
     end
   end
 end

data/lib/jaina/version.rb

@@ -5,5 +5,5 @@ module Jaina
   #
   # @api public
   # @since 0.0.0
-  VERSION = '0.3.0'
+  VERSION = '0.4.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: jaina
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Rustam Ibragimov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-06-21 00:00:00.000000000 Z
+date: 2019-06-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: coveralls
@@ -154,6 +154,9 @@ files:
 - lib/jaina/parser/expression/registry.rb
 - lib/jaina/parser/expression/registry/access_interface_mixin.rb
 - lib/jaina/parser/tokenizer.rb
+- lib/jaina/parser/tokenizer/token.rb
+- lib/jaina/parser/tokenizer/token_builder.rb
+- lib/jaina/parser/tokenizer/token_builder/argument_type_caster.rb
 - lib/jaina/version.rb
 homepage: https://github.com/0exp/jaina
 licenses: []