jrr 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a8d6ef127fac8457e34a4a262b330795ca9bd4ff
+  data.tar.gz: 4384f180fc3da1d524c39dd2c67cddbd916dc744
+SHA512:
+  metadata.gz: 3374d28083b994db49d6a771cc62eeaf776e33e645094c5aa1478202f1eab2d3cdc081fb04272efea06f159761f8582b10003e9374ca0df18ae1be9083c8dd63
+  data.tar.gz: d951c93717278842d14e5bf1b0ee1a1018335112f59fe214f0b75214d89d8a6d84f86dfc240fedf2f84a2892996ac2ad6fca3ed4dd7124cfd7ac638ba1ce17d5

data/.gitignore

@@ -0,0 +1,12 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.4.3
+before_install: gem install bundler -v 1.15.4

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in jrr.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Solomon White
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,25 @@
+# JRR Tokenizer
+
+Parses text into fantastic tokens!
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'jrr'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install jrr
+
+## Usage
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "jrr"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/jrr.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "jrr/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "jrr"
+  spec.version       = Jrr::VERSION
+  spec.authors       = ["Solomon White"]
+  spec.email         = ["rubysolo@gmail.com"]
+
+  spec.summary       = %q{parse strings into fantastic tokens}
+  spec.homepage      = "https://github.com/rubysolo/jrr_tokenizer"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.15"
+  spec.add_development_dependency "pry"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+end

data/lib/jrr.rb

@@ -0,0 +1,7 @@
+require "jrr/version"
+
+module Jrr
+  def self.tokenize(input)
+    Tokenizer.new.tokenize(input)
+  end
+end

data/lib/jrr/default_scanners.rb

@@ -0,0 +1,117 @@
+require 'bigdecimal'
+require 'time'
+require 'jrr/token'
+
+module Jrr
+  module DefaultScanners
+    def null
+      new(:null, 'null\b', ->(*) { [nil] })
+    end
+
+    def whitespace
+      new(:whitespace, '\s+', ->(*) { ' ' })
+    end
+
+    def datetime
+      new(:datetime, /\d{2}\d{2}?-\d{1,2}-\d{1,2}( \d{1,2}:\d{1,2}:\d{1,2})? ?(Z|((\+|\-)\d{2}\:?\d{2}))?/, ->(raw) {
+        Time.parse(raw).to_datetime
+      })
+    end
+
+    def numeric
+      new(:numeric, '((?:\d+(\.\d+)?|\.\d+)(?:(e|E)(\+|-)?\d+)?)\b', ->(raw) {
+        raw =~ /\./ ? BigDecimal.new(raw) : raw.to_i
+      })
+    end
+
+    def hexadecimal
+      new(:numeric, '(0x[0-9a-f]+)\b', ->(raw) { raw[2..-1].to_i(16) })
+    end
+
+    def double_quoted_string
+      new(:string, '"[^"]*"', ->(raw) { raw.gsub(/^"|"$/, '') })
+    end
+
+    def single_quoted_string
+      new(:string, "'[^']*'", ->(raw) { raw.gsub(/^'|'$/, '') })
+    end
+
+    def negate
+      new(:arithmetic_operator, '-', ->(raw) { :negate }, ->(previous_token) {
+        previous_token.nil?                      ||
+        previous_token.is?(:arithmetic_operator) ||
+        previous_token.is?(:comparison_operator) ||
+        previous_token.is?(:boolean_operator)    ||
+        previous_token.value == :open            ||
+        previous_token.value == :comma
+      })
+    end
+
+    def boolean_operator
+      names = { and: '&&', or: '||' }.invert
+
+      new(:boolean_operator, '(and|or|&&|\|\|)\s+', ->(raw) {
+        norm = raw.strip.downcase
+        names.fetch(norm) { norm.to_sym }
+      })
+    end
+
+    def arithmetic_operator
+      names = {
+        pow: '^', add: '+', subtract: '-', multiply: '*', divide: '/', mod: '%',
+        bitor: '|', bitand: '&'
+      }.invert
+
+      new(:arithmetic_operator, '\^|\+|-|\*|\/|%|\||&', ->(raw) { names.fetch(raw) })
+    end
+
+    def grouping
+      names = { open: '(', close: ')', comma: ',' }.invert
+      new(:grouping, '\(|\)|,', ->(raw) { names.fetch(raw) })
+    end
+
+    def access
+      names = { lbracket: '[', rbracket: ']' }.invert
+      new(:access, '\[|\]', ->(raw) { names.fetch(raw) })
+    end
+
+    def case_statement
+      names = { open: 'case', close: 'end', then: 'then', when: 'when', else: 'else' }.invert
+      new(:case, '(case|end|then|when|else)\b', ->(raw) { names.fetch(raw.downcase) })
+    end
+
+    def comparison_operator
+      names = { le: '<=', ge: '>=', ne: '!=', lt: '<', gt: '>', eq: '=' }.invert
+      alternate = { ne: '<>', eq: '==' }.invert
+      new(:comparison_operator, '<=|>=|!=|<>|<|>|==|=', ->(raw) {
+        names.fetch(raw) { alternate.fetch(raw) }
+      })
+    end
+
+    def boolean
+      new(:boolean, '(true|false)\b', ->(raw) { raw.strip.downcase == 'true' })
+    end
+
+    def function
+      new(:function, '\w+!?\s*\(', ->(raw) do
+        function_name = raw.gsub('(', '')
+        [
+          Token.new(:function, function_name.strip.downcase.to_sym, function_name),
+          Token.new(:grouping, :open, '(')
+        ]
+      end)
+    end
+
+    def identifier
+      new(:identifier, '[\w\.]+\b', ->(raw) { standardize_case(raw.strip) })
+    end
+
+    def standardize_case(value)
+      if @case_sensitive
+        value
+      else
+        value.downcase
+      end
+    end
+  end
+end

data/lib/jrr/exceptions.rb

@@ -0,0 +1,25 @@
+module Jrr
+  class TokenizerError < StandardError
+    attr_reader :reason, :meta
+
+    def initialize(reason, **meta)
+      @reason = reason
+      @meta = meta
+    end
+
+    private_class_method :new
+
+    VALID_REASONS = %i[
+      parse_error too_many_opening_parentheses too_many_closing_parentheses
+      unexpected_zero_width_match
+    ].freeze
+
+    def self.for(reason, **meta)
+      unless VALID_REASONS.include?(reason)
+        raise ::ArgumentError, "Unhandled #{reason}"
+      end
+
+      new reason, meta
+    end
+  end
+end

data/lib/jrr/matcher.rb

@@ -0,0 +1,138 @@
+require 'jrr/token'
+
+module Jrr
+  class Matcher
+    attr_reader :children, :categories, :values
+
+    def initialize(categories = nil, values = nil, children = [])
+      # store categories and values as hash to optimize key lookup, h/t @jan-mangs
+      @categories = [categories].compact.flatten.each_with_object({}) { |c, h| h[c] = 1 }
+      @values     = [values].compact.flatten.each_with_object({}) { |v, h| h[v] = 1 }
+      @children   = children.compact
+      @invert     = false
+
+      @min = 1
+      @max = 1
+      @range = (@min..@max)
+    end
+
+    def | (other_matcher)
+      self.class.new(:nomatch, :nomatch, leaf_matchers + other_matcher.leaf_matchers)
+    end
+
+    def invert
+      @invert = ! @invert
+      self
+    end
+
+    def ==(token)
+      leaf_matcher? ? matches_token?(token) : any_child_matches_token?(token)
+    end
+
+    def match(token_stream, offset = 0)
+      matched_tokens = []
+      matched = false
+
+      while self == token_stream[matched_tokens.length + offset] && matched_tokens.length < @max
+        matched_tokens << token_stream[matched_tokens.length + offset]
+      end
+
+      if @range.cover?(matched_tokens.length)
+        matched = true
+      end
+
+      [matched, matched_tokens]
+    end
+
+    def caret
+      @caret = true
+      self
+    end
+
+    def caret?
+      @caret
+    end
+
+    def star
+      @min = 0
+      @max = Float::INFINITY
+      @range = (@min..@max)
+      self
+    end
+
+    def plus
+      @max = Float::INFINITY
+      @range = (@min..@max)
+      self
+    end
+
+    def leaf_matcher?
+      children.empty?
+    end
+
+    def leaf_matchers
+      leaf_matcher? ? [self] : children
+    end
+
+    private
+
+    def any_child_matches_token?(token)
+      children.any? { |child| child == token }
+    end
+
+    def matches_token?(token)
+      return false if token.nil?
+      (category_match(token.category) && value_match(token.value)) ^ @invert
+    end
+
+    def category_match(category)
+      @categories.empty? || @categories.key?(category)
+    end
+
+    def value_match(value)
+      @values.empty? || @values.key?(value)
+    end
+
+    def self.datetime;       new(:datetime);                       end
+    def self.numeric;        new(:numeric);                        end
+    def self.string;         new(:string);                         end
+    def self.logical;        new(:logical);                        end
+    def self.value
+      new(:datetime) | new(:numeric) | new(:string) | new(:logical)
+    end
+
+    def self.addsub;         new(:operator, [:add, :subtract]);    end
+    def self.subtract;       new(:operator, :subtract);            end
+    def self.anchored_minus; new(:operator, :subtract).caret;      end
+    def self.muldiv;         new(:operator, [:multiply, :divide]); end
+    def self.pow;            new(:operator, :pow);                 end
+    def self.mod;            new(:operator, :mod);                 end
+    def self.combinator;     new(:combinator);                     end
+
+    def self.comparator;     new(:comparator);                     end
+    def self.comp_gt;        new(:comparator, [:gt, :ge]);         end
+    def self.comp_lt;        new(:comparator, [:lt, :le]);         end
+
+    def self.open;           new(:grouping, :open);                end
+    def self.close;          new(:grouping, :close);               end
+    def self.comma;          new(:grouping, :comma);               end
+    def self.non_group;      new(:grouping).invert;                end
+    def self.non_group_star; new(:grouping).invert.star;           end
+    def self.non_close_plus; new(:grouping, :close).invert.plus;   end
+    def self.arguments;      (value | comma).plus;                 end
+
+    def self.if;             new(:function, :if);                  end
+    def self.round;          new(:function, :round);               end
+    def self.roundup;        new(:function, :roundup);             end
+    def self.rounddown;      new(:function, :rounddown);           end
+    def self.not;            new(:function, :not);                 end
+
+    def self.method_missing(name, *args, &block)
+      new(:function, name)
+    end
+
+    def self.respond_to_missing?(name, include_priv)
+      true
+    end
+  end
+end

data/lib/jrr/scanner.rb

@@ -0,0 +1,83 @@
+require 'jrr/default_scanners'
+require 'jrr/token'
+
+module Jrr
+  class Scanner
+    attr_reader :category, :regex, :converter, :condition
+    extend DefaultScanners
+
+    def initialize(category, regex, converter=nil, condition=nil)
+      @category  = category
+      @regex     = %r[\A(#{ regex })]i
+      @converter = converter
+      @condition = condition || ->(*) { true }
+    end
+
+    def scan(input, previous_token=nil)
+      if (m = regex.match(input)) && continue?(previous_token)
+        raw = m.to_s
+
+        return Array(convert(raw)).map do |value|
+          case value
+          when Token then value
+          else Token.new(category, value, raw)
+          end
+        end
+      end
+
+      false
+    end
+
+    def continue?(previous_token)
+      condition.call(previous_token)
+    end
+
+    def convert(raw_value)
+      if converter
+        converter.call(raw_value)
+      else
+        raw_value
+      end
+    end
+
+    def self.default_scanners
+      [
+        :null,
+        :whitespace,
+        :datetime,
+        :numeric,
+        :hexadecimal,
+        :double_quoted_string,
+        :single_quoted_string,
+        :negate,
+        :boolean_operator,
+        :arithmetic_operator,
+        :grouping,
+        :access,
+        :case_statement,
+        :comparison_operator,
+        :boolean,
+        :function,
+        :identifier
+      ]
+    end
+
+    def self.register_default_scanners
+      @scanners = default_scanners.map { |key| [key, self.send(key)] }
+    end
+
+    def self.scanners=(keys)
+      @scanners.select! { |(key,_)| keys.include?(key) }
+    end
+
+    def self.register_scanner(key, scanner)
+      @scanners.push([key, scanner])
+    end
+
+    def self.scanners(options={})
+      @scanners.map { |(_, scanner)| scanner }
+    end
+
+    register_default_scanners
+  end
+end

data/lib/jrr/token.rb

@@ -0,0 +1,31 @@
+module Jrr
+  class Token
+    attr_reader :category, :value, :raw_value
+
+    def initialize(category, value, raw_value=nil)
+      @category = category
+      @value = value
+      @raw_value = raw_value
+    end
+
+    def to_s
+      (raw_value || value).to_s
+    end
+
+    def length
+      raw_value.to_s.length
+    end
+
+    def empty?
+      length.zero?
+    end
+
+    def is?(some_category)
+      category == some_category
+    end
+
+    def ==(other_token)
+      is?(other_token.category) && value == other_token.value
+    end
+  end
+end

data/lib/jrr/tokenizer.rb

@@ -0,0 +1,85 @@
+require 'jrr/exceptions'
+require 'jrr/matcher'
+require 'jrr/scanner'
+require 'jrr/token'
+
+module Jrr
+  class Tokenizer
+    attr_reader :case_sensitive
+
+    LPAREN = Matcher.new(:grouping, :open)
+    RPAREN = Matcher.new(:grouping, :close)
+
+    def tokenize(string, options = {})
+      @nesting = 0
+      @tokens  = []
+      input    = strip_comments(string.to_s.dup)
+      @case_sensitive = options.fetch(:case_sensitive, false)
+
+      until input.empty?
+        scanned = Scanner.scanners(case_sensitive: case_sensitive).any? do |scanner|
+          scanned, input = scan(input, scanner)
+          scanned
+        end
+
+        unless scanned
+          fail! :parse_error, at: input
+        end
+      end
+
+      fail! :too_many_opening_parentheses if @nesting > 0
+
+      @tokens
+    end
+
+    def last_token
+      @tokens.last
+    end
+
+    def scan(string, scanner)
+      if tokens = scanner.scan(string, last_token)
+        tokens.each do |token|
+          if token.empty?
+            fail! :unexpected_zero_width_match,
+                  token_category: token.category, at: string
+          end
+
+          @nesting += 1 if LPAREN == token
+          @nesting -= 1 if RPAREN == token
+          fail! :too_many_closing_parentheses if @nesting < 0
+
+          @tokens << token unless token.is?(:whitespace)
+        end
+
+        match_length = tokens.map(&:length).reduce(:+)
+        [true, string[match_length..-1]]
+      else
+        [false, string]
+      end
+    end
+
+    def strip_comments(input)
+      input.gsub(/\/\*[^*]*\*+(?:[^*\/][^*]*\*+)*\//, '')
+    end
+
+    private
+
+    def fail!(reason, **meta)
+      message =
+        case reason
+        when :parse_error
+          "parse error at: '#{meta.fetch(:at)}'"
+        when :too_many_opening_parentheses
+          "too many opening parentheses"
+        when :too_many_closing_parentheses
+          "too many closing parentheses"
+        when :unexpected_zero_width_match
+          "unexpected zero-width match (:#{meta.fetch(:category)}) at '#{meta.fetch(:at)}'"
+        else
+          raise ::ArgumentError, "Unhandled #{reason}"
+        end
+
+      raise TokenizerError.for(reason, meta), message
+    end
+  end
+end

data/lib/jrr/version.rb

@@ -0,0 +1,3 @@
+module Jrr
+  VERSION = "0.0.1"
+end

metadata

@@ -0,0 +1,118 @@
+--- !ruby/object:Gem::Specification
+name: jrr
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Solomon White
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-12-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: 
+email:
+- rubysolo@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- jrr.gemspec
+- lib/jrr.rb
+- lib/jrr/default_scanners.rb
+- lib/jrr/exceptions.rb
+- lib/jrr/matcher.rb
+- lib/jrr/scanner.rb
+- lib/jrr/token.rb
+- lib/jrr/tokenizer.rb
+- lib/jrr/version.rb
+homepage: https://github.com/rubysolo/jrr_tokenizer
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.13
+signing_key: 
+specification_version: 4
+summary: parse strings into fantastic tokens
+test_files: []