search_lingo 2.0.0.pre2 → 3.0.0.pre1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 48dc212ce20b6ca6d502ad3776598375e9aadf5aba67448a0db0833ff9ae9b22
-  data.tar.gz: 9110f9d208a47032ded854d4541eb2d212359599f8b6ddc13bdb6add5c4ee6ab
+  metadata.gz: 06b21144b96415a618c188261eca38aab055ca218d294a966f96f543c41759a2
+  data.tar.gz: eff6897cd99f4abf444a9dd9c8c5d1d545b1da5277d507a8fb8b7a685756ca44
 SHA512:
-  metadata.gz: a9bfec6dad76471dedab609b8eef445c3b61bfb3eaded66d362849a1f42192de117485b7a9c4157aa8277a7a8a34df5f9c9f1aa15997d001751c92a3259172fe
-  data.tar.gz: 38a894d13baec1cb47a68bf805243fe238355dcf65cc35effae2d5c3d304082883fa73e60de07340d2f29c74ad15e89b5b1ca6d20235a63bfd6cd545b07d67af
+  metadata.gz: fbd45adde84bf9a73d73c15cb91545e1e46b7ddbc2b0647506a087dcf77cf1aa7777ed29e57c8329a101b98b01b647308ed5657157fc60cc9858bd5281a25ca0
+  data.tar.gz: a05213f9fd31a2049f649bc935ac9d5edf0e4c8cd4bc390ee423bdc660180d78fb1a9802ffe7ff428b886b70911f93c98d469edb1dfb59b5c9e43c13c4eb1a74

data/.gitignore

@@ -8,3 +8,4 @@
 /pkg/
 /spec/reports/
 /tmp/
+.tool-versions

data/.gitlab-ci.yml

@@ -0,0 +1,27 @@
+before_script:
+  - gem install bundler --no-document -v '~> 2.3.11'
+  - bundle install --jobs="$(nproc)" --retry=3
+
+build:rubocop:
+  stage: build
+  image: "ruby:3.1"
+  script:
+    - bundle exec rubocop
+
+test:ruby-2.7:
+  stage: test
+  image: "ruby:2.7"
+  script:
+    - bundle exec rake test
+
+test:ruby-3.0:
+  stage: test
+  image: "ruby:3.0"
+  script:
+    - bundle exec rake test
+
+test:ruby-3.1:
+  stage: test
+  image: "ruby:3.1"
+  script:
+    - bundle exec rake test

data/.rubocop.yml

@@ -0,0 +1,8 @@
+AllCops:
+  NewCops: enable
+  SuggestExtensions: false
+  TargetRubyVersion: 2.7.0
+Style/TrailingCommaInArrayLiteral:
+  EnforcedStyleForMultiline: consistent_comma
+Style/TrailingCommaInHashLiteral:
+  EnforcedStyleForMultiline: consistent_comma

data/.travis.yml

@@ -1,4 +1,7 @@
 language: ruby
 rvm:
-  - 2.4.4
-  - 2.5.1
+  - "2.7"
+  - "3.0"
+  - "3.1"
+before_install:
+  - gem install bundler

data/Gemfile

@@ -1,3 +1,5 @@
+# frozen-string-literal: true
+
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in search_lingo.gemspec

data/Rakefile

@@ -1,9 +1,15 @@
-require "bundler/gem_tasks"
+# frozen-string-literal: true
+
+require 'bundler/gem_tasks'
 require 'rake/testtask'
 
 Rake::TestTask.new do |t|
   t.libs << 'test'
-  t.pattern = 'test/**/test_*.rb'
+  t.pattern = 'test/**/*_test.rb'
 end
 
 task default: :test
+
+task :rubocop do
+  system 'rubocop'
+end

data/bin/console

@@ -1,13 +1,14 @@
 #!/usr/bin/env ruby
+# frozen-string-literal: true
 
-require "bundler/setup"
-require "search_lingo"
+require 'bundler/setup'
+require 'search_lingo'
 
 # You can add fixtures and/or initialization code here to make experimenting
 # with your gem easier. You can also use a different console, if you like.
 
 # (If you use this, don't forget to add pry to your Gemfile!)
-require "pry"
+require 'pry'
 Pry.start
 
 # require "irb"

data/examples/complex.rb

@@ -1,9 +1,19 @@
+# frozen-string-literal: true
+
 class Job < ActiveRecord::Base # :nodoc:
-  # Assume this model has attributes: :id, :date, :name
+  # Attributes:
+  #   :id
+  #   :date
+  #   :name
 end
 
 class Receipt < ActiveRecord::Base # :nodoc:
-  # Assume this model has attributes: :id, :check_no, :check_date, :post_date, :amount
+  # Attributes:
+  #   :id
+  #   :check_no
+  #   :check_date
+  #   :post_date
+  #   :amount
 end
 
 module Parsers # :nodoc:
@@ -13,7 +23,7 @@ module Parsers # :nodoc:
     end
 
     def call(token, chain)
-      token.match /\Aid:\s*([[:digit:]]+)\z/ do |m|
+      token.match(/\Aid:\s*([[:digit:]]+)\z/) do |m|
         chain.where @table => { id: m[1] }
       end
     end
@@ -30,9 +40,11 @@ class JobSearch < SearchLingo::AbstractSearch # :nodoc:
 end
 
 class ReceiptSearch < SearchLingo::AbstractSearch # :nodoc:
-  parser SearchLingo::Parsers::DateParser.new Receipt.arel_table[:check_date]
-  parser SearchLingo::Parsers::DateParser.new Receipt.arel_table[:post_date],
-    modifier: 'posted'
+  # You might prefer to include SearchLingo::Parsers if you you are going to
+  # instantiate multiple DateParsers.
+  include SearchLingo::Parsers
+  parser DateParser.new Receipt.arel_table[:check_date]
+  parser DateParser.new Receipt.arel_table[:post_date], modifier: 'posted'
 
   parser do |token, chain|
     token.match(/\Aamount: (\d+(?:\.\d+)?)\z/) do |m|

data/examples/sequel_example.rb

@@ -1,3 +1,5 @@
+# frozen-string-literal: true
+
 require 'sequel'
 require 'sqlite3'
 
@@ -17,12 +19,13 @@ require 'sqlite3'
 
 class CategoryParser # :nodoc:
   def call(token, chain)
-    if token.modifier == 'cat'
-      # This is kind of broken. The categories table will be joined once each
-      # time this parser matches a token.
-      chain.join(:categories, id: :category_id)
-        .where Sequel.qualify('categories', 'name') => token.term
-    end
+    return nil unless token.modifier == 'cat'
+
+    # This is not an ideal example. Sequel will join the categories table for
+    # each token that matches. I'm ignoring the problem since this is only an
+    # example.
+    category_name = Sequel.qualify :categories, :name
+    chain.join(:categories, id: :category_id).where category_name => token.term
   end
 end
 
@@ -53,12 +56,11 @@ class TaskSearch < SearchLingo::AbstractSearch # :nodoc:
   # 7/4/17   => Tasks with due_date == Date.new(2017, 7, 4)
   parser do |token, chain|
     token.match %r{\A(?<m>\d{1,2})/(?<d>\d{1,2})/(?<y>\d{2}\d{2}?)\z} do |m|
-      begin
-        date = Date.parse "#{m[:y]}/#{m[:m]}/#{m[:d]}"
-        chain.where due_date: date
-      rescue ArgumentError
-        # Date.parse raised an ArgumentError
-      end
+      date = Date.parse "#{m[:y]}/#{m[:m]}/#{m[:d]}"
+      chain.where due_date: date
+    rescue ArgumentError
+      # Fail if Date.parse raises an ArgumentError
+      nil
     end
   end
 

data/lib/search_lingo/abstract_search.rb

@@ -1,17 +1,51 @@
+# frozen-string-literal: true
+
 require 'search_lingo/tokenizer'
 
 module SearchLingo
+  ##
+  # AbstractSearch is an abstract implementation from which search classes
+  # should inherit.
+  #
+  # Search classes are instantiated with a query string and a default scope on
+  # which to perform the search.
+  #
+  # Child classes must implement the #default_parse instance method, and they
+  # may optionally register one or more parsers.
+  #
+  #   class MySearch < SearchLingo::AbstractSearch
+  #     def default_parse(token, chain)
+  #       chain.where attribute: token.term
+  #     end
+  #   end
+  #
+  #   class MyOtherSearch < SearchLingo::AbstractSearch
+  #     parser SearchLingo::Parsers::DateParser.new Job.arel_table[:date]
+  #
+  #     parser do |token, chain|
+  #       token.match(/\Aid: [[:space:]]* (?<id>[[:digit:]]+)\z/x) do |m|
+  #         chain.where id: m[:id]
+  #       end
+  #     end
+  #
+  #     def default_parse(token, chain)
+  #       chain.where Job.arel_table[:name].matches "%#{token.term}%"
+  #     end
+  #   end
   class AbstractSearch
-    attr_reader :query, :scope
+    attr_reader :query, :scope, :logger
 
     ##
     # Instantiates a new search object. +query+ is the string that is to be
     # parsed and compiled into an actual query. If +query+ is falsey, an empty
     # string will be used. +scope+ is the object to which the compiled query
-    # should be sent, e.g., an +ActiveRecord+ model.
-    def initialize(query, scope)
-      @query = query || ''
-      @scope = scope
+    # should be sent, e.g., an +ActiveRecord::Relation+.
+    #
+    #   MySearchClass.new 'foo bar: baz "froz quux"', Task.all
+    def initialize(query, scope, logger: nil)
+      @query  = query || ''
+      @scope  = scope
+      @logger = logger
     end
 
     ##
@@ -27,8 +61,8 @@ module SearchLingo
     # responds to +#call+. The parser will be send +#call+ with a single
     # argument which will be a token from the query string.
     #
-    # If both a callable object and a block are given, or if neither a callable
-    # object nor a block are given, an +ArgumentError+ will be raised.
+    # Raises +ArgumentError+ if +parser+ does not respond to +#call+ and no
+    # block is given.
     #
     #   class MyParser
     #     def call(token)
@@ -43,10 +77,13 @@ module SearchLingo
     #     end
     #   end
     def self.parser(parser = nil, &block)
-      unless block_given? ^ parser.respond_to?(:call)
-        raise ArgumentError, 'parse must be called with callable OR block'
+      if parser.respond_to? :call
+        parsers << parser
+      elsif block_given?
+        parsers << block
+      else
+        raise ArgumentError, 'parse must be called with block or callable object'
       end
-      parsers << (parser || block)
     end
 
     ##
@@ -64,26 +101,12 @@ module SearchLingo
     ##
     # Load search results by composing query string tokens into a query chain.
     #
-    # @query is borken down into tokens, and each token is passed through the
-    # list of defined parsers. If a parser is successful, +:match+ is thrown,
-    # processing moves on to the next token. If none of the parsers succeed and
-    # the token is compound, the token is simplified and reprocessed as before.
-    # If still no parser succeeds, fall back on +#default_parse+.
+    # @query is broken down into tokens and parses each one in turn. The
+    # results of parsing each token are chained onto the end of +scope+ to
+    # compose the query.
     def load_results
       tokenizer.reduce(scope) do |chain, token|
-        catch(:match) do
-          # 1. Try each parser with token until :match is thrown.
-          parse token, chain
-
-          # 2. If :match not thrown and token is compund, simplify and retry.
-          if token.compound?
-            token = tokenizer.simplify
-            parse token, chain
-          end
-
-          # 3. If :match still not thrown, fall back on default parser.
-          default_parse token, chain
-        end
+        parse token, chain
       end
     end
 
@@ -93,6 +116,26 @@ module SearchLingo
       @tokenizer ||= Tokenizer.new query
     end
 
+    ##
+    # Passes +token+ and +chain+ through the array of parsers until +:match+ is
+    # thrown. If none of the parsers match and the token is compound,
+    # simplifies the token and reruns the parsers. If no parsers match after
+    # the second pass or if the token was not compound, falls back on
+    # `#default_parse`.
+    def parse(token, chain)
+      catch(:match) do
+        run_parsers token, chain
+
+        if token.compound?
+          token = tokenizer.simplify
+          run_parsers token, chain
+        end
+
+        logger&.debug "default_parse token=#{token.inspect}"
+        default_parse token, chain
+      end
+    end
+
     ##
     # Passes +token+ to each parser in turn. If a parser succeeds, throws
     # +:match+ with the result.
@@ -100,10 +143,13 @@ module SearchLingo
     # A parser succeeds if +call+ returns a truthy value. A successful parser
     # will typically send something to +chain+ and return the result. In this
     # way, the tokens of the search are reduced into a composed query.
-    def parse(token, chain)
+    def run_parsers(token, chain)
       parsers.each do |parser|
         result = parser.call token, chain
-        throw :match, result if result
+        if result
+          logger&.debug "parser:#{parser.inspect} token=#{token.inspect}"
+          throw :match, result
+        end
       end
       nil
     end
@@ -115,9 +161,9 @@ module SearchLingo
     # This is a skeletal implementation that raises +NotImplementedError+.
     # Child classes should provide their own implementation. At a minimum, that
     # implementation should return +chain+. (Doing so would ignore +token+.)
-    def default_parse(token, chain)
+    def default_parse(_token, _chain)
       raise NotImplementedError,
-        "#default_parse must be implemented by #{self.class}"
+            "#default_parse must be implemented by #{self.class}"
     end
   end
 end

data/lib/search_lingo/constants.rb

@@ -1,9 +1,24 @@
+# frozen-string-literal: true
+
 module SearchLingo
   ##
   # Pattern for matching modifiers within a token.
-  MODIFIER = /[[:alnum:]]+/
+  MODIFIER = /[[:alnum:]]+/.freeze
+
+  ##
+  # Pattern for matching a simple token.
+  SIMPLE_TOKEN = /"[^"]+"|[[:graph:]]+/.freeze
+
+  ##
+  # Pattern for matching a simple or compound token.
+  SIMPLE_OR_COMPOUND_TOKEN = /(?:#{MODIFIER}:[[:space:]]*)?#{SIMPLE_TOKEN}/.freeze
+
+  ##
+  # Pattern for matching a simple or compound token, with regex grouping to aid
+  # in decomposing the token into its modifier and term components.
+  SIMPLE_OR_COMPOUND_TOKEN_WITH_GROUPING = /\A(?:(#{MODIFIER}):[[:space:]]*)?(#{SIMPLE_TOKEN})\z/.freeze
 
   ##
-  # Pattern for matching terms within a token.
-  TERM = /"[^"]+"|[[:graph:]]+/
+  # Pattern for matching the delimiter between tokens.
+  DELIMITER = /[[:space:]]*/.freeze
 end

data/lib/search_lingo/parsers/date_parser.rb

@@ -1,36 +1,53 @@
+# frozen-string-literal: true
+
 require 'search_lingo/parsers/mdy'
 
 module SearchLingo
   module Parsers # :nodoc:
+    ##
+    # DateParser is an example parser which handles dates that adhere to the
+    # MDY format used in the US. It uses `SearchLingo::Parsers::MDY.parse` to
+    # parse the date. It handles simple dates as well as closed and open-ended
+    # date ranges.
+    #
+    # Examples of single dates are 7/14, 7/14/17, and 7/14/2017.
+    # Examples of closed date ranges are 1/1-6/30 and 7/1/16-6/30/18.
+    # Examples of open date ranges are -6/30 and 7/1/17-.
     class DateParser
       include MDY
 
+      attr_reader :column, :prefix, :decorator
+
       ##
       # Instantiates a new DateParser object.
       #
       # The required argument +column+ should be an Arel attribute.
       #
       # If present, the optional argument +modifier+ will be used as the
-      # operator which precedes the date term.
+      # token operator which precedes the date term.
       #
-      # DateParser.new Booking.arel_table[:date]
-      # DateParser.new Contract.arel_table[:date], modifier: 'contract'
-      def initialize(column, modifier: nil)
+      # If a block is provided, it will be called with the filter chain. This
+      # is useful if you need to send additional messages to the filter chain
+      # which are independent of the content of the token, e.g., if you need to
+      # join another table.
+      #
+      # DateParser.new Model.arel_table[:date]
+      # DateParser.new Model.arel_table[:date], modifier: 'contract'
+      # DateParser.new Model.arel_table[:date] do |chain|
+      #   chain.joins(:relation)
+      # end
+      def initialize(column, modifier: nil, &block)
         @column = column
-        @prefix = %r{#{modifier}:[[:space:]]*} if modifier
+        @prefix = /#{modifier}:[[:space:]]*/ if modifier
+        @decorator = block_given? ? block : ->(chain) { chain }
       end
 
-      attr_reader :column, :prefix
-
       ##
-      # Attempts to parse the token as a date, closed date range, or open date
-      # range.
-      #
-      # Examples of single dates are 7/14, 7/14/17, and 7/14/2017.
-      # Examples of closed date ranges are 1/1-6/30 and 7/1/16-6/30/18.
-      # Examples of open date ranges are -6/30 and 7/1/17-.
+      # Attempts to parse +token+ as a single date, closed date range, or open
+      # date range. If parsing succeeds, the parser sends #where to +chain+
+      # with the appropriate Arel node and returns the result.
       def call(token, chain)
-        catch :stop do
+        catch :halt do
           parse_single_date token, chain
           parse_date_range  token, chain
           parse_lte_date    token, chain
@@ -39,40 +56,46 @@ module SearchLingo
       end
 
       def inspect # :nodoc:
-        '#<%s:0x%x @prefix=%s @column=%s>' %
-          [self.class, object_id << 1, prefix.inspect, column.inspect]
+        format '#<%<cls>s @prefix=%<prefix>s @column=%<column>s>',
+               cls: self.class,
+               prefix: prefix.inspect,
+               column: "#{column.relation.name}.#{column.name}".inspect
       end
 
       private
 
-      def parse_single_date(token, chain)
+      def parse_single_date(token, chain) # :nodoc:
         token.match(/\A#{prefix}(?<date>#{US_DATE})\z/) do |m|
           date = parse(m[:date]) or return nil
-          throw :stop, chain.where(column.eq(date))
+          throw :halt, decorate(chain).where(column.eq(date))
         end
       end
 
-      def parse_date_range(token, chain)
+      def parse_date_range(token, chain) # :nodoc:
         token.match(/\A#{prefix}(?<min>#{US_DATE})-(?<max>#{US_DATE})\z/) do |m|
           min = parse(m[:min]) or return nil
           max = parse(m[:max], relative_to: min.next_year) or return nil
-          throw :stop, chain.where(column.in(min..max))
+          throw :halt, decorate(chain).where(column.between(min..max))
         end
       end
 
-      def parse_lte_date(token, chain)
+      def parse_lte_date(token, chain) # :nodoc:
         token.match(/\A#{prefix}-(?<date>#{US_DATE})\z/) do |m|
           date = parse(m[:date]) or return nil
-          throw :stop, chain.where(column.lteq(date))
+          throw :halt, decorate(chain).where(column.lteq(date))
         end
       end
 
-      def parse_gte_date(token, chain)
+      def parse_gte_date(token, chain) # :nodoc:
         token.match(/\A#{prefix}(?<date>#{US_DATE})-\z/) do |m|
           date = parse(m[:date]) or return nil
-          throw :stop, chain.where(column.gteq(date))
+          throw :halt, decorate(chain).where(column.gteq(date))
         end
       end
+
+      def decorate(chain)
+        decorator.call chain
+      end
     end
   end
 end

data/lib/search_lingo/parsers/mdy.rb

@@ -1,13 +1,20 @@
+# frozen-string-literal: true
+
 require 'date'
 
 module SearchLingo
   module Parsers # :nodoc:
+    ##
+    # MDY provides a parser for dates that adhere to the M/D/Y format used in
+    # the US.
     module MDY
       ##
       # Pattern for matching US-formatted date strings.
       #
       # The year may be two or four digits, or it may be omitted.
-      US_DATE = %r{(?<m>\d{1,2})/(?<d>\d{1,2})(?:/(?<y>\d{2}\d{2}?))?}
+      US_DATE = %r{(?<m>\d{1,2})/(?<d>\d{1,2})(?:/(?<y>\d{2}\d{2}?))?}.freeze
+
+      module_function
 
       ##
       # Returns a +Date+ object for the date represented by +term+. Returns
@@ -19,28 +26,32 @@ module SearchLingo
       # If the year is omitted, it will be inferred using +relative_to+ as a
       # reference date. In this scenario, the resulting date will always be
       # less than or equal to the reference date. If +relative_to+ omitted, it
-      # defaults to today's date.
+      # defaults to +Date.today+.
       #
       # Available as both a class method and an instance method.
       def parse(term, relative_to: Date.today)
-        term.match /\A#{US_DATE}\z/ do |m|
-          return Date.parse "#{m[:y]}/#{m[:m]}/#{m[:d]}" if m[:y]
-
-          ref   = relative_to
-          day   = Integer(m[:d])
-          month = Integer(m[:m])
-          year  = if month < ref.month || month == ref.month && day <= ref.day
-                    ref.year
-                  else
-                    ref.year - 1
-                  end
-
-          Date.new year, month, day
+        term.match(/\A#{US_DATE}\z/) do |m|
+          date = reformat_date m, relative_to
+          Date.parse date
         end
       rescue ArgumentError
+        # Fail if Date.parse or Date.new raise ArgumentError.
+        nil
       end
 
-      module_function :parse
+      def reformat_date(match, today) # :nodoc:
+        return match.values_at(:y, :m, :d).join('/') if match[:y]
+
+        month = Integer match[:m]
+        day = Integer match[:d]
+        year = if month < today.month || (month == today.month && day <= today.day)
+                 today.year
+               else
+                 today.year - 1
+               end
+
+        "#{year}/#{month}/#{day}"
+      end
     end
   end
 end

data/lib/search_lingo/parsers.rb

@@ -1 +1,3 @@
+# frozen-string-literal: true
+
 require 'search_lingo/parsers/date_parser'

data/lib/search_lingo/token.rb

@@ -1,33 +1,45 @@
+# frozen-string-literal: true
+
 require 'delegate'
 require 'search_lingo/constants'
 
 module SearchLingo
+  ##
+  # Single token from a query string. A token consists of a term an an optional
+  # modifier. The term may be a word or multiple words contained within double
+  # quotes. The modifier is one or more alphanumeric characters. The modifier
+  # and term and separated by a colon followed by zero or more whitespace
+  # characters.
+  #
+  # The following are examples of tokens:
+  #
+  #   Token.new('foo')
+  #   Token.new('"foo bar"')
+  #   Token.new('foo: bar')
+  #   Token.new('foo: "bar baz"')
   class Token < DelegateClass(String)
-    ##
-    # Pattern for decomposing a token into a modifier and a term.
-    STRUCTURE = /\A(?:(#{MODIFIER}):[[:space:]]*)?"?(.+?)"?\z/
-
     ##
     # Returns the modifier portion of the token. Returns +nil+ if token does
     # not have a modifier.
     #
-    #   Token.new('foo: bar').modifier # => 'foo'
+    #   Token.new('foo: bar').modifier # => "foo"
     #   Token.new('bar').modifier      # => nil
     def modifier
-      self[STRUCTURE, 1]
+      self[SIMPLE_OR_COMPOUND_TOKEN_WITH_GROUPING, 1]
     end
 
-    alias_method :operator, :modifier
+    alias operator modifier
 
     ##
     # Returns the term portion of the token. If the term is wrapped in quotes,
     # they are removed.
     #
-    #   Token.new('foo: bar').term  # => 'bar'
-    #   Token.new('bar').term       # => 'bar'
-    #   Token.new('"bar baz"').term # => 'bar baz'
+    #   Token.new('foo: bar').term  # => "bar"
+    #   Token.new('bar').term       # => "bar"
+    #   Token.new('"bar baz"').term # => "bar baz"
+    #   Token.new('""').term        # => ""
     def term
-      self[STRUCTURE, 2]
+      self[SIMPLE_OR_COMPOUND_TOKEN_WITH_GROUPING, 2].delete_prefix('"').delete_suffix('"')
     end
 
     ##
@@ -36,12 +48,15 @@ module SearchLingo
     #   Token.new('foo: bar').compound? # => true
     #   Token.new('bar').compound?      # => false
     def compound?
-      !!modifier
+      !modifier.nil? && !modifier.empty?
     end
 
     def inspect # :nodoc:
-      '#<%s String(%s) modifier=%s term=%s>' %
-        [self.class, super, modifier.inspect, term.inspect]
+      format '#<%<cls>s String(%<str>s) modifier=%<mod>s term=%<term>s>',
+             cls: self.class,
+             str: super,
+             mod: modifier.inspect,
+             term: term.inspect
     end
   end
 end

data/lib/search_lingo/tokenizer.rb

@@ -1,25 +1,21 @@
+# frozen-string-literal: true
+
 require 'forwardable'
 require 'strscan'
 require 'search_lingo/constants'
 require 'search_lingo/token'
 
 module SearchLingo
+  ##
+  # Tokenizer breaks down a query string into individual tokens.
+  #
+  #   Tokenizer.new 'foo'
+  #   Tokenizer.foo 'foo "bar baz"'
+  #   Tokenizer.foo 'foo "bar baz" froz: quux'
   class Tokenizer
     include Enumerable
     extend Forwardable
 
-    ##
-    # Pattern for matching a simple token (a term without a modifier).
-    SIMPLE_TOKEN   = /#{TERM}/
-
-    ##
-    # Pattern for matching a compound token (a term with an optional modifier).
-    COMPOUND_TOKEN = /(?:#{MODIFIER}:[[:space:]]*)?#{TERM}/
-
-    ##
-    # Pattern for matching the delimiter between tokens.
-    DELIMITER      = /[[:space:]]*/
-
     def initialize(query) # :nodoc:
       @scanner = StringScanner.new query.strip
     end
@@ -30,18 +26,17 @@ module SearchLingo
     def each
       return to_enum(__callee__) unless block_given?
 
-      until scanner.eos?
-        yield self.next
-      end
+      yield self.next until scanner.eos?
     end
 
     ##
-    # Returns a Token for the next token in the query string. When the end of
+    # Returns a +Token+ for the next token in the query string. When the end of
     # the query string is reached raises +StopIteration+.
     def next
       scanner.skip DELIMITER
-      token = scanner.scan COMPOUND_TOKEN
+      token = scanner.scan SIMPLE_OR_COMPOUND_TOKEN
       raise StopIteration unless token
+
       Token.new token
     end
 

data/lib/search_lingo/version.rb

@@ -1,3 +1,5 @@
+# frozen-string-literal: true
+
 module SearchLingo
-  VERSION = '2.0.0.pre2'
+  VERSION = '3.0.0.pre1'
 end

data/lib/search_lingo.rb

@@ -1,3 +1,5 @@
+# frozen-string-literal: true
+
 require 'search_lingo/version'
 require 'search_lingo/abstract_search'
 require 'search_lingo/parsers'

data/search_lingo.gemspec

@@ -1,34 +1,43 @@
-# coding: utf-8
-lib = File.expand_path('../lib', __FILE__)
+# frozen-string-literal: true
+
+lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'search_lingo/version'
 
 Gem::Specification.new do |spec|
-  spec.name          = "search_lingo"
+  spec.name          = 'search_lingo'
   spec.version       = SearchLingo::VERSION
-  spec.authors       = ["John Parker"]
-  spec.email         = ["jparker@urgetopunt.com"]
+  spec.authors       = ['John Parker']
+  spec.email         = ['jparker@urgetopunt.com']
 
-  spec.summary       = %q{Framework for defining and parsing search queries.}
-  spec.description   = %q{SearchLingo is a simple framework for defining simple query languages and translating them into application-specific queries.}
-  spec.homepage      = "https://github.com/jparker/search_lingo"
-  spec.license       = "MIT"
+  spec.summary       = 'Framework for defining and parsing search queries.'
+  spec.description   = <<~DESCRIPTION
+    SearchLingo is a simple framework for defining simple query languages and
+    translating them into application-specific queries.
+  DESCRIPTION
+  spec.homepage      = 'https://github.com/jparker/search_lingo'
+  spec.license       = 'MIT'
 
-  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
-  spec.bindir        = "exe"
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = 'exe'
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
+  spec.require_paths = ['lib']
 
   spec.rdoc_options += ['-x', 'examples/', '-x', 'test/']
 
-  spec.required_ruby_version = '>= 2.1'
+  spec.required_ruby_version = '>= 2.7'
 
-  spec.add_development_dependency "bundler", "~> 1.9"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency 'bundler', '~> 2.0'
   spec.add_development_dependency 'minitest'
   spec.add_development_dependency 'minitest-focus'
   spec.add_development_dependency 'mocha'
   spec.add_development_dependency 'pry'
+  spec.add_development_dependency 'rake', '~> 13.0'
+  spec.add_development_dependency 'rubocop', '~> 1.12'
+  spec.add_development_dependency 'rubocop-minitest'
   spec.add_development_dependency 'sequel', '~> 5.0'
   spec.add_development_dependency 'sqlite3'
+  spec.metadata['rubygems_mfa_required'] = 'true'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: search_lingo
 version: !ruby/object:Gem::Version
-  version: 2.0.0.pre2
+  version: 3.0.0.pre1
 platform: ruby
 authors:
 - John Parker
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-10-02 00:00:00.000000000 Z
+date: 2022-04-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -16,30 +16,30 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.9'
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.9'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: minitest
+  name: minitest-focus
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -53,7 +53,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: minitest-focus
+  name: mocha
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -67,7 +67,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: mocha
+  name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -81,7 +81,35 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: pry
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+- !ruby/object:Gem::Dependency
+  name: rubocop-minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -122,8 +150,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: SearchLingo is a simple framework for defining simple query languages
-  and translating them into application-specific queries.
+description: |
+  SearchLingo is a simple framework for defining simple query languages and
+  translating them into application-specific queries.
 email:
 - jparker@urgetopunt.com
 executables: []
@@ -131,6 +160,8 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".gitlab-ci.yml"
+- ".rubocop.yml"
 - ".travis.yml"
 - Gemfile
 - LICENSE.txt
@@ -153,7 +184,8 @@ files:
 homepage: https://github.com/jparker/search_lingo
 licenses:
 - MIT
-metadata: {}
+metadata:
+  rubygems_mfa_required: 'true'
 post_install_message: 
 rdoc_options:
 - "-x"
@@ -166,15 +198,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.1'
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">"
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 3.3.7
 signing_key: 
 specification_version: 4
 summary: Framework for defining and parsing search queries.