lucene_query_parser 0.0.1

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/.rspec

@@ -0,0 +1,2 @@
+-f doc
+--colour

data/Gemfile

@@ -0,0 +1,14 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in lucene_query_parser.gemspec
+gemspec
+
+group "development" do
+  gem "guard"
+  gem "guard-rspec"
+
+  if PLATFORM =~ /darwin/
+    gem "rb-fsevent"
+    gem "growl_notify"
+  end
+end

data/Guardfile

@@ -0,0 +1,9 @@
+guard 'rspec', :version => 2, :cli => "--format doc --colour" do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec/" }
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})                           { |m| "spec/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')                        { "spec/" }
+end
+

data/README.md

@@ -0,0 +1,37 @@
+# Lucene Query Parser
+
+## Synopsis
+
+This library provides a basic parser that implements the majority of the
+[Lucene Query Syntax](http://lucene.apache.org/java/2_9_1/queryparsersyntax.html)
+"specification". Additionally, it includes a `check_lucene_query` script
+to check for errors in a given query.
+
+## Requirements
+
+* Ruby 1.8.7 (hasn't been tested elsewhere)
+* [parslet](http://kschiess.github.com/parslet/)
+* [rainbow](https://github.com/sickill/rainbow)
+* Rspec 2 for development
+
+## Install
+
+    gem install lucene_query_parser
+
+## Usage
+
+    check_lucene_query --help
+
+    check_lucene_query query.txt
+
+    pbpaste | check_lucene_query -
+
+## Development
+
+    bundle
+    rake
+
+## Contributing
+
+Fork, patch, test, and send a pull request.
+

data/Rakefile

@@ -0,0 +1,14 @@
+require "bundler/gem_tasks"
+
+require 'rspec/core/rake_task'
+
+desc 'Default: run specs.'
+task :default => :spec
+
+desc "Run specs"
+RSpec::Core::RakeTask.new do |t|
+  # t.pattern = "./spec/**/*_spec.rb" # don't need this, it's default.
+  # Put spec opts in a file named .rspec in root
+end
+
+

data/bin/check_lucene_query

@@ -0,0 +1,75 @@
+#!/usr/bin/env ruby
+require "lucene_query_parser"
+require "rainbow"
+
+if ARGV.include?("-h") || ARGV.include?("--help")
+  name = __FILE__
+  puts <<-help
+NAME
+
+    #{name} - check lucene query syntax
+
+SYNOPSIS
+
+    #{name} <filename>
+
+ARGUMENTS
+
+    <filename>
+        a file containing a lucene query to parse and validate
+        use '-' to read from STDIN
+
+EXAMPLES
+
+    Check a file containing a query:
+
+      #{name} file_with_query.txt
+
+    Check the contents of the clipboard (mac):
+
+      pbpaste | #{name} -
+
+  help
+  exit -1
+end
+
+input = nil
+
+if input_file = ARGV[0]
+  if input_file == "-"
+    input = STDIN.read
+  else
+    input = File.read input_file
+  end
+else
+  abort "please provide a filename to check or '-' for STDIN"
+end
+
+parser = LuceneQueryParser::Parser.new
+
+
+location = parser.error_location(input)
+if location
+  puts location[:message].color(:yellow)
+  puts
+
+  lines = input.split("\n")
+  lines.each_with_index do |line, i|
+    if i + 1 == location[:line]
+      col = location[:column]
+      print line[0,col-1]
+      print line[col-1, 1].color(:red).background(:yellow)
+      puts line[col..-1]
+    else
+      puts line
+    end
+  end
+
+  puts
+
+  exit -1 # query not ok
+
+else
+  puts "query ok".color(:green)
+end
+

data/lib/lucene_query_parser/parser.rb

@@ -0,0 +1,122 @@
+module LuceneQueryParser
+  class Parser < Parslet::Parser
+
+    # Public: find and explain errors in a query, if any
+    #
+    # query - the query to check
+    #
+    # Returns nil if the query is parseable, or a hash containing information
+    # about the invalid query if not.
+    def error_location(query)
+      parse query
+      nil
+    rescue Parslet::ParseFailed => error
+      cause = find_cause root.error_tree
+      cause =~ /line (\d+) char (\d+)/
+      {:line => $1.to_i, :column => $2.to_i, :message => cause}
+    end
+
+    # Recursively find a "real" cause within a Parslet error tree. "Real"
+    # causes contain line/column positions.
+    def find_cause(node)
+      if node.parslet.cause
+        node.cause
+      else
+        # go in reverse to find the last thing that failed rather than the first
+        node.children.reverse.each do |child|
+          if cause = find_cause(child)
+            return cause
+          end
+        end
+      end
+    end
+
+    # ----- grammar definition -----
+
+    root :expr
+
+    rule :expr do
+      space.maybe >>
+      operand >> (space >> (operator >> space >> operand | operand)).repeat >>
+      space.maybe
+    end
+
+    rule :operator do
+      str('AND').as(:op) | str('OR').as(:op)
+    end
+
+    rule :operand do
+      unary_operator.maybe >> (
+        group |
+        field |
+        term |
+        phrase
+      )
+    end
+
+    rule :term do
+      match["\\w'"].repeat(1).as(:term) >> (fuzzy | boost).maybe
+    end
+
+    rule :phrase do
+      str('"') >> match['^"'].repeat(1).as(:phrase) >> str('"') >>
+      (distance | boost).maybe
+    end
+
+    rule :distance do
+      str('~') >> match['0-9'].repeat(1).as(:distance)
+    end
+
+    rule :group do
+      str('(') >> space.maybe >> expr.as(:group) >> space.maybe >> str(')')
+    end
+
+    rule :field do
+      match["\\w"].repeat(1).as(:field) >> str(':') >>
+      (
+        term | phrase | group |
+        inclusive_range.as(:inclusive_range) |
+        exclusive_range.as(:exclusive_range)
+      )
+    end
+
+    rule :inclusive_range do
+      str('[') >> space.maybe >>
+      word.as(:from) >> space >> str('TO') >> space >> word.as(:to) >>
+      space.maybe >> str(']')
+    end
+
+    rule :exclusive_range do
+      str('{') >> space.maybe >>
+      word.as(:from) >> space >> str('TO') >> space >> word.as(:to) >>
+      space.maybe >> str('}')
+    end
+
+    rule :unary_operator do
+      str('+').as(:required) |
+      str('-').as(:prohibited) |
+      (str('NOT').as(:op) >> space)
+    end
+
+    rule :fuzzy do
+      str('~') >>
+      ( str('0.') >> match['0-9'].repeat(1) | match['01'] ).maybe.as(:similarity)
+    end
+
+    rule :boost do
+      str('^') >> (
+        str('0.') >> match['0-9'].repeat(1) |
+        match['0-9'].repeat(1)
+      ).as(:boost)
+    end
+
+    rule :word do
+      match["\\w"].repeat(1)
+    end
+
+    rule :space do
+      match["\n \t"].repeat(1)
+    end
+
+  end
+end

data/lib/lucene_query_parser/version.rb

@@ -0,0 +1,3 @@
+module LuceneQueryParser
+  VERSION = "0.0.1"
+end

data/lib/lucene_query_parser.rb

@@ -0,0 +1,9 @@
+require "lucene_query_parser/version"
+
+require "parslet"
+
+module LuceneQueryParser
+  # Your code goes here...
+
+  require "lucene_query_parser/parser"
+end

data/lucene_query_parser.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "lucene_query_parser/version"
+
+Gem::Specification.new do |s|
+  s.name        = "lucene_query_parser"
+  s.version     = LuceneQueryParser::VERSION
+  s.authors     = ["Nathan Witmer"]
+  s.email       = ["nwitmer@gmail.com"]
+  s.homepage    = "https://github.com/aniero/lucene_query_parser"
+  s.summary     = %q{Lucene query parser and syntax checker}
+  s.description = %q{Parser class and syntax checking script for validating Lucene queries}
+
+  s.rubyforge_project = "lucene_query_parser"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_runtime_dependency "parslet"
+  s.add_runtime_dependency "rainbow"
+  s.add_development_dependency "rspec", "~> 2.5.0"
+end

data/spec/lucene_query_parser/parser_spec.rb

@@ -0,0 +1,195 @@
+require "spec_helper"
+
+describe LuceneQueryParser::Parser do
+  let(:parser) { LuceneQueryParser::Parser.new }
+
+  describe "#parse" do
+    it "parses a term" do
+      should parse("foo").as({:term => "foo"})
+    end
+
+    it "parses a phrase" do
+      should parse('"foo bar"').as({:phrase => "foo bar"})
+    end
+
+    it "parses a term and a phrase" do
+      parse(%q(foo "stuff and things")).as [
+        {:term => "foo"},
+        {:phrase => "stuff and things"}
+      ]
+    end
+
+    it "parses a phrase and two terms" do
+      should parse(%q("foo bar" isn't one)).as [
+        {:phrase => "foo bar"},
+        {:term => "isn't"},
+        {:term => "one"}
+      ]
+    end
+
+    it "parses multiple phrases" do
+      should parse(%q("foo bar"~3 "mumble stuff"~5 "blah blah")).as [
+        {:phrase => "foo bar", :distance => "3"},
+        {:phrase => "mumble stuff", :distance => "5"},
+        {:phrase => "blah blah"}
+      ]
+    end
+
+    it "parses a nearness query" do
+      should parse(%q("foo bar"~2)).as(
+        {:phrase => "foo bar", :distance => "2"}
+      )
+    end
+
+    it "parses a paren grouping" do
+      should parse(%q((foo bar))).as(
+        {:group => [{:term => "foo"}, {:term => "bar"}]}
+      )
+    end
+
+    it "parses nested paren groups" do
+      should parse(%q((foo (bar (baz))))).as(
+        {:group => [
+          {:term => "foo"},
+          {:group => [
+            {:term => "bar"},
+            {:group => {:term => "baz"}}
+          ]}
+        ]}
+      )
+    end
+
+    it "parses a required term" do
+      should parse("+foo").as({:term => "foo", :required => "+"})
+    end
+
+    it "parses a prohibited term" do
+      should parse("-foo").as({:term => "foo", :prohibited => "-"})
+    end
+
+    it "parses prohibited groups and phrases" do
+      should parse(%q(+(foo bar) -"mumble stuff")).as [
+        {:group => [{:term => "foo"}, {:term => "bar"}], :required => "+"},
+        {:phrase => "mumble stuff", :prohibited => "-"}
+      ]
+    end
+
+    it "ignores leading spaces" do
+      should parse("   foo bar").as [{:term => "foo"}, {:term => "bar"}]
+    end
+
+    it "ignores trailing spaces" do
+      should parse("foo bar   ").as [{:term => "foo"}, {:term => "bar"}]
+    end
+
+    it "ignores trailing spaces" do
+
+    end
+
+    it "parses AND groupings" do
+      should parse(%q(foo AND bar)).as [
+        {:term => "foo"},
+        {:op => "AND", :term => "bar"}
+      ]
+    end
+
+    it "parses a sequence of AND and OR" do
+      should parse(%q(foo AND bar OR baz OR mumble)).as [
+        {:term => "foo"},
+        {:op => "AND", :term => "bar"},
+        {:op => "OR", :term => "baz"},
+        {:op => "OR", :term => "mumble"}
+      ]
+    end
+
+    it "parses NOTs" do
+      should parse("foo NOT bar").as [
+        {:term => "foo"},
+        {:term => "bar", :op => "NOT"}
+      ]
+    end
+
+    it "parses field:value" do
+      should parse("title:foo").as(
+        {:field => "title", :term => "foo"}
+      )
+    end
+
+    it 'parses field:"a phrase"' do
+      should parse('title:"a phrase"').as(
+        {:field => "title", :phrase => "a phrase"}
+      )
+    end
+
+    it "parses field:(foo AND bar)" do
+      should parse('title:(foo AND bar)').as(
+        {:field => "title", :group => [
+          {:term => "foo"},
+          {:op => "AND", :term => "bar"}
+        ]}
+      )
+    end
+
+    it "parses fuzzy terms" do
+      should parse('fuzzy~').as(
+        {:term => "fuzzy", :similarity => nil}
+      )
+    end
+
+    it "parses a fuzzy similarity of 0" do
+      should parse('fuzzy~0').as(
+        {:term => "fuzzy", :similarity => "0"}
+      )
+    end
+
+    it "parses a fuzzy similarity of 1" do
+      should parse('fuzzy~1').as(
+        {:term => "fuzzy", :similarity => "1"}
+      )
+    end
+
+    it "parses a fuzzy similarity of 0.8" do
+      should parse('fuzzy~0.8').as(
+        {:term => "fuzzy", :similarity => "0.8"}
+      )
+    end
+
+    it { should parse('year:[2010 TO 2011]').as(
+      {:field => "year", :inclusive_range => {:from => "2010", :to => "2011"}}
+    ) }
+
+    it { should parse('year:{2009 TO 2012}').as(
+      {:field => "year", :exclusive_range => {:from => "2009", :to => "2012"}}
+    ) }
+
+    it { should parse('boosted^1').as({:term => "boosted", :boost => "1"})}
+    it { should parse('boosted^0.1').as({:term => "boosted", :boost => "0.1"})}
+
+    it { should parse('boosted^10 normal').as([
+      {:term => "boosted", :boost => "10"},
+      {:term => "normal"}
+    ])}
+
+    it { should parse('"boosted phrase"^10 "normal phrase"').as([
+      {:phrase => "boosted phrase", :boost => "10"},
+      {:phrase => "normal phrase"}
+    ])}
+
+  end
+
+  describe "#error_location" do
+    let(:parser) { LuceneQueryParser::Parser.new }
+
+    it "returns nil for a valid query" do
+      parser.error_location("valid query").should be_nil
+    end
+
+    it "returns a hash with the line and column for an invalid query" do
+      error = parser.error_location("invalid^ query")
+      error[:line].should == 1
+      error[:column].should == 8
+      error[:message].should =~ /Expected/
+    end
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,6 @@
+require "lucene_query_parser"
+
+require "parslet/rig/rspec"
+
+RSpec.configure do |config|
+end

metadata

@@ -0,0 +1,122 @@
+--- !ruby/object:Gem::Specification 
+name: lucene_query_parser
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Nathan Witmer
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-08-24 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: parslet
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: rainbow
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 27
+        segments: 
+        - 2
+        - 5
+        - 0
+        version: 2.5.0
+  type: :development
+  version_requirements: *id003
+description: Parser class and syntax checking script for validating Lucene queries
+email: 
+- nwitmer@gmail.com
+executables: 
+- check_lucene_query
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- .rspec
+- Gemfile
+- Guardfile
+- README.md
+- Rakefile
+- bin/check_lucene_query
+- lib/lucene_query_parser.rb
+- lib/lucene_query_parser/parser.rb
+- lib/lucene_query_parser/version.rb
+- lucene_query_parser.gemspec
+- spec/lucene_query_parser/parser_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/aniero/lucene_query_parser
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: lucene_query_parser
+rubygems_version: 1.8.7
+signing_key: 
+specification_version: 3
+summary: Lucene query parser and syntax checker
+test_files: 
+- spec/lucene_query_parser/parser_spec.rb
+- spec/spec_helper.rb