slingshot-rb 0.0.1

data/.gitignore

@@ -0,0 +1,7 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+rdoc/
+coverage/
+scratch/

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in slingshot.gemspec
+gemspec

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2011 Karel Minarik
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.markdown

@@ -0,0 +1,164 @@
+Slingshot
+=========
+
+_Slingshot_ aims to provide a rich Ruby API and DSL for the
+[ElasticSearch](http://www.elasticsearch.org/) search engine/database.
+
+_ElasticSearch_ is a scalable, distributed, highly-available,
+RESTful database communicating by JSON over HTTP, based on [Lucene](http://lucene.apache.org/),
+written in Java. It manages to very simple and very powerful at the same time.
+You should seriously consider it to power search in your Ruby applications:
+it will deliver all the features you want — and many more you may have not
+imagined yet (native geo search? histogram facets?)
+
+_Slingshot_ currently allow basic operation with the index and searching. See chapters below.
+
+
+Installation
+------------
+
+First, you need a running _ElasticSearch_ server. Thankfully, it's easy. Let's define easy:
+
+    $ curl -L -o elasticsearch-0.14.4.tar.gz http://github.com/downloads/elasticsearch/elasticsearch/elasticsearch-0.14.4.tar.gz
+    $ tar -zxvf elasticsearch-0.14.4.tar.gz
+    $ ./elasticsearch-0.14.4/bin/elasticsearch -f
+
+OK, easy. Now, install the gem via Rubygems:
+
+    $ gem install slingshot
+
+or from source:
+
+    $ git clone git://github.com/karmi/slingshot.git
+    $ rake install
+
+
+Usage
+-----
+
+Currently, you can use _Slingshot_ via the DSL (eg. by extending your class with it).
+Plans for full ActiveModel integration (and other convenience layers) are in progress.
+
+To kick the tiers, require the gem in an IRB session or a Ruby script
+(note that you can run the full example from [`examples/dsl.rb`](https://github.com/karmi/slingshot/blob/master/examples/dsl.rb)):
+
+    require 'rubygems'
+    require 'slingshot'
+
+First, let's create an index named `articles` and store/index some documents:
+
+    Slingshot.index 'articles' do
+      delete
+      create
+
+      store :title => 'One',   :tags => ['ruby']
+      store :title => 'Two',   :tags => ['ruby', 'python']
+      store :title => 'Three', :tags => ['java']
+      store :title => 'Four',  :tags => ['ruby', 'php']
+
+      refresh
+    end
+
+Now, let's query the database:
+
+We are searching for articles tagged _ruby_, sorted by `title` in `descending` order,
+and also retrieving some [_facets_](http://www.lucidimagination.com/Community/Hear-from-the-Experts/Articles/Faceted-Search-Solr)
+from the database:
+
+    s = Slingshot.search 'articles' do
+      query do
+        terms :tags, ['ruby']
+      end
+
+      sort do
+        title 'desc'
+      end
+
+      facet 'global-tags' do
+        terms :tags, :global => true
+      end
+
+      facet 'current-tags' do
+        terms :tags
+      end
+    end
+
+Let's display the results:
+
+    s.results.each do |document|
+      puts "* #{ document['_source']['title'] }"
+    end
+
+    # * Two
+    # * One
+    # * Four
+
+Let's display the facets (distribution of tags across the whole database):
+
+    s.results.facets['global-tags']['terms'].each do |f|
+      puts "#{f['term'].ljust(10)} #{f['count']}"
+    end
+
+    # ruby       3
+    # python     1
+    # php        1
+    # java       1
+    
+We can display the full query JSON:
+
+    puts s.to_json
+    # {"facets":{"current-tags":{"terms":{"field":"tags"}},"global-tags":{"global":true,"terms":{"field":"tags"}}},"sort":[{"title":"desc"}],"query":{"terms":{"tags":["ruby"]}}}
+
+See, a Ruby DSL for this thing is kinda handy? We can query _ElasticSearch_ manually with `curl`, simply:
+
+    puts s.to_curl
+    # curl -X POST "http://localhost:9200/articles/_search?pretty=true" -d '{"facets":{"current-tags":{"terms":{"field":"tags"}},"global-tags":{"global":true,"terms":{"field":"tags"}}},"sort":[{"title":"desc"}],"query":{"terms":{"tags":["ruby"]}}}'
+
+
+Features
+--------
+
+Currently, _Slingshot_ supports only a limited subset of vast _ElasticSearch_ [Search API](http://www.elasticsearch.org/guide/reference/api/search/request-body.html) and it's [Query DSL](http://www.elasticsearch.org/guide/reference/query-dsl/):
+
+* Creating, deleting and refreshing the index
+* Storing a document in the index
+* [Querying](https://github.com/karmi/slingshot/blob/master/examples/dsl.rb) the index with the `query_string`, `term` and `terms` types of queries
+* Sorting the results by `fields`
+* Retrieving a _terms_ type of [facets](http://www.elasticsearch.org/guide/reference/api/search/facets/index.html) -- other types are high priority
+* Returning just specific fields from documents
+* Paging with `from` and `size` query options
+
+See the [`examples/dsl.rb`](blob/master/examples/dsl.rb).
+
+Todo & Plans
+------------
+
+In order of importance:
+
+* Basic wrapper class for _hits_ in results, so we could write `results.first.document.title` instead of using the raw Hash
+* Getting document [by ID](http://www.elasticsearch.org/guide/reference/api/get.html)
+* Seamless _ActiveModel_ compatibility for easy usage in _Rails_ applications (this also means nearly full _ActiveRecord_ compatibility)
+* Allowing to set custom non-ActiveModel wrapper class (your own)
+* Seamless [will_paginate](https://github.com/mislav/will_paginate) compatibility for easy pagination
+* [Histogram](http://www.elasticsearch.org/guide/reference/api/search/facets/histogram-facet.html) facets
+* Seamless support for [auto-updating _river_ index](http://www.elasticsearch.org/guide/reference/river/couchdb.html) for _CouchDB_ `_changes` feed
+* [Mapping](http://www.elasticsearch.org/guide/reference/mapping/) management
+* Infrastructure for query filters
+* [Range](http://www.elasticsearch.org/guide/reference/query-dsl/range-filter.html) filters and queries
+* [Geo Filters](http://www.elasticsearch.org/blog/2010/08/16/geo_location_and_search.html) for queries
+* [Statistical](http://www.elasticsearch.org/guide/reference/api/search/facets/statistical-facet.html) facets
+* [Geo Distance](http://www.elasticsearch.org/guide/reference/api/search/facets/geo-distance-facet.html) facets
+* [Index aliases](http://www.elasticsearch.org/guide/reference/api/admin-indices-aliases.html) management
+* [Analyze](http://www.elasticsearch.org/guide/reference/api/admin-indices-analyze.html) API support
+* [Highligting](http://www.elasticsearch.org/guide/reference/api/search/highlighting.html) support
+* [Bulk](http://www.elasticsearch.org/guide/reference/api/bulk.html) API
+* Embedded webserver to display cluster statistics and allow easy searches
+
+Feedback
+--------
+
+You can send feedback via [e-mail](mailto:karmi@karmi.cz) or via [Github Issues](https://github.com/karmi/slingshot/issues).
+
+-----
+
+[Karel Minarik](http://karmi.cz)

data/Rakefile

@@ -0,0 +1,52 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+task :default => :test
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/*_test.rb'
+  test.verbose = true
+end
+
+namespace :test do
+  Rake::TestTask.new(:unit) do |test|
+    test.libs << 'lib' << 'test'
+    test.pattern = 'test/unit/*_test.rb'
+    test.verbose = true
+  end
+  Rake::TestTask.new(:integration) do |test|
+    test.libs << 'lib' << 'test'
+    test.pattern = 'test/integration/*_test.rb'
+    test.verbose = true
+  end
+end
+
+# Generate documentation
+begin
+  require 'sdoc'
+rescue LoadError
+end
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "Slingshot"
+  rdoc.rdoc_files.include('README.rdoc')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+# Generate coverage reports
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |test|
+    test.libs << 'test'
+    test.rcov_opts = ['--exclude', 'gems/*']
+    test.pattern = 'test/**/*_test.rb'
+    test.verbose = true
+  end
+rescue LoadError
+  task :rcov do
+    abort "RCov is not available. In order to run rcov, you must: sudo gem install rcov"
+  end
+end

data/examples/dsl.rb

@@ -0,0 +1,70 @@
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+
+require 'rubygems'
+require 'slingshot'
+
+extend Slingshot::DSL
+
+configure do
+  url "http://localhost:9200"
+end
+
+index 'articles' do
+  delete
+  create
+
+  puts "Documents:", "-"*80
+  [
+    { :title => 'One',   :tags => ['ruby'] },
+    { :title => 'Two',   :tags => ['ruby', 'python'] },
+    { :title => 'Three', :tags => ['java'] },
+    { :title => 'Four',  :tags => ['ruby', 'php'] }
+  ].each do |article|
+    puts "Indexing article: #{article.to_json}"
+    store article
+  end  
+
+  refresh
+end
+
+s = search 'articles' do
+  query do
+    terms :tags, ['ruby']
+  end
+
+  sort do
+    title 'desc'
+  end
+
+  facet 'global-tags' do
+    terms :tags, :global => true
+  end
+
+  facet 'current-tags' do
+    terms :tags
+  end
+end
+
+puts "", "Query:", "-"*80
+puts s.to_json
+
+puts "", "Raw JSON result:", "-"*80
+puts JSON.pretty_generate(s.response)
+
+puts "", "Try the query in Curl:", "-"*80
+puts s.to_curl
+
+puts "", "Results:", "-"*80
+s.results.each_with_index do |document, i|
+  puts "#{i+1}. #{ document['_source']['title'].ljust(20) } [id] #{document['_id']}"
+end
+
+puts "", "Facets: tags distribution across the whole database:", "-"*80
+s.results.facets['global-tags']['terms'].each do |f|
+  puts "#{f['term'].ljust(10)} #{f['count']}"
+end
+
+puts "", "Facets: tags distribution for the current query", "-"*80
+s.results.facets['current-tags']['terms'].each do |f|
+  puts "#{f['term'].ljust(10)} #{f['count']}"
+end

data/lib/slingshot/client.rb

@@ -0,0 +1,25 @@
+module Slingshot
+
+  module Client
+
+    class Base
+      def post(url, data)
+        raise NoMethodError, "Implement this method in your client class"
+      end
+      def delete(url)
+        raise NoMethodError, "Implement this method in your client class"
+      end
+    end
+
+    class RestClient < Base
+      def self.post(url, data)
+        ::RestClient.post url, data
+      end
+      def self.delete(url)
+        ::RestClient.delete url rescue nil
+      end
+    end
+
+  end
+
+end

data/lib/slingshot/configuration.rb

@@ -0,0 +1,15 @@
+module Slingshot
+
+  class Configuration
+
+    def self.url(value=nil)
+      @url    = value || @url || "http://localhost:9200"
+    end
+
+    def self.client(klass=nil)
+      @client = klass || @client || Client::RestClient
+    end
+
+  end
+
+end

data/lib/slingshot/dsl.rb

@@ -0,0 +1,17 @@
+module Slingshot
+  module DSL
+
+    def configure(&block)
+      Configuration.class_eval(&block)
+    end
+
+    def search(indices, &block)
+      Search::Search.new(indices, &block).perform
+    end
+
+    def index(name, &block)
+      Index.new(name, &block)
+    end
+
+  end
+end

data/lib/slingshot/index.rb

@@ -0,0 +1,42 @@
+module Slingshot
+  class Index
+
+    def initialize(name, &block)
+      @name = name
+      instance_eval(&block) if block_given?
+    end
+
+    def delete
+      response = Configuration.client.delete "#{Configuration.url}/#{@name}"
+      return response =~ /error/ ? false : true
+    rescue
+      false
+    end
+
+    def create
+      Configuration.client.post "#{Configuration.url}/#{@name}", ''
+    rescue
+      false
+    end
+
+    def store(*args)
+      if args.size > 1
+        (type, document = args)
+      else
+        (document = args.pop; type = :document)
+      end
+      document = case true
+        when document.is_a?(String) then document
+        when document.respond_to?(:to_indexed_json) then document.to_indexed_json
+        else raise ArgumentError, "Please pass a JSON string or object with a 'to_indexed_json' method"
+      end
+      result = Configuration.client.post "#{Configuration.url}/#{@name}/#{type}/", document
+      JSON.parse(result)
+    end
+
+    def refresh
+      Configuration.client.post "#{Configuration.url}/#{@name}/_refresh", ''
+    end
+
+  end
+end

data/lib/slingshot/results/collection.rb

@@ -0,0 +1,22 @@
+module Slingshot
+  module Results
+
+    class Collection
+      include Enumerable
+      attr_reader :time, :total, :results, :facets
+
+      def initialize(response)
+        @time    = response['took']
+        @total   = response['hits']['total']
+        @results = response['hits']['hits']
+        @facets  = response['facets']
+      end
+
+      def each(&block)
+        @results.each(&block)
+      end
+
+    end
+
+  end
+end

data/lib/slingshot/rubyext/hash.rb

@@ -0,0 +1,3 @@
+class Hash
+  alias_method :to_indexed_json, :to_json if respond_to?(:to_json)
+end

data/lib/slingshot/search/facet.rb

@@ -0,0 +1,34 @@
+module Slingshot
+  module Search
+
+    #--
+    # TODO: Implement all elastic search facets (geo, histogram, range, etc)
+    # https://github.com/elasticsearch/elasticsearch/wiki/Search-API-Facets
+    #++
+
+    class Facet
+
+      def initialize(name, options={}, &block)
+        @name    = name
+        @options = options
+        self.instance_eval(&block) if block_given?
+      end
+
+      def terms(field, options={})
+        @value = { :terms => { :field => field } }.update(options)
+        self
+      end
+
+      def to_json
+        to_hash.to_json
+      end
+
+      def to_hash
+        h = { @name => @value }
+        h[@name].update @options
+        return h
+      end
+    end
+
+  end
+end

data/lib/slingshot/search/query.rb

@@ -0,0 +1,33 @@
+module Slingshot
+  module Search
+
+    class Query
+      def initialize(&block)
+        self.instance_eval(&block) if block_given?
+      end
+
+      def term(field, value)
+        @value = { :term => { field => value } }
+      end
+
+      def terms(field, value, options={})
+        @value = { :terms => { field => value } }
+        @value[:terms].update( { :minimum_match => options[:minimum_match] } ) if options[:minimum_match]
+        @value
+      end
+
+      def string(value, options={})
+        @value = { :query_string => { :query => value } }
+        @value[:query_string].update( { :default_field => options[:default_field] } ) if options[:default_field]
+        # TODO: https://github.com/elasticsearch/elasticsearch/wiki/Query-String-Query
+        @value
+      end
+
+      def to_json
+        @value.to_json
+      end
+
+    end
+
+  end
+end

data/lib/slingshot/search/sort.rb

@@ -0,0 +1,24 @@
+module Slingshot
+  module Search
+
+    class Sort
+      def initialize(&block)
+        @value = []
+        self.instance_eval(&block) if block_given?
+      end
+
+      def method_missing(id, *args, &block)
+        case arg = args.shift
+          when String, Symbol, Hash then @value << { id => arg }
+          else @value << id
+        end
+        self
+      end
+
+      def to_json
+        @value.to_json
+      end
+    end
+
+  end
+end

data/lib/slingshot/search.rb

@@ -0,0 +1,70 @@
+module Slingshot
+  module Search
+  
+    class Search
+
+      attr_reader :indices, :url, :results, :response, :query, :facets
+
+      def initialize(*indices, &block)
+        raise ArgumentError, 'Please pass index or indices to search' if indices.empty?
+        @indices = indices
+        instance_eval(&block) if block_given?
+      end
+
+      def query(&block)
+        @query = Query.new(&block)
+        self
+      end
+
+      def sort(&block)
+        @sort = Sort.new(&block)
+        self
+      end
+
+      def facet(name, options={}, &block)
+        @facets ||= {}
+        @facets.update Facet.new(name, options, &block).to_hash
+        self
+      end
+
+      def from(value)
+        @from = value
+        self
+      end
+
+      def size(value)
+        @size = value
+        self
+      end
+
+      def fields(fields=[])
+        @fields = fields
+        self
+      end
+
+      def perform
+        @url     = "#{Configuration.url}/#{indices.join(',')}/_search"
+        @response = JSON.parse( Configuration.client.post(@url, self.to_json) )
+        @results = Results::Collection.new(@response)
+        self
+      end
+
+      def to_curl
+        %Q|curl -X POST "http://localhost:9200/#{indices}/_search?pretty=true" -d '#{self.to_json}'|
+      end
+
+      def to_json
+        request = {}
+        request.update( { :query  => @query } )
+        request.update( { :sort   => @sort } )   if @sort
+        request.update( { :facets => @facets } ) if @facets
+        request.update( { :size => @size } )     if @size
+        request.update( { :from => @from } )     if @from
+        request.update( { :fields => @fields } ) if @fields
+        request.to_json
+      end
+
+    end
+
+  end
+end

data/lib/slingshot/slingshot-rb.rb

@@ -0,0 +1 @@
+require 'slingshot'

data/lib/slingshot/version.rb

@@ -0,0 +1,3 @@
+module Slingshot
+  VERSION = "0.0.1"
+end

data/lib/slingshot.rb

@@ -0,0 +1,18 @@
+require 'rest_client'
+require 'yajl/json_gem'
+
+require 'slingshot/rubyext/hash'
+require 'slingshot/configuration'
+require 'slingshot/client'
+require 'slingshot/client'
+require 'slingshot/search'
+require 'slingshot/search/query'
+require 'slingshot/search/sort'
+require 'slingshot/search/facet'
+require 'slingshot/results/collection'
+require 'slingshot/index'
+require 'slingshot/dsl'
+
+module Slingshot
+  extend DSL
+end

data/slingshot.gemspec

@@ -0,0 +1,41 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "slingshot/version"
+
+Gem::Specification.new do |s|
+  s.name        = "slingshot-rb"
+  s.version     = Slingshot::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.summary       = "Ruby API for ElasticSearch"
+  s.homepage      = "http://github.com/karmi/slingshot"
+  s.authors       = [ 'Karel Minarik' ]
+  s.email         = 'karmi@karmi.cz'
+
+  s.rubyforge_project = "slingshot"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+
+  s.require_paths = ["lib"]
+
+  s.extra_rdoc_files  = [ "README.markdown", "MIT-LICENSE" ]
+  s.rdoc_options      = [ "--charset=UTF-8" ]
+
+  s.required_rubygems_version = ">= 1.3.6"
+
+  s.add_dependency "bundler",     "~> 1.0.0"
+  s.add_dependency "rest-client", "~> 1.6.0"
+  s.add_dependency "yajl-ruby",   "> 0.7.9"
+
+  s.add_development_dependency "turn"
+  s.add_development_dependency "shoulda"
+  s.add_development_dependency "mocha"
+  s.add_development_dependency "sdoc"
+  s.add_development_dependency "rcov"
+
+  s.description = <<-DESC
+   Ruby API for the ElasticSearch search engine/database.
+   A work in progress, currently.
+  DESC
+end