elasticsearch-dsl 0.0.0 → 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 8f022719f270cffd657795f61f6f90a4b6b5cece
+  data.tar.gz: 6ad90bcc1c4088c814290d11aa6ce8e30c1b7bed
+SHA512:
+  metadata.gz: 33d9acad04bf367d192fbe1f75d69e56cdc2cf656e27889c5653f91e1ca24b9040d3d5afad0f5ef2638291c958a79dd659b9a48f5cb06ccc87c2b9f4143a014e
+  data.tar.gz: c4a2b3afc45f69f75ba6483484feac3340f8069f6149e6a69f506869e593f53b922b0fcb61837c74cb585bf5afcf15b540592e41d36418bc0a429695d807752e

data/Gemfile

@@ -2,3 +2,19 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in elasticsearch-dsl.gemspec
 gemspec
+
+if File.exists? File.expand_path("../../elasticsearch/elasticsearch.gemspec", __FILE__)
+  gem 'elasticsearch', :path => File.expand_path("../../elasticsearch", __FILE__), :require => false
+end
+
+if File.exists? File.expand_path("../../elasticsearch-transport", __FILE__)
+  gem 'elasticsearch-transport', :path => File.expand_path("../../elasticsearch-transport", __FILE__), :require => true
+end
+
+if File.exists? File.expand_path("../../elasticsearch-api", __FILE__)
+  gem 'elasticsearch-api', :path => File.expand_path("../../elasticsearch-api", __FILE__), :require => false
+end
+
+if File.exists? File.expand_path("../../elasticsearch-extensions", __FILE__)
+  gem 'elasticsearch-extensions', :path => File.expand_path("../../elasticsearch-extensions", __FILE__), :require => false
+end

data/LICENSE.txt

@@ -1,22 +1,13 @@
-Copyright (c) 2013 Karel Minarik
+Copyright (c) 2014 Elasticsearch
 
-MIT License
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
 
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
+    http://www.apache.org/licenses/LICENSE-2.0
 
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -1,29 +1,258 @@
-# Elasticsearch::Dsl
+# Elasticsearch::DSL
 
-TODO: Write a gem description
+The `elasticsearch-dsl` library provides a Ruby API for
+the [Elasticsearch Query DSL](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl.html).
+
+The library is compatible with Ruby 1.9 or higher and Elasticsearch 1.0 and higher.
 
 ## Installation
 
-Add this line to your application's Gemfile:
+Install the package from [Rubygems](https://rubygems.org):
 
-    gem 'elasticsearch-dsl'
+    gem install elasticsearch-dsl
 
-And then execute:
+To use an unreleased version, either add it to your `Gemfile` for [Bundler](http://gembundler.com):
 
-    $ bundle
+    gem 'elasticsearch-dsl', git: 'git://github.com/elasticsearch/elasticsearch-ruby.git'
 
-Or install it yourself as:
+or install it from a source code checkout:
 
-    $ gem install elasticsearch-dsl
+    git clone https://github.com/elasticsearch/elasticsearch-ruby.git
+    cd elasticsearch-ruby/elasticsearch-dsl
+    bundle install
+    rake install
 
 ## Usage
 
-TODO: Write usage instructions here
+The library is designed as a group of standalone Ruby modules, classes and DSL methods,
+which provide an idiomatic way to build complex
+[search definitions](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/search-request-body.html).
+
+Let's have a simple example using the declarative variant:
+
+```ruby
+require 'elasticsearch/dsl'
+include Elasticsearch::DSL
+
+definition = search do
+  query do
+    match title: 'test'
+  end
+end
+
+definition.to_hash
+# => { query: { match: { title: "test"} } }
+
+require 'elasticsearch'
+client = Elasticsearch::Client.new trace: true
+
+client.search body: definition
+# curl -X GET 'http://localhost:9200/test/_search?pretty' -d '{
+#   "query":{
+#     "match":{
+#       "title":"test"
+#     }
+#   }
+# }'
+# ...
+# => {"took"=>10, "hits"=> {"total"=>42, "hits"=> [...] } }
+```
+
+Let's build the same definition in a more imperative fashion:
+
+```ruby
+require 'elasticsearch/dsl'
+include Elasticsearch::DSL
+
+definition = Search::Search.new
+definition.query = Search::Queries::Match.new title: 'test'
+
+definition.to_hash
+# => { query: { match: { title: "test"} } }
+```
+
+The library doesn't depend on an Elasticsearch client -- its sole purpose is to facilitate
+building search definitions in Ruby. This makes it possible to use it with any Elasticsearch client:
+
+```ruby
+require 'elasticsearch/dsl'
+include Elasticsearch::DSL
+
+definition = search { query { match title: 'test' } }
+
+require 'json'
+require 'faraday'
+client   = Faraday.new(url: 'http://localhost:9200')
+response = JSON.parse(
+              client.post(
+                '/_search',
+                JSON.dump(definition.to_hash),
+                { 'Accept' => 'application/json', 'Content-Type' => 'application/json' }
+              ).body
+            )
+# => {"took"=>10, "hits"=> {"total"=>42, "hits"=> [...] } }
+```
+
+## Features Overview
+
+The library allows to programatically build complex search definitions for Elasticsearch in Ruby,
+which are translated to Hashes, and ultimately, JSON, the language of Elasticsearch.
+
+All Elasticsearch DSL features are supported, namely:
+
+* [Queries](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl-queries.html)
+* [Filters](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl-filters.html)
+* [Aggregations](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/search-aggregations.html)
+* [Suggestions](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/search-suggesters.html)
+* [Sorting](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/search-request-sort.html)
+* [Pagination](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/search-request-from-size.html)
+* [Options](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/search-request-body.html) (source filtering, highlighting, etc)
+
+An example of a complex search definition would look like this:
+
+```ruby
+require 'awesome_print'
+
+require 'elasticsearch'
+require 'elasticsearch/dsl'
+
+include Elasticsearch::DSL
+
+client = Elasticsearch::Client.new transport_options: { request: { timeout: 3600, open_timeout: 3600 } }
+
+# Restore an index from a snapshot
+#
+client.indices.delete index: 'bicycles.stackexchange.com', ignore: 404
+
+puts "Recovering the 'bicycles.stackexchange.com' index...".gray
+client.snapshot.create_repository repository: 'data.elasticsearch.org', body: { type: 'url', settings: { url: 'https://s3.amazonaws.com/data.elasticsearch.org/bicycles.stackexchange.com/' } }
+client.snapshot.restore repository: 'data.elasticsearch.org', snapshot: 'bicycles.stackexchange.com', body: { indices: 'bicycles.stackexchange.com' }
+until client.cluster.health(level: 'indices')['indices']['bicycles.stackexchange.com']['status'] == 'green'
+  r = client.indices.status(index: 'bicycles.stackexchange.com', human: true, recovery: true)['indices']['bicycles.stackexchange.com']['shards']['0'][0]
+  print "\r#{r['index']['size']} of #{r['gateway_recovery']['index']['expected_recovered_size']}".ljust(52).gray
+  sleep 1
+end
+
+# The search definition
+#
+definition = search {
+  query do
+
+    # Use a `function_score` query to modify the default score
+    #
+    function_score do
+      query do
+        filtered do
+
+          # Use a `multi_match` query for the fulltext part of the search
+          #
+          query do
+            multi_match do
+              query    'fixed fixie'
+              operator 'or'
+              fields   %w[ title^10 body ]
+            end
+          end
+
+          # Use a `range` filter on the `creation_date` field
+          #
+          filter do
+            range :creation_date do
+              gte '2013-01-01'
+            end
+          end
+        end
+      end
+
+      # Multiply the default `_score` by a (slightly normalized) document rating
+      #
+      functions << { script_score: { script: '_score * log10( doc["rating"].value )' } }
+    end
+  end
+
+  # Calculate the most frequently used tags
+  #
+  aggregation :tags do
+    terms do
+      field 'tags'
+    end
+  end
+
+  # Calculate the posting frequency
+  #
+  aggregation :frequency do
+    date_histogram do
+      field    'creation_date'
+      interval 'month'
+      format   'yyyy-MM'
+    end
+  end
+
+  # Calculate the statistical information about the number of comments
+  #
+  aggregation :comment_count_stats do
+    stats field: 'comment_count'
+  end
+
+  # Highlight the `title` and `body` fields
+  #
+  highlight fields: {
+    title: { fragment_size: 50 },
+    body:  { fragment_size: 50 }
+  }
+
+  # Return only a selection of the fields
+  #
+  source ['title', 'tags', 'creation_date', 'rating', 'user.location', 'user.display_name']
+}
+
+puts "Search definition #{'-'*63}\n".gray
+ap   definition.to_hash
+
+# Execute the search request
+#
+response = client.search index: 'bicycles.stackexchange.com', type: ['question','answer'], body: definition
+
+puts "\nSearch results #{'-'*66}\n".gray
+ap   response
+```
+
+**Please see the extensive RDoc examples in the source code and the integration tests.**
+
+## Development
+
+To work on the code, clone the repository and install the dependencies:
+
+```
+git clone https://github.com/elasticsearch/elasticsearch-ruby.git
+cd elasticsearch-ruby/elasticsearch-dsl/
+bundle install
+```
+
+Use the Rake tasks to run the test suites:
+
+```
+bundle exec rake test:unit
+bundle exec rake test:integration
+```
+
+To launch a separate Elasticsearch server for integration tests,
+see instructions in the main [README](../README.md#development).
+
+## License
+
+This software is licensed under the Apache 2 license, quoted below.
+
+    Copyright (c) 2015 Elasticsearch <http://www.elasticsearch.org>
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
 
-## Contributing
+       http://www.apache.org/licenses/LICENSE-2.0
 
-1. Fork it
-2. Create your feature branch (`git checkout -b my-new-feature`)
-3. Commit your changes (`git commit -am 'Add some feature'`)
-4. Push to the branch (`git push origin my-new-feature`)
-5. Create new Pull Request
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.

data/Rakefile

@@ -1 +1,213 @@
 require "bundler/gem_tasks"
+
+task(:default) { system "rake --tasks" }
+task  :test    => 'test:unit'
+
+# ----- Test tasks ------------------------------------------------------------
+
+require 'rake/testtask'
+namespace :test do
+
+  Rake::TestTask.new(:unit) do |test|
+    Rake::Task['test:ci_reporter'].invoke if ENV['CI']
+    test.libs << 'lib' << 'test'
+    test.test_files = FileList["test/unit/**/*_test.rb"]
+    # test.verbose = true
+    # test.warning = true
+  end
+
+  Rake::TestTask.new(:integration) do |test|
+    Rake::Task['test:ci_reporter'].invoke if ENV['CI']
+    test.libs << 'lib' << 'test'
+    test.test_files = FileList["test/integration/**/*_test.rb"]
+  end
+
+  desc "Run unit and integration tests"
+  task :all do
+    Rake::Task['test:ci_reporter'].invoke if ENV['CI']
+    Rake::Task['test:unit'].invoke
+    Rake::Task['test:integration'].invoke
+  end
+
+  namespace :cluster do
+    desc "Start Elasticsearch nodes for tests"
+    task :start do
+      $LOAD_PATH << File.expand_path('../../elasticsearch-transport/lib', __FILE__) << File.expand_path('../test', __FILE__)
+      require 'elasticsearch/extensions/test/cluster'
+      Elasticsearch::Extensions::Test::Cluster.start
+    end
+
+    desc "Stop Elasticsearch nodes for tests"
+    task :stop do
+      $LOAD_PATH << File.expand_path('../../elasticsearch-transport/lib', __FILE__) << File.expand_path('../test', __FILE__)
+      require 'elasticsearch/extensions/test/cluster'
+      Elasticsearch::Extensions::Test::Cluster.stop
+    end
+  end
+end
+
+# ----- Documentation tasks ---------------------------------------------------
+
+require 'yard'
+YARD::Rake::YardocTask.new(:doc) do |t|
+  t.options = %w| --embed-mixins --markup=markdown |
+end
+
+# ----- Code analysis tasks ---------------------------------------------------
+
+require 'cane/rake_task'
+Cane::RakeTask.new(:quality) do |cane|
+  cane.abc_max = 15
+  cane.no_style = true
+end
+
+# ----- Generating the source code --------------------------------------------
+
+require 'net/http'
+require 'json'
+require 'coderay'
+
+namespace :generate do
+  desc <<-DESC.gsub(/^    /, '')
+    Generate Ruby source and tests for query/filter/aggregation
+
+    Pass the type of the component, the name, and any option methods as Rake task arguments.
+
+    Example:
+
+        $ rake generate:source[query,boosting]
+        Source: /.../elasticsearch-ruby/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/boosting.rb
+        ...
+        Test: /.../elasticsearch-ruby/elasticsearch-dsl/test/unit/queries/boosting_test.rb
+        ...
+
+        $ rake generate:source[query,common,query/cutoff_frequency/low_freq_operator/...]
+        Source: /.../elasticsearch-ruby/elasticsearch-dsl/lib/elasticsearch/dsl/search/queries/common.rb
+        ...
+        Test: /.../elasticsearch-ruby/elasticsearch-dsl/test/unit/queries/common_test.rb
+        ...
+
+  DESC
+  task :source, [:type, :name, :option_methods] do |task, options|
+    begin
+      query    = URI.escape("#{options[:name]} #{options[:type]}")
+      response = Net::HTTP.get('search.elasticsearch.org', "/search/?q=#{query}")
+      hits     = JSON.load(response)['hits']['hits']
+
+      if hit = hits.first
+        doc_url = ("http://elasticsearch.org" + hit['fields']['url']).gsub(/#.+$/, '') if hit['_score'] > 0.2
+      end
+    rescue Exception => e
+      puts "[!] ERROR: #{e.inspect}"
+    end unless ENV['NOCRAWL']
+
+    case options[:type]
+      when /query/
+        module_name = 'Queries'
+        path_name   = 'queries'
+        include_module = 'BaseComponent'
+      when /filter/
+        module_name = 'Filters'
+        path_name   = 'filters'
+        include_module = 'BaseComponent'
+      when /agg/
+        module_name = 'Aggregations'
+        path_name   = 'aggregations'
+        include_module = 'BaseAggregationComponent'
+      else raise ArgumentError, "Unknown DSL type [#{options[:type]}]"
+    end
+
+    name = options[:name].downcase
+
+    class_name = options[:name].split('_').map(&:capitalize).join
+
+    option_methods = options[:option_methods].to_s.split('/').reduce('') do |sum, item|
+      sum << "                "
+      sum << "option_method :#{item}"
+      sum << "\n" unless item == options[:option_methods].to_s.split('/').last
+      sum
+    end
+
+    option_methods = "\n\n#{option_methods}" unless option_methods.empty?
+
+    source = <<-RUBY.gsub(/^      /, '')
+      module Elasticsearch
+        module DSL
+          module Search
+            module #{module_name}
+
+              # #{class_name} #{options[:type]}
+              #
+              # @example
+              #
+              # @see #{doc_url}
+              #
+              class #{class_name}
+                include #{include_module}#{option_methods}
+              end
+
+            end
+          end
+        end
+      end
+    RUBY
+
+    if options[:option_methods].to_s.empty?
+      test_option_methods = ''
+    else
+      setup = "\n" + options[:option_methods].to_s.split('/').reduce('') do |sum,item|
+        sum << "          subject.#{item} 'bar'\n"; sum
+      end
+      asserts = "\n          assert_equal %w[ #{options[:option_methods].to_s.split('/').sort.join(' ')} ],\n                       subject.to_hash[:#{name}][:foo].keys.map(&:to_s).sort"
+      asserts << "\n          assert_equal 'bar', subject.to_hash[:#{name}][:foo][:#{options[:option_methods].to_s.split('/').first}]"
+
+      test_option_methods =  <<-RUBY.gsub(/^        /, '')
+
+        should "have option methods" do
+          subject = #{class_name}.new :foo
+          #{setup}#{asserts}
+        end
+
+        should "take a block" do
+          subject = #{class_name}.new :foo do
+            #{options[:option_methods].to_s.split('/').first} 'bar'
+          end
+          assert_equal({#{name}: { foo: { #{options[:option_methods].to_s.split('/').first}: 'bar' } }}, subject.to_hash)
+        end
+      RUBY
+    end
+
+    test = <<-RUBY.gsub(/^      /, '')
+      require 'test_helper'
+
+      module Elasticsearch
+        module Test
+          module #{module_name}
+            class #{class_name}Test < ::Test::Unit::TestCase
+              include Elasticsearch::DSL::Search::#{module_name}
+
+              context "#{class_name} #{options[:type]}" do
+                subject { #{class_name}.new }
+
+                should "be converted to a Hash" do
+                  assert_equal({ #{name}: {} }, subject.to_hash)
+                end
+                #{test_option_methods.empty? ? '' : test_option_methods.split("\n").map { |l| '                ' + l }.join("\n")}
+              end
+            end
+          end
+        end
+      end
+    RUBY
+
+    source_full_path = File.expand_path("../lib/elasticsearch/dsl/search/#{path_name}/#{name}.rb", __FILE__)
+    test_full_path   = File.expand_path("../test/unit/#{path_name}/#{name}_test.rb", __FILE__)
+
+    puts '-'*80, "Source: #{source_full_path}", '-'*80, "\n", CodeRay.scan(source, :ruby).terminal, "\n\n"
+    puts '-'*80, "Test: #{test_full_path}",     '-'*80, "\n", CodeRay.scan(test,   :ruby).terminal, "\n"
+
+    File.open(source_full_path, 'w') { |file| file << source }
+
+    File.open(test_full_path, 'w') { |file| file << test }
+  end
+end