opensearch-ruby 2.1.0 → 3.0.0

data/opensearch-ruby.gemspec

@@ -24,58 +24,46 @@
 # specific language governing permissions and limitations
 # under the License.
 
-lib = File.expand_path('../lib', __FILE__)
-$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'opensearch/version'
-
-signing_key_path = File.expand_path("../gem-private_key.pem")
+require_relative './lib/opensearch/version'
 
 Gem::Specification.new do |s|
   s.name          = 'opensearch-ruby'
   s.version       = OpenSearch::VERSION
-  s.authors       = ['Jayesh Hathila', 'Vamshi Vijay Nakkirtha', 'Vijayan Balasubramanian' , 'Yuvraj Jaiswal']
-  s.email         = ['jayehh@amazon.com', 'vamshin@amazon.com', 'balasvij@amazon.com', 'jaiyuvra@amazon.com']
-  s.summary       = 'Ruby integrations for OpenSearch'
-  s.homepage      = 'https://opensearch.org/docs/latest'
+  s.authors       = ['Elastic', 'OpenSearch Contributors']
+  s.email         = 'opensearch@amazon.com'
+  s.summary       = 'Ruby Client for OpenSearch'
+  s.homepage      = 'https://github.com/opensearch-project/opensearch-ruby'
   s.license       = 'Apache-2.0'
+  s.description   = <<~DESC
+    OpenSearch Ruby is a Ruby client for OpenSearch. You can use the client to
+    execute OpenSearch API commands, and build OpenSearch queries and aggregations
+    using the included OpenSearch DSL.
+  DESC
   s.metadata = {
-    'homepage_uri' => 'https://opensearch.org/docs/latest/',
+    'homepage_uri' => 'https://github.com/opensearch-project/opensearch-ruby',
+    'documentation_uri' => 'https://rubydoc.info/gems/opensearch-ruby',
     'source_code_uri' => 'https://github.com/opensearch-project/opensearch-ruby/tree/main',
-    'bug_tracker_uri' => 'https://github.com/opensearch-project/opensearch-ruby/issues'
+    'bug_tracker_uri' => 'https://github.com/opensearch-project/opensearch-ruby/issues',
+    'changelog_uri' => 'https://github.com/opensearch-project/opensearch-ruby/blob/main/CHANGELOG.md'
   }
-  s.files         = `git ls-files`.split($/)
-  s.executables   = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
-  s.executables   << 'opensearch_ruby_console'
-  s.test_files    = s.files.grep(%r{^(test|spec|features)/})
-
-  if $PROGRAM_NAME.end_with?("gem") && ARGV == ["build", __FILE__] && File.exist?(signing_key_path)
-    s.signing_key = signing_key_path
-    s.cert_chain  = ['../certs/opensearch-rubygems.pem']
-  end
 
+  s.files         = Dir['lib/**/*', '*.gemspec']
+  s.test_files    = Dir['spec**/*', 'test/**/*', 'features/**/*']
   s.require_paths = ['lib']
-  s.bindir = 'bin'
+  s.bindir        = 'bin'
+  s.executables   = 'opensearch_ruby_console'
 
-  s.extra_rdoc_files  = [ 'README.md', 'LICENSE' ]
-  s.rdoc_options      = [ '--charset=UTF-8' ]
+  s.extra_rdoc_files  = %w[README.md USER_GUIDE.md LICENSE.txt]
+  s.rdoc_options      = ['--charset=UTF-8']
 
-  s.required_ruby_version = '>= 2.4'
-
-  s.add_dependency 'opensearch-transport', '~> 2.0'
-  s.add_dependency 'opensearch-api',       '~> 2.1'
+  signing_key = File.expand_path('gem-private_key.pem')
+  if $PROGRAM_NAME.end_with?('gem') && ARGV.first == 'build' && File.exist?(signing_key)
+    s.signing_key = signing_key
+    s.cert_chain  = ['.github/opensearch-rubygems.pem']
+  end
 
-  s.add_development_dependency 'bundler'
-  s.add_development_dependency 'byebug' unless defined?(JRUBY_VERSION) || defined?(Rubinius)
-  s.add_development_dependency 'pry'
-  s.add_development_dependency 'rake', '~> 13'
-  s.add_development_dependency 'require-prof' unless defined?(JRUBY_VERSION) || defined?(Rubinius)
-  s.add_development_dependency 'rspec'
-  s.add_development_dependency 'ruby-prof' unless defined?(JRUBY_VERSION) || defined?(Rubinius)
-  s.add_development_dependency 'simplecov'
-  s.add_development_dependency 'webmock'
-  s.add_development_dependency 'yard'
+  s.required_ruby_version = '>= 2.5'
 
-  s.description = <<-DESC.gsub(/^    /, '')
-    Ruby integrations for OpenSearch (client, API, etc.)
-  DESC
+  s.add_dependency 'faraday', '>= 1.0', '< 3'
+  s.add_dependency 'multi_json', '>= 1.0'
 end

data/spec/spec_helper.rb

@@ -24,12 +24,103 @@
 # specific language governing permissions and limitations
 # under the License.
 
+if ENV.fetch('COVERAGE', nil) && ENV['CI'].nil?
+  require 'simplecov'
+  SimpleCov.start { add_filter %r{^/test|spec/} }
+end
+
+require 'ansi'
 require 'opensearch'
-require 'rspec'
+require 'opensearch-dsl'
+require 'jbuilder'
+require 'jsonify'
+require 'yaml'
+require 'hashie/mash'
+
+require 'faraday/httpclient'
+require 'faraday/net_http_persistent'
+require 'logger'
+require 'ansi/code'
+
+if defined?(JRUBY_VERSION)
+  require 'opensearch/transport/transport/http/manticore'
+  require 'pry-nav'
+else
+  require 'faraday/patron'
+  require 'faraday/typhoeus' if Gem::Version.new(Faraday::VERSION) >= Gem::Version.new('2')
+  require 'opensearch/transport/transport/http/curb'
+  require 'curb'
+end
+
+tracer = ::Logger.new(STDERR)
+tracer.formatter = ->(_s, _d, _p, m) { "#{m.gsub(/^.*$/) { |n| '   ' + n }.ansi(:faint)}\n" }
 
-OPENSEARCH_URL = ENV['TEST_OPENSEARCH_SERVER'] || "http://localhost:#{(ENV['PORT'] || 9200)}"
-raise URI::InvalidURIError unless OPENSEARCH_URL =~ /\A#{URI::DEFAULT_PARSER.make_regexp}\z/
+unless defined?(OPENSEARCH_URL)
+  OPENSEARCH_URL = ENV.fetch('OPENSEARCH_URL', nil) ||
+                   ENV.fetch('TEST_OPENSEARCH_SERVER', nil) ||
+                   "http://localhost:#{ENV.fetch('TEST_CLUSTER_PORT', nil) || 9200}"
+end
+
+DEFAULT_CLIENT = OpenSearch::Client.new(host: OPENSEARCH_URL,
+                                        tracer: (tracer unless ENV['QUIET']))
+
+# The hosts to use for creating a opensearch client.
+hosts = ENV['TEST_OPENSEARCH_SERVER'] || ENV['OPENSEARCH_HOSTS']
+OPENSEARCH_HOSTS = if hosts
+                     hosts.split(',').map do |host|
+                       /(http\:\/\/)?(\S+)/.match(host)[2]
+                     end
+                   else
+                     ['localhost:9200']
+                   end.freeze
+
+# Are we testing on JRuby?
+# @return [ true, false ] Whether JRuby is being used.
+def jruby?
+  RUBY_PLATFORM =~ /\bjava\b/
+end
+
+# The names of the connected nodes.
+# @return [ Array<String> ] The node names.
+def node_names
+  node_stats = default_client.perform_request('GET', '_nodes/stats').body
+  $node_names ||= node_stats['nodes'].collect do |name, stats|
+    stats['name']
+  end
+end
+
+# The default client.
+# @return [ OpenSearch::Client ] The default client.
+def default_client
+  $client ||= OpenSearch::Client.new(hosts: OPENSEARCH_HOSTS)
+end
+
+class NotFound < StandardError; end
+
+module HelperModule
+  def self.included(context)
+    context.let(:client_double) do
+      Class.new { include OpenSearch::API }.new.tap do |client|
+        expect(client).to receive(:perform_request).with(*expected_args).and_return(response_double)
+      end
+    end
+
+    context.let(:client) do
+      Class.new { include OpenSearch::API }.new.tap do |client|
+        expect(client).to receive(:perform_request).with(*expected_args).and_return(response_double)
+      end
+    end
+
+    context.let(:response_double) do
+      double('response', status: 200, body: {}, headers: {})
+    end
+
+    context.let(:hosts) { OPENSEARCH_HOSTS }
+  end
+end
 
 RSpec.configure do |config|
-  config.formatter = :documentation
+  config.include(HelperModule)
+  config.formatter = 'documentation'
+  config.color = true
 end

data/test/dsl/integration/search_aggregation_children_test.rb

@@ -0,0 +1,91 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+#
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require_relative '../test_helper'
+
+module OpenSearch
+  module Test
+    class ChildrenAggregationIntegrationTest < ::OpenSearch::Test::IntegrationTestCase
+      include OpenSearch::DSL::Search
+
+      context 'A children aggregation' do
+        setup do
+          @client.indices.create index: 'articles-and-comments', body: {
+            mappings: {
+              properties: {
+                title: { type: 'text' },
+                category: { type: 'keyword' },
+                join_field: { type: 'join', relations: { article: 'comment' } },
+                author: { type: 'keyword' }
+              }
+            }
+          }
+
+          @client.index index: 'articles-and-comments', id: 1,
+                        body: { title: 'A', category: 'one', join_field: 'article' }
+          @client.index index: 'articles-and-comments', id: 2,
+                        body: { title: 'B', category: 'one', join_field: 'article' }
+          @client.index index: 'articles-and-comments', id: 3,
+                        body: { title: 'C', category: 'two', join_field: 'article' }
+
+          @client.index index: 'articles-and-comments', routing: '1',
+                        body: { author: 'John', join_field: { name: 'comment', parent: 1 } }
+          @client.index index: 'articles-and-comments', routing: '1',
+                        body: { author: 'Mary', join_field: { name: 'comment', parent: 1 } }
+          @client.index index: 'articles-and-comments', routing: '2',
+                        body: { author: 'John', join_field: { name: 'comment', parent: 2 } }
+          @client.index index: 'articles-and-comments', routing: '2',
+                        body: { author: 'Dave', join_field: { name: 'comment', parent: 2 } }
+          @client.index index: 'articles-and-comments', routing: '3',
+                        body: { author: 'Ruth', join_field: { name: 'comment', parent: 3 } }
+          @client.indices.refresh index: 'articles-and-comments'
+        end
+
+        should 'return the top commenters per article category' do
+          response = @client.search index: 'articles-and-comments', size: 0, body: search {
+            aggregation :top_categories do
+              terms field: 'category' do
+                aggregation :comments do
+                  children type: 'comment' do
+                    aggregation :top_authors do
+                      terms field: 'author'
+                    end
+                  end
+                end
+              end
+            end
+          }.to_hash
+
+          assert_equal 'one', response['aggregations']['top_categories']['buckets'][0]['key']
+          assert_equal 3,
+                       response['aggregations']['top_categories']['buckets'][0]['comments']['top_authors']['buckets'].size
+          assert_equal 'John',
+                       response['aggregations']['top_categories']['buckets'][0]['comments']['top_authors']['buckets'][0]['key']
+        end
+      end
+    end
+  end
+end

data/test/dsl/integration/search_aggregation_geo_test.rb

@@ -0,0 +1,112 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+#
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require_relative '../test_helper'
+
+module OpenSearch
+  module Test
+    class GeoAggregationIntegrationTest < ::OpenSearch::Test::IntegrationTestCase
+      include OpenSearch::DSL::Search
+
+      context 'A geo aggregation' do
+        setup do
+          @client.indices.create index: 'venues-test', body: {
+            mappings: {
+              properties: {
+                location: { type: 'geo_point' }
+              }
+            }
+          }
+          @client.index index: 'venues-test',
+                        body: { name: 'Space', location: '38.886214,1.403889' }
+          @client.index index: 'venues-test',
+                        body: { name: 'Pacha', location: '38.9184427,1.4433646' }
+          @client.index index: 'venues-test',
+                        body: { name: 'Amnesia', location: '38.948045,1.408341' }
+          @client.index index: 'venues-test',
+                        body: { name: 'Privilege', location: '38.958082,1.408288' }
+          @client.index index: 'venues-test',
+                        body: { name: 'Es Paradis', location: '38.979071,1.307394' }
+          @client.indices.refresh index: 'venues-test'
+        end
+
+        should 'return the geo distances from a location' do
+          response = @client.search index: 'venues-test', size: 0, body: search {
+            aggregation :venue_distances do
+              geo_distance do
+                field  :location
+                origin '38.9126352,1.4350621'
+                unit   'km'
+                ranges [{ to: 1 }, { from: 1, to: 5 }, { from: 5, to: 10 }, { from: 10 }]
+
+                aggregation :top_venues do
+                  top_hits _source: { include: 'name' }
+                end
+              end
+            end
+          }.to_hash
+
+          result = response['aggregations']['venue_distances']
+
+          assert_equal 4,       result['buckets'].size
+          assert_equal 1,       result['buckets'][0]['doc_count']
+          assert_equal 'Pacha', result['buckets'][0]['top_venues']['hits']['hits'][0]['_source']['name']
+
+          assert_equal 2,       result['buckets'][1]['top_venues']['hits']['total']['value']
+        end
+
+        should 'return the geohash grid distribution' do
+          #
+          # See the geohash plot eg. at http://openlocation.org/geohash/geohash-js/
+          # See the locations visually eg. at http://geohash.org/sncj8h17r2
+          #
+          response = @client.search index: 'venues-test', size: 0, body: search {
+            aggregation :venue_distributions do
+              geohash_grid do
+                field     :location
+                precision 5
+
+                aggregation :top_venues do
+                  top_hits _source: { include: 'name' }
+                end
+              end
+            end
+          }.to_hash
+
+          result = response['aggregations']['venue_distributions']
+
+          assert_equal 4,       result['buckets'].size
+          assert_equal 'sncj8', result['buckets'][0]['key']
+          assert_equal 2,       result['buckets'][0]['doc_count']
+
+          assert_same_elements %w[Privilege Amnesia], result['buckets'][0]['top_venues']['hits']['hits'].map { |h|
+                                                        h['_source']['name']
+                                                      }
+        end
+      end
+    end
+  end
+end

data/test/dsl/integration/search_aggregation_nested_test.rb

@@ -0,0 +1,129 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+#
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require_relative '../test_helper'
+
+module OpenSearch
+  module Test
+    class NestedAggregationIntegrationTest < ::OpenSearch::Test::IntegrationTestCase
+      include OpenSearch::DSL::Search
+
+      context 'A nested aggregation' do
+        setup do
+          @client.indices.create index: 'products-test', body: {
+            mappings: {
+              properties: {
+                title: { type: 'text' },
+                category: { type: 'keyword' },
+                offers: {
+                  type: 'nested',
+                  properties: {
+                    name: { type: 'text' },
+                    price: { type: 'double' }
+                  }
+                }
+              }
+            }
+          }
+
+          @client.index index: 'products-test',
+                        body: { title: 'A',
+                                category: 'audio',
+                                offers: [{ name: 'A1', price: 100 }, { name: 'A2', price: 120 }] }
+          @client.index index: 'products-test',
+                        body: { title: 'B',
+                                category: 'audio',
+                                offers: [{ name: 'B1', price: 200 }, { name: 'B2', price: 180 }] }
+          @client.index index: 'products-test',
+                        body: { title: 'C',
+                                category: 'video',
+                                offers: [{ name: 'C1', price: 300 }, { name: 'C2', price: 350 }] }
+          @client.indices.refresh index: 'products-test'
+        end
+
+        should 'return the minimal price from offers' do
+          response = @client.search index: 'products-test', body: search {
+            query { match title: 'A' }
+
+            aggregation :offers do
+              nested do
+                path 'offers'
+                aggregation :min_price do
+                  min field: 'offers.price'
+                end
+              end
+            end
+          }.to_hash
+
+          assert_equal 100, response['aggregations']['offers']['min_price']['value'].to_i
+        end
+
+        should 'return the top categories for offer price range' do
+          response = @client.search index: 'products-test', body: search {
+            query do
+              bool do
+                must do
+                  nested do
+                    path 'offers'
+                    query do
+                      bool do
+                        filter do
+                          range 'offers.price' do
+                            gte 100
+                            lte 300
+                          end
+                        end
+                      end
+                    end
+                  end
+                end
+              end
+            end
+
+            aggregation :offers do
+              nested do
+                path 'offers'
+                aggregation :top_categories do
+                  reverse_nested do
+                    aggregation :top_category_per_offer do
+                      terms field: 'category'
+                    end
+                  end
+                end
+              end
+            end
+          }.to_hash
+
+          assert_equal 2, response['aggregations']['offers']['top_categories']['top_category_per_offer']['buckets'].size
+          assert_equal 'audio',
+                       response['aggregations']['offers']['top_categories']['top_category_per_offer']['buckets'][0]['key']
+          assert_equal 'video',
+                       response['aggregations']['offers']['top_categories']['top_category_per_offer']['buckets'][1]['key']
+        end
+      end
+    end
+  end
+end