elasticsearch-test-runner 0.1.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 259a2010d6c8c55a9f565bee24e7c50b3b2fea08714766797985b8eaa5db03fa
-  data.tar.gz: 9d54c97d90b659a510589a3fd8513e9647d92492971f4d33db055f2b2fc95bde
+  metadata.gz: 736ee5a6506aad22d554ba00880706b6641192297e69c42a815050e4e6d784fd
+  data.tar.gz: 2faea7eacfc8c421675cbae4783fcce40866daf380a8d5fd417b297cd9de7e51
 SHA512:
-  metadata.gz: 1b030edc69c376dd401c4f16f796fb35bd310754fa42bb509a0b8298d69de6d18a24bf1404e201b733c277f565daab98a4325758b95972fa453b60d8be03004e
-  data.tar.gz: be60c13c2a9f49b2de32eb4cb1784bf2f6de6d14a049585ed4d8736fb4469ed81f2485b48744419789567ad5b9782303b625923318cc22a8542f0054c51972ba
+  metadata.gz: '0529c7247fe886df1ac6f99f95b45f2fd576dce42bcc80e01d032d57e3adda91f0556da62000201fa447fe7895b7799844c4479077f7136df382bcabb69f5ab8'
+  data.tar.gz: 90f5be281f003fb63ad11bb1180d47d5b3d50ab3db4914ff7652e2eb89cbfc828a78f2118ebf665e60e688f0e065011154b3bf7c4703b562d9f7a3f3256cc3e1

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 ## [Unreleased]
 
+## [0.3.0] - 2024-06-27
+
+- Fixes exit code.
+- Refactors error handling for more detailed information.
+
+## [0.2.0] - 2024-06-25
+
+- Renames gemspec from elasticsearch-tests to elasticsearch-test-runner.
+- Adds ability to run individual test files or directories.
+
 ## [0.1.1] - 2024-06-20
 
 - Require 'fileutils' in Downloader.

data/README.md

@@ -35,9 +35,30 @@ You can optionally pass in an object that implements Ruby's Logger to the `TestR
 logger = Logger.new($stdout)
 logger.level = Logger::WARN unless ENV['DEBUG']
 
-Elasticsearch::Tests::TestRunner.new(client, tests_path, logger).run
+runner = Elasticsearch::Tests::TestRunner.new(client, tests_path, logger)
+runner.run
 ```
 
+When you run the tests, you can pass in the name of a particular test or a whole test folder, to run only those tests. Tests in the clients project are located [in the `tests` directory](https://github.com/elastic/elasticsearch-clients-tests/tree/main/tests) either as single yaml files or inside a specific directory, referring to a specific namespace. For example [`tests/get.yml`](https://github.com/elastic/elasticsearch-clients-tests/blob/main/tests/get.yml) and [`tests/bulk/10_basic.yml`](https://github.com/elastic/elasticsearch-clients-tests/blob/main/tests/bulk/10_basic.yml). If you want to run the `get.yml` test, you can pass in the file name to `run`:
+
+```ruby
+runner.run('get.yml')
+```
+
+If you want to run the basic bulk tests, you can run:
+
+```ruby
+runner.run('bulk/10_basic.yml')
+```
+
+If you want to run all the tests in a directory, you can pass in the directory:
+
+```ruby
+runner.run('indices')
+```
+
+This will run all the tests in [`tests/indices`](https://github.com/elastic/elasticsearch-clients-tests/tree/main/tests/indices) such as `alias.yml`, `analyze.yml`, and so on.
+
 You can **download the YAML test files** from [the clients tests project](https://github.com/elastic/elasticsearch-clients-tests) with the following code:
 
 ```ruby

data/lib/elasticsearch/tests/errors.rb

@@ -19,5 +19,16 @@ module Elasticsearch
   module Tests
     class TestFailure < StandardError; end
     class TestError < StandardError; end
+
+    # Class to track exactly which action errored
+    class ActionError < StandardError
+      attr_reader :test_file, :action
+
+      def initialize(message, test_file, action)
+        super(message)
+        @test_file = test_file
+        @action = action
+      end
+    end
   end
 end

data/lib/elasticsearch/tests/printer.rb

@@ -40,7 +40,7 @@ module Elasticsearch
         else
           pp response
         end
-        raise Elasticsearch::Tests::TestFailure
+        raise Elasticsearch::Tests::ActionError.new('ERROR', @file, action)
       end
 
       def print_match_failure(action)
@@ -65,7 +65,9 @@ module Elasticsearch
       def self.display_errors(errors)
         puts "+++ ❌ Errors/Failures: #{errors.count}"
         errors.map do |error|
-          puts "* Test: #{error[:file]}\n #{error[:error].message}"
+          puts "🧪 Test: #{error[:file]}"
+          puts "▶ Action: #{error[:action]['do']}" if error[:action]
+          puts "🔬 #{error[:error].message}"
           pp error[:error].backtrace.join("$/\n") if ENV['DEBUG']
           puts
         end

data/lib/elasticsearch/tests/test.rb

@@ -16,6 +16,7 @@
 # under the License.
 
 require_relative 'code_runner'
+require_relative 'errors'
 
 module Elasticsearch
   module Tests
@@ -70,6 +71,8 @@ module Elasticsearch
         when 'gt', 'gte', 'lt', 'lte'
           compare(action)
         end
+      rescue StandardError => e
+        raise ActionError.new(e.message, @file, action)
       end
 
       def run_setup

data/lib/elasticsearch/tests/test_runner.rb

@@ -29,7 +29,7 @@ module Elasticsearch
 
       def initialize(client, path = nil, logger = nil)
         @client = client
-        @serverless = (defined?(::ElasticsearchServerless) && client.is_a?(::ElasticsearchServerless::Client))
+        @serverless = defined?(::ElasticsearchServerless) && client.is_a?(::ElasticsearchServerless::Client)
         @path = path || File.expand_path('./tmp', __dir__)
         @logger = logger || LOGGER
       end
@@ -41,6 +41,11 @@ module Elasticsearch
         run_the_tests
         Elasticsearch::Tests::Printer::display_errors(@errors) unless @errors.empty?
         Elasticsearch::Tests::Printer::display_summary(@tests_count, @errors.count, @start_time)
+        if @errors.empty?
+          exit 0
+        else
+          exit 1
+        end
       end
 
       def run_the_tests
@@ -54,6 +59,9 @@ module Elasticsearch
           @errors << { error: e, file: test_file }
           @logger.warn("YAML error in #{test_file}")
           @logger.warn e
+        rescue ActionError => e
+          @errors << { error: e, file: test_file, action: e.action }
+          @logger.debug e
         rescue StandardError => e
           @errors << { error: e, file: test_file }
           @logger.debug e
@@ -74,13 +82,14 @@ module Elasticsearch
       end
 
       def select_test_files(test_files)
-        if test_files.empty?
-          Dir.glob("#{@path}/**/*.yml")
-        else
-          test_files.map do |test_file|
-            "#{@path}/tests/#{test_file}"
-          end
-        end
+        tests_path = if test_files.empty?
+                       "#{@path}/**/*.yml"
+                     elsif test_files.include?('yml')
+                       "#{@path}/#{test_files}"
+                     else
+                       "#{@path}/#{test_files}/*.yml"
+                     end
+        Dir.glob(tests_path)
       end
 
       def extract_requires!(yaml)

data/lib/elasticsearch/tests/version.rb

@@ -19,6 +19,6 @@
 
 module Elasticsearch
   module Tests
-    VERSION = "0.1.1"
+    VERSION = '0.3.0'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: elasticsearch-test-runner
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Elastic Client Library Maintainers
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-06-20 00:00:00.000000000 Z
+date: 2024-06-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: elasticsearch
@@ -53,7 +53,6 @@ files:
 - NOTICE
 - README.md
 - Rakefile
-- elasticsearch-tests.gemspec
 - lib/elasticsearch/tasks/download_test_suite.rake
 - lib/elasticsearch/tests.rb
 - lib/elasticsearch/tests/code_runner.rb

data/elasticsearch-tests.gemspec

@@ -1,51 +0,0 @@
-# Licensed to Elasticsearch B.V. under one or more contributor
-# license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright
-# ownership. Elasticsearch B.V. licenses this file to you under
-# the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# frozen_string_literal: true
-
-require_relative 'lib/elasticsearch/tests/version'
-
-Gem::Specification.new do |spec|
-  spec.name = 'elasticsearch-test-runner'
-  spec.email = ['client-libs@elastic.co']
-  spec.version = Elasticsearch::Tests::VERSION
-  spec.authors = ['Elastic Client Library Maintainers']
-  spec.licenses = ['Apache-2.0']
-  spec.summary = 'Tool to test Elasticsearch clients against the YAML clients test suite.'
-  spec.homepage = 'https://www.elastic.co/guide/en/elasticsearch/client/ruby-api/current/index.html'
-  spec.description = 'A test runner for the Elasticsearch clients YAML test suite, used in the elasticsearch and elasticsearch-serverless gems.'
-  spec.metadata['homepage_uri'] = spec.homepage
-  spec.metadata['changelog_uri'] = 'https://github.com/elastic/es-test-runner-ruby/blob/main/CHANGELOG.md'
-  spec.metadata['source_code_uri'] = 'https://github.com/elastic/es-test-runner-ruby/tree/main'
-
-  spec.required_ruby_version = '>= 3.0'
-
-  # Specify which files should be added to the gem when it is released.
-  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
-  spec.files = Dir.chdir(__dir__) do
-    `git ls-files -z`.split("\x0").reject do |f|
-      (File.expand_path(f) == __FILE__) ||
-        f.start_with?(*%w[bin/ test/ spec/ features/ .git appveyor Gemfile])
-    end
-  end
-  spec.bindir = 'exe'
-  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
-  spec.require_paths = ['lib']
-
-  spec.add_development_dependency 'elasticsearch'
-  spec.add_development_dependency 'elasticsearch-serverless'
-end