elasticsearch-api 7.11.2 → 8.6.0

data/api-spec-testing/test_file.rb

@@ -0,0 +1,179 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require_relative 'test_file/action'
+require_relative 'test_file/test'
+require_relative 'test_file/task_group'
+require_relative 'logging'
+
+module Elasticsearch
+  module RestAPIYAMLTests
+    # Custom exception to raise when a test file needs to be skipped. This is
+    # captured as soon as possible so the test runners can move on to the next test.
+    class SkipTestsException < StandardError
+    end
+
+    # Class representing a single test file, containing a setup, teardown, and multiple tests.
+    #
+    # @since 6.2.0
+    class TestFile
+      include Elasticsearch::RestAPIYAMLTests::Logging
+      attr_reader :features_to_skip, :name, :client
+
+      # Initialize a single test file.
+      #
+      # @example Create a test file object.
+      #   TestFile.new(file_name)
+      #
+      # @param [ String ] file_name The name of the test file.
+      # @param [ Client] An instance of the client
+      # @param [ Array<Symbol> ] skip_features The names of features to skip.
+      #
+      # @since 6.1.0
+      def initialize(file_name, client, features_to_skip = [])
+        @name = file_name
+        @client = client
+        begin
+          documents = YAML.load_stream(File.new(file_name))
+        rescue Psych::SyntaxError => e
+          raise e unless e.message.include?('found unexpected \':\'')
+
+          message = "Exception found when parsing YAML in #{file_name}: #{e.message}"
+          logger.error message
+          raise SkipTestsException, message
+        rescue StandardError => e
+          logger.error e
+          logger.error "Filename : #{@name}"
+        end
+        @test_definitions = documents.reject { |doc| doc['setup'] || doc['teardown'] }
+        @setup = documents.find { |doc| doc['setup'] }
+        skip_entire_test_file?(file_name) if @setup
+        @teardown = documents.find { |doc| doc['teardown'] }
+        @features_to_skip = REST_API_YAML_SKIP_FEATURES + features_to_skip
+      end
+
+      def skip_entire_test_file?(file_name)
+        @skip = @setup['setup']&.select { |a| a['skip'] }
+        return false if @skip.empty?
+
+        raise SkipTestsException, "Skipping #{file_name} due to 'skip all'." if skip_version?(@client, @skip.first['skip'])
+      end
+
+      def skip_version?(client, skip_definition)
+        return true if skip_definition['version'] == 'all'
+
+        range_partition = /\s*-\s*/
+        return unless (versions = skip_definition['version'])
+
+        low, high = __parse_versions(versions.partition(range_partition))
+        range = low..high
+        begin
+          server_version = client.info['version']['number']
+        rescue
+          warn('Could not determine Elasticsearch version when checking if test should be skipped.')
+        end
+        range.cover?(Gem::Version.new(server_version))
+      end
+
+      def __parse_versions(versions)
+        versions = versions.split('-') if versions.is_a? String
+
+        low = (['', nil].include? versions[0]) ? '0' : versions[0]
+        high = (['', nil].include? versions[2]) ? '9999' : versions[2]
+        [Gem::Version.new(low), Gem::Version.new(high)]
+      end
+
+      # Get a list of tests in the test file.
+      #
+      # @example Get the list of tests
+      #   test_file.tests
+      #
+      # @return [ Array<Test> ] A list of Test objects.
+      #
+      # @since 6.2.0
+      def tests
+        @test_definitions.collect do |test_definition|
+          Test.new(self, test_definition)
+        end
+      end
+
+      # Run the setup tasks defined for a single test file.
+      #
+      # @example Run the setup tasks.
+      #   test_file.setup
+      #
+      # @param [ Elasticsearch::Client ] client The client to use to perform the setup tasks.
+      #
+      # @return [ self ]
+      #
+      # @since 6.2.0
+      def setup
+        return unless @setup
+
+        actions = @setup['setup'].select { |action| action['do'] }.map { |action| Action.new(action['do']) }
+        run_actions_and_retry(actions)
+        self
+      end
+
+      # Run the teardown tasks defined for a single test file.
+      #
+      # @example Run the teardown tasks.
+      #   test_file.teardown
+      #
+      # @param [ Elasticsearch::Client ] client The client to use to perform the teardown tasks.
+      #
+      # @return [ self ]
+      #
+      # @since 6.2.0
+      def teardown
+        return unless @teardown
+
+        actions = @teardown['teardown'].select { |action| action['do'] }.map { |action| Action.new(action['do']) }
+        run_actions_and_retry(actions)
+        self
+      end
+
+      # Helper function to run actions. If the server returns an error, give it some time and retry
+      # for a few times.
+      def run_actions_and_retry(actions)
+        count = 0
+        loop do
+          actions.delete_if do |action|
+            begin
+              action.execute(client)
+              true
+            rescue Elastic::Transport::Transport::Errors::RequestTimeout,
+                   Elastic::Transport::Transport::Errors::ServiceUnavailable => e
+              # The action sometimes gets the cluster in a recovering state, so we
+              # retry a few times and then raise an exception if it's still
+              # happening
+              count += 1
+              sleep 10
+              logger.debug(
+                "The server responded with an #{e.class} error. Retrying action - (#{count})"
+              )
+              raise e if count > 11
+
+              false
+            end
+          end
+          break if actions.empty?
+        end
+      end
+    end
+  end
+end

data/api-spec-testing/wipe_cluster.rb

@@ -0,0 +1,424 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require_relative 'logging'
+require_relative 'custom_cleanup'
+include Elasticsearch::RestAPIYAMLTests::Logging
+
+module Elasticsearch
+  module RestAPIYAMLTests
+    module WipeCluster
+      PRESERVE_ILM_POLICY_IDS = [
+        'ilm-history-ilm-policy', 'slm-history-ilm-policy', 'watch-history-ilm-policy',
+        'ml-size-based-ilm-policy', 'logs', 'metrics', 'synthetics', '7-days-default',
+        '30-days-default', '90-days-default', '180-days-default', '365-days-default',
+        '.fleet-actions-results-ilm-policy', '.deprecation-indexing-ilm-policy',
+        'watch-history-ilm-policy-16', '.monitoring-8-ilm-policy'
+      ].freeze
+
+      PLATINUM_TEMPLATES = [
+        '.watches', 'logstash-index-template', '.logstash-management',
+        'security_audit_log', '.slm-history', '.async-search',
+        'saml-service-provider', 'ilm-history', 'logs', 'logs-settings',
+        'logs-mappings', 'metrics', 'metrics-settings', 'metrics-mappings',
+        'synthetics', 'synthetics-settings', 'synthetics-mappings',
+        '.snapshot-blob-cache', '.deprecation-indexing-template',
+        '.deprecation-indexing-mappings', '.deprecation-indexing-settings',
+        'security-index-template', 'data-streams-mappings'
+      ].freeze
+
+      # Wipe Cluster, based on PHP's implementation of ESRestTestCase.java:wipeCluster()
+      # https://github.com/elastic/elasticsearch-php/blob/7.10/tests/Elasticsearch/Tests/Utility.php#L97
+      def self.run(client)
+        ensure_no_initializing_shards(client)
+        wipe_cluster(client)
+        wait_for_cluster_tasks(client)
+        check_for_unexpectedly_recreated_objects(client)
+      end
+
+      def self.create_xpack_rest_user(client)
+        client.security.put_user(
+          username: 'x_pack_rest_user',
+          body: { password: 'x-pack-test-password', roles: ['superuser'] }
+        )
+      end
+
+      class << self
+        private
+
+        def wipe_cluster(client)
+          read_plugins(client)
+          if @has_rollups
+            wipe_rollup_jobs(client)
+            wait_for_pending_rollup_tasks(client)
+          end
+          delete_all_slm_policies(client)
+          wipe_searchable_snapshot_indices(client) if @has_xpack
+          wipe_snapshots(client)
+          wipe_datastreams(client)
+          wipe_all_indices(client)
+          if platinum?
+            wipe_templates_for_xpack(client)
+          else
+            wipe_all_templates(client)
+          end
+          wipe_cluster_settings(client)
+          if platinum?
+            clear_ml_jobs(client)
+            clear_datafeeds(client)
+            delete_data_frame_analytics(client)
+            clear_ml_filters(client)
+            delete_trained_models(client)
+          end
+          delete_all_ilm_policies(client) if @has_ilm
+          delete_all_follow_patterns(client) if @has_ccr
+          delete_all_node_shutdown_metadata(client)
+          # clear_tasks(client)
+          # clear_transforms(client)
+
+          # Custom implementations
+          CustomCleanup::wipe_calendars(client)
+        end
+
+        def ensure_no_initializing_shards(client)
+          client.cluster.health(wait_for_no_initializing_shards: true, timeout: '70s', level: 'shards')
+        end
+
+        def check_for_unexpectedly_recreated_objects(client)
+          unexpected_ilm_policies = client.index_lifecycle_management.get_lifecycle
+          unexpected_ilm_policies.reject! { |k, _| PRESERVE_ILM_POLICY_IDS.include? k }
+          unless unexpected_ilm_policies.empty?
+            logger.info(
+              "Expected no ILM policies after deletions, but found #{unexpected_ilm_policies.keys.join(',')}"
+            )
+          end
+          return unless platinum?
+
+          templates = client.indices.get_index_template
+          unexpected_templates = templates['index_templates'].reject do |t|
+            # reject platinum templates
+            PLATINUM_TEMPLATES.include? t['name']
+          end.map { |t| t['name'] } # only keep the names
+          legacy_templates = client.indices.get_template
+          unexpected_templates << legacy_templates.keys.reject { |t| PLATINUM_TEMPLATES.include? t }
+
+          unless unexpected_templates.empty?
+            logger.info(
+              "Expected no templates after deletions, but found #{unexpected_templates.join(',')}"
+            )
+          end
+        end
+
+        def platinum?
+          ENV['TEST_SUITE'] == 'platinum'
+        end
+
+        def read_plugins(client)
+          response = client.perform_request('GET', '_nodes/plugins').body
+
+          response['nodes'].each do |node|
+            node[1]['modules'].each do |mod|
+              @has_xpack = true if mod['name'].include?('x-pack')
+              @has_ilm = true if mod['name'].include?('x-pack-ilm')
+              @has_rollups = true if mod['name'].include?('x-pack-rollup')
+              @has_ccr = true if mod['name'].include?('x-pack-ccr')
+              @has_shutdown = true if mod['name'].include?('x-pack-shutdown')
+            end
+          end
+        end
+
+        def wipe_rollup_jobs(client)
+          client.rollup.get_jobs(id: '_all')['jobs'].each do |d|
+            client.rollup.stop_job(id: d['config']['id'], wait_for_completion: true, timeout: '10s', ignore: 404)
+            client.rollup.delete_job(id: d['config']['id'], ignore: 404)
+          end
+        end
+
+        def wait_for_pending_rollup_tasks(client)
+          filter = 'xpack/rollup/job'
+          loop do
+            results = client.cat.tasks(detailed: true).split("\n")
+            count = 0
+
+            time = Time.now.to_i
+            results.each do |task|
+              next if task.empty? || skippable_task?(task)
+
+              logger.debug("Pending task: #{task}")
+              count += 1 if task.include?(filter)
+            end
+            break unless count.positive? && Time.now.to_i < (time + 30)
+          end
+        end
+
+        def delete_all_slm_policies(client)
+          policies = client.snapshot_lifecycle_management.get_lifecycle
+
+          policies.each do |name, _|
+            client.snapshot_lifecycle_management.delete_lifecycle(policy_id: name)
+          end
+        end
+
+        def wipe_searchable_snapshot_indices(client)
+          indices = client.cluster.state(metric: 'metadata', filter_path: 'metadata.indices.*.settings.index.store.snapshot')
+          return unless indices.dig('metadata', 'indices')
+
+          indices['metadata']['indices'].each do |index|
+            index_name = if index.is_a?(Array)
+                           index[0]
+                         elsif index.is_a?(Hash)
+                           index.keys.first
+                         end
+            client.indices.delete(index: index_name, ignore: 404)
+          end
+        end
+
+        def wipe_snapshots(client)
+          # Repeatedly delete the snapshots until there aren't any
+          loop do
+            repositories = client.snapshot.get_repository(repository: '_all')
+            break if repositories.empty?
+
+            repositories.each_key do |repository|
+              if ['fs', 'source'].include? repositories[repository]['type']
+                response = client.snapshot.get(repository: repository, snapshot: '_all', ignore_unavailable: true)
+                response['snapshots'].each do |snapshot|
+                  if snapshot['state'] != 'SUCCESS'
+                    logger.debug("Found snapshot that did not succeed #{snapshot}")
+                  end
+                  client.snapshot.delete(repository: repository, snapshot: snapshot['snapshot'], ignore: 404)
+                end
+              end
+              client.snapshot.delete_repository(repository: repository, ignore: 404)
+            end
+          end
+        end
+
+        def wipe_datastreams(client)
+          begin
+            client.indices.delete_data_stream(name: '*', expand_wildcards: 'all')
+          rescue StandardError => e
+            logger.error "Caught exception attempting to delete data streams: #{e}"
+            client.indices.delete_data_stream(name: '*')
+          end
+        end
+
+        def wipe_all_indices(client)
+          client.indices.delete(index: '*,-.ds-ilm-history-*', expand_wildcards: 'open,closed,hidden', ignore: 404)
+        end
+
+        def wipe_templates_for_xpack(client)
+          templates = client.indices.get_index_template
+
+          templates['index_templates'].each do |template|
+            next if platinum_template? template['name']
+
+            begin
+              client.indices.delete_index_template(name: template['name'], ignore: 404)
+            end
+          end
+          # Delete component template
+          result = client.cluster.get_component_template
+
+          result['component_templates'].each do |template|
+            next if platinum_template? template['name']
+
+            client.cluster.delete_component_template(name: template['name'], ignore: 404)
+          end
+
+          # Always check for legacy templates
+          templates = client.indices.get_template
+          templates.each do |name, _|
+            next if platinum_template? name
+
+            begin
+              client.indices.delete_template(name: name)
+            rescue StandardError => e
+              logger.info("Unable to remove index template #{name}")
+            end
+          end
+        end
+
+        def wipe_all_templates(client)
+          client.indices.delete_template(name: '*')
+          begin
+            client.indices.delete_index_template(name: '*')
+            client.cluster.delete_component_template(name: '*')
+          rescue StandardError => e
+            logger.info('Using a version of ES that doesn\'t support index templates v2 yet, so it\'s safe to ignore')
+          end
+        end
+
+        def platinum_template?(template)
+          platinum_prefixes = ['.monitoring', '.watch', '.triggered-watches', '.data-frame', '.ml-', '.transform', 'deprecation', 'data-streams-mappings'].freeze
+          platinum_prefixes.map { |a| return true if a.include? template }
+
+          PLATINUM_TEMPLATES.include? template
+        end
+
+        def wait_for_cluster_tasks(client)
+          time = Time.now.to_i
+          count = 0
+
+          loop do
+            results = client.cluster.pending_tasks
+            results['tasks'].each do |task|
+
+              next if task.empty? || skippable_task?(task)
+
+              logger.debug "Pending cluster task: #{task}"
+              count += 1
+            end
+            break unless count.positive? && Time.now.to_i < (time + 30)
+          end
+        end
+
+        def skippable_task?(task)
+          names = ['health-node', 'cluster:monitor/tasks/lists', 'create-index-template-v2',
+                   'remove-component-template']
+          if task.is_a?(String)
+            names.select { |n| task.match? n }.any?
+          elsif task.is_a?(Hash)
+            names.select { |n| task['source'].match? n }.any?
+          end
+        end
+
+        def delete_all_ilm_policies(client)
+          policies = client.ilm.get_lifecycle
+          policies.each do |policy|
+            client.ilm.delete_lifecycle(policy: policy[0]) unless PRESERVE_ILM_POLICY_IDS.include? policy[0]
+          end
+        end
+
+        def wipe_cluster_settings(client)
+          settings = client.cluster.get_settings
+          new_settings = []
+          settings.each do |name, value|
+            next unless !value.empty? && value.is_a?(Array)
+
+            new_settings[name] = [] if new_settings[name].empty?
+            value.each do |key, _v|
+              new_settings[name]["#{key}.*"] = nil
+            end
+          end
+          client.cluster.put_settings(body: new_settings) unless new_settings.empty?
+        end
+
+        def delete_all_follow_patterns(client)
+          patterns = client.cross_cluster_replication.get_auto_follow_pattern
+
+          patterns['patterns'].each do |pattern|
+            client.cross_cluster_replication.delete_auto_follow_pattern(name: pattern)
+          end
+        end
+
+        def clear_roles(client)
+          client.security.get_role.each do |role, _|
+            begin; client.security.delete_role(name: role); rescue; end
+          end
+        end
+
+        def clear_users(client)
+          client.security.get_user.each do |user, _|
+            begin; client.security.delete_user(username: user); rescue; end
+          end
+        end
+
+        def clear_privileges(client)
+          client.security.get_privileges.each do |privilege, _|
+            begin; client.security.delete_privileges(name: privilege); rescue; end
+          end
+        end
+
+        def clear_ml_jobs(client)
+          client.ml.close_job(job_id: '_all', force: true)
+          client.ml.get_jobs['jobs'].each do |d|
+            client.ml.delete_job(job_id: d['job_id'])
+          end
+        end
+
+        def clear_datafeeds(client)
+          client.ml.stop_datafeed(datafeed_id: '_all', force: true)
+          client.ml.get_datafeeds['datafeeds'].each do |d|
+            client.ml.delete_datafeed(datafeed_id: d['datafeed_id'])
+          end
+        end
+
+        def clear_tasks(client)
+          tasks = client.tasks.get['nodes'].values.first['tasks'].values.select do |d|
+            d['cancellable']
+          end.map do |d|
+            "#{d['node']}:#{d['id']}"
+          end
+          tasks.each { |t| client.tasks.cancel task_id: t }
+        end
+
+        def clear_machine_learning_indices(client)
+          client.indices.delete(index: '.ml-*', ignore: 404)
+        end
+
+        def clear_index_templates(client)
+          client.indices.delete_template(name: '*')
+          templates = client.indices.get_index_template
+          templates['index_templates'].each do |template|
+            client.indices.delete_index_template(name: template['name'])
+          end
+        end
+
+        def clear_transforms(client)
+          client.transform.get_transform(transform_id: '*')['transforms'].each do |transform|
+            client.transform.delete_transform(transform_id: transform[:id])
+          end
+        end
+
+        def clear_ml_filters(client)
+          filters = client.ml.get_filters['filters']
+          filters.each do |filter|
+            client.ml.delete_filter(filter_id: filter['filter_id'])
+          end
+        end
+
+        def delete_all_node_shutdown_metadata(client)
+          nodes = client.shutdown.get_node
+          return unless nodes['nodes']
+
+          nodes['nodes'].each do |node|
+            client.shutdown.delete_node(node['node_id'])
+          end
+        end
+
+        def delete_data_frame_analytics(client)
+          dfs = client.ml.get_data_frame_analytics
+          return unless dfs['data_frame_analytics']
+
+          dfs['data_frame_analytics'].each do |df|
+            client.ml.delete_data_frame_analytics(id: df['id'], force: true)
+          end
+        end
+
+        def delete_trained_models(client)
+          models = client.ml.get_trained_models
+          return unless models['trained_model_configs']
+
+          models['trained_model_configs'].each do |model|
+            client.ml.delete_trained_model(model_id: model['model_id'], force: true, ignore: 400)
+          end
+        end
+      end
+    end
+  end
+end

data/elasticsearch-api.gemspec

@@ -26,12 +26,12 @@ Gem::Specification.new do |s|
   s.authors       = ['Karel Minarik']
   s.email         = ['karel.minarik@elasticsearch.org']
   s.summary       = 'Ruby API for Elasticsearch.'
-  s.homepage      = 'https://www.elastic.co/guide/en/elasticsearch/client/ruby-api/7.x/index.html'
+  s.homepage      = 'https://www.elastic.co/guide/en/elasticsearch/client/ruby-api/current/index.html'
   s.license       = 'Apache-2.0'
   s.metadata = {
-    'homepage_uri' => 'https://www.elastic.co/guide/en/elasticsearch/client/ruby-api/7.x/index.html',
-    'changelog_uri' => 'https://github.com/elastic/elasticsearch-ruby/blob/7.x/CHANGELOG.md',
-    'source_code_uri' => 'https://github.com/elastic/elasticsearch-ruby/tree/7.x/elasticsearch-api',
+    'homepage_uri' => 'https://www.elastic.co/guide/en/elasticsearch/client/ruby-api/current/index.html',
+    'changelog_uri' => 'https://github.com/elastic/elasticsearch-ruby/blob/main/CHANGELOG.md',
+    'source_code_uri' => 'https://github.com/elastic/elasticsearch-ruby/tree/main/elasticsearch-api',
     'bug_tracker_uri' => 'https://github.com/elastic/elasticsearch-ruby/issues'
   }
   s.files         = `git ls-files`.split($/)
@@ -39,38 +39,43 @@ Gem::Specification.new do |s|
   s.test_files    = s.files.grep(%r{^(test|spec|features)/})
   s.require_paths = ['lib']
 
-  s.extra_rdoc_files  = [ 'README.md', 'LICENSE' ]
-  s.rdoc_options      = [ '--charset=UTF-8' ]
+  s.extra_rdoc_files  = ['README.md', 'LICENSE.txt']
+  s.rdoc_options      = ['--charset=UTF-8']
 
-  s.required_ruby_version = '>= 2.4'
+  s.required_ruby_version = '>= 2.5'
 
   s.add_dependency 'multi_json'
 
   s.add_development_dependency 'ansi'
   s.add_development_dependency 'bundler'
   s.add_development_dependency 'elasticsearch'
-  s.add_development_dependency 'elasticsearch-extensions'
-  s.add_development_dependency 'elasticsearch-transport'
   s.add_development_dependency 'minitest'
   s.add_development_dependency 'minitest-reporters'
   s.add_development_dependency 'mocha'
   s.add_development_dependency 'pry'
-  s.add_development_dependency 'rake', '~> 13'
+  s.add_development_dependency 'rake'
   s.add_development_dependency 'shoulda-context'
   s.add_development_dependency 'yard'
 
   # Gems for testing integrations
   s.add_development_dependency 'jsonify'
   s.add_development_dependency 'hashie'
+  # Temporary support for Ruby 2.6, since it's EOL March 2022:
+  if RUBY_VERSION < '2.7.0'
+    s.add_development_dependency 'jbuilder', '< 7.0.0'
+  else
+    s.add_development_dependency 'activesupport'
+    s.add_development_dependency 'jbuilder'
+  end
 
   s.add_development_dependency 'cane'
   s.add_development_dependency 'escape_utils' unless defined? JRUBY_VERSION
-  s.add_development_dependency 'jbuilder'
+
   s.add_development_dependency 'require-prof' unless defined?(JRUBY_VERSION) || defined?(Rubinius)
-  s.add_development_dependency 'simplecov', '~> 0.17', '< 0.18'
-  s.add_development_dependency 'simplecov-rcov'
-  s.add_development_dependency 'test-unit', '~> 2'
   s.add_development_dependency 'ruby-prof' unless defined?(JRUBY_VERSION) || defined?(Rubinius)
+  s.add_development_dependency 'simplecov'
+
+  s.add_development_dependency 'test-unit', '~> 2'
 
   s.description = <<-DESC.gsub(/^    /, '')
     Ruby API for Elasticsearch. See the `elasticsearch` gem for full integration.