elastic-enterprise-search 8.6.0 → 8.7.0

data/.ci/docker-compose.yml

@@ -1,166 +0,0 @@
-
-version: "2.2"
-
-services:
-  setup:
-    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
-    volumes:
-      - certs:/usr/share/elasticsearch/config/certs
-    user: "0"
-    command: >
-      bash -c '
-        if [ x${ELASTIC_PASSWORD} == x ]; then
-          echo "Set the ELASTIC_PASSWORD environment variable in the .env file";
-          exit 1;
-        elif [ x${KIBANA_PASSWORD} == x ]; then
-          echo "Set the KIBANA_PASSWORD environment variable in the .env file";
-          exit 1;
-        fi;
-        if [ ! -f certs/ca.zip ]; then
-          echo "Creating CA";
-          bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip;
-          unzip config/certs/ca.zip -d config/certs;
-        fi;
-        if [ ! -f certs/certs.zip ]; then
-          echo "Creating certs";
-          echo -ne \
-          "instances:\n"\
-          "  - name: es01\n"\
-          "    dns:\n"\
-          "      - es01\n"\
-          "      - localhost\n"\
-          "    ip:\n"\
-          "      - 127.0.0.1\n"\
-          > config/certs/instances.yml;
-          bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key;
-          unzip config/certs/certs.zip -d config/certs;
-        fi;
-        echo "Setting file permissions"
-        chown -R root:root config/certs;
-        find . -type d -exec chmod 750 \{\} \;;
-        find . -type f -exec chmod 640 \{\} \;;
-        echo "Waiting for Elasticsearch availability";
-        until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done;
-        echo "Setting kibana_system password";
-        until curl -s -X POST --cacert config/certs/ca/ca.crt -u elastic:${ELASTIC_PASSWORD} -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done;
-        echo "All done!";
-      '
-    healthcheck:
-      test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"]
-      interval: 1s
-      timeout: 5s
-      retries: 120
-
-  es01:
-    depends_on:
-      setup:
-        condition: service_healthy
-    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
-    volumes:
-      - certs:/usr/share/elasticsearch/config/certs
-      - esdata01:/usr/share/elasticsearch/data
-    ports:
-      - ${ES_PORT}:9200
-    environment:
-      - node.name=es01
-      - cluster.name=${CLUSTER_NAME}
-      - cluster.initial_master_nodes=es01
-      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
-      - bootstrap.memory_lock=true
-      - xpack.security.enabled=true
-      - xpack.security.http.ssl.enabled=true
-      - xpack.security.http.ssl.key=certs/es01/es01.key
-      - xpack.security.http.ssl.certificate=certs/es01/es01.crt
-      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
-      - xpack.security.http.ssl.verification_mode=certificate
-      - xpack.security.transport.ssl.enabled=true
-      - xpack.security.transport.ssl.key=certs/es01/es01.key
-      - xpack.security.transport.ssl.certificate=certs/es01/es01.crt
-      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
-      - xpack.security.transport.ssl.verification_mode=certificate
-      - xpack.license.self_generated.type=trial
-    mem_limit: ${MEM_LIMIT}
-    ulimits:
-      memlock:
-        soft: -1
-        hard: -1
-    healthcheck:
-      test:
-        [
-            "CMD-SHELL",
-            "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
-        ]
-      interval: 10s
-      timeout: 10s
-      retries: 120
-
-  kibana:
-    depends_on:
-      es01:
-        condition: service_healthy
-    image: docker.elastic.co/kibana/kibana:${STACK_VERSION}
-    volumes:
-      - certs:/usr/share/kibana/config/certs
-      - kibanadata:/usr/share/kibana/data
-    ports:
-      - ${KIBANA_PORT}:5601
-    environment:
-      - SERVERNAME=kibana
-      - ELASTICSEARCH_HOSTS=https://es01:9200
-      - ELASTICSEARCH_USERNAME=kibana_system
-      - ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD}
-      - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt
-      - ENTERPRISESEARCH_HOST=http://enterprisesearch:${ENTERPRISE_SEARCH_PORT}
-    mem_limit: ${MEM_LIMIT}
-    healthcheck:
-      test:
-        [
-            "CMD-SHELL",
-            "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'",
-        ]
-      interval: 10s
-      timeout: 10s
-      retries: 120
-
-  enterprisesearch:
-    depends_on:
-      es01:
-        condition: service_healthy
-      kibana:
-        condition: service_healthy
-    image: docker.elastic.co/enterprise-search/enterprise-search:${STACK_VERSION}
-    volumes:
-      - certs:/usr/share/enterprise-search/config/certs
-      - enterprisesearchdata:/usr/share/enterprise-search/config
-    ports:
-      - ${ENTERPRISE_SEARCH_PORT}:3002
-    environment:
-      - SERVERNAME=enterprisesearch
-      - secret_management.encryption_keys=[${ENCRYPTION_KEYS}]
-      - allow_es_settings_modification=true
-      - elasticsearch.host=https://es01:9200
-      - elasticsearch.username=elastic
-      - elasticsearch.password=${ELASTIC_PASSWORD}
-      - elasticsearch.ssl.enabled=true
-      - elasticsearch.ssl.certificate_authority=/usr/share/enterprise-search/config/certs/ca/ca.crt
-      - kibana.external_url=http://kibana:5601
-    mem_limit: ${MEM_LIMIT}
-    healthcheck:
-      test:
-        [
-            "CMD-SHELL",
-            "curl -s -I http://localhost:3002 | grep -q 'HTTP/1.1 302 Found'",
-        ]
-      interval: 10s
-      timeout: 10s
-      retries: 120
-
-volumes:
-  certs:
-    driver: local
-  enterprisesearchdata:
-    driver: local
-  esdata01:
-    driver: local
-  kibanadata:
-    driver: local

data/.ci/functions/cleanup.sh

@@ -1,67 +0,0 @@
-#!/usr/bin/env bash
-#
-# Shared cleanup routines between different steps
-#
-# Please source .ci/functions/imports.sh as a whole not just this file
-#
-# Version 1.0.0
-# - Initial version after refactor
-
-function cleanup_volume {
-  if [[ "$(docker volume ls -q -f name=$1)" ]]; then
-    echo -e "\033[34;1mINFO:\033[0m Removing volume $1\033[0m"
-    (docker volume rm "$1") || true
-  fi
-}
-function container_running {
-  if [[ "$(docker ps -q -f name=$1)" ]]; then
-    return 0;
-    else return 1;
-  fi
-}
-function cleanup_node {
-  if container_running "$1"; then
-    echo -e "\033[34;1mINFO:\033[0m Removing container $1\033[0m"
-    (docker container rm --force --volumes "$1") || true
-  fi
-  if [[ -n "$1" ]]; then
-    echo -e "\033[34;1mINFO:\033[0m Removing volume $1-${suffix}-data\033[0m"
-    cleanup_volume "$1-${suffix}-data"
-  fi
-}
-function cleanup_network {
-  if [[ "$(docker network ls -q -f name=$1)" ]]; then
-    echo -e "\033[34;1mINFO:\033[0m Removing network $1\033[0m"
-    (docker network rm "$1") || true
-  fi
-}
-
-function cleanup_trap {
-  status=$?
-  set +x
-  if [[ "$DETACH" != "true" ]]; then
-    echo -e "\033[34;1mINFO:\033[0m clean the network if not detached (start and exit)\033[0m"
-    cleanup_all_in_network "$1"
-  fi
-  # status is 0 or SIGINT
-  if [[ "$status" == "0" || "$status" == "130" ]]; then
-    echo -e "\n\033[32;1mSUCCESS run-tests\033[0m"
-    exit 0
-  else
-    echo -e "\n\033[31;1mFAILURE during run-tests\033[0m"
-    exit ${status}
-  fi
-};
-function cleanup_all_in_network {
-
-  if [[ -z "$(docker network ls -q -f name="^$1\$")" ]]; then
-    echo -e "\033[34;1mINFO:\033[0m $1 is already deleted\033[0m"
-    return 0
-  fi
-  containers=$(docker network inspect -f '{{ range $key, $value := .Containers }}{{ printf "%s\n" .Name}}{{ end }}' $1)
-  while read -r container; do
-    cleanup_node "$container"
-  done <<< "$containers"
-  cleanup_network $1
-  echo -e "\033[32;1mSUCCESS:\033[0m Cleaned up and exiting\033[0m"
-};

data/.ci/functions/imports.sh

@@ -1,60 +0,0 @@
-#!/usr/bin/env bash
-#
-# Sets up all the common variables and imports relevant functions
-#
-# Version 1.0.1
-# - Initial version after refactor
-# - Validate STACK_VERSION asap
-
-function require_stack_version() {
-  if [[ -z $STACK_VERSION ]]; then
-    echo -e "\033[31;1mERROR:\033[0m Required environment variable [STACK_VERSION] not set\033[0m"
-    exit 1
-  fi
-}
-
-require_stack_version
-
-if [[ -z $es_node_name ]]; then
-  # only set these once
-  set -euo pipefail
-  export TEST_SUITE=${TEST_SUITE-free}
-  export SERVICE=${SERVICE-}
-  export RUNSCRIPTS=${RUNSCRIPTS-}
-  export DETACH=${DETACH-false}
-  export CLEANUP=${CLEANUP-false}
-
-  export es_node_name=instance
-  export elastic_password=changeme
-  export elasticsearch_image=elasticsearch
-  export elasticsearch_url=https://elastic:${elastic_password}@${es_node_name}:9200
-  if [[ $TEST_SUITE != "platinum" ]]; then
-    export elasticsearch_url=http://${es_node_name}:9200
-  fi
-  export external_elasticsearch_url=${elasticsearch_url/$es_node_name/localhost}
-  export elasticsearch_container="${elasticsearch_image}:${STACK_VERSION}"
-
-  export suffix=rest-test
-  export moniker=$(echo "$elasticsearch_container" | tr -C "[:alnum:]" '-')
-  export network_name=${moniker}${suffix}
-
-  export ssl_cert="${script_path}/certs/testnode.crt"
-  export ssl_key="${script_path}/certs/testnode.key"
-  export ssl_ca="${script_path}/certs/ca.crt"
-
-fi
-
-  export script_path=$(dirname $(realpath -s $0))
-  source $script_path/functions/cleanup.sh
-  source $script_path/functions/wait-for-container.sh
-  trap "cleanup_trap ${network_name}" EXIT
-
-
-if [[ "$CLEANUP" == "true" ]]; then
-  cleanup_all_in_network $network_name
-  exit 0
-fi
-
-echo -e "\033[34;1mINFO:\033[0m Creating network $network_name if it does not exist already \033[0m"
-docker network inspect "$network_name" > /dev/null 2>&1 || docker network create "$network_name"
-

data/.ci/functions/wait-for-container.sh

@@ -1,36 +0,0 @@
-#!/usr/bin/env bash
-#
-# Exposes a routine scripts can call to wait for a container if that container set up a health command
-#
-# Please source .ci/functions/imports.sh as a whole not just this file
-#
-# Version 1.0.1
-# - Initial version after refactor
-# - Make sure wait_for_contiainer is silent
-
-function wait_for_container {
-  set +x
-  until ! container_running "$1" || (container_running "$1" && [[ "$(docker inspect -f "{{.State.Health.Status}}" ${1})" != "starting" ]]); do
-    echo ""
-    docker inspect -f "{{range .State.Health.Log}}{{.Output}}{{end}}" ${1}
-    echo -e "\033[34;1mINFO:\033[0m waiting for node $1 to be up\033[0m"
-    sleep 20;
-  done;
-
-  # Always show logs if the container is running, this is very useful both on CI as well as while developing
-  if container_running $1; then
-    docker logs $1
-  fi
-
-  if ! container_running $1 || [[ "$(docker inspect -f "{{.State.Health.Status}}" ${1})" != "healthy" ]]; then
-    cleanup_all_in_network $2
-    echo
-    echo -e "\033[31;1mERROR:\033[0m Failed to start $1 in detached mode beyond health checks\033[0m"
-    echo -e "\033[31;1mERROR:\033[0m dumped the docker log before shutting the node down\033[0m"
-    return 1
-  else
-    echo
-    echo -e "\033[32;1mSUCCESS:\033[0m Detached and healthy: ${1} on docker network: ${network_name}\033[0m"
-    return 0
-  fi
-}

data/.ci/jobs/defaults.yml

@@ -1,73 +0,0 @@
----
-
-##### GLOBAL METADATA
-
-- meta:
-    cluster: clients-ci
-
-##### JOB DEFAULTS
-
-- job:
-    project-type: matrix
-    logrotate:
-      daysToKeep: 30
-      numToKeep: 100
-    properties:
-    - github:
-        url: https://github.com/elastic/enterprise-search-ruby/
-    - inject:
-        properties-content: HOME=$JENKINS_HOME
-    concurrent: true
-    node: flyweight
-    scm:
-    - git:
-        name: origin
-        credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
-        reference-repo: /var/lib/jenkins/.git-references/enterprise-search-ruby.git
-        branches:
-        - ${branch_specifier}
-        url: git@github.com:elastic/enterprise-search-ruby.git
-        basedir: ''
-        wipe-workspace: 'True'
-    triggers:
-    - github
-    - timed: 'H 5,17 * * *'
-    axes:
-    - axis:
-        type: slave
-        name: label
-        values:
-        - linux
-    - axis:
-        type: yaml
-        filename: .ci/test-matrix.yml
-        name: STACK_VERSION
-    - axis:
-        type: yaml
-        filename: .ci/test-matrix.yml
-        name: RUBY_VERSION
-    - axis:
-        type: yaml
-        filename: .ci/test-matrix.yml
-        name: SERVICE
-    yaml-strategy:
-      exclude-key: exclude
-      filename: .ci/test-matrix.yml
-    wrappers:
-      - ansicolor
-      - timeout:
-          type: absolute
-          timeout: 120
-          fail: true
-      - timestamps
-      - workspace-cleanup
-    builders:
-      - shell: |-
-          #!/usr/local/bin/runbld
-          .ci/run-tests
-    publishers:
-      - email:
-          recipients: build-lang-clients@elastic.co
-      - junit:
-          results: "*-junit.xml"
-          allow-empty-results: true

data/.ci/jobs/elastic+enterprise-search-ruby+7.17.yml

@@ -1,12 +0,0 @@
----
-- job:
-    name: elastic+enterprise-search-ruby+7.17
-    display-name: 'elastic / enterprise-search-ruby # 7.17'
-    description: Testing the enterprise-search-ruby 7.17 branch.
-    junit_results: "*-junit.xml"
-    parameters:
-    - string:
-        name: branch_specifier
-        default: refs/heads/7.17
-        description: the Git branch specifier to build (<branchName>, <tagName>,
-          <commitId>, etc.)

data/.ci/jobs/elastic+enterprise-search-ruby+8.3.yml

@@ -1,12 +0,0 @@
----
-- job:
-    name: elastic+enterprise-search-ruby+8.3
-    display-name: 'elastic / enterprise-search-ruby # 8.3'
-    description: Testing the enterprise-search-ruby 8.3 branch.
-    junit_results: "*-junit.xml"
-    parameters:
-    - string:
-        name: branch_specifier
-        default: refs/heads/8.3
-        description: the Git branch specifier to build (<branchName>, <tagName>,
-          <commitId>, etc.)

data/.ci/jobs/elastic+enterprise-search-ruby+8.4.yml

@@ -1,12 +0,0 @@
----
-- job:
-    name: elastic+enterprise-search-ruby+8.4
-    display-name: 'elastic / enterprise-search-ruby # 8.4'
-    description: Testing the enterprise-search-ruby 8.4 branch.
-    junit_results: "*-junit.xml"
-    parameters:
-    - string:
-        name: branch_specifier
-        default: refs/heads/8.4
-        description: the Git branch specifier to build (<branchName>, <tagName>,
-          <commitId>, etc.)

data/.ci/jobs/elastic+enterprise-search-ruby+8.5.yml

@@ -1,12 +0,0 @@
----
-- job:
-    name: elastic+enterprise-search-ruby+8.5
-    display-name: 'elastic / enterprise-search-ruby # 8.5'
-    description: Testing the enterprise-search-ruby 8.5 branch.
-    junit_results: "*-junit.xml"
-    parameters:
-    - string:
-        name: branch_specifier
-        default: refs/heads/8.5
-        description: the Git branch specifier to build (<branchName>, <tagName>,
-          <commitId>, etc.)

data/.ci/jobs/elastic+enterprise-search-ruby+main.yml

@@ -1,12 +0,0 @@
----
-- job:
-    name: elastic+enterprise-search-ruby+main
-    display-name: 'elastic / enterprise-search-ruby # main'
-    description: Testing the enterprise-search-ruby main branch.
-    junit_results: "*-junit.xml"
-    parameters:
-    - string:
-        name: branch_specifier
-        default: refs/heads/main
-        description: the Git branch specifier to build (<branchName>, <tagName>,
-          <commitId>, etc.)

data/.ci/jobs/elastic+enterprise-search-ruby+pull-request.yml

@@ -1,19 +0,0 @@
----
-- job:
-    name: elastic+enterprise-search-ruby+pull-request
-    display-name: 'elastic / enterprise-search-ruby # pull-request'
-    description: Testing of enterprise-search-ruby pull requests.
-    junit_results: "*-junit.xml"
-    scm:
-    - git:
-        branches:
-        - ${ghprbActualCommit}
-        refspec: +refs/pull/*:refs/remotes/origin/pr/*
-    triggers:
-    - github-pull-request:
-        org-list:
-        - elastic
-        allow-whitelist-orgs-as-admins: true
-        github-hooks: true
-        status-context: clients-ci
-        cancel-builds-on-update: true

data/.ci/run-elasticsearch.sh

@@ -1,127 +0,0 @@
-#!/usr/bin/env bash
-#
-# Launch one or more Elasticsearch nodes via the Docker image,
-# to form a cluster suitable for running the REST API tests.
-#
-# Export the STACK_VERSION variable, eg. '8.0.0-SNAPSHOT'.
-# Export the TEST_SUITE variable, eg. 'free' or 'platinum' defaults to 'free'.
-# Export the NUMBER_OF_NODES variable to start more than 1 node
-
-# Version 1.2.0
-# - Initial version of the run-elasticsearch.sh script
-# - Deleting the volume should not dependent on the container still running
-# - Fixed `ES_JAVA_OPTS` config
-# - Moved to STACK_VERSION and TEST_VERSION
-# - Refactored into functions and imports
-# - Support NUMBER_OF_NODES
-# - Added 5 retries on docker pull for fixing transient network errors
-
-script_path=$(dirname $(realpath -s $0))
-source $script_path/functions/imports.sh
-set -euo pipefail
-
-echo -e "\033[34;1mINFO:\033[0m Take down node if called twice with the same arguments (DETACH=true) or on seperate terminals \033[0m"
-cleanup_node $es_node_name
-
-master_node_name=${es_node_name}
-cluster_name=${moniker}${suffix}
-
-declare -a volumes
-environment=($(cat <<-END
-  --env node.name=$es_node_name
-  --env cluster.name=$cluster_name
-  --env cluster.initial_master_nodes=$master_node_name
-  --env discovery.seed_hosts=$master_node_name
-  --env cluster.routing.allocation.disk.threshold_enabled=false
-  --env bootstrap.memory_lock=true
-  --env node.attr.testattr=test
-  --env path.repo=/tmp
-  --env repositories.url.allowed_urls=http://snapshot.test*
-END
-))
-if [[ "$TEST_SUITE" == "platinum" ]]; then
-  environment+=($(cat <<-END
-    --env ELASTIC_PASSWORD=$elastic_password
-    --env xpack.license.self_generated.type=trial
-    --env xpack.security.enabled=true
-    --env xpack.security.http.ssl.enabled=true
-    --env xpack.security.http.ssl.verification_mode=certificate
-    --env xpack.security.http.ssl.key=certs/testnode.key
-    --env xpack.security.http.ssl.certificate=certs/testnode.crt
-    --env xpack.security.http.ssl.certificate_authorities=certs/ca.crt
-    --env xpack.security.transport.ssl.enabled=true
-    --env xpack.security.transport.ssl.key=certs/testnode.key
-    --env xpack.security.transport.ssl.certificate=certs/testnode.crt
-    --env xpack.security.transport.ssl.certificate_authorities=certs/ca.crt
-END
-))
-  volumes+=($(cat <<-END
-    --volume $ssl_cert:/usr/share/elasticsearch/config/certs/testnode.crt
-    --volume $ssl_key:/usr/share/elasticsearch/config/certs/testnode.key
-    --volume $ssl_ca:/usr/share/elasticsearch/config/certs/ca.crt
-END
-))
-fi
-
-cert_validation_flags=""
-if [[ "$TEST_SUITE" == "platinum" ]]; then
-  cert_validation_flags="--insecure --cacert /usr/share/elasticsearch/config/certs/ca.crt --resolve ${es_node_name}:443:127.0.0.1"
-fi
-
-# Pull the container, retry on failures up to 5 times with
-# short delays between each attempt. Fixes most transient network errors.
-docker_pull_attempts=0
-until [ "$docker_pull_attempts" -ge 5 ]
-do
-   docker pull docker.elastic.co/elasticsearch/"$elasticsearch_container" && break
-   docker_pull_attempts=$((docker_pull_attempts+1))
-   echo "Failed to pull image, retrying in 10 seconds (retry $docker_pull_attempts/5)..."
-   sleep 10
-done
-
-NUMBER_OF_NODES=${NUMBER_OF_NODES-1}
-http_port=9200
-for (( i=0; i<$NUMBER_OF_NODES; i++, http_port++ )); do
-  node_name=${es_node_name}$i
-  node_url=${external_elasticsearch_url/9200/${http_port}}$i
-  if [[ "$i" == "0" ]]; then node_name=$es_node_name; fi
-  environment+=($(cat <<-END
-    --env node.name=$node_name
-END
-))
-  echo "$i: $http_port $node_url "
-  volume_name=${node_name}-${suffix}-data
-  volumes+=($(cat <<-END
-    --volume $volume_name:/usr/share/elasticsearch/data${i}
-END
-))
-
-  # make sure we detach for all but the last node if DETACH=false (default) so all nodes are started
-  local_detach="true"
-  if [[ "$i" == "$((NUMBER_OF_NODES-1))" ]]; then local_detach=$DETACH; fi
-  echo -e "\033[34;1mINFO:\033[0m Starting container $node_name \033[0m"
-  set -x
-  docker run \
-    --name "$node_name" \
-    --network "$network_name" \
-    --env "ES_JAVA_OPTS=-Xms1g -Xmx1g" \
-    "${environment[@]}" \
-    "${volumes[@]}" \
-    --publish "$http_port":9200 \
-    --ulimit nofile=65536:65536 \
-    --ulimit memlock=-1:-1 \
-    --detach="$local_detach" \
-    --health-cmd="curl $cert_validation_flags --fail $elasticsearch_url/_cluster/health || exit 1" \
-    --health-interval=2s \
-    --health-retries=20 \
-    --health-timeout=2s \
-    --rm \
-    docker.elastic.co/elasticsearch/"$elasticsearch_container";
-
-  set +x
-  if wait_for_container "$es_node_name" "$network_name"; then
-    echo -e "\033[32;1mSUCCESS:\033[0m Running on: $node_url\033[0m"
-  fi
-
-done
-