@elastic/elasticsearch 8.7.0 → 8.8.1

package/.buildkite/Dockerfile

@@ -0,0 +1,14 @@
+ARG NODE_VERSION=${NODE_VERSION:-18}
+FROM node:$NODE_VERSION
+
+# Install required tools
+RUN apt-get clean -y && \
+    apt-get -qy update && \
+    apt-get -y install zip && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+WORKDIR /usr/src/app
+
+COPY . .
+RUN npm install --production=false

package/.buildkite/functions/cleanup.sh

@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+#
+# Shared cleanup routines between different steps
+#
+# Please source .ci/functions/imports.sh as a whole not just this file
+#
+# Version 1.0.0
+# - Initial version after refactor
+
+function cleanup_volume {
+  if [[ "$(docker volume ls -q -f name=$1)" ]]; then
+    echo -e "\033[34;1mINFO:\033[0m Removing volume $1\033[0m"
+    (docker volume rm "$1") || true
+  fi
+}
+function container_running {
+  if [[ "$(docker ps -q -f name=$1)" ]]; then
+    return 0;
+    else return 1;
+  fi
+}
+function cleanup_node {
+  if container_running "$1"; then
+    echo -e "\033[34;1mINFO:\033[0m Removing container $1\033[0m"
+    (docker container rm --force --volumes "$1") || true
+  fi
+  if [[ -n "$1" ]]; then
+    echo -e "\033[34;1mINFO:\033[0m Removing volume $1-${suffix}-data\033[0m"
+    cleanup_volume "$1-${suffix}-data"
+  fi
+}
+function cleanup_network {
+  if [[ "$(docker network ls -q -f name=$1)" ]]; then
+    echo -e "\033[34;1mINFO:\033[0m Removing network $1\033[0m"
+    (docker network rm "$1") || true
+  fi
+}
+
+function cleanup_trap {
+  status=$?
+  set +x
+  if [[ "$DETACH" != "true" ]]; then
+    echo -e "\033[34;1mINFO:\033[0m clean the network if not detached (start and exit)\033[0m"
+    cleanup_all_in_network "$1"
+  fi
+  # status is 0 or SIGINT
+  if [[ "$status" == "0" || "$status" == "130" ]]; then
+    echo -e "\n\033[32;1mSUCCESS run-tests\033[0m"
+    exit 0
+  else
+    echo -e "\n\033[31;1mFAILURE during run-tests\033[0m"
+    exit ${status}
+  fi
+};
+function cleanup_all_in_network {
+
+  if [[ -z "$(docker network ls -q -f name="^$1\$")" ]]; then
+    echo -e "\033[34;1mINFO:\033[0m $1 is already deleted\033[0m"
+    return 0
+  fi
+  containers=$(docker network inspect -f '{{ range $key, $value := .Containers }}{{ printf "%s\n" .Name}}{{ end }}' $1)
+  while read -r container; do
+    cleanup_node "$container"
+  done <<< "$containers"
+  cleanup_network $1
+  echo -e "\033[32;1mSUCCESS:\033[0m Cleaned up and exiting\033[0m"
+};

package/.buildkite/functions/imports.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+#
+# Sets up all the common variables and imports relevant functions
+#
+# Version 1.0.1
+# - Initial version after refactor
+# - Validate STACK_VERSION asap
+
+function require_stack_version() {
+  if [[ -z $STACK_VERSION ]]; then
+    echo -e "\033[31;1mERROR:\033[0m Required environment variable [STACK_VERSION] not set\033[0m"
+    exit 1
+  fi
+}
+
+require_stack_version
+
+if [[ -z $es_node_name ]]; then
+  # only set these once
+  set -euo pipefail
+  export TEST_SUITE=${TEST_SUITE-free}
+  export RUNSCRIPTS=${RUNSCRIPTS-}
+  export DETACH=${DETACH-false}
+  export CLEANUP=${CLEANUP-false}
+
+  export es_node_name=instance
+  export elastic_password=changeme
+  export elasticsearch_image=elasticsearch
+  export elasticsearch_scheme="https"
+  if [[ $TEST_SUITE != "platinum" ]]; then
+    export elasticsearch_scheme="http"
+  fi
+  export elasticsearch_url=${elasticsearch_scheme}://elastic:${elastic_password}@${es_node_name}:9200
+  export external_elasticsearch_url=${elasticsearch_url/$es_node_name/localhost}
+  export elasticsearch_container="${elasticsearch_image}:${STACK_VERSION}"
+
+  export suffix=rest-test
+  export moniker=$(echo "$elasticsearch_container" | tr -C "[:alnum:]" '-')
+  export network_name=${moniker}${suffix}
+
+  export ssl_cert="${script_path}/certs/testnode.crt"
+  export ssl_key="${script_path}/certs/testnode.key"
+  export ssl_ca="${script_path}/certs/ca.crt"
+
+fi
+
+  export script_path=$(dirname $(realpath -s $0))
+  source $script_path/functions/cleanup.sh
+  source $script_path/functions/wait-for-container.sh
+  trap "cleanup_trap ${network_name}" EXIT
+
+
+if [[ "$CLEANUP" == "true" ]]; then
+  cleanup_all_in_network $network_name
+  exit 0
+fi
+
+echo -e "\033[34;1mINFO:\033[0m Creating network $network_name if it does not exist already \033[0m"
+docker network inspect "$network_name" > /dev/null 2>&1 || docker network create "$network_name"
+

package/.buildkite/functions/wait-for-container.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+#
+# Exposes a routine scripts can call to wait for a container if that container set up a health command
+#
+# Please source .ci/functions/imports.sh as a whole not just this file
+#
+# Version 1.0.1
+# - Initial version after refactor
+# - Make sure wait_for_contiainer is silent
+
+function wait_for_container {
+  set +x
+  until ! container_running "$1" || (container_running "$1" && [[ "$(docker inspect -f "{{.State.Health.Status}}" ${1})" != "starting" ]]); do
+    echo ""
+    docker inspect -f "{{range .State.Health.Log}}{{.Output}}{{end}}" ${1}
+    echo -e "\033[34;1mINFO:\033[0m waiting for node $1 to be up\033[0m"
+    sleep 2;
+  done;
+
+  # Always show logs if the container is running, this is very useful both on CI as well as while developing
+  if container_running $1; then
+    docker logs $1
+  fi
+
+  if ! container_running $1 || [[ "$(docker inspect -f "{{.State.Health.Status}}" ${1})" != "healthy" ]]; then
+    cleanup_all_in_network $2
+    echo
+    echo -e "\033[31;1mERROR:\033[0m Failed to start $1 in detached mode beyond health checks\033[0m"
+    echo -e "\033[31;1mERROR:\033[0m dumped the docker log before shutting the node down\033[0m"
+    return 1
+  else
+    echo
+    echo -e "\033[32;1mSUCCESS:\033[0m Detached and healthy: ${1} on docker network: ${network_name}\033[0m"
+    return 0
+  fi
+}

package/.buildkite/pipeline.yml

@@ -0,0 +1,30 @@
+steps:
+  - label: ":elasticsearch: :javascript: ES JavaScript ({{ matrix.nodejs }}) Test Suite: {{ matrix.suite }}"
+    agents:
+      provider: "gcp"
+    env:
+      NODE_VERSION: "{{ matrix.nodejs }}"
+      TEST_SUITE: "{{ matrix.suite }}"
+      STACK_VERSION: 8.8.0-SNAPSHOT
+    matrix:
+      setup:
+        suite:
+          - "free"
+          - "platinum"
+        nodejs:
+          - "16"
+          - "18"
+          - "20"
+    command: ./.buildkite/run-tests.sh
+    artifact_paths: "./junit-output/junit-*.xml"
+  - wait: ~
+    continue_on_failure: true
+  - label: ":junit: Test results"
+    agents:
+      provider: "gcp"
+      image: family/core-ubuntu-2204
+    plugins:
+      - junit-annotate#v2.4.1:
+          artifacts: "junit-output/junit-*.xml"
+          job-uuid-file-pattern: 'junit-(.*).xml'
+          fail-build-on-error: true

package/.buildkite/run-client.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+#
+# Once called Elasticsearch should be up and running
+#
+script_path=$(dirname "$(realpath -s "$0")")
+set -euo pipefail
+repo=$(pwd)
+
+export NODE_VERSION=${NODE_VERSION:-18}
+
+echo "--- :javascript: Building Docker image"
+docker build \
+       --file "$script_path/Dockerfile" \
+       --tag elastic/elasticsearch-js \
+       --build-arg NODE_VERSION="$NODE_VERSION" \
+       .
+
+echo "--- :javascript: Running $TEST_SUITE tests"
+mkdir -p "$repo/junit-output"
+docker run \
+       --network="${network_name}" \
+       --env "TEST_ES_SERVER=${elasticsearch_url}" \
+       --env "ELASTIC_PASSWORD=${elastic_password}" \
+       --env "TEST_SUITE=${TEST_SUITE}" \
+       --env "ELASTIC_USER=elastic" \
+       --env "BUILDKITE=true" \
+       --volume "$repo/junit-output:/junit-output" \
+       --name elasticsearch-js \
+       --rm \
+       elastic/elasticsearch-js \
+       bash -c "npm run test:integration; [ -f ./$TEST_SUITE-report-junit.xml ] && mv ./$TEST_SUITE-report-junit.xml /junit-output/junit-$BUILDKITE_JOB_ID.xml || echo 'No JUnit artifact found'"

package/.buildkite/run-elasticsearch.sh

@@ -0,0 +1,152 @@
+#!/usr/bin/env bash
+#
+# Launch one or more Elasticsearch nodes via the Docker image,
+# to form a cluster suitable for running the REST API tests.
+#
+# Export the STACK_VERSION variable, eg. '8.0.0-SNAPSHOT'.
+# Export the TEST_SUITE variable, eg. 'free' or 'platinum' defaults to 'free'.
+# Export the NUMBER_OF_NODES variable to start more than 1 node
+
+# Version 1.6.1
+# - Initial version of the run-elasticsearch.sh script
+# - Deleting the volume should not dependent on the container still running
+# - Fixed `ES_JAVA_OPTS` config
+# - Moved to STACK_VERSION and TEST_VERSION
+# - Refactored into functions and imports
+# - Support NUMBER_OF_NODES
+# - Added 5 retries on docker pull for fixing transient network errors
+# - Added flags to make local CCR configurations work
+# - Added action.destructive_requires_name=false as the default will be true in v8
+# - Added ingest.geoip.downloader.enabled=false as it causes false positives in testing
+# - Moved ELASTIC_PASSWORD and xpack.security.enabled to the base arguments for "Security On by default"
+# - Use https only when TEST_SUITE is "platinum", when "free" use http
+# - Set xpack.security.enabled=false for "free" and xpack.security.enabled=true for "platinum"
+
+script_path=$(dirname $(realpath -s $0))
+source $script_path/functions/imports.sh
+set -euo pipefail
+
+echo -e "\033[34;1mINFO:\033[0m Take down node if called twice with the same arguments (DETACH=true) or on separate terminals \033[0m"
+cleanup_node $es_node_name
+
+master_node_name=${es_node_name}
+cluster_name=${moniker}${suffix}
+
+# Set vm.max_map_count kernel setting to 262144
+sudo sysctl -w vm.max_map_count=262144
+
+declare -a volumes
+environment=($(cat <<-END
+  --env ELASTIC_PASSWORD=$elastic_password
+  --env node.name=$es_node_name
+  --env cluster.name=$cluster_name
+  --env cluster.initial_master_nodes=$master_node_name
+  --env discovery.seed_hosts=$master_node_name
+  --env cluster.routing.allocation.disk.threshold_enabled=false
+  --env bootstrap.memory_lock=true
+  --env node.attr.testattr=test
+  --env path.repo=/tmp
+  --env repositories.url.allowed_urls=http://snapshot.test*
+  --env action.destructive_requires_name=false
+  --env ingest.geoip.downloader.enabled=false
+  --env cluster.deprecation_indexing.enabled=false
+END
+))
+if [[ "$TEST_SUITE" == "platinum" ]]; then
+  environment+=($(cat <<-END
+    --env xpack.security.enabled=true
+    --env xpack.license.self_generated.type=trial
+    --env xpack.security.http.ssl.enabled=true
+    --env xpack.security.http.ssl.verification_mode=certificate
+    --env xpack.security.http.ssl.key=certs/testnode.key
+    --env xpack.security.http.ssl.certificate=certs/testnode.crt
+    --env xpack.security.http.ssl.certificate_authorities=certs/ca.crt
+    --env xpack.security.transport.ssl.enabled=true
+    --env xpack.security.transport.ssl.verification_mode=certificate
+    --env xpack.security.transport.ssl.key=certs/testnode.key
+    --env xpack.security.transport.ssl.certificate=certs/testnode.crt
+    --env xpack.security.transport.ssl.certificate_authorities=certs/ca.crt
+END
+))
+  volumes+=($(cat <<-END
+    --volume $ssl_cert:/usr/share/elasticsearch/config/certs/testnode.crt
+    --volume $ssl_key:/usr/share/elasticsearch/config/certs/testnode.key
+    --volume $ssl_ca:/usr/share/elasticsearch/config/certs/ca.crt
+END
+))
+else
+  environment+=($(cat <<-END
+    --env node.roles=data,data_cold,data_content,data_frozen,data_hot,data_warm,ingest,master,ml,remote_cluster_client,transform
+    --env xpack.security.enabled=false
+    --env xpack.security.http.ssl.enabled=false
+END
+))
+fi
+
+cert_validation_flags=""
+if [[ "$TEST_SUITE" == "platinum" ]]; then
+  cert_validation_flags="--insecure --cacert /usr/share/elasticsearch/config/certs/ca.crt --resolve ${es_node_name}:443:127.0.0.1"
+fi
+
+echo "--- :elasticsearch: Environment setup"
+echo "TEST_SUITE: $TEST_SUITE"
+echo "Elasticsearch URL: $elasticsearch_url"
+echo "Elasticsearch External URL: $external_elasticsearch_url"
+
+
+echo "--- :elasticsearch: Running container"
+# Pull the container, retry on failures up to 5 times with
+# short delays between each attempt. Fixes most transient network errors.
+docker_pull_attempts=0
+until [ "$docker_pull_attempts" -ge 5 ]
+do
+   docker pull docker.elastic.co/elasticsearch/"$elasticsearch_container" && break
+   docker_pull_attempts=$((docker_pull_attempts+1))
+   echo "Failed to pull image, retrying in 10 seconds (retry $docker_pull_attempts/5)..."
+   sleep 10
+done
+
+NUMBER_OF_NODES=${NUMBER_OF_NODES-1}
+http_port=9200
+for (( i=0; i<$NUMBER_OF_NODES; i++, http_port++ )); do
+  node_name=${es_node_name}$i
+  node_url=${external_elasticsearch_url/9200/${http_port}}$i
+  if [[ "$i" == "0" ]]; then node_name=$es_node_name; fi
+  environment+=($(cat <<-END
+    --env node.name=$node_name
+END
+))
+  echo "$i: $http_port $node_url "
+  volume_name=${node_name}-${suffix}-data
+  volumes+=($(cat <<-END
+    --volume $volume_name:/usr/share/elasticsearch/data${i}
+END
+))
+
+  # make sure we detach for all but the last node if DETACH=false (default) so all nodes are started
+  local_detach="true"
+  if [[ "$i" == "$((NUMBER_OF_NODES-1))" ]]; then local_detach=$DETACH; fi
+  echo -e "\033[34;1mINFO:\033[0m Starting container $node_name \033[0m"
+  set -x
+  docker run \
+    --name "$node_name" \
+    --network "$network_name" \
+    --env "ES_JAVA_OPTS=-Xms1g -Xmx1g -da:org.elasticsearch.xpack.ccr.index.engine.FollowingEngineAssertions" \
+    "${environment[@]}" \
+    "${volumes[@]}" \
+    --publish "$http_port":9200 \
+    --ulimit nofile=65536:65536 \
+    --ulimit memlock=-1:-1 \
+    --detach="$local_detach" \
+    --health-cmd="curl $cert_validation_flags --fail $elasticsearch_url/_cluster/health || exit 1" \
+    --health-interval=2s \
+    --health-retries=20 \
+    --health-timeout=2s \
+    --rm \
+    docker.elastic.co/elasticsearch/"$elasticsearch_container";
+
+  set +x
+  if wait_for_container "$es_node_name" "$network_name"; then
+    echo -e "\033[32;1mSUCCESS:\033[0m Running on: $node_url\033[0m"
+  fi
+done

package/.buildkite/run-tests.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+#
+# Script to run Elasticsearch container and Elasticsearch client integration tests on Buildkite
+#
+# Version 0.1
+#
+script_path=$(dirname "$(realpath -s "$0")")
+source "$script_path/functions/imports.sh"
+
+set -euo pipefail
+
+echo "--- :elasticsearch: Starting Elasticsearch"
+DETACH=true bash "$script_path/run-elasticsearch.sh"
+
+echo "+++ :javascript: Run Client"
+bash "$script_path/run-client.sh"

package/catalog-info.yaml

@@ -0,0 +1,49 @@
+---
+# yaml-language-server: $schema=https://json.schemastore.org/catalog-info.json
+apiVersion: backstage.io/v1alpha1
+kind: Component
+metadata:
+  name: elasticsearch-js
+spec:
+  type: library
+  owner: group:clients-team
+  lifecycle: production
+
+---
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: elasticsearch-js-integration-tests
+  description: Elasticsearch JavaScript client integration tests
+spec:
+  type: buildkite-pipeline
+  owner: group:clients-team
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: Elasticsearch JavaScript client integration tests
+    spec:
+      repository: elastic/elasticsearch-js
+      pipeline_file: .buildkite/pipeline.yml
+      teams:
+        clients-team:
+          access_level: MANAGE_BUILD_AND_READ
+        everyone:
+          access_level: READ_ONLY
+      provider_settings:
+        build_pull_requests: false
+      cancel_intermediate_builds: true
+      cancel_intermediate_builds_branch_filter: '!main'
+      schedules:
+        main_semi_daily:
+          branch: 'main'
+          cronline: '*/12 * * *'
+        8_8_semi_daily:
+          branch: '8.8'
+          cronline: '*/12 * * *'
+        8_7_daily:
+          branch: '8.7'
+          cronline: '@daily'