elastic-enterprise-search 0.2.1 → 7.12.0

data/.ci/jobs/elastic+enterprise-search-ruby+7.11.yml

@@ -0,0 +1,12 @@
+---
+- job:
+    name: elastic+enterprise-search-ruby+7.11
+    display-name: 'elastic / enterprise-search-ruby # 7.11'
+    description: Testing the enterprise-search-ruby 7.11 branch.
+    junit_results: "*-junit.xml"
+    parameters:
+    - string:
+        name: branch_specifier
+        default: refs/heads/7.11
+        description: the Git branch specifier to build (<branchName>, <tagName>,
+          <commitId>, etc.)

data/.ci/jobs/elastic+enterprise-search-ruby+master.yml

@@ -0,0 +1,12 @@
+---
+- job:
+    name: elastic+enterprise-search-ruby+master
+    display-name: 'elastic / enterprise-search-ruby # master'
+    description: Testing the enterprise-search-ruby master branch.
+    junit_results: "*-junit.xml"
+    parameters:
+    - string:
+        name: branch_specifier
+        default: refs/heads/master
+        description: the Git branch specifier to build (<branchName>, <tagName>,
+          <commitId>, etc.)

data/.ci/jobs/elastic+enterprise-search-ruby+pull-request.yml

@@ -0,0 +1,19 @@
+---
+- job:
+    name: elastic+enterprise-search-ruby+pull-request
+    display-name: 'elastic / enterprise-search-ruby # pull-request'
+    description: Testing of enterprise-search-ruby pull requests.
+    junit_results: "*-junit.xml"
+    scm:
+    - git:
+        branches:
+        - ${ghprbActualCommit}
+        refspec: +refs/pull/*:refs/remotes/origin/pr/*
+    triggers:
+    - github-pull-request:
+        org-list:
+        - elastic
+        allow-whitelist-orgs-as-admins: true
+        github-hooks: true
+        status-context: clients-ci
+        cancel-builds-on-update: true

data/.ci/run-elasticsearch.sh

@@ -0,0 +1,127 @@
+#!/usr/bin/env bash
+#
+# Launch one or more Elasticsearch nodes via the Docker image,
+# to form a cluster suitable for running the REST API tests.
+#
+# Export the STACK_VERSION variable, eg. '8.0.0-SNAPSHOT'.
+# Export the TEST_SUITE variable, eg. 'free' or 'platinum' defaults to 'free'.
+# Export the NUMBER_OF_NODES variable to start more than 1 node
+
+# Version 1.2.0
+# - Initial version of the run-elasticsearch.sh script
+# - Deleting the volume should not dependent on the container still running
+# - Fixed `ES_JAVA_OPTS` config
+# - Moved to STACK_VERSION and TEST_VERSION
+# - Refactored into functions and imports
+# - Support NUMBER_OF_NODES
+# - Added 5 retries on docker pull for fixing transient network errors
+
+script_path=$(dirname $(realpath -s $0))
+source $script_path/functions/imports.sh
+set -euo pipefail
+
+echo -e "\033[34;1mINFO:\033[0m Take down node if called twice with the same arguments (DETACH=true) or on seperate terminals \033[0m"
+cleanup_node $es_node_name
+
+master_node_name=${es_node_name}
+cluster_name=${moniker}${suffix}
+
+declare -a volumes
+environment=($(cat <<-END
+  --env node.name=$es_node_name
+  --env cluster.name=$cluster_name
+  --env cluster.initial_master_nodes=$master_node_name
+  --env discovery.seed_hosts=$master_node_name
+  --env cluster.routing.allocation.disk.threshold_enabled=false
+  --env bootstrap.memory_lock=true
+  --env node.attr.testattr=test
+  --env path.repo=/tmp
+  --env repositories.url.allowed_urls=http://snapshot.test*
+END
+))
+if [[ "$TEST_SUITE" == "platinum" ]]; then
+  environment+=($(cat <<-END
+    --env ELASTIC_PASSWORD=$elastic_password
+    --env xpack.license.self_generated.type=trial
+    --env xpack.security.enabled=true
+    --env xpack.security.http.ssl.enabled=true
+    --env xpack.security.http.ssl.verification_mode=certificate
+    --env xpack.security.http.ssl.key=certs/testnode.key
+    --env xpack.security.http.ssl.certificate=certs/testnode.crt
+    --env xpack.security.http.ssl.certificate_authorities=certs/ca.crt
+    --env xpack.security.transport.ssl.enabled=true
+    --env xpack.security.transport.ssl.key=certs/testnode.key
+    --env xpack.security.transport.ssl.certificate=certs/testnode.crt
+    --env xpack.security.transport.ssl.certificate_authorities=certs/ca.crt
+END
+))
+  volumes+=($(cat <<-END
+    --volume $ssl_cert:/usr/share/elasticsearch/config/certs/testnode.crt
+    --volume $ssl_key:/usr/share/elasticsearch/config/certs/testnode.key
+    --volume $ssl_ca:/usr/share/elasticsearch/config/certs/ca.crt
+END
+))
+fi
+
+cert_validation_flags=""
+if [[ "$TEST_SUITE" == "platinum" ]]; then
+  cert_validation_flags="--insecure --cacert /usr/share/elasticsearch/config/certs/ca.crt --resolve ${es_node_name}:443:127.0.0.1"
+fi
+
+# Pull the container, retry on failures up to 5 times with
+# short delays between each attempt. Fixes most transient network errors.
+docker_pull_attempts=0
+until [ "$docker_pull_attempts" -ge 5 ]
+do
+   docker pull docker.elastic.co/elasticsearch/"$elasticsearch_container" && break
+   docker_pull_attempts=$((docker_pull_attempts+1))
+   echo "Failed to pull image, retrying in 10 seconds (retry $docker_pull_attempts/5)..."
+   sleep 10
+done
+
+NUMBER_OF_NODES=${NUMBER_OF_NODES-1}
+http_port=9200
+for (( i=0; i<$NUMBER_OF_NODES; i++, http_port++ )); do
+  node_name=${es_node_name}$i
+  node_url=${external_elasticsearch_url/9200/${http_port}}$i
+  if [[ "$i" == "0" ]]; then node_name=$es_node_name; fi
+  environment+=($(cat <<-END
+    --env node.name=$node_name
+END
+))
+  echo "$i: $http_port $node_url "
+  volume_name=${node_name}-${suffix}-data
+  volumes+=($(cat <<-END
+    --volume $volume_name:/usr/share/elasticsearch/data${i}
+END
+))
+
+  # make sure we detach for all but the last node if DETACH=false (default) so all nodes are started
+  local_detach="true"
+  if [[ "$i" == "$((NUMBER_OF_NODES-1))" ]]; then local_detach=$DETACH; fi
+  echo -e "\033[34;1mINFO:\033[0m Starting container $node_name \033[0m"
+  set -x
+  docker run \
+    --name "$node_name" \
+    --network "$network_name" \
+    --env "ES_JAVA_OPTS=-Xms1g -Xmx1g" \
+    "${environment[@]}" \
+    "${volumes[@]}" \
+    --publish "$http_port":9200 \
+    --ulimit nofile=65536:65536 \
+    --ulimit memlock=-1:-1 \
+    --detach="$local_detach" \
+    --health-cmd="curl $cert_validation_flags --fail $elasticsearch_url/_cluster/health || exit 1" \
+    --health-interval=2s \
+    --health-retries=20 \
+    --health-timeout=2s \
+    --rm \
+    docker.elastic.co/elasticsearch/"$elasticsearch_container";
+
+  set +x
+  if wait_for_container "$es_node_name" "$network_name"; then
+    echo -e "\033[32;1mSUCCESS:\033[0m Running on: $node_url\033[0m"
+  fi
+
+done
+

data/.ci/run-enterprise-search.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+#
+# Launch one App Search node via the Docker image,
+# to form a cluster suitable for running the REST API tests.
+#
+# Export the STACK_VERSION variable, eg. '8.0.0-SNAPSHOT'.
+
+# Version 1.1.0
+# - Initial version of the run-app-search.sh script
+# - Refactored .ci version
+
+script_path=$(dirname $(realpath -s $0))
+source $script_path/functions/imports.sh
+set -euo pipefail
+
+CONTAINER_NAME=${CONTAINER_NAME-enterprise-search}
+APP_SEARCH_SECRET_SESSION_KEY=${APP_SEARCH_SECRET_SESSION_KEY-int_test_secret}
+
+echo -e "\033[34;1mINFO:\033[0m Take down node if called twice with the same arguments (DETACH=true) or on seperate terminals \033[0m"
+cleanup_node $CONTAINER_NAME
+
+http_port=3002
+url=http://127.0.0.1:${http_port}
+
+# Pull the container, retry on failures up to 5 times with
+# short delays between each attempt. Fixes most transient network errors.
+docker_pull_attempts=0
+until [ "$docker_pull_attempts" -ge 5 ]
+do
+   docker pull docker.elastic.co/enterprise-search/enterprise-search:"$STACK_VERSION" && break
+   docker_pull_attempts=$((docker_pull_attempts+1))
+   echo "Failed to pull image, retrying in 10 seconds (retry $docker_pull_attempts/5)..."
+   sleep 10
+done
+
+echo -e "\033[34;1mINFO:\033[0m Starting container $CONTAINER_NAME \033[0m"
+set -x
+docker run \
+  --name "$CONTAINER_NAME" \
+  --network "$network_name" \
+  --env "elasticsearch.host=$elasticsearch_url" \
+  --env "elasticsearch.username=elastic" \
+  --env "elasticsearch.password=$elastic_password" \
+  --env "ENT_SEARCH_DEFAULT_PASSWORD=$elastic_password" \
+  --env "secret_management.encryption_keys=[$APP_SEARCH_SECRET_SESSION_KEY]" \
+  --env "enterprise_search.listen_port=$http_port" \
+  --env "log_level=info" \
+  --env "hide_version_info=false" \
+  --env "worker.threads=2" \
+  --env "allow_es_settings_modification=true" \
+  --env "JAVA_OPTS=-Xms1g -Xmx2g" \
+  --env "elasticsearch.ssl.enabled=true" \
+  --env "elasticsearch.ssl.verify=true" \
+  --env "elasticsearch.ssl.certificate=/usr/share/app-search/config/certs/testnode.crt" \
+  --env "elasticsearch.ssl.certificate_authority=/usr/share/app-search/config/certs/ca.crt" \
+  --env "elasticsearch.ssl.key=/usr/share/app-search/config/certs/testnode.key" \
+  --volume $ssl_cert:/usr/share/app-search/config/certs/testnode.crt \
+  --volume $ssl_key:/usr/share/app-search/config/certs/testnode.key \
+  --volume $ssl_ca:/usr/share/app-search/config/certs/ca.crt \
+  --publish "$http_port":3002 \
+  --detach="$DETACH" \
+  --health-cmd="curl --insecure --fail ${url} || exit 1" \
+  --health-interval=30s \
+  --health-retries=50 \
+  --health-timeout=10s \
+  --rm \
+  docker.elastic.co/enterprise-search/enterprise-search:"$STACK_VERSION";
+
+if wait_for_container "$CONTAINER_NAME" "$network_name"; then
+  echo -e "\033[32;1mSUCCESS:\033[0m Running on: ${url}\033[0m"
+fi

data/.ci/run-local.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+#
+export TEST_SUITE=platinum
+export CONTAINER_NAME=enterprise-search
+
+script_path=$(dirname $(realpath -s $0))
+source $script_path/functions/imports.sh
+set -euo pipefail
+
+echo -e "\033[1m>>>>> Start [$STACK_VERSION container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
+DETACH=true bash .ci/run-elasticsearch.sh
+
+echo -e "\033[1m>>>>> Running run-enterprise-search.sh >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
+bash .ci/run-enterprise-search.sh

data/.ci/run-repository.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+#
+# Called by entry point `run-test` use this script to add your repository specific test commands
+#
+# Once called Elasticsearch is up and running
+#
+# Its recommended to call `imports.sh` as defined here so that you get access to all variables defined there
+#
+# Any parameters that test-matrix.yml defines should be declared here with appropiate defaults
+
+script_path=$(dirname $(realpath -s $0))
+source $script_path/functions/imports.sh
+set -euo pipefail
+
+echo -e "\033[34;1mINFO:\033[0m VERSION: ${STACK_VERSION}\033[0m"
+echo -e "\033[34;1mINFO:\033[0m TEST_SUITE: ${TEST_SUITE}\033[0m"
+echo -e "\033[34;1mINFO:\033[0m RUNSCRIPTS: ${RUNSCRIPTS}\033[0m"
+echo -e "\033[34;1mINFO:\033[0m URL: ${elasticsearch_url}\033[0m"
+
+echo -e "\033[34;1mINFO:\033[0m pinging Elasticsearch ..\033[0m"
+curl --insecure --fail $external_elasticsearch_url/_cluster/health?pretty
+
+if [[ "$RUNSCRIPTS" = *"enterprise-search"* ]]; then
+  enterprise_search_url="http://localhost:3002"
+  echo -e "\033[34;1mINFO:\033[0m pinging Enterprise Search ..\033[0m"
+  curl -I --fail $enterprise_search_url
+fi
+
+echo -e "\033[32;1mSUCCESS:\033[0m successfully started the ${STACK_VERSION} stack.\033[0m"
+
+echo -e "\033[32;1mBUILD: \033[31mRuby \e[0m container.\033[0m"
+
+docker build \
+       --file .ci/Dockerfile \
+       --tag elastic/enterprise-search-ruby \
+       .
+
+echo -e "\033[32;1mRUN: \033[31mRuby \e[0m container.\033[0m"
+
+docker run \
+       --network ${network_name} \
+       --name enterprise-search-ruby \
+       --env "ELASTIC_ENTERPRISE_HOST=http://${CONTAINER_NAME}:3002" \
+       --rm \
+       --volume `pwd`:/code/enterprise-search-ruby \
+       elastic/enterprise-search-ruby \
+       rake spec:integration

data/.ci/run-tests

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+#
+# Version 1.1
+# - Moved to .ci folder and seperated out `run-repository.sh`
+# - Add `$RUNSCRIPTS` env var for running Elasticsearch dependent products
+export TEST_SUITE=platinum
+export CONTAINER_NAME=enterprise-search
+export RUNSCRIPTS=enterprise-search
+
+script_path=$(dirname $(realpath -s $0))
+source $script_path/functions/imports.sh
+set -euo pipefail
+
+echo -e "\033[1m>>>>> Start [$STACK_VERSION container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
+DETACH=true bash .ci/run-elasticsearch.sh
+
+if [[ -n "$RUNSCRIPTS" ]]; then
+  for RUNSCRIPT in ${RUNSCRIPTS//,/ } ; do
+    echo -e "\033[1m>>>>> Running run-$RUNSCRIPT.sh >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
+    CONTAINER_NAME=${RUNSCRIPT} \
+      DETACH=true \
+      bash .ci/run-${RUNSCRIPT}.sh
+  done
+fi
+
+echo -e "\033[1m>>>>> Repository specific tests >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
+bash .ci/run-repository.sh

data/.ci/test-matrix.yml

@@ -0,0 +1,14 @@
+---
+RUBY_VERSION:
+  - 3.0
+  - 2.7
+  - 2.6
+  - 2.5
+
+STACK_VERSION:
+  - 7.x-SNAPSHOT
+
+TEST_SUITE:
+  - platinum
+
+exclude: ~

data/.github/workflows/rubocop.yml

@@ -0,0 +1,15 @@
+name: rubocop
+on: [push, pull_request]
+
+jobs:
+  rubocop:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: 2.7
+    - name: Run Rubocop
+      run: |
+        bundle install
+        bundle exec rubocop

data/.github/workflows/testing.yml

@@ -0,0 +1,40 @@
+name: master
+on: [push, pull_request]
+
+jobs:
+  test-ruby:
+    env:
+      ENDPOINT: http://localhost:8080
+    strategy:
+      matrix:
+        ruby: [ 2.5, 2.6, 2.7, 3.0 ]
+
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - name: Build and test with Rake
+      run: |
+        ruby -v
+        bundle install
+        bundle exec rake spec:client
+  test-jruby:
+    env:
+      ENDPOINT: http://localhost:3002/api/ws/v1
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ jruby-9.2 ]
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - name: Build and test with Rake
+      run: |
+        ruby -v
+        bundle install
+        bundle exec rake spec:client

data/.gitignore

@@ -8,3 +8,7 @@ pkg/*
 .ruby-gemset
 .yardoc
 doc
+.#*
+*junit.xml
+.byebug_history
+coverage

data/.rubocop.yml

@@ -0,0 +1,52 @@
+AllCops:
+  TargetRubyVersion: 2.5
+Layout/EndAlignment:
+  AutoCorrect: true
+Style/StringLiterals:
+  EnforcedStyle: single_quotes
+Layout/LineLength:
+  Max: 120
+Style/SymbolArray:
+  EnforcedStyle: brackets
+Style/WordArray:
+  EnforcedStyle: brackets
+Layout/EmptyLinesAroundAttributeAccessor:
+  Enabled: true
+Layout/SpaceAroundMethodCallOperator:
+  Enabled: true
+Lint/DeprecatedOpenSSLConstant:
+  Enabled: true
+Lint/MixedRegexpCaptureTypes:
+  Enabled: true
+Lint/RaiseException:
+  Enabled: true
+Lint/StructNewOverride:
+  Enabled: true
+Naming/FileName:
+  Enabled: false
+Style/ExponentialNotation:
+  Enabled: true
+Style/HashEachMethods:
+  Enabled: true
+Style/HashTransformKeys:
+  Enabled: true
+Style/HashTransformValues:
+  Enabled: true
+Style/RedundantFetchBlock:
+  Enabled: true
+Style/RedundantRegexpCharacterClass:
+  Enabled: true
+Style/RedundantRegexpEscape:
+  Enabled: true
+Style/SlicingWithRange:
+  Enabled: true
+Metrics/BlockLength:
+  Exclude:
+    - 'spec/**/*'
+Metrics/MethodLength:
+  Max: 15
+Metrics/AbcSize:
+  Max: 20
+Style/RegexpLiteral:
+  EnforcedStyle: mixed
+  AllowInnerSlashes: true