instana 1.215.1 → 1.216.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7087be480ee9027905afc8e29781253aa34e13fda02dec5607840d61cb4699c0
-  data.tar.gz: c73ad67e15ec9a21051f888ac443f35a39f2e9e9e7c1efeb3bd0824a41bdaad3
+  metadata.gz: d537c1bb144d87403ff4f77159269e31e8af893fd31f6c942d5d6c00a2f89a4a
+  data.tar.gz: 81d978a21bd5bd84c1305851a34570a20c69dd794ccaef7081490db78037c5d4
 SHA512:
-  metadata.gz: 2d09eb59acb9156a0cd5bbd85390406b7da1a72e9bc8692b2084feee186b6c723de72f34218520909226b0b7aa48eeb96708673fd514f8e7ebdc9ad24c84542a
-  data.tar.gz: 1b79ce75d861bcb005ccf46d18d61673faa194be298e5cbfb95ce6fb0a9f503a60c4709fed85b5ab295468c19674005d3ec7e3afda57a03704ab89788013999d
+  metadata.gz: 423d36b04c2e7b6035b566d0bd9617dfd9c2d7700e8d0c8ae934ce2631703cdc4e875ac73a64b136aed2f511a07bb5cfdd257c881a293a2cb85b66bc1d347bd9
+  data.tar.gz: ef472837810e93803daef7776c0b0e664093ddb72cb44f149be2798c9f8ddc0d26eeefe25a9d01a5114e42fb578d050ddc75e03470d1735d1d34eecd5226baa5

data/.circleci/config.yml

@@ -183,6 +183,43 @@ executors:
         environment:
           POSTGRES_PASSWORD: 'test'
           POSTGRES_DB: 'ci_test'
+  ruby_34:
+    docker:
+      - image: ruby:3.4.0-preview1-bookworm
+        environment:
+          MEMCACHED_HOST: '127.0.0.1:11211'
+          REDIS_URL: 'redis://127.0.0.1:6379'
+          DATABASE_URL: 'sqlite3::memory:'
+      - image: memcached
+      - image: redis
+      - image: amazon/dynamodb-local
+      - image: minio/minio:latest
+        command: ["server", "/data"]
+      - image: s12v/sns
+      - image: softwaremill/elasticmq-native
+      - image: mongo:5-focal
+  ruby_34_mysql2:
+    docker:
+      - image: ruby:3.4.0-preview1-bookworm
+        environment:
+          DATABASE_URL: "mysql2://root@127.0.0.1:3306/ci_test"
+      - image: mariadb
+        environment:
+          MYSQL_DATABASE: 'ci_test'
+          MYSQL_USER: 'root'
+          MYSQL_PASSWORD: ''
+          MYSQL_ALLOW_EMPTY_PASSWORD: 'yes'
+          MYSQL_ROOT_PASSWORD: ''
+          MYSQL_ROOT_HOST: '%'
+  ruby_34_postgres:
+    docker:
+      - image: ruby:3.4.0-preview1-bookworm
+        environment:
+          DATABASE_URL: "postgres://postgres:test@127.0.0.1:5432/ci_test"
+      - image: postgres
+        environment:
+          POSTGRES_PASSWORD: 'test'
+          POSTGRES_DB: 'ci_test'
   ruby_33_lint:
     docker:
       - image: cimg/ruby:3.3-node
@@ -319,12 +356,12 @@ jobs:
             bundle check || bundle install
       - run_rubocop
   report_coverage:
-    executor: ruby_32
+    executor: ruby_33
     steps:
       - checkout
       - run_sonarqube
   publish:
-    executor: ruby_30
+    executor: ruby_33
     steps:
       - checkout
       - setup
@@ -350,7 +387,8 @@ workflows:
                 - ruby_31
                 - ruby_32
                 - ruby_33
-  libraries_ruby_3:
+                - ruby_34
+  libraries_ruby_30_32:
     jobs:
       - test_apprisal:
           matrix:
@@ -359,8 +397,26 @@ workflows:
                 - ruby_30
                 - ruby_31
                 - ruby_32
+              <<: *library_gemfile
+  libraries_ruby_33_34:
+    jobs:
+      - test_apprisal:
+          matrix:
+            parameters:
+              stack:
                 - ruby_33
+                - ruby_34
               <<: *library_gemfile
+            # Currently compiling the native extensions for `grpc`
+            # exceeds an internal RAM and/or CPU limit of CircleCI executor,
+            # and the job gets terminated with:
+            # `Received "killed" signal`
+            # Hence we exclude it here for the time being.
+            # TODO: Remove this exclusion as soon as pre-built binaries are available
+            # on https://rubygems.org/gems/grpc/versions
+            exclude:
+            - stack: ruby_34
+              gemfile: "./gemfiles/grpc_10.gemfile"
   rails_ruby_3:
     jobs:
       - test_apprisal:
@@ -376,12 +432,36 @@ workflows:
                 - ruby_32
                 - ruby_32_postgres
                 - ruby_32_mysql2
+                - ruby_33
                 - ruby_33_postgres
                 - ruby_33_mysql2
+                - ruby_34
+                - ruby_34_postgres
+                - ruby_34_mysql2
               gemfile:
                 - "./gemfiles/rails_61.gemfile"
                 - "./gemfiles/rails_70.gemfile"
                 - "./gemfiles/rails_71.gemfile"
+  sequel:
+      jobs:
+      - test_apprisal:
+          matrix:
+            parameters:
+              stack:
+                - ruby_30
+                - ruby_30_mysql2
+                - ruby_31
+                - ruby_31_mysql2
+                - ruby_32
+                - ruby_32_mysql2
+                - ruby_33
+                - ruby_33_mysql2
+                - ruby_34
+                - ruby_34_mysql2
+              gemfile:
+                - "./gemfiles/sequel_56.gemfile"
+                - "./gemfiles/sequel_57.gemfile"
+                - "./gemfiles/sequel_58.gemfile"
   report_coverage:
     jobs:
       - report_coverage

data/.tekton/.currency/resources/requirements.txt

@@ -2,4 +2,3 @@ requests
 pandas
 beautifulsoup4
 tabulate
-kubernetes

data/.tekton/.currency/scripts/generate_report.py

@@ -1,4 +1,5 @@
 # Standard Libraries
+import glob
 import re
 from json import load
 
@@ -6,143 +7,49 @@ from json import load
 from requests import get
 from pandas import DataFrame
 from bs4 import BeautifulSoup
-from kubernetes import client, config
 
 JSON_FILE = "resources/table.json"
 REPORT_FILE = "docs/report.md"
 API_V1_ENDPOINT = "https://rubygems.org/api/v1/versions/"
 
-def filter_taskruns(taskrun_filter, taskruns):
-    filtered_taskruns = list(filter(taskrun_filter, taskruns))
-    filtered_taskruns.sort(
-        key=lambda tr: tr["metadata"]["creationTimestamp"], reverse=True
-    )
-
-    return filtered_taskruns
-
-
-def get_taskruns(namespace, task_name):
-    group = "tekton.dev"
-    version = "v1"
-    plural = "taskruns"
-
-    # access the custom resource from tekton
-    tektonV1 = client.CustomObjectsApi()
-    taskruns = tektonV1.list_namespaced_custom_object(
-        group,
-        version,
-        namespace,
-        plural,
-        label_selector=f"{group}/task={task_name}, triggers.tekton.dev/trigger=ruby-tracer-scheduled-pipeline-triggger",
-    )["items"]
-
-    return taskruns
-
-
-def process_taskrun_logs(
-    taskruns, core_v1_client, namespace, library, tekton_ci_output
-):
-    for tr in taskruns:
-        pod_name = tr["status"]["podName"]
-        taskrun_name = tr["metadata"]["name"]
-        logs = core_v1_client.read_namespaced_pod_log(
-            pod_name, namespace, container="step-unittest"
-        )
-        if "Installing" not in logs:
-            print(
-                f"Unable to retrieve logs from taskrun pod {pod_name} of taskrun {taskrun_name} for gem {library}."
-            )
-            continue
-
-        print(
-            f"Retrieving logs from taskrun pod {pod_name} of taskrun {taskrun_name} for gem {library}.."
-        )
-
-        match = re.search(f"Installing ({library} [^\s]+)", logs)
-        tekton_ci_output += f"{match[1]}\n"
-        break
-
-    return tekton_ci_output
-
-
-def get_tekton_ci_output():
-    config.load_incluster_config()
-
-    namespace = "default"
-    core_v1_client = client.CoreV1Api()
-
-    ruby_33_prefix = "unittest-default-ruby-33-"
-    ruby_31_prefix = "unittest-default-ruby-31-"
-
-    default_libraries_dict = {
-        "cuba": f"{ruby_33_prefix}1",
-        "excon": f"{ruby_33_prefix}4",
-        "graphql": f"{ruby_33_prefix}6",
-        "grpc": f"{ruby_33_prefix}7",
-        "rack": f"{ruby_33_prefix}10",
-        "rest-client": f"{ruby_33_prefix}11",
-        "roda": f"{ruby_33_prefix}13",
-        "sinatra": f"{ruby_33_prefix}16",
-        "net-http": f"{ruby_31_prefix}8",
+def get_bundle_install_output():
+
+    libraries_from_logs = {
+        "cuba": "cuba_40_ruby_3.3.",
+        "excon": "excon_100_ruby_3.3.",
+        "graphql": "graphql_20_ruby_3.3.",
+        "grpc": "grpc_10_ruby_3.3.",
+        "rack": "rack_30_ruby_3.3.",
+        "rest-client": "rest_client_20_ruby_3.3.",
+        "roda": "roda_30_ruby_3.3.",
+        "sinatra": "sinatra_40_ruby_3.3.",
+        "net-http": "net_http_01_ruby_3.1.",
+        "rails": "rails_71_sqlite3_ruby_3.3.",
+        "dalli": "dalli_32_ruby_3.3.",
+        "resque": "resque_20_ruby_3.3.",
+        "sidekiq": "sidekiq_70_ruby_3.3."
     }
 
-    tekton_ci_output = ""
-    task_name = "ruby-tracer-unittest-default-libraries-task"
-    default_taskruns = get_taskruns(namespace, task_name)
+    bundle_install_output = ""
 
-    for library, pattern in default_libraries_dict.items():
-        taskrun_filter = (
-            lambda tr: tr["metadata"]["name"].endswith(pattern)
-            and tr["status"]["conditions"][0]["type"] == "Succeeded"
-        )
-        filtered_default_taskruns = filter_taskruns(taskrun_filter, default_taskruns)
-
-        tekton_ci_output = process_taskrun_logs(
-            filtered_default_taskruns,
-            core_v1_client,
-            namespace,
-            library,
-            tekton_ci_output,
-        )
+    for library, pattern in libraries_from_logs.items():
+        glob_result = glob.glob(f"../../dep_{pattern}*")
+        if not glob_result:
+            print(f"Could not find bundle install log for gem '{library}'.")
+            continue
 
-    other_libraries_dict = {
-        "rails": {
-            "pattern": "rails-postgres-11",
-            "task_name": "ruby-tracer-unittest-rails-postgres-task",
-        },
-        "dalli": {
-            "pattern": "memcached-11",
-            "task_name": "ruby-tracer-unittest-memcached-libraries-task",
-        },
-        "resque": {
-            "pattern": "unittest-redis-ruby-32-33-9",
-            "task_name": "ruby-tracer-unittest-redis-libraries-task",
-        },
-        "sidekiq": {
-            "pattern": "unittest-redis-ruby-32-33-18",
-            "task_name": "ruby-tracer-unittest-redis-libraries-task",
-        },
-    }
+        with open(glob_result[0], 'r') as file:
+            logs = file.read().replace('\n', ' ')
 
-    for library, inner_dict in other_libraries_dict.items():
-        pattern = inner_dict["pattern"]
-        task_name = inner_dict["task_name"]
-        taskrun_filter = (
-            lambda tr: tr["metadata"]["name"].endswith(pattern)
-            and tr["status"]["conditions"][0]["type"] == "Succeeded"
-        )
-        other_taskruns = get_taskruns(namespace, task_name)
-        filtered_other_taskruns = filter_taskruns(taskrun_filter, other_taskruns)
-
-        tekton_ci_output = process_taskrun_logs(
-            filtered_other_taskruns,
-            core_v1_client,
-            namespace,
-            library,
-            tekton_ci_output
-        )
+        if "Installing" not in logs:
+            print( f"Unable to retrieve logs from for gem '{library}'.")
+            continue
+
+        print(f"Retrieving currency for gem '{library}'.")
+        match = re.search(f"Installing ({library} [^\s]+)", logs)
+        bundle_install_output += f"{match[1]}\n"
 
-    return tekton_ci_output
+    return bundle_install_output
 
 
 def get_upstream_version(dependency):
@@ -161,11 +68,11 @@ def get_upstream_version(dependency):
     return latest_version
 
 
-def get_last_supported_version(tekton_ci_output, dependency):
+def get_last_supported_version(bundle_install_output, dependency):
     """get up-to-date supported version"""
     pattern = r" ([^\s]+)"
 
-    last_supported_version = re.search(dependency + pattern, tekton_ci_output, flags=re.I | re.M)
+    last_supported_version = re.search(dependency + pattern, bundle_install_output, flags=re.I | re.M)
 
     return last_supported_version[1]
 
@@ -183,7 +90,7 @@ def main():
     with open(JSON_FILE) as file:
         data = load(file)
 
-    tekton_ci_output = get_tekton_ci_output()
+    bundle_install_output = get_bundle_install_output()
 
     items = data["table"]
 
@@ -194,7 +101,7 @@ def main():
         latest_version = get_upstream_version(package)
 
         if not package in ["rails lts", "rails-api"]:
-            last_supported_version = get_last_supported_version(tekton_ci_output, package)
+            last_supported_version = get_last_supported_version(bundle_install_output, package)
         else:
             last_supported_version = latest_version
 

data/.tekton/README.md

@@ -0,0 +1,278 @@
+# Tekton CI for Instana Ruby Tracer
+
+## Basic Tekton setup
+
+### Get a cluster
+
+What you will need:
+* Full administrator access
+* Enough RAM and CPU on a cluster node to run all the pods of a single Pipelinerun on a single node.
+  Multiple nodes increase the number of parallel `PipelineRun` instances.
+  Currently one `PipelineRun` instance is capable of saturating a 8vCPU - 16GB RAM worker node.
+
+### Setup Tekton on your cluster
+
+1. Install latest stable Tekton Pipeline release
+```bash
+   kubectl apply --filename https://storage.googleapis.com/tekton-releases/pipeline/latest/release.yaml
+```
+
+2. Install Tekton Dashboard Full (the normal is read only, and doesn't allow for example to re-run).
+
+````bash
+   kubectl apply --filename https://storage.googleapis.com/tekton-releases/dashboard/latest/release-full.yaml
+````
+
+3. Access the dashboard
+
+```bash
+kubectl proxy
+```
+
+Once the proxy is active, navigate your browser to the [dashboard url](
+http://localhost:8001/api/v1/namespaces/tekton-pipelines/services/tekton-dashboard:http/proxy/)
+
+### Setup the ruby-tracer-ci-pipeline
+
+````bash
+   kubectl apply --filename task.yaml && kubectl apply --filename pipeline.yaml
+````
+
+### Run the pipeline manually
+
+#### From the Dashboard
+Navigate your browser to the [pipelineruns section of the dashboard](
+http://localhost:8001/api/v1/namespaces/tekton-pipelines/services/tekton-dashboard:http/proxy/#/pipelineruns)
+
+1. Click `Create`
+2. Select the `Namespace` (where the `Pipeline` resource is created by default it is `default`)
+3. Select the `Pipeline` created in the `pipeline.yaml` right now it is `ruby-tracer-ci-pipeline`
+4. Fill in `Params`. The `revision` should be `master` for the `master` branch
+4. Select the `ServiceAccount` set to `default`
+5. Optionally, enter a `PipelineRun name` for example `my-master-test-pipeline`,
+   but if you don't then the Dashboard will generate a unique one for you.
+6. As long as [the known issue with Tekton Dashboard Workspace binding](
+   https://github.com/tektoncd/dashboard/issues/1283), is not resolved.
+   You have to go to `YAML Mode` and insert the workspace definition at the end of the file,
+   with the exact same indentation:
+
+````yaml
+  workspaces:
+  - name: ruby-tracer-ci-pipeline-pvc-$(params.revision)
+    volumeClaimTemplate:
+      spec:
+        accessModes:
+          - ReadWriteOnce
+        resources:
+          requests:
+            storage: 200Mi
+
+````
+7. Click `Create` at the bottom of the page
+
+
+#### From kubectl CLI
+As an alternative to using the Dashboard, you can manually edit `pipelinerun.yaml` and create it with:
+````bash
+   kubectl apply --filename pipelinerun.yaml
+````
+
+### Clanup PipelineRun and associated PV resources
+
+`PipelineRuns` and workspace `PersistentVolume` resources by default are kept indefinitely,
+and repeated runs might exhaust the available resources, therefore they need to be cleaned up either
+automatically or manually.
+
+#### Manully from the Dashboard
+
+Navigate to `PipelineRuns` and check the checkbox next to the pipelinerun
+and then click `Delete` in the upper right corner.
+
+#### Manually from the CLI
+
+You can use either `kubectl`
+````bash
+kubectl get pipelinerun
+kubectl delete pipelinerun <selected-pipelinerun-here>
+````
+
+or `tkn` cli
+````bash
+tkn pipelinerun list
+tkn pipelinerun delete <selected-pipelinerun-here>
+````
+
+#### Automatic cleanup with a cronjob
+
+Install and configure resources from https://github.com/3scale-ops/tekton-pipelinerun-cleaner
+
+
+## Integrate with GitHub
+
+### GitHub PR Trigger & PR Check API integration
+
+The GitHub integration requires further Tekton Triggers and Interceptors to be installed
+````bash
+kubectl apply --filename \
+https://storage.googleapis.com/tekton-releases/triggers/latest/release.yaml
+kubectl apply --filename \
+https://storage.googleapis.com/tekton-releases/triggers/latest/interceptors.yaml
+````
+#### Create a ServiceAccount
+
+Our future GitHub PR Event listener needs a service account,
+`tekton-triggers-eventlistener-serviceaccount` which authorizes it to
+perform operations specified in eventlistener `Role` and `ClusteRole`.
+Create the service account with the needed role bindings:
+
+````bash
+   kubectl apply --filename tekton-triggers-eventlistener-serviceaccount.yaml
+````
+
+#### Create the Secret for the GitHub repository webhook
+
+In order to authorize the incoming webhooks into our cluster, we need to share
+a secret between our webhook listener, and the GitHub repo.
+Generate a long, strong and random generated token, put it into `github-interceptor-secret.yaml`.
+Create the secret resource:
+````bash
+   kubectl apply --filename github-interceptor-secret.yaml
+````
+
+#### Create the Task and token to report PR Check status to GitHub
+
+The GitHub PR specific Tekton pipeline will want to send data to report the `PR Check Status`.
+That [GitHub API](https://docs.github.com/en/rest/commits/statuses?apiVersion=2022-11-28#create-a-commit-status
+) requires authentication, and therefore we need a token.
+The user which generates the token has to have `Write` access in the target repo,
+as part of the organisation. Check the repo access for this repo under
+https://github.com/instana/ruby-sensor/settings/access.
+
+With the proper user:
+1. Navigate to https://github.com/settings/tokens
+2. Click on `Generate new token` dropdown `Generate new token (classic)`.
+3. Fill in `Note` with for example `Tekton commit status`,
+4. Make sure if you set an expiration, than you remember to renew the token after expiry.
+5. Under `Select scopes` find `repo` and below that only select the checkbox next to `repo:status` - `Access commit status`.
+ click `Generate token`
+6. Create the kubernetes secret with the token:
+
+````bash
+   kubectl create secret generic githubtoken --from-literal token="MY_TOKEN"
+````
+
+And we also make an HTTP POST with the status update data to GitHub.
+This is done in a `Task` called `github-set-status`, create it as such:
+````bash
+   kubectl apply -f github-set-status-task.yaml
+````
+
+#### Create the GitHub PR pipeline
+
+Create the new pipeline, which executes the previously created `ruby-tracer-ci-pipeline`,
+removes the currency report tasks and wraps the unittest jobs with GitHub Check status reporting tasks.
+As long as [Pipelines in Pipelines](
+https://tekton.dev/docs/pipelines/pipelines-in-pipelines/), remains an
+unimplemented `alpha` feature in Tekton,
+we will need the [yq](https://github.com/mikefarah/yq) (at least `4.0`)
+to pull the tasks from our previous `ruby-tracer-ci-pipeline` into the
+new pipeline `github-pr-ruby-tracer-ci-pipeline`.
+
+````bash
+   (cat github-pr-pipeline.yaml.part && yq '{"a": {"b": .spec.tasks}}' pipeline.yaml| tail --lines=+3| head --lines=-16) | kubectl apply -f -
+````
+
+#### Create the GitHub PR Event Listener, TriggerTemplate and TriggerBinding
+
+Once the new GitHub specific pipeline is created, we need a listener which starts
+a new `PipelineRun` based on GitHub events.
+For security reasons the listener will only trigger if the PR owner is a 
+repo owner. PRs from non-repo owners can be allowed in by owners after an initial review
+with a comment on the pull request that contains `/ok-to-test`.
+More abou this feature in the [tekton triggers documentsion](
+https://github.com/tektoncd/triggers/blob/main/docs/interceptors.md#owners-validation-for-pull-requests).
+
+````bash
+   kubectl apply --filename github-pr-eventlistener.yaml
+````
+
+After this ensure that there is a pod and a service created:
+
+````bash
+   kubectl get pod | grep -i el-github-pr-eventlistener
+   kubectl get svc | grep -i el-github-pr-eventlistener
+````
+
+Do not continue if any of these missing.
+
+#### Create the Ingress for the GitHub Webhook to come through
+
+You will need an ingress controller for this.
+On IKS you might want to read these resources:
+* [managed ingress](https://cloud.ibm.com/docs/containers?topic=containers-managed-ingress-about)
+* Or unmanaged [ingress controller howto](
+https://github.com/IBM-Cloud/iks-ingress-controller/blob/master/docs/installation.md
+).
+
+1. Check the available `ingressclass` resources on your cluster
+
+````bash
+   kubectl get ingressclass
+````
+
+* On `IKS` it will be `public-iks-k8s-nginx`.
+* On `EKS` with the `ALB` ingress controller, it might be just `alb`
+* On self hosted [nginx controller](https://kubernetes.github.io/ingress-nginx/deploy/)
+  this might just be `nginx`.
+
+Edit and save the value of `ingressClassName:` in `github-webhook-ingress.yaml`.
+
+2. Find out your Ingress domain or subdomain name.
+
+* On `IKS`, go to `Clusters` select your cluster and then click `Overview`.
+  The domain name is listed under `Ingress subdomain`.
+
+and create the resource:
+
+````bash
+   kubectl apply --filename github-webhook-ingress.yaml
+````
+
+Make sure that you can use the ingress with the `/hooks` path via `https`:
+````bash
+   curl https://<INGRESS_DOMAIN_NAME>/hooks
+````
+
+At this point this should respond this:
+```json
+  {
+   "eventListener":"github-pr-eventlistener",
+   "namespace":"default",
+   "eventListenerUID":"",
+   "errorMessage":"Invalid event body format : unexpected end of JSON input"
+   }
+```
+
+#### Setup the webhook on GitHub
+
+In the GitHub repo go to `Settings` -> `Webhooks` and click `Add Webhook`.
+The fields we need to set are:
+* `Payload URL`: `https://<INGRESS_DOMAIN_NAME>/hooks`
+* `Content type`: application/json
+* `Secret`: XXXXXXX (the secret token from github-interceptor-secret.yaml)
+
+Under `SSL verification` select the radio button for `Enable SSL verification`.
+Under `Which events would you like to trigger this webhook?` select
+the radio button for `Let me select individual events.` and thick the checkbox next to
+`Pull requests` and ensure that the rest are unthicked.
+
+Click `Add webhook`.
+
+If the webhook has been set up correctly, then GitHub sends a ping message.
+Ensure that the ping is received from GitHub, and that it is filtered out so
+a simple ping event does not trigger any `PipelineRun` unnecessarily.
+
+````bash
+eventlistener_pod=$(kubectl get pods -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | grep el-github-pr-ruby)
+kubectl logs "${eventlistener_pod}" | grep 'event type ping is not allowed'
+````

data/.tekton/github-pr-eventlistener.yaml

@@ -38,7 +38,7 @@ spec:
                 - ReadWriteOnce
               resources:
                 requests:
-                  storage: 100Mi
+                  storage: 200Mi
 ---
 apiVersion: triggers.tekton.dev/v1beta1
 kind: TriggerBinding