mtr_monitor 0.18.2 → 0.19.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 747f1b3e515f1cfa0fe1a872a8b5078c11c3d625
-  data.tar.gz: 405a074875ea2635ce3496e10fe7769ccb2bf3b4
+  metadata.gz: aeabbe48387119e1600b78288faf7a8020eabb01
+  data.tar.gz: bfd527edfa5ff5269ef72763d7521e431de43624
 SHA512:
-  metadata.gz: 384338753adc39791c06f67d6ef6ba403e2c17a08a35ad90cb05d3196c1643058dc841eea113b7d5c6781792605a1ea1e2b1e5d2b7b4a22987f38be7cb02f0ad
-  data.tar.gz: dab41ed813f7ebec31de3b11fe39d1f7fd3aa5f15446cb8b8a2caa3adebb120f7de4488d599f67f571508cc77d547094d669b28a6dd764b4c0b272b64caef8b3
+  metadata.gz: edbf53bf490b0714e699b6f57cc40fb3f01058822cc5ea1ab0f15dbbf0c5ab142fb14261dd1c61f5cb09f08dd0190a17c4393320af5d589b89ce11bc5887ddfc
+  data.tar.gz: 7b98dd952a32caeadf513029bc9364164c103a6f39140a47b816dc033fedd5b81e742dd1ad749de113fe98130f77d222de757c7cd1ee3dc79063b8a79802b9a1

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    mtr_monitor (0.18.2)
+    mtr_monitor (0.19.0)
       logdna
       rt-watchman (~> 0.10.0)
 

data/Makefile

@@ -0,0 +1,43 @@
+.PHONY: publish-gem gem-publish monitor-build monitor-push http-endpoint-build http-endpoint-push deploy-us-east-1 deploy-us-west-1 deploy-us-west-2
+
+gem-build:
+	gem install bundler
+	bundle install
+	chmod 0600 /home/runner/.gem/credentials
+	gem build mtr_monitor.gemspec
+
+gem-publish: gem-build
+	gem push mtr_monitor-*
+
+monitor-build:
+	docker build -t renderedtext/mtr-monitor .
+
+monitor-push: monitor-build
+	docker push renderedtext/mtr-monitor
+
+http-endpoint-build:
+	cd http_endpoint && docker build -t renderedtext/mtr-nginx .
+
+http-endpoint-push: http-endpoint-build
+	docker push renderedtext/mtr-nginx
+
+deploy-us-east-1:
+	NAME="us-east-1-to-hetzner"                         \
+	SERVER="ec2.mtr-monitor.us-east-1.semaphoreci.com"  \
+	HOSTNAME="mtr-monitor.us-east-1.semaphoreci.com"    \
+	MTR_OPTIONS="-z -c 50 -f 12"                           \
+	bundle exec rake deploy
+
+deploy-us-west-1:
+	NAME="us-west-1-to-hetzner"                         \
+	SERVER="ec2.mtr-monitor.us-west-1.semaphoreci.com"  \
+	HOSTNAME="mtr-monitor.us-west-1.semaphoreci.com"    \
+	MTR_OPTIONS="-z -c 50 -f 12"                           \
+	bundle exec rake deploy
+
+deploy-us-west-2:
+	NAME="us-west-2-to-hetzner"                         \
+	SERVER="ec2.mtr-monitor.us-west-2.semaphoreci.com"  \
+	HOSTNAME="mtr-monitor.us-west-2.semaphoreci.com"    \
+	MTR_OPTIONS="-z -c 50 -f 12"                           \
+	bundle exec rake deploy

data/README.md

@@ -6,6 +6,9 @@ In December 2017, Hetzner, our hosting provider for the Build Platform, had a
 major network incident that lasted for almost a whole week. Our users were
 rightly frustrated.
 
+You can find more information about the incident
+[in our public Post Mortem](http://semaphoreci.com/blog/2018/01/18/dec-12-17-network-incident-report.html).
+
 To prevent and monitor these situation in the future, we have set up a
 transatlantic monitoring system based on MTR reports and Curl-ing important
 vendors for our platform such are GitHub and DockerHub. This system should
@@ -15,34 +18,42 @@ DockerHub).
 This project is part of the effort to have a readily available MTR reports
 before, during and after incidents, that we can send to Hetzner.
 
-The MTR monitor is an application that generates MTR reports every 5 minutes and
-uploads them to an S3 Bucket. It is available as a standalone Docker container,
-and as a Ruby gem that can be injected into other Ruby applications.
+The project consists of two parts. A MTR monitor that continiously tests the
+quality of the network by running `mtr` from both sides of the Atlantic, and
+CURL monitor that continiously tries to eastablish a HTTPS connection to the
+other side of the Atlantic.
+
+MTR reports are generated every 5 minutes and uploaded to an S3 bucket. Results
+of CURL tests are displayed on the
+[Platform — Network](https://semaphore.grafana.net/dashboard/db/platform-network?refresh=10s&orgId=1)
+Grafana dashboard and are connected to PagerDuty based alerts.
 
 Currently, we have the following routes covered:
 
 - Germany(Hetzner) -> AWS US East 1 (part of Job Runner)
 - Germany(Hetzner) -> AWS US West 1 (part of Job Runner)
 - Germany(Hetzner) -> AWS US West 2 (part of Job Runner)
-- Germany(Hetzner) -> GitHub (part of Job Runner)
-- Germany(Hetzner) -> DockerHub (part of Job Runner)
-- Germany(Hetzner) -> Stripe (part of Job Runner)
-- Germany(Hetzner) -> SemaphoreCI (part of Job Runner)
 - AWS US East 1 -> Builder sb1 in Hetzner (standalone AWS instance with Docker container)
 - AWS US West 1 -> Builder sb1 in Hetzner (standalone AWS instance with Docker container)
 - AWS US West 2 -> Builder sb1 in Hetzner (standalone AWS instance with Docker container)
 
-Dashboards for the MTR monitor can be found on the
-[Platform — Network](https://semaphore.grafana.net/dashboard/db/platform-network?refresh=10s&orgId=1)
-dashboard on Grafana.
+The tests from Germany are executed from every Builder machine, where this
+project is injected as a gem.
 
-The US based MTR monitors have the following DNS addresses:
+The DNS records of the US based MTR monitors are the following:
 
 - `mtr-monitor.us-east-1.semaphoreci.com`
 - `mtr-monitor.us-west-1.semaphoreci.com`
 - `mtr-monitor.us-west-2.semaphoreci.com`
 
-To SSH into the server, run `ssh ubuntu@<address>`
+These records point to the Load Balancer. If you want to SSH into the machines,
+use the following commands:
+
+- `ssh ubuntu@ec2.mtr-monitor.us-east-1.semaphoreci.com`
+- `ssh ubuntu@ec2.mtr-monitor.us-west-1.semaphoreci.com`
+- `ssh ubuntu@ec2.mtr-monitor.us-west-2.semaphoreci.com`
+
+To create a new MTR monitor follow [this guide](docs/provision-new-ec2-machine.md).
 
 ## Location of the generated MTR reports
 
@@ -163,16 +174,10 @@ To update the version used in Job Runner:
 The MTR monitor can be used as a standalone Docker container. This is our
 current approach for monitors that are hitting Germany from the United States.
 
-To run a standalone MTR monitor, run the following command:
-
-``` bash
-docker run --name mtr-monitor -d -v /var/log/mtr:/var/log/mtr -e NAME=<> -e DOMAIN=<> -e MTR_OPTIONS=<> -e S3_BUCKET=<> -e AWS_ACCESS_KEY_ID=<> -e AWS_SECRET_ACCESS_KEY=<> -e SLEEP_TIME=<> renderedtext/mtr_monitor
-```
-
 By default, the containers running on us-east-1, us-west-1, and us-west-2 are
 automatically deployed on every merge into master in for this repository.
 
-The new container on the machine will trigger a MTR report generation every 5
+The container on the ec2 machines will trigger a MTR report generation every 5
 minutes. Every time a Report is generated the following is executed:
 
 - a new MTR report is generate on your local system under the `/var/log/mtr` directory
@@ -192,49 +197,9 @@ MTR hops are also submitted to Grafana. Based on these metrics you can observe
 the packet loss, avg, best, and worst latency on the network. For more
 information read the code in `lib/mtr_monitor/metrics.rb`.
 
-#### Setting up a new EC2 machine for a MTR monitor
-
-1. Buy a new EC2 machine on AWS. Choose, a `t2-nano` instance type with Ubuntu
-   14.04 operating system.
-
-2. SSH into the machine with the newly generated SSH keypair.
-
-3. Add RT developers to the authorized keys file. For a list of public keys,
-   refer to `s3://renderedtext-secrets/stg1-semaphore/authorized-keys`.
-
-4. Install docker. Run `curl https://get.docker.com | curl`.
-
-5. Add the `ubuntu` user to docker group. `sudo usermod -aG docker ubuntu`
-
-6. Re-login into the SSH session.
-
-7. Pull and Run the MTR monitor:
-
-``` bash
-docker run --name mtr-monitor -d -v /var/log/mtr:/var/log/mtr -e NAME=<> -e DOMAIN=<> -e MTR_OPTIONS=<> -e S3_BUCKET=<> -e AWS_ACCESS_KEY_ID=<> -e AWS_SECRET_ACCESS_KEY=<> -e SLEEP_TIME=<> renderedtext/mtr_monitor
-```
-
-If you want to keep this machine permanently, add it to the list of continuously
-deployed servers.
-
-#### Continuously deploying MTR monitor to a EC2 machine
-
-To deploy MTR monitor as docker container within EC2 machine:
-1. Make sure Docker is installed:
-```
-curl https://get.docker.com | sudo bash
-```
-
-2. Run deploy rake task in [Rakefile](https://github.com/renderedtext/mtr-monitor/blob/master/Rakefile#L8) by passing server IP address like:
-```
-SERVER=1.2.3.4 bundle exec rake deploy
-```
-
-#### Set up Alerts and Monitoring for a MTR monitor
-
-Each deployed MTR monitor needs to has its own panel on grafana [Platform - Network](https://semaphore.grafana.net/dashboard/db/platform-network?orgId=1) dashboard.
-
-Make sure to setup BuildServers alert.
+This is deployed as a docker-compose group of docker images. One docker images
+generates the MTR reports, while the other onw exposes an nginx server that
+responsd yes to incomming requests.
 
 ### MTR incident tracing
 

data/Rakefile

@@ -5,35 +5,38 @@ RSpec::Core::RakeTask.new(:spec)
 
 task :default => :spec
 
-desc "Deploys mtr-monitor to servers"
+desc "Generic way to deploy mtr-monitor to a server"
 task :deploy do
   server = ENV.fetch("SERVER")
 
-  output =
-    `ssh -o StrictHostKeychecking=no ubuntu@#{server} <<'ENDSSH'
-       docker stop mtr-monitor || true
-       docker rm mtr-monitor || true
-       docker system prune --force
-       docker pull renderedtext/mtr-monitor
-       docker run -d \
-         --name mtr-monitor \
-         --hostname "#{server}" \
-         -v /var/log/mtr:/var/log/mtr \
-         -e DIG_IP_ADDRESS="#{ENV['DIG_IP_ADDRESS']}" \
-         -e WATCHMAN_HOST="#{ENV['WATCHMAN_HOST']}" \
-         -e NAME="#{ENV['NAME']}" \
-         -e DOMAIN="#{ENV['DOMAIN']}" \
-         -e MTR_OPTIONS="#{ENV['MTR_OPTIONS']}" \
-         -e S3_BUCKET="#{ENV['S3_BUCKET']}" \
-         -e AWS_ACCESS_KEY_ID="#{ENV['AWS_ACCESS_KEY_ID']}" \
-         -e AWS_SECRET_ACCESS_KEY="#{ENV['AWS_SECRET_ACCESS_KEY']}" \
-         -e SLEEP_TIME="#{ENV['SLEEP_TIME']}" \
-         -e LOGDNA_INGESTION_KEY="#{ENV['LOGDNA_INGESTION_KEY']}" \
-         renderedtext/mtr-monitor
-
-       docker ps | grep mtr-monitor
-ENDSSH`
+  File.open(".env", "w") do |f|
+    f.write("LOGDNA_INGESTION_KEY=#{ENV.fetch("LOGDNA_INGESTION_KEY")}\n")
+    f.write("DOMAIN=#{ENV.fetch("DOMAIN")}\n")
+    f.write("WATCHMAN_HOST=#{ENV.fetch("WATCHMAN_HOST")}\n")
+    f.write("SLEEP_TIME=#{ENV.fetch("SLEEP_TIME")}\n")
+    f.write("AWS_SECRET_ACCESS_KEY=#{ENV.fetch("AWS_SECRET_ACCESS_KEY")}\n")
+    f.write("AWS_ACCESS_KEY_ID=#{ENV.fetch("AWS_ACCESS_KEY_ID")}\n")
+    f.write("S3_BUCKET=#{ENV.fetch("S3_BUCKET")}\n")
+    f.write("MTR_OPTIONS=#{ENV.fetch("MTR_OPTIONS")}\n")
+    f.write("NAME=#{ENV.fetch("NAME")}\n")
+    f.write("DIG_IP_ADDRESS=#{ENV.fetch("DIG_IP_ADDRESS")}\n")
+    f.write("HOSTNAME=#{ENV.fetch("HOSTNAME")}\n")
+  end
+
+  run "scp -o StrictHostKeyChecking=no .env ubuntu@#{server}:/home/ubuntu/"
+  run "scp -o StrictHostKeyChecking=no docker-compose.yml ubuntu@#{server}:/home/ubuntu/"
+  run "ssh -o StrictHostKeychecking=no ubuntu@#{server} 'docker-compose down --rmi all && docker-compose up -d'"
+  run "rm .env"
+end
+
+def run(command)
+  puts "\nExecuting: #{command}"
+
+  output = `#{command}`
 
   puts output
-  exit 1 if $?.exitstatus != 0
+
+  abort if $?.exitstatus != 0
+
+  output
 end

data/docker-compose.yml

@@ -1,5 +1,24 @@
 version: "2.0"
 
 services:
-  app:
-    build: .
+  mtr-monitor:
+    image: renderedtext/mtr-monitor:latest
+    hostname: ${HOSTNAME}
+    volumes:
+      - /var/log/mtr:/var/log/mtr
+    environment:
+      - LOGDNA_INGESTION_KEY
+      - DOMAIN
+      - WATCHMAN_HOST
+      - SLEEP_TIME
+      - AWS_SECRET_ACCESS_KEY
+      - AWS_ACCESS_KEY_ID
+      - S3_BUCKET
+      - MTR_OPTIONS
+      - NAME
+      - DIG_IP_ADDRESS
+
+  nginx:
+    image: renderedtext/mtr-nginx
+    ports:
+      - 80:80

data/docs/provision-new-ec2-machine.md

@@ -0,0 +1,53 @@
+# Setting up a new EC2 machine for a MTR monitor
+
+1. Buy a new EC2 machine on AWS. Choose, a `t2-nano` instance type with Ubuntu
+   14.04 operating system. Set up security group that allows access to SSH and
+   HTTP ports. Generated an SSH keypair.
+
+2. In the ec2 console, name your server by following this scheme
+   `mtr-monitor-<location>`. For example, in Oregon, you should call it
+   `mtr-monitor-us-west-2`.
+
+3. Create a Route 53 entry for the new server. Use the Public DNS of the server
+   and a CNAME rule in Route 53. Use the following scheme to name your server
+   `ec2.mtr-monitor.<location>.semaphoreci.com`. For example, in oregon, you
+   should use `ec2.mtr-monitor.us-west-2.semaphoreci.com`.
+
+4. SSH into the machine. `chmod 0400 <pemfile> && ssh -i <pemfile> ubuntu@ec2.mtr-monitor.<location>.semaphoreci.com`
+
+5. Add RT developers to the authorized keys file. For a list of public keys,
+   refer to `s3://renderedtext-secrets/stg1-semaphore/authorized-keys`.
+
+6. Install docker. Run `curl https://get.docker.com | sh`.
+   Add the `ubuntu` user to docker group. `sudo usermod -aG docker ubuntu`.
+   Re-login into the SSH session.
+
+7. Install docker-compose. Run:
+
+``` bash
+sudo curl -L https://github.com/docker/compose/releases/download/1.18.0/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose
+sudo chmod +x /usr/local/bin/docker-compose
+docker-compose -v
+```
+
+8. Provide docker credentials for docker pull. `docker login` as rtrobot.
+
+9. Add a new deployment target to the Makefile, and trigger the deploy from
+   Semaphore.
+
+10. Set up a load balancer that exposes the HTTP endpoint of the server to the
+   public. Set the name of the load balancer to `mtr-monitor-<location>`. Choose
+   HTTPS to HTTP configuration. Create a new security group with the same name
+   that allows 443 port from all sources. Choose the standard semaphoreci.com
+   certificate from the list, and set up the default health check. Finally, add
+   the ec2 machine to the load balancer
+
+11. Set up a DNS enpoint in Route 53 for the new load balancer `mtr-monitor.<location>.semaphoreci.com`.
+
+#### Set up Alerts and Monitoring for a MTR monitor
+
+Each deployed MTR monitor needs to has its own panel on grafana
+[Platform - Network](https://semaphore.grafana.net/dashboard/db/platform-network?orgId=1)
+dashboard.
+
+Make sure to setup BuildServers alert.

data/http_endpoint/Dockerfile

@@ -0,0 +1,3 @@
+FROM nginx:latest
+
+COPY nginx.conf /etc/nginx/nginx.conf

data/http_endpoint/nginx.conf

@@ -0,0 +1,26 @@
+worker_processes 1;
+
+events {
+  worker_connections  1024;
+}
+
+http {
+    server {
+        listen   80;
+        listen   [::]:80;
+
+        client_body_timeout 12;
+        client_header_timeout 12;
+        keepalive_timeout 15;
+        send_timeout 10;
+
+        client_body_buffer_size 10K;
+        client_header_buffer_size 1k;
+        client_max_body_size 1m;
+        large_client_header_buffers 2 1k;
+
+        location / {
+          return 200 'yes';
+        }
+    }
+}

data/lib/mtr_monitor.rb

@@ -89,6 +89,8 @@ module MtrMonitor
         gtt
       elsif mtr_log.include?("ntt.net")
         ntt
+      elsif mtr_log.include?("core-backbone.com")
+        core_backbone
       else
         unknown
       end
@@ -109,6 +111,11 @@ module MtrMonitor
       @logger.info("Network provider: NTT -- Emitted to LogDNA")
     end
 
+    def core_backbone
+      @logdna.log("Core Backbone")
+      @logger.info("Network provider: Core Backbone -- Emitted to LogDNA")
+    end
+
     def unknown
       @logdna.log("Unknown network provider ")
       @logger.info("Network provider: Unknown -- Emitted to LogDNA")

data/lib/mtr_monitor/version.rb

@@ -1,3 +1,3 @@
 module MtrMonitor
-  VERSION = "0.18.2"
+  VERSION = "0.19.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mtr_monitor
 version: !ruby/object:Gem::Version
-  version: 0.18.2
+  version: 0.19.0
 platform: ruby
 authors:
 - RenderedText DevOps Team
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-01-11 00:00:00.000000000 Z
+date: 2018-01-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logdna
@@ -96,13 +96,17 @@ files:
 - Gemfile
 - Gemfile.lock
 - LICENSE.txt
+- Makefile
 - README.md
 - Rakefile
 - bin/console
 - bin/setup
 - docker-compose.yml
+- docs/provision-new-ec2-machine.md
 - exe/mtr-incident-trace
 - exe/mtr_report
+- http_endpoint/Dockerfile
+- http_endpoint/nginx.conf
 - lib/mtr_monitor.rb
 - lib/mtr_monitor/cleaner.rb
 - lib/mtr_monitor/hop.rb