mtr_monitor 0.13.2 → 0.13.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b53b90dc83a1ef94404e83554ab2f3e7a9cf6452
-  data.tar.gz: 617c0477ee74365fc37ca38ec81dbdc596363af7
+  metadata.gz: 011e48e8082ba84320bb8cd5fd46accdd903b0a4
+  data.tar.gz: 258ff5f5ca66c2394d968430f9b0538541b3521f
 SHA512:
-  metadata.gz: e12d0e58f8843b0a3ebaf80d94dbb03544b0442a76d1c295a805b311fa3e28dfb0970ae58152e619fc3ae886a50c5ddc96a83f667f6dc413ba5a5a86f40d8fd6
-  data.tar.gz: 00f9dfbdb1ba60f162f3d294b18bad0031703e3051ae7a13f411ffa13b344ea7d7f68eacad27fa476c41da30802babc8d265e9f59d99c6049c153fe6372033dd
+  metadata.gz: 5321c6d62702880ba38db94ee1cac1df6394417807062334cbc7d14fe157ebaf4a3260993c2f720b12df9b5c488b64763b12c009e27383e1340ddd176500b172
+  data.tar.gz: a6dc9538ddf761c7ea47427aca2034227ef7d883984113571a6e018c7dd7b06548d3d9f572c983fdb39af3c7d2472f5bea3890a5a0cc645d8f19b9c48e817946

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    mtr_monitor (0.13.2)
+    mtr_monitor (0.13.3)
       rt-watchman (~> 0.10.0)
 
 GEM

data/README.md

@@ -36,6 +36,63 @@ Dashboards for the MTR monitor can be found on the
 [Platform — Network](https://semaphore.grafana.net/dashboard/db/platform-network?refresh=10s&orgId=1)
 dashboard on Grafana.
 
+The US based MTR monitors have the following DNS addresses:
+
+- `mtr-monitor.us-east-1.semaphoreci.com`
+- `mtr-monitor.us-west-1.semaphoreci.com`
+- `mtr-monitor.us-west-2.semaphoreci.com`
+
+To SSH into the, run `ssh ubuntu@<address>`
+
+## Location of the generated MTR reports
+
+The MTR monitor generate and stores MTR reports both on the local machine, and
+uploads them to S3.
+
+Local reports on the machine are located in the `/var/log/mtr` directory, and
+the following structure:
+
+``` txt
+/var/log/mtr/<name>-<YYYY-DD-MM>-<host-ip-address>-<HH-MM>.log
+```
+
+For example, if you call your report `hetzner-to-us-east-1` and run it at
+`2017-12-18 12:33:06`, the log will be generated in:
+
+``` txt
+/var/log/mtr/hetzner-to-us-east-1-2017-12-18-142-21-43-11-12-33.log
+```
+
+On S3, the path will follow the same convention, but will use a nested directory
+structure:
+
+``` txt
+s3://<bucket-name>/<name>/<YYYY-DD-MM>/<host-ip-address>/<HH-MM>.log
+```
+
+``` txt
+s3://<bucket-name>/hetzner-to-us-east-1/2017-12-18/142-21-43-11/12-33.log
+```
+
+## Report Name
+
+The name of the report is used to group reports with the same purpose on S3 and
+on the local file system.
+
+We use the following naming convention:
+
+``` txt
+<from>-to-<destination>
+```
+
+Examples:
+
+``` txt
+hetzner-to-github
+us-east-1-to-hetzner-sb1
+hetzner-to-us-west-2
+```
+
 ## Using MTR Monitor as a gem
 
 The MTR monitor can be used as a gem and injected into existing Ruby
@@ -85,12 +142,67 @@ information read the code in `lib/mtr_monitor/metrics.rb`.
 
 ## Using MTR Monitor as a standalone Docker container
 
+The MTR monitor can be used as a standalone Docker container. This is our
+current approach for monitors that are hitting Germany from the United States.
+
+To run a standalone MTR monitor, run the following command:
+
 ``` bash
-docker run -d -v /var/log/mtr:/var/log/mtr -e NAME=<> -e DOMAIN=<> -e MTR_OPTIONS=<> -e S3_BUCKET=<> -e AWS_ACCESS_KEY_ID=<> -e AWS_SECRET_ACCESS_KEY=<> -e SLEEP_TIME=<> renderedtext/mtr_monitor
+docker run --name mtr-monitor -d -v /var/log/mtr:/var/log/mtr -e NAME=<> -e DOMAIN=<> -e MTR_OPTIONS=<> -e S3_BUCKET=<> -e AWS_ACCESS_KEY_ID=<> -e AWS_SECRET_ACCESS_KEY=<> -e SLEEP_TIME=<> renderedtext/mtr_monitor
 ```
 
-### Generate an MTR report from Ruby
+By default, the containers running on us-east-1, us-west-1, and us-west-2 are
+automatically deployed on every merge into master in for this repository.
+
+The new container on the machine will trigger a MTR report generation every 5
+minutes. Every time a Report is generated the following is executed:
+
+- a new MTR report is generate on your local system under the `/var/log/mtr` directory
+- the report is uploaded to the provided S3 bucket
+- metrics are submitted via Watchman and a pulse is generated
+- the MTR cleaner is uninitiated that cleans all reports from the local system
+  that are older then 2 weeks
+
+To monitor if the CRON task is running as expected, you should
+set up an alert on Grafana based on the "pulse" metric.
+
+The pulse metric has the format `network.mtr.pulse` and is tagged with the
+hostname of the server where the MTR monitor is running and with the name of the
+metric.
+
+MTR hops are also submitted to Grafana. Based on these metrics you can observe
+the packet loss, avg, best, and worst latency on the network. For more
+information read the code in `lib/mtr_monitor/metrics.rb`.
+
+#### Setting up a new EC2 machine for a MTR monitor
+
+1. Buy a new EC2 machine on AWS. Choose, a `t2-nano` instance type with Ubuntu
+   14.04 operating system.
+
+2. SSH into the machine with the newly generated SSH keypair.
+
+3. Add RT developers to the authorized keys file. For a list of public keys,
+   refer to `s3://renderedtext-secrets/stg1-semaphore/authorized-keys`.
+
+4. Install docker. Run `curl https://get.docker.com | curl`.
+
+5. Add the `ubuntu` user to docker group. `sudo usermod -aG docker ubuntu`
+
+6. Re-login into the SSH session.
+
+7. Pull and Run the MTR monitor:
+
+``` bash
+docker run --name mtr-monitor -d -v /var/log/mtr:/var/log/mtr -e NAME=<> -e DOMAIN=<> -e MTR_OPTIONS=<> -e S3_BUCKET=<> -e AWS_ACCESS_KEY_ID=<> -e AWS_SECRET_ACCESS_KEY=<> -e SLEEP_TIME=<> renderedtext/mtr_monitor
+```
+
+If you want to keep this machine permanently, add it to the list of continuously
+deployed servers.
+
+#### Continuously deploying MTR monitor to a EC2 machine
 
+TODO @bmarkons
 
-Invoke mtr generation:
+#### Set up Alerts and Monitoring for a MTR monitor
 
+TODO @bmarkons

data/lib/mtr_monitor.rb

@@ -7,9 +7,19 @@ require "mtr_monitor/metrics"
 require "mtr_monitor/cleaner"
 
 module MtrMonitor
+
+  #
+  # MtrMonitor generates a MTR report for a domain and saves the report to:
+  # /var/log/mtr/<name>-<YYYY-DD-MM>-<host-ip-address>-<HH-MM>.log
+  #
+  # After that, it uploads the report to S3:
+  # s3://<bucket-name>/<name>/<YYYY-DD-MM>/<host-ip-address>/<HH-MM>.log
   #
-  # Runs a mtr trace toward a domain and saves the report
-  # to /var/log/mtr/<domain>_timestamp.log , uploads to s3 and reports metrics to statsd.
+  # Parameters:
+  #
+  # Name:        The name of this report.    Example: "hetzner-to-us-east-1"
+  # Domain:      The domain to hit with MTR. Example: "github.com"
+  # MTR Options: Options to pass to the mtr command line tool.
   #
 
   REPORTS_PATH = "/var/log/mtr"
@@ -27,14 +37,15 @@ module MtrMonitor
       @aws_access_key_id     = aws_access_key_id
       @aws_secret_access_key = aws_secret_access_key
       @mtr_options           = mtr_options
-      @host_ip_address       = `curl http://ifconfig.co`.strip.gsub(".", "-")
+      @host_ip_address       = `dig +short myip.opendns.com @resolver1.opendns.com`.strip.gsub(".", "-")
       @hostname              = `hostname`.strip.gsub(".", "-")
       @dig_ip_address        = dig_ip_address
       @logger                = logger || Logger.new(STDOUT)
 
-      @log_name  = "#{@name}/#{Time.now.strftime("%Y-%m-%d")}/#{@host_ip_address}/#{Time.now.strftime("%H-%M")}.log"
-      @log_path  = "#{MtrMonitor::REPORTS_PATH}/#{@log_name.gsub("/", "-")}"
-      @s3_path   = "s3://#{@s3_bucket}/#{@log_name}"
+      path = "#{@name}/#{Time.now.strftime("%Y-%m-%d")}/#{@host_ip_address}/#{Time.now.strftime("%H-%M")}.log"
+
+      @log_path  = "#{MtrMonitor::REPORTS_PATH}/#{path.gsub("/", "-")}"
+      @s3_path   = "s3://#{@s3_bucket}/#{path}"
     end
 
     def generate

data/lib/mtr_monitor/version.rb

@@ -1,3 +1,3 @@
 module MtrMonitor
-  VERSION = "0.13.2"
+  VERSION = "0.13.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mtr_monitor
 version: !ruby/object:Gem::Version
-  version: 0.13.2
+  version: 0.13.3
 platform: ruby
 authors:
 - RenderedText DevOps Team
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-01-03 00:00:00.000000000 Z
+date: 2018-01-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rt-watchman