vagrant-g5k 0.0.15 → 0.0.16

data/results/files/heka/lua_decoders/os_rabbitmq_log.lua

@@ -0,0 +1,79 @@
+-- Copyright 2015-2016 Mirantis, Inc.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+local dt     = require "date_time"
+local l      = require 'lpeg'
+l.locale(l)
+
+local patt   = require 'os_patterns'
+local utils  = require 'os_utils'
+
+
+-- Allow custom type
+t = read_config('type')
+if t == nil or t == '' then
+  t = 'log'
+end
+
+local msg = {
+    Timestamp   = nil,
+    Type        = t,
+    Hostname    = nil,
+    Payload     = nil,
+    Pid         = nil,
+    Fields      = nil,
+    Severity    = nil,
+}
+
+-- RabbitMQ message logs are formatted like this:
+--   =ERROR REPORT==== 2-Jan-2015::09:17:22 ===
+--   Blabla
+--   Blabla
+--
+local message   = l.Cg(patt.Message / utils.chomp, "Message")
+-- The token before 'REPORT' isn't standardized so it can be a valid severity
+-- level as 'INFO' or 'ERROR' but also 'CRASH' or 'SUPERVISOR'.
+local severity  = l.Cg(l.R"AZ"^1, "SeverityLabel")
+local day = l.R"13" * l.R"09" + l.R"19"
+local datetime = l.Cg(day, "day") * patt.dash * dt.date_mabbr * patt.dash * dt.date_fullyear *
+                 "::" * dt.rfc3339_partial_time
+local timestamp = l.Cg(l.Ct(datetime)/ dt.time_to_ns, "Timestamp")
+
+local grammar = l.Ct("=" * severity * " REPORT==== " * timestamp * " ===" * l.P'\n' * message)
+
+function process_message ()
+    local log = read_message("Payload")
+
+    local m = grammar:match(log)
+    if not m then
+        return -1
+    end
+
+    msg.Timestamp = m.Timestamp
+    msg.Payload = m.Message
+    msg.Logger = read_message("Logger")
+
+    if utils.label_to_severity_map[m.SeverityLabel] then
+        msg.Severity = utils.label_to_severity_map[m.SeverityLabel]
+    elseif m.SeverityLabel == 'CRASH' then
+        msg.Severity = 2 -- CRITICAL
+    else
+        msg.Severity = 5 -- NOTICE
+    end
+
+    msg.Fields = {}
+    msg.Fields.severity_label = utils.severity_to_label_map[msg.Severity]
+    msg.Fields.programname = 'rabbitmq'
+
+    return utils.safe_inject_message(msg)
+end

data/results/files/kibana/all_objects.json

@@ -0,0 +1,81 @@
+[
+  {
+    "_id": "Logs",
+    "_type": "dashboard",
+    "_source": {
+      "title": "Logs",
+      "hits": 0,
+      "description": "",
+      "panelsJSON": "[{\"id\":\"Types\",\"type\":\"visualization\",\"panelIndex\":1,\"size_x\":4,\"size_y\":3,\"col\":1,\"row\":1},{\"id\":\"progname\",\"type\":\"visualization\",\"panelIndex\":2,\"size_x\":4,\"size_y\":3,\"col\":5,\"row\":1},{\"id\":\"http_status\",\"type\":\"visualization\",\"panelIndex\":3,\"size_x\":4,\"size_y\":3,\"col\":9,\"row\":1},{\"id\":\"raw\",\"type\":\"search\",\"panelIndex\":4,\"size_x\":12,\"size_y\":6,\"col\":1,\"row\":4,\"columns\":[\"_source\"],\"sort\":[\"Timestamp\",\"desc\"]}]",
+      "optionsJSON": "{\"darkTheme\":false}",
+      "uiStateJSON": "{}",
+      "version": 1,
+      "timeRestore": false,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}}}]}"
+      }
+    }
+  },
+  {
+    "_id": "raw",
+    "_type": "search",
+    "_source": {
+      "title": "raw",
+      "description": "",
+      "hits": 0,
+      "columns": [
+        "_source"
+      ],
+      "sort": [
+        "Timestamp",
+        "desc"
+      ],
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"heka*\",\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647},\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}}}"
+      }
+    }
+  },
+  {
+    "_id": "progname",
+    "_type": "visualization",
+    "_source": {
+      "title": "progname",
+      "visState": "{\"title\":\"progname\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"programname\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
+      "uiStateJSON": "{}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"heka*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "http_status",
+    "_type": "visualization",
+    "_source": {
+      "title": "http_status",
+      "visState": "{\"title\":\"http_status\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"http_status\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
+      "uiStateJSON": "{}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"heka*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "Types",
+    "_type": "visualization",
+    "_source": {
+      "title": "Types",
+      "visState": "{\"title\":\"Types\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"Type\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
+      "uiStateJSON": "{}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"heka*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
+      }
+    }
+  }
+]

data/results/files/nginx.conf

@@ -0,0 +1,34 @@
+user  nginx;
+worker_processes  1;
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+events {
+  worker_connections  1024;
+}
+
+http {
+  include       /etc/nginx/mime.types;
+  # prevent chrome to guess the content-type
+  add_header X-Content-Type-Options nosniff;
+  default_type  text/plain;
+  log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                    '$status $body_bytes_sent "$http_referer" '
+                    '"$http_user_agent" "$http_x_forwarded_for"';
+  access_log  /var/log/nginx/access.log  main;
+  sendfile        on;
+  keepalive_timeout  65;
+  server {
+    listen       80;
+    server_name  localhost;
+    location / {
+        root   /usr/share/nginx/html;
+        autoindex on;
+    }
+
+    error_page   500 502 503 504  /50x.html;
+    location = /50x.html {
+        root   /usr/share/nginx/html;
+    }
+  }
+}

data/results/templates/heka-elasticsearch.toml.j2

@@ -0,0 +1,18 @@
+[elasticsearch_json_encoder]
+type = "ESJsonEncoder"
+#index = "{{ item }}"
+index = "heka"
+es_index_from_timestamp = true
+fields = ["Timestamp", "Type", "Logger", "Severity", "Payload", "Pid", "Hostname", "DynamicFields"]
+
+[elasticsearch_output]
+type = "ElasticSearchOutput"
+# elastic container is likned to heka one
+server = "http://elasticsearch:9200"
+message_matcher = "Type =~ /^log.*/"
+encoder = "elasticsearch_json_encoder"
+use_buffering = true
+    [elasticsearch_output.buffering]
+    max_buffer_size = 1073741824  # 1024 * 1024 * 1024
+    max_file_size = 134217728  # 128 * 1024 * 1024
+    full_action = "drop"

data/results/templates/heka-haproxy.toml.j2

@@ -0,0 +1,10 @@
+[haproxy_log_decoder]
+type = "SandboxDecoder"
+filename = "lua_decoders/haproxy_log.lua"
+
+[haproxy_logstreamer_input]
+type = "LogstreamerInput"
+decoder = "haproxy_log_decoder"
+log_directory = "/var/log/kolla"
+file_match = 'haproxy/haproxy\.log'
+

data/results/templates/heka-keystone.toml.j2

@@ -0,0 +1,14 @@
+[keystone_apache_log_decoder]
+type = "SandboxDecoder"
+filename = "lua_decoders/os_keystone_apache_log.lua"
+    [keystone_apache_log_decoder.config]
+    type = "log-{{ item[1] }}"
+    apache_log_pattern = '%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b %D \"%{Referer}i\" \"%{User-Agent}i\"'
+
+[keystone_apache_logstreamer_input]
+type = "LogstreamerInput"
+decoder = "keystone_apache_log_decoder"
+log_directory = "/var/log/kolla"
+file_match = 'keystone/keystone-apache-(?P<Service>.+)-access\.log\.?(?P<Seq>\d*)$'
+priority = ["^Seq"]
+differentiator = ["keystone-apache-", "Service"]

data/results/templates/heka-mariadb.toml.j2

@@ -0,0 +1,14 @@
+[mariadb_log_decoder]
+type = "SandboxDecoder"
+filename = "lua_decoders/os_mysql_log.lua"
+
+[mariadb_log_decoder.config]
+type = "log-{{ item[1] }}"
+
+[mariadb_logstreamer_input]
+type = "LogstreamerInput"
+decoder = "mariadb_log_decoder"
+log_directory = "/var/log/kolla"
+file_match = 'mariadb/mariadb\.log\.?(?P<Seq>\d*)$'
+priority = ["^Seq"]
+differentiator = ['mariadb']

data/results/templates/heka-openstack.toml.j2

@@ -0,0 +1,15 @@
+[openstack_log_decoder]
+type = "SandboxDecoder"
+filename = "lua_decoders/os_openstack_log.lua"
+
+[openstack_log_decoder.config]
+type = "log-{{ item[1] }}"
+
+[openstack_logstreamer_input]
+type = "LogstreamerInput"
+decoder = "openstack_log_decoder"
+log_directory = "/var/log/kolla"
+file_match = '(?P<Service>cloudkitty|nova|glance|keystone|neutron|ceph|cinder|heat|murano|magnum|mistral|manila|senlin)/(?P<Program>.*)\.log\.?(?P<Seq>\d*)$'
+priority = ["^Seq"]
+differentiator = ["Service", "_", "Program"]
+

data/results/templates/heka-rabbitmq.toml.j2

@@ -0,0 +1,21 @@
+[rabbitmq_log_decoder]
+type = "SandboxDecoder"
+filename = "lua_decoders/os_rabbitmq_log.lua"
+
+[rabbitmq_log_decoder.config]
+type = "log-{{ item[1] }}"
+
+[rabbitmq_log_splitter]
+type = "RegexSplitter"
+delimiter = '\n\n(=[^=]+====)'
+delimiter_eol = false
+deliver_incomplete_final = true
+
+[rabbitmq_logstreamer_input]
+type = "LogstreamerInput"
+decoder = "rabbitmq_log_decoder"
+splitter = "rabbitmq_log_splitter"
+log_directory = "/var/log/kolla"
+file_match = 'rabbitmq/(?P<Service>rabbit.*)\.log\.?(?P<Seq>\d*)$'
+priority = ["^Seq"]
+differentiator = ["Service"]

data/results/test.rb

@@ -0,0 +1,32 @@
+require 'net/ssh/gateway'
+require 'net/http'
+include Process
+
+$gateway = Net::SSH::Gateway.new('access.grid5000.fr', 'msimonin')
+ports = []
+
+def close()
+  $gateway.shutdown!
+end
+
+
+# Trap ^C 
+Signal.trap("INT") { 
+  close()
+  exit
+}
+
+# Trap ^C 
+Signal.trap("TERM") { 
+  close()
+  exit
+}
+
+remote_host = 'parapide-9.rennes.grid5000.fr'
+remote_ports = [5601, 3000]
+remote_ports.each do |remote_port| 
+port = $gateway.open(remote_host, remote_port) 
+  puts "localhost:#{port} forwards to #{remote_host}:#{remote_port}"
+end
+
+wait

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: vagrant-g5k
 version: !ruby/object:Gem::Version
-  version: 0.0.15
+  version: 0.0.16
 platform: ruby
 authors:
 - Matthieu Simonin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-08 00:00:00.000000000 Z
+date: 2016-10-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: iniparse
@@ -127,6 +127,7 @@ files:
 - LICENSE
 - README.md
 - Vagrantfile
+- Vagrantfile.multisite
 - lib/vagrant-g5k.rb
 - lib/vagrant-g5k/.config.rb.swp
 - lib/vagrant-g5k/action.rb
@@ -151,6 +152,31 @@ files:
 - lib/vagrant-g5k/util/launch_vm_fwd.sh
 - lib/vagrant-g5k/version.rb
 - locales/en.yml
+- results/.gitignore
+- results/README.md
+- results/Vagrantfile
+- results/ansible.cfg
+- results/boilerplate.retry
+- results/boilerplate.yml
+- results/files/grafana/dashboard.json
+- results/files/grafana/dedicated.json
+- results/files/grafana/haproxy.json
+- results/files/heka/config.json
+- results/files/heka/heka-globals.toml
+- results/files/heka/lua_decoders/haproxy_log.lua
+- results/files/heka/lua_decoders/os_keystone_apache_log.lua
+- results/files/heka/lua_decoders/os_mysql_log.lua
+- results/files/heka/lua_decoders/os_openstack_log.lua
+- results/files/heka/lua_decoders/os_rabbitmq_log.lua
+- results/files/kibana/all_objects.json
+- results/files/nginx.conf
+- results/templates/heka-elasticsearch.toml.j2
+- results/templates/heka-haproxy.toml.j2
+- results/templates/heka-keystone.toml.j2
+- results/templates/heka-mariadb.toml.j2
+- results/templates/heka-openstack.toml.j2
+- results/templates/heka-rabbitmq.toml.j2
+- results/test.rb
 - spec/vagrant-g5k/config_spec.rb
 - vagrant-g5k.gemspec
 homepage: https://github.com/msimonin/vagrant-g5k