vagrant-g5k 0.0.15 → 0.0.16

data/results/files/heka/config.json

@@ -0,0 +1,47 @@
+{
+    "command": "/usr/bin/hekad -config=/etc/heka/",
+    "config_files": [
+        {
+            "source": "/var/lib/kolla/config_files/heka-elasticsearch.toml",
+            "dest": "/etc/heka/heka-elasticsearch.toml",
+            "owner": "heka",
+            "perm": "0600",
+            "optional": true
+        },
+        {
+            "source": "/var/lib/kolla/config_files/heka-openstack.toml",
+            "dest": "/etc/heka/heka-openstack.toml",
+            "owner": "heka",
+            "perm": "0600",
+            "optional": true
+        },
+        {
+            "source": "/var/lib/kolla/config_files/heka-keystone.toml",
+            "dest": "/etc/heka/heka-keystone.toml",
+            "owner": "heka",
+            "perm": "0600",
+            "optional": true
+        },
+        {
+            "source": "/var/lib/kolla/config_files/heka-mariadb.toml",
+            "dest": "/etc/heka/heka-mariadb.toml",
+            "owner": "heka",
+            "perm": "0600",
+            "optional": true
+        },
+        {
+            "source": "/var/lib/kolla/config_files/heka-rabbitmq.toml",
+            "dest": "/etc/heka/heka-rabbitmq.toml",
+            "owner": "heka",
+            "perm": "0600",
+            "optional": true
+        },
+        {
+            "source": "/var/lib/kolla/config_files/heka-globals.toml",
+            "dest": "/etc/heka/heka-globals.toml",
+            "owner": "heka",
+            "perm": "0600",
+            "optional": true
+        }
+    ]
+}

data/results/files/heka/heka-globals.toml

@@ -0,0 +1,2 @@
+[hekad]
+maxprocs=16

data/results/files/heka/lua_decoders/haproxy_log.lua

@@ -0,0 +1,162 @@
+--[[
+
+This is a decoder for custom haproxy logs.
+e.g.
+Jun 13 15:29:06 localhost.localdomain haproxy[33168]: 10.10.10.10 - - [13/Jun/2016:07:29:05 +0000] "POST /graph/pie HTTP/1.1" 200 848 "" "" 52591 218 "111.111.111.111_80" "my_app-demo.server.com_http_rr" "inst_13.136" 2 0 2 938 942 ---- 0 0 0 0 0 0 0 "" "" ""lb-node.server.com""
+
+Config:
+  log_type = "haproxy.run.decoded", default value is "haproxy.run.default"
+
+--]]
+
+local dt = require "date_time"
+local ip = require "ip_address"
+local l = require 'lpeg'
+
+l.locale(l)
+
+local syslog = require "syslog"
+
+require "cjson"
+
+
+--[ grammar ]
+local sp                        = l.space
+local timestamp                 = "[" * l.Cg(dt.build_strftime_grammar("%d/%b/%Y:%H:%M:%S %z") * dt.time_secfrac / dt.time_to_ns, "Timestamp") * "]"
+local log_date                  = l.Cg(dt.build_strftime_grammar("%b %d %H:%M:%S"), "LogDate")
+local host                      = (l.alnum^1 + l.S("-_"))^1
+local fqdn                      = (l.alnum^1 + l.S("-_."))^1
+local slash                     = l.P"/"
+local integer                   = (l.S("+-")^-1) * (l.digit^1)
+local printusascii              = l.R"!~"
+local nilvalue                  = l.P"-"
+
+local http_post                 = l.P"POST"
+local http_get                  = l.P"GET"
+local http_delete               = l.P"DELETE"
+local http_update               = l.P"UPDATE"
+local url                       = l.P"/"^1 * l.alnum^0
+local http_method               = l.Cg((http_post + http_get + http_delete + http_update)^1, "http_method")
+local request_url               = l.Cg(url^1, "request_url")
+local http_version              = l.Cg((l.P("HTTP") + l.P("HTTPS"))^1 * "/" * l.digit * "." * l.digit, "http_version")
+
+local client_ip                 = l.Cg(ip.v4, "client_ip")
+local gmt_date_time             = "[" * l.Cg(dt.build_strftime_grammar("%d/%b/%Y:%H:%M:%S %z") / dt.time_to_ns, "gmt_date_time") * "]"
+local http_request              = l.Cg("\"" * l.Cg(http_method * sp * request_url * sp * http_version, "http_request") * "\"")
+local status_code               = l.Cg(l.digit * l.digit * l.digit, "status_code")
+local bytes_read                = l.Cg(l.digit^1 / tonumber, "bytes_read")
+local client_port               = l.Cg(l.digit^1, "client_port")
+local accept_date_milliseconds  = l.Cg(l.digit^1 / tonumber, "accept_date_milliseconds")
+local frontend_name             = "\"" * l.Cg(ip.v4 * "_" * l.digit^1, "frontend_name") * "\""
+local backend_name              = "\"" * l.Cg((l.alnum + l.S"-_.")^1, "backend_name") * "\""
+local server_name               = "\"" * l.Cg((l.alnum + l.S"-_.")^1, "server_name") * "\""
+local Tq                        = l.Cg(integer / tonumber, "Tq")
+local Tw                        = l.Cg(integer / tonumber, "Tw")
+local Tc                        = l.Cg(integer / tonumber, "Tc")
+local Tr                        = l.Cg(integer / tonumber, "Tr")
+local Tt                        = l.Cg(integer / tonumber, "Tq")
+local termination_state_with_cookie_status = l.Cg(( nilvalue + l.upper ) * ( nilvalue + l.upper ) * ( nilvalue + l.upper ) * ( nilvalue + l.upper ), "termination_state_with_cookie_status")
+local actconn                   = l.Cg(integer / tonumber, "actconn")
+local feconn                    = l.Cg(integer / tonumber, "feconn")
+local beconn                    = l.Cg(integer / tonumber, "beconn")
+local srv_conn                  = l.Cg(integer / tonumber, "srv_conn")
+local retries                   = l.P"+"^-1 * l.Cg(l.digit^1 / tonumber, "retries")
+local srv_queue                 = l.Cg(integer / tonumber, "srv_queue")
+local backend_queue             = l.Cg(integer / tonumber, "backend_queue")
+local captured_request_cookie   = l.Cg(printusascii^-255, "captured_request_cookie")
+local captured_response_cookie  = l.Cg(printusascii^-255, "captured_response_cookie")
+local domain_name               = "\"" * l.Cg((l.alnum + l.S"-_.")^1, "domain_name") * "\""
+
+-- Returns an LPeg grammar based on the Haproxy log_format configuration string.
+local function build_grammar()
+    local grm =
+        client_ip * sp * "- -" * sp
+      * gmt_date_time * sp
+      * "\"" * http_method * sp
+      * request_url * sp
+      * http_version * "\"" * sp
+      * status_code * sp
+      * bytes_read * sp
+      * "\"\"" * sp * "\"\"" * sp
+      * client_port * sp
+      * accept_date_milliseconds * sp
+      * frontend_name * sp
+      * backend_name * sp
+      * server_name * sp
+      * Tq * sp
+      * Tw * sp
+      * Tc * sp
+      * Tr * sp
+      * Tt * sp
+      * termination_state_with_cookie_status * sp
+      * actconn * sp
+      * feconn * sp
+      * beconn * sp
+      * srv_conn * sp
+      * retries * sp
+      * srv_queue * sp
+      * backend_queue * sp
+      * captured_request_cookie * sp
+      * captured_response_cookie * sp
+
+    return l.Ct(grm)
+end
+
+local msg = {
+    Timestamp   = nil,
+    Hostname    = nil,
+    Payload     = nil,
+    Pid         = nil,
+    Fields      = nil
+}
+
+local default_log_type = "haproxy.run.default"
+local default_syslog_format = "%TIMESTAMP% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
+
+--[ read config variables ]
+local log_type = read_config("log_type") or default_log_type
+local syslog_format = read_config("syslog_format") or default_syslog_format
+
+local syslog_grammar = syslog.build_rsyslog_grammar(syslog_format)
+local halog_grammar = build_grammar()
+
+local msg = {
+    Timestamp = nil,
+    Type      = nil,
+    Payload   = nil,
+    Fields    = nil
+}
+
+function process_message ()
+  local log = read_message("Payload")
+  local fields = syslog_grammar:match(log)
+  if not fields then return -1 end
+
+  msg.Timestamp = fields.timestamp
+  fields.timestamp = nil
+
+  msg.Type = log_type
+  msg.Payload   = nil
+
+  msg.Pid = fields.syslogtag.pid or nil
+  fields.syslogtag = nil
+
+  msg.Hostname = fields.hostname
+  fields.hostname = nil
+
+  local m = halog_grammar:match(fields.msg)
+  if m then
+    -- fill all fields
+    for k,v in pairs(m) do
+      fields[k] = v
+    end
+  else
+    -- not matched, break
+    return -1
+  end
+
+  fields.msg = nil
+  msg.Fields = fields
+  inject_message(msg)
+  return 0
+end

data/results/files/heka/lua_decoders/os_keystone_apache_log.lua

@@ -0,0 +1,78 @@
+-- Copyright 2015 Mirantis, Inc.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+local l      = require 'lpeg'
+l.locale(l)
+
+local common_log_format = require 'common_log_format'
+local patt = require 'os_patterns'
+local utils  = require 'os_utils'
+
+-- Allow custom type
+t = read_config('type')
+if t == nil or t == '' then
+  t = 'log'
+end
+
+local msg = {
+    Timestamp   = nil,
+    Type        = t,
+    Hostname    = nil,
+    Payload     = nil,
+    Pid         = nil,
+    Fields      = nil,
+    Severity    = 6,
+}
+
+local severity_label = utils.severity_to_label_map[msg.Severity]
+
+local apache_log_pattern = read_config("apache_log_pattern") or error(
+    "apache_log_pattern configuration must be specificed")
+local apache_grammar = common_log_format.build_apache_grammar(apache_log_pattern)
+local request_grammar = l.Ct(patt.http_request)
+
+function process_message ()
+
+    -- logger is either "keystone-apache-public" or "keystone-apache-admin"
+    local logger = read_message("Logger")
+
+    local log = read_message("Payload")
+
+    local m
+
+    m = apache_grammar:match(log)
+    if m then
+        msg.Logger = 'openstack.keystone'
+        msg.Payload = log
+        msg.Timestamp = m.time
+
+        msg.Fields = {}
+        msg.Fields.http_status = m.status
+        msg.Fields.http_response_time = m.request_time.value / 1e6 -- us to sec
+        msg.Fields.programname = logger
+        msg.Fields.severity_label = severity_label
+
+        local request = m.request
+        m = request_grammar:match(request)
+        if m then
+            msg.Fields.http_method = m.http_method
+            msg.Fields.http_url = m.http_url
+            msg.Fields.http_version = m.http_version
+        end
+
+        return utils.safe_inject_message(msg)
+    end
+
+    return -1, string.format("Failed to parse %s log: %s", logger, string.sub(log, 1, 64))
+end

data/results/files/heka/lua_decoders/os_mysql_log.lua

@@ -0,0 +1,56 @@
+-- Copyright 2015-2016 Mirantis, Inc.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+require "string"
+local l      = require 'lpeg'
+l.locale(l)
+
+local patt   = require 'os_patterns'
+local utils  = require 'os_utils'
+
+-- Allow custom type
+t = read_config('type')
+if t == nil or t == '' then
+  t = 'log'
+end
+
+local msg = {
+    Timestamp   = nil,
+    Type        = t,
+    Hostname    = nil,
+    Payload     = nil,
+    Pid         = nil,
+    Fields      = {},
+    Severity    = nil,
+}
+
+local sp    = patt.sp
+local colon = patt.colon
+
+-- mysqld logs are cranky,the date is YYMMMDD, the hours have no leading zero and the "real" severity level is enclosed by square brackets...
+local mysql_grammar = l.Ct(l.digit^-6 * sp^1 *  l.digit^-2 * colon * l.digit^-2 * colon * l.digit^-2 * sp^1 * l.P"[" * l.Cg(l.R("az", "AZ")^0 / string.upper, "SeverityLabel") * l.P"]" * sp^1 * l.Cg(patt.Message, "Message"))
+
+
+function process_message ()
+    local log = read_message("Payload")
+    local logger = read_message("Logger")
+
+    local m = mysql_grammar:match(log)
+    if not m then return -1 end
+
+    msg.Logger = logger
+    msg.Payload = m.Message
+    msg.Fields.severity_label = m.SeverityLabel
+
+    return utils.safe_inject_message(msg)
+end

data/results/files/heka/lua_decoders/os_openstack_log.lua

@@ -0,0 +1,146 @@
+-- Copyright 2015-2016 Mirantis, Inc.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+require "string"
+require "table"
+local l      = require 'lpeg'
+l.locale(l)
+
+local patt = require 'os_patterns'
+local utils = require 'os_utils'
+
+-- Allow custom type
+t = read_config('type')
+if t == nil or t == '' then
+  t = 'log'
+end
+
+local msg = {
+    Timestamp   = nil,
+    Type        = t,
+    Hostname    = nil,
+    Payload     = nil,
+    Pid         = nil,
+    Fields      = nil,
+    Severity    = nil,
+}
+
+-- traceback_lines is a reference to a table used to accumulate lines of
+-- a Traceback. traceback_key represent the key of the Traceback lines
+-- being accumulated in traceback_lines. This is used to know when to
+-- stop accumulating and inject the Heka message.
+local traceback_key = nil
+local traceback_lines = nil
+
+function prepare_message (service, timestamp, pid, severity_label,
+        python_module, programname, payload)
+    msg.Logger = 'openstack.' .. service
+    msg.Timestamp = timestamp
+    msg.Payload = payload
+    msg.Pid = pid
+    msg.Severity = utils.label_to_severity_map[severity_label] or 7
+    msg.Fields = {}
+    msg.Fields.severity_label = severity_label
+    msg.Fields.python_module = python_module
+    msg.Fields.programname = programname
+    msg.Payload = payload
+end
+
+-- OpenStack log messages are of this form:
+-- 2015-11-30 08:38:59.306 3434 INFO oslo_service.periodic_task [-] Blabla...
+--
+-- [-] is the "request" part, it can take multiple forms.
+
+function process_message ()
+
+    -- Logger is of form "<service>_<program>" (e.g. "nova_nova-api",
+    -- "neutron_l3-agent").
+    local logger = read_message("Logger")
+    local service, program = string.match(logger, '([^_]+)_(.+)')
+
+    local log = read_message("Payload")
+    local m
+
+    m = patt.openstack:match(log)
+    if not m then
+        return -1, string.format("Failed to parse %s log: %s", logger, string.sub(log, 1, 64))
+    end
+
+    local key = {
+        Timestamp     = m.Timestamp,
+        Pid           = m.Pid,
+        SeverityLabel = m.SeverityLabel,
+        PythonModule  = m.PythonModule,
+        service       = service,
+        program       = program,
+    }
+
+    if traceback_key ~= nil then
+        -- If traceback_key is not nil then it means we've started accumulated
+        -- lines of a Python traceback. We keep accumulating the traceback
+        -- lines util we get a different log key.
+        if utils.table_equal(traceback_key, key) then
+            table.insert(traceback_lines, m.Message)
+            return 0
+        else
+            prepare_message(traceback_key.service, traceback_key.Timestamp,
+                traceback_key.Pid, traceback_key.SeverityLabel,
+                traceback_key.PythonModule, traceback_key.program,
+                table.concat(traceback_lines, ''))
+            traceback_key = nil
+            traceback_lines = nil
+            -- Ignore safe_inject_message status code here to still get a
+            -- chance to inject the current log message.
+            utils.safe_inject_message(msg)
+        end
+    end
+
+    if patt.traceback:match(m.Message) then
+        -- Python traceback detected, begin accumulating the lines making
+        -- up the traceback.
+        traceback_key = key
+        traceback_lines = {}
+        table.insert(traceback_lines, m.Message)
+        return 0
+    end
+
+    prepare_message(service, m.Timestamp, m.Pid, m.SeverityLabel, m.PythonModule,
+        program, m.Message)
+
+    m = patt.openstack_request_context:match(msg.Payload)
+    if m then
+        msg.Fields.request_id = m.RequestId
+        if m.UserId then
+          msg.Fields.user_id = m.UserId
+        end
+        if m.TenantId then
+          msg.Fields.tenant_id = m.TenantId
+        end
+    end
+
+    m = patt.openstack_http:match(msg.Payload)
+    if m then
+        msg.Fields.http_method = m.http_method
+        msg.Fields.http_status = m.http_status
+        msg.Fields.http_url = m.http_url
+        msg.Fields.http_version = m.http_version
+        msg.Fields.http_response_size = m.http_response_size
+        msg.Fields.http_response_time = m.http_response_time
+        m = patt.ip_address:match(msg.Payload)
+        if m then
+            msg.Fields.http_client_ip_address = m.ip_address
+        end
+    end
+
+    return utils.safe_inject_message(msg)
+end