consul-templaterb 1.0.3

data/samples/consul_template.xml.erb

@@ -0,0 +1,70 @@
+<%
+  require 'rexml/document'
+
+  def build_node(e, name, attributes, text = nil)
+    xe = REXML::Element.new(name)
+    attributes.each do |name|
+      xe.add_attribute name, e[name]
+    end
+    xe.text = text if text
+    xe
+  end
+
+  def create_datacenters
+    xdatacenters = REXML::Element.new 'datacenters'
+    datacenters.each do |dc|
+      xdc = REXML::Element.new 'datacenter'
+      xdc.add_attribute 'name', dc
+      xdc.add_attribute 'services', services(dc:dc).keys.count
+      xdc.add_attribute 'nodes', nodes(dc:dc).count
+      xdatacenters.add_element xdc
+    end
+    xdatacenters
+  end
+
+  def create_services
+    xservices = REXML::Element.new 'services'
+    xservices.add_attribute 'count', services.count
+    services.each do |service_name, tags|
+      xserv = REXML::Element.new 'service'
+      xserv.add_attribute 'name', service_name
+      xserv.add_attribute 'tags', tags.sort.join(',')
+      xnodes = REXML::Element.new 'instances'
+      xnodes.add_attribute 'count', service(service_name).count
+      service(service_name).sort {|a,b| a['Node']['Node'] <=> b['Node']['Node'] }.each do |snode|
+        xnode = REXML::Element.new 'service-instance'
+        xnode.add_attribute 'id', snode['Node']['ID'] if snode['Node']['ID']
+        xnode.add_attribute 'address', snode['Node']['Address']
+        xnode.add_attribute 'name', snode['Node']['Name']
+        xnode.add_attribute 'port', snode['Service']['Port'].to_i
+        xnode.add_attribute 'tags', snode['Service']['Tags'].sort.join(',')
+        xnode.add_attribute 'statuses', (snode['Checks'].map { |c| c['Status'] }.join(' '))
+        xnodes.add_element xnode
+      end
+      xserv.add_element xnodes
+      xservices.add_element xserv
+    end
+    xservices
+  end
+
+  def create_nodes
+    xnodes = REXML::Element.new 'nodes'
+    xnodes.add_attribute 'count', nodes.count
+    nodes.sort {|a,b| a['Node'] <=> b['Node'] }.each do |snode|
+      xnode = REXML::Element.new 'node'
+      xnode.add_attribute 'id', snode['ID']
+      xnode.add_attribute 'address', snode['Address']
+      xnode.add_attribute 'name', snode['Node']
+      xnodes.add xnode
+    end
+    xnodes
+  end
+  doc = REXML::Document.new '<consul-info/>'
+  doc << REXML::XMLDecl.new(version='1.0', 'UTF-8')
+
+  doc.root.add_element create_datacenters
+  doc.root.add_element create_services
+  doc.root.add_element create_nodes
+  result = ''
+  doc.write(result, 2)
+%><%= result %>

data/samples/criteo/haproxy.cfg.erb

@@ -0,0 +1,163 @@
+<%
+# This template can be configure the following way with environment variables
+# Environment variables to filter services/instances
+#   SERVICES_TAG_FILTER: basic tag filter for service (default HTTP)
+#   INSTANCE_MUST_TAG: Second level of filtering (optional, default to SERVICES_TAG_FILTER)
+#   INSTANCE_EXCLUDE_TAG: Exclude instances having the given tag (default: canary)
+#   EXCLUDE_SERVICES: comma-separated services to exclude (default: consul-agent-http,mesos-slave,mesos-agent-watcher)
+# Environment variables to configure HaProxy
+#   PORT0 : HaProxy main port (default to 8000)
+#   PORT1 : HaProxy admin port (default to 8001)
+#   HAPROXY_ADMIN_PORT: Port to listen
+#   HAPROXY_ADMIN_USER_PASSWORD: User:Password to access to stats (default: none)
+#   HAPROXY_SOCKET_FILE: HaProxy socket to control HaProxy
+
+  service_tag_filter = ENV['SERVICES_TAG_FILTER'] || 'http'
+  instance_must_tag = ENV['INSTANCE_MUST_TAG'] || service_tag_filter
+  instance_exclude_tag = ENV['INSTANCE_EXCLUDE_TAG'] || 'canary'
+  kv_root = ENV['KV_ROOT'] || "services-data"
+  optional_dns_suffix = ENV['DNS_SUFFIX'] || ''
+
+  # Services to hide
+  services_blacklist = (ENV['EXCLUDE_SERVICES'] || 'consul-agent-http,mesos-slave,mesos-agent-watcher,mesos-exporter-slave').split(',')
+  # Compute the health of a Service
+  def compute_state(node)
+    states = ['passing', []]
+    node['Checks'].each do |chk|
+      st = chk['Status']
+      states[1] << st
+      states[0] = st if st == 'critical' || (st == 'warning' && states[0] == 'passing')
+    end
+    states
+  end
+  def compute_attributes(node)
+    w = 100
+    node['Service']['Tags'].each do |tag|
+      match = /^weight-([1-9][0-9])*$/.match(tag)
+      w = match[1].to_i if match
+    end
+    attributes = ""
+    states = compute_state(node)
+    attributes = "#{attributes} disabled" if states[0] == 'critical'
+    if states[0] == 'warning'
+      w = w / 8
+    end
+    attributes = "#{attributes} weight #{w}" if w.positive?
+  end
+  backends = {}
+  all_kv = {}
+  services(tag: service_tag_filter).each do |service_name, tags|
+    if !services_blacklist.include?(service_name) && tags.include?(instance_must_tag)
+      the_backends = []
+      kv_data = kv("#{kv_root}/#{service_name}/network-service").get_value_json
+      all_kv[service_name] = kv_data if kv_data
+      service(service_name, tag:'http').sort {|a,b| a['Node']['Node'] <=> b['Node']['Node'] }.each do |node|
+        tags_of_instance = node['Service']['Tags']
+        if tags_of_instance.include?(instance_must_tag) && !tags_of_instance.include?(instance_exclude_tag)
+          the_backends << node if node['Service']['Port']
+        end
+      end
+      # We add the backend ONLY if at least one valid instance does exists
+      backends[service_name] = the_backends
+    end
+  end
+
+  @all_kv = all_kv
+  def find_alternative_fqdns(service_name)
+    json_kv = @all_kv[service_name]
+    return nil unless json_kv
+    return nil unless json_kv['alternative_fqdn']
+    # Add spaces before
+    " #{json_kv['alternative_fqdn'].join(' ')}"
+  end
+%>global
+     log 127.0.0.1 local0 info
+     maxconn 262144
+     user haproxy
+     group haproxy
+     nbproc 1
+     daemon
+     stats socket /var/lib/haproxy/stats level admin mode 644 expose-fd listeners
+     stats timeout 2m
+     tune.bufsize 33792
+     ssl-default-bind-options no-sslv3 no-tlsv10 no-tlsv11 no-tls-tickets
+     ssl-default-bind-ciphers ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256
+     ssl-default-server-options no-sslv3 no-tlsv10 no-tlsv11 no-tls-tickets
+     ssl-default-server-ciphers ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256
+     ssl-server-verify required
+
+defaults
+     log global
+     mode http
+     retries 3
+     timeout connect 10s
+     timeout client 30s
+     timeout server 30s
+     timeout http-keep-alive 10s
+     timeout http-request 10s
+     timeout queue 1s
+     timeout check 5s
+     option httplog
+     option dontlognull
+     option redispatch
+     option prefer-last-server
+     option dontlog-normal
+     option http-keep-alive
+     option forwardfor except 127.0.0.0/8
+     balance roundrobin
+     maxconn 262134
+     http-reuse safe
+     default-server inter 5s fastinter 1s fall 3 slowstart 20s
+
+listen stats
+     bind *:8080
+     stats enable
+     stats uri /haproxy_stats
+
+# ==== Managed by consul-templaterb ====
+
+frontend fe_main # HTTP(S) Service
+    bind *:80 name http
+
+    # Monitoring
+    monitor-uri /delivery/monitor/mesos-haproxy/lb-check
+    errorfile 200 /etc/haproxy/200-criteo-ok.txt
+
+    # Adding a header indicating client side is using HTTPS
+    acl http  ssl_fc,not
+    acl https ssl_fc
+    http-request set-header X-Forwarded-Protocol https if https
+
+    # Disable CONNECT globally (identified as vulnerability by audit tools such as Qualys)
+    acl ::disabled_http_methods method CONNECT
+    http-request block if ::disabled_http_methods
+
+    # Consul-agent ACL
+    acl consul-agent-http-acl hdr_dom(host) -i  consul-agent-http.lb-ty5.crto.in consul-agent-http.lb-ty5.central.criteo.prod
+    use_backend bestatic_consul-agent-http if consul-agent-http-acl
+
+    # ACLs, HTTP Host header based, following the pattern <consul_app_name>.<fqdn_suffix>
+<% backends.each_pair do |service_name, nodes|
+%>
+   acl <%= service_name %>-acl hdr_beg(host) -i <%= service_name %>.<%= optional_dns_suffix %><%= find_alternative_fqdns(service_name)%>
+   use_backend be_<%= service_name %> if <%= service_name%>-acl
+<% end %>
+
+# Backends
+backend bestatic_consul-agent-http
+    http-request deny if !METH_GET # Deny writes
+    server srv0 127.0.0.1:8500
+
+<% backends.each_pair do |service_name, nodes|
+%>
+backend be_<%= service_name %>
+<% if all_kv[service_name] %>
+  # data2: <%= all_kv[service_name] %>
+<% end %>
+  mode http
+  balance leastconn
+<%   nodes.each_with_index do |node, idx|
+  %>  server srv<%= idx %> <%= node['Node']['Address']%>:<%= node['Service']['Port'] %><%= compute_attributes(node) %>
+<%   end
+   end
+%>

data/samples/criteo_choregraphies.html.erb

@@ -0,0 +1,91 @@
+<% require 'base64'
+   require 'json'
+   require 'date'
+   @current_time = Time.now.utc
+%>
+<%= render_file 'common/header.html.erb' %>
+<main role="main" class="container">
+  <div>
+  <h1>Show all choregraphie information</h1>
+  <div>This page only show choregraphie when at least one holder exists.</div>
+<%
+  def display_holder(holder, value)
+    begin
+      holder_date = Time.parse(value)
+      status = 'success'
+      diff = (@current_time - holder_date).round(0)
+      status = 'warning' if diff > 3600
+      status = 'danger' if diff > 7200
+      diff_txt = "#{diff % 3600} seconds"
+      diff_txt = "#{(diff % 86400) / 3600} hours, #{diff_txt}" if diff > 3600
+      diff_txt = "#{diff / 86400} days, #{diff_txt}" if diff > 86400
+    rescue StandardError => e
+      status = 'info'
+      holder_date = "Cannot parse date: #{e}"
+      diff_txt = "Error Parsing date #{value}"
+    end
+   ["<li class=\"list-group-item list-group-item-#{status}\">#{holder}: #{holder_date} : #{diff_txt} ago", status]
+  end
+%>
+
+<div id="accordion">
+<% kv('choregraphie', recurse:true).each do |tuple| %>
+  <!-- <%= tuple['Key'] %> -->
+  <%
+    key = tuple['Key'].gsub('/', '-')
+    if tuple['Value'].nil?
+      json = []
+      holders = []
+    %>
+    <div class="alert alert-warning" role="alert">
+      Invalid Choregraphie data: <%= tuple %>
+    </div>
+  <%
+    else
+      json = JSON.parse(Base64.decode64(tuple['Value']))
+      holders = json['holders']
+    end
+  %>
+  <% if holders.count > 0 %>
+
+    <div class="card">
+      <div class="card-header" id="heading-<%= key %>">
+        <span class="badge badge-pill badge-primary float-right"><%= holders.count%>/<%= json['concurrency'] %></span>
+        <h5 class="mb-0">
+          <a href="#<%= key %>" class="btn btn-link collapsed" data-toggle="collapse" data-target="#collapse-<%= key %>" aria-expanded="true" aria-controls="collapse-<%= key %>">
+            <%= tuple['Key'] %>
+          </a>
+        </h5>
+      </div>
+      <%
+        text_result = ''
+        clazz = 'collapse'
+        holders.each_pair do |key,value|
+          if value.is_a?(Hash)
+            text_result+= "<li><u>#{key}:</u><ul class=\"list-group\">"
+            value.each_pair do |k, v|
+              res, status = display_holder(k, v)
+              clazz = 'collapse show' unless status == 'success'
+              text_result += res
+            end
+            text_result += '</ul></li>'
+          else
+            res, status = display_holder(key, value)
+            clazz = 'collapse show' unless status == 'success'
+            text_result << res
+          end
+        end
+      %>
+      <div id="collapse-<%= key %>" class="<%= clazz %>" aria-labelledby="heading-<%= key %>" data-parent="#accordion">
+        <div class="card-body">
+          <pre class="pre-scrollable"><code><%= JSON.pretty_generate(json) %></code></pre>
+          <ul class="list-group">
+            <%= text_result %>
+          </ul>
+        </div>
+      </div>
+    </div>
+  <% end %>
+<% end %>
+</div>
+<%= render_file 'common/footer.html.erb' %>

data/samples/ha_proxy.cfg.erb

@@ -0,0 +1,127 @@
+<%
+# This template can be configure the following way with environment variables
+# Environment variables to filter services/instances
+#   SERVICES_TAG_FILTER: basic tag filter for service (default HTTP)
+#   INSTANCE_MUST_TAG: Second level of filtering (optional, default to SERVICES_TAG_FILTER)
+#   INSTANCE_EXCLUDE_TAG: Exclude instances having the given tag (default: canary)
+#   EXCLUDE_SERVICES: comma-separated services to exclude (default: consul-agent-http,mesos-slave,mesos-agent-watcher)
+# Environment variables to configure HaProxy
+#   PORT0 : HaProxy main port (default to 8000)
+#   PORT1 : HaProxy admin port (default to 8001)
+#   HAPROXY_ADMIN_PORT: Port to listen
+#   HAPROXY_ADMIN_USER_PASSWORD: User:Password to access to stats (default: none)
+#   HAPROXY_SOCKET_FILE: HaProxy socket to control HaProxy
+
+  service_tag_filter = ENV['SERVICES_TAG_FILTER'] || 'http'
+  instance_must_tag = ENV['INSTANCE_MUST_TAG'] || service_tag_filter
+  instance_exclude_tag = ENV['INSTANCE_EXCLUDE_TAG'] || 'canary'
+
+  # Services to hide
+  services_blacklist = (ENV['EXCLUDE_SERVICES'] || 'consul-agent-http,mesos-slave,mesos-agent-watcher,mesos-exporter-slave').split(',')
+  # Compute the health of a Service
+  def compute_state(node)
+    states = ['passing', []]
+    node['Checks'].each do |chk|
+      st = chk['Status']
+      states[1] << st
+      if st == 'critical'
+        states[0] = st
+      elsif st == 'warning' && states[0] == 'passing'
+        states[0] = st
+      end
+    end
+    states
+  end
+  def compute_attributes(node)
+    w = 100
+    node['Service']['Tags'].each do |tag|
+      match = /^weight-([1-9][0-9])*$/.match(tag)
+      w = match[1].to_i if match
+    end
+    attributes = ""
+    states = compute_state(node)
+    attributes = "#{attributes} disabled" if states[0] == 'critical'
+    if states[0] == 'warning'
+      w = w / 8
+    end
+    attributes = "#{attributes} weight #{w}" if w.positive?
+  end
+  backends = {}
+  services(tag: service_tag_filter).each do |service_name, tags|
+    if !services_blacklist.include?(service_name) && tags.include?(instance_must_tag)
+      the_backends = []
+      service(service_name, tag:'http').sort {|a,b| a['Node']['Node'] <=> b['Node']['Node'] }.each do |node|
+        tags_of_instance = node['Service']['Tags']
+        if tags_of_instance.include?(instance_must_tag) && !tags_of_instance.include?(instance_exclude_tag)
+          the_backends << node if node['Service']['Port']
+        end
+      end
+      # We add the backend ONLY if at least one valid instance does exists
+      backends[service_name] = the_backends
+    end
+  end
+%>
+# This HaProxy configuration is generated by consul-templaterb
+# Service/Instances filtering
+# Service tag MUST be <%= service_tag_filter %>
+# Instance tag MUST be <%= instance_must_tag %>
+# Instance tag excluded: <%= instance_exclude_tag %>
+
+# Found <%= backends.keys.count %> different services with <%= backends.values.flatten.count %> nodes
+
+global
+  maxconn 1024
+<%
+  socket_file = ENV['HAPROXY_SOCKET_FILE']
+  if socket_file
+%>
+  stats socket <%= socket_file %> mode 600 level admin
+<% end %>
+
+defaults
+  timeout connect 1000
+  timeout client 5000
+  timeout server 15000
+<% backends.each_pair do |service_name, nodes|
+%>
+backend backend_http__<%= service_name %>
+  mode http
+  balance leastconn
+<%   nodes.each do |node|
+  %>  server <%= node['Node']['Node']%>_<%= node['Service']['Port'] %> <%= node['Node']['Address']%>:<%= node['Service']['Port'] %><%= compute_attributes(node) %>
+<%   end
+   end
+%>
+frontend fontend_http
+   mode http
+   bind *:<%= ENV['PORT0'] || 8000 %>
+   option httplog
+   log 127.0.0.1:534 local5
+   option  dontlognull
+   option forwardfor
+<% backends.each_pair do |service_name, nodes|
+%>
+   acl host__<%= service_name %> hdr_beg(host) -i <%= service_name %>.
+   use_backend backend_http__<%= service_name %> if host__<%= service_name%>
+<% end %>
+
+listen stats
+  bind    :<%= ENV['PORT1'] || 8001 %>
+  mode            http
+  log             global
+  maxconn 10
+  timeout client  100s
+  timeout server  100s
+  timeout connect 100s
+  timeout queue   100s
+  stats enable
+  stats hide-version
+  stats refresh 30s
+  stats show-node
+  stats uri  /haproxy?stats
+<%
+   user_password = ENV['HAPROXY_ADMIN_USER_PASSWORD']
+   if user_password
+%>
+  stats auth <%= user_password %>
+<% end %>