comap 1.0.0.pre.a

data/lib/comap/app.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+# Copyright (c) 2017-2018 Make.org
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'faraday'
+require 'openssl'
+
+# Main module
+module COMAP
+  # Rack App
+  class App
+    # rubocop:disable  Metrics/ParameterLists
+    def initialize(
+      docker_host: 'http://127.0.0.1',
+      docker_port: '2375',
+      docker_ssl: '',
+      metrics_path: '',
+      network: '',
+      services: []
+    )
+      raise ArgumentError, 'No services given' if services.empty?
+      raise ArgumentError, 'No network given' if network.nil?
+      ssl_hash = parse_ssl(docker_ssl)
+      @client = DockerClient.new(docker_host, docker_port, ssl_hash)
+      @network = network
+      @services = services
+      @metrics_path = metrics_path
+      error_msg = 'Could not retrieve Docker API version from '\
+        "#{docker_host}:#{docker_port} ssl:#{docker_ssl}"
+      raise StandardError, error_msg if @client.call('version').nil?
+    end
+    # rubocop:enable  Metrics/ParameterLists
+
+    def call(_env)
+      ['200', { 'Content-Type' => 'text/plain' }, [body]]
+    end
+
+    private
+
+    def body
+      @services.map do |s|
+        service, port = s.split(':')
+        service_metrics(service, port)
+      end.compact.map(&flat_hash).join
+    end
+
+    def service_metrics(service, port)
+      containers(service).sort_by { |c| c['slot'] }.map do |container|
+        next puts "#{service} service not found" if container.empty?
+        url = "http://#{container['ip']}:#{port}/#{@metrics_path}"
+        container_metrics(url, service, container['slot'])
+      end.reduce(&merge_proc)
+    end
+
+    def container_metrics(url, service, slot)
+      data = scrape_metrics(url, service)
+      return {} if data.nil?
+      data.lines.each_with_object({}) do |value, hash|
+        next hash[value] = nil if value.start_with?('#')
+        (hash[hash.keys.last] ||= []) << add_label(value, "#{service}.#{slot}")
+      end
+    end
+
+    def scrape_metrics(url, service)
+      Faraday.get(url) do |req|
+        req.options.timeout = 2
+        req.options.open_timeout = 1
+      end.body
+    rescue StandardError
+      puts "Could not connect to #{service} endpoint #{url}"
+    end
+
+    def add_label(line, label)
+      replace = %(container="#{label}")
+      line.gsub(/ /, "{#{replace}} ").gsub(/}{/, ',')
+    end
+
+    def containers(service)
+      running_tasks(service).map do |task|
+        {
+          'slot' => task['Slot'],
+          'ip' => ip(task).split('/').first
+        }
+      end.uniq
+    end
+
+    def running_tasks(service)
+      filters = {
+        'service' => { service => true },
+        'desired-state' => { 'running' => true }
+      }
+      @client.call('tasks', filters).keep_if do |task|
+        task['Status']['State'] == 'running'
+      end
+    end
+
+    def ip(task)
+      task['NetworksAttachments'].select do |h|
+        h['Network']['Spec']['Name'] == @network
+      end.first['Addresses'].first
+    end
+
+    # helpers
+
+    def parse_ssl(docker_ssl)
+      docker_ssl.split(',').map do |opt|
+        k, v = opt.split('=')
+        if k.include?('cert')
+          v = OpenSSL::X509::Certificate.new(File.read(v))
+        elsif k.include? 'key'
+          v = OpenSSL::PKey::RSA.new(File.read(v))
+        end
+        [k, v]
+      end.to_h
+    end
+
+    def merge_proc
+      proc do |old, new|
+        old.merge(new) { |_, value1, value2| value1 + value2 }
+      end
+    end
+
+    def flat_hash
+      proc do |item|
+        item.map { |key, value| "#{key}#{value.join}" }
+      end
+    end
+  end
+end

data/lib/comap/docker_client.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+# Copyright (c) 2017-2018 Make.org
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'faraday'
+require 'json'
+
+# Module Docker
+module COMAP
+  # Simple Engine API client
+  class DockerClient
+    def initialize(url, port, ssl = {})
+      @conn = Faraday.new(url: "#{url}:#{port}", ssl: ssl) do |faraday|
+        faraday.options.params_encoder = Faraday::FlatParamsEncoder
+        faraday.adapter Faraday.default_adapter
+      end
+    end
+
+    def call(route, filters = {})
+      response = @conn.get(route) do |req|
+        req.params['filters'] = filters.to_json
+      end
+      JSON.parse(response.body)
+    rescue StandardError => e
+      puts "Could not connect to Docker API #{@conn.url_prefix}"
+      puts e.message
+    end
+  end
+end

data/lib/comap/version.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+# Copyright (c) 2017-2018 Make.org
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Define version
+module COMAP
+  VERSION = '1.0.0-a'
+end

data/metadata.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+name 'comap-test'
+maintainer 'Make.org'
+maintainer_email 'sre@make.org'
+license 'Apache-2.0'
+description 'Integration tests for Comap'
+long_description 'Integration tests for Comap'
+source_url 'https://gitlab.com/sre-gems/comap'
+issues_url 'https://gitlab.com/sre-gems/comap/issues'
+version '1.0.0'
+
+chef_version '>= 13.0'
+
+supports 'centos', '>= 7.4'
+
+depends 'docker-platform'

data/recipes/build_comap.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+# Copyright (c) 2018 Make.org
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Build comap docker image and launch it
+directory '/root/pkg'
+
+cookbook_file '/root/Dockerfile' do
+  source 'Dockerfile'
+  notifies :run, 'execute[docker build comap]', :delayed
+end
+
+cookbook_file '/root/pkg/comap.gem' do
+  source 'comap.gem'
+  notifies :run, 'execute[docker build comap]', :delayed
+end
+
+execute 'docker build comap' do
+  command 'docker build . -t makeorg/comap'
+  cwd '/root'
+  action :nothing
+end

data/recipes/launch_comap.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+# Copyright (c) 2018 Make.org
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Create secrets
+dir = '/root/.docker'
+%w[ca.pem cert.pem key.pem].each do |secret|
+  execute "docker secret create #{secret} #{dir}/#{secret}" do
+    not_if "docker secret ls | grep #{secret}"
+  end
+end
+
+metrics_content = <<~CONTENT
+  # TYPE metrics_sample_seconds gauge
+  metrics_sample_seconds 42
+  # TYPE metrics_sample_2_seconds gauge
+  metrics_sample_2_seconds 24
+CONTENT
+
+docker_platform_service 'metrics' do
+  options(
+    mode: 'replicated',
+    replicas: 2,
+    network: 'kitchen',
+    publish: '5678:5678'
+  )
+  image 'hashicorp/http-echo'
+  command "-text '#{metrics_content.chomp("\n")}'"
+  action :create
+end
+
+server = 'https://comap-swarm-manager-centos-7'
+ssl = {
+  'verify' => false, # https://github.com/berkshelf/berkshelf/issues/1458
+  'client_cert' => '/run/secrets/cert.pem',
+  'client_key' => '/run/secrets/key.pem',
+  'ca_file' => '/run/secrets/ca.pem'
+}
+ssl_opt = ssl.map { |k, v| "#{k}=#{v}" }.join(',')
+puts "-H #{server} -p 2376 -s #{ssl_opt} -n kitchen metrics:5678"
+
+# TODO: give options as an array in docker-platform
+docker_platform_service 'comap' do
+  options(
+    mode: 'replicated',
+    network: 'kitchen',
+    publish: '9100:9100',
+    secret: %w[cert.pem key.pem ca.pem]
+  )
+  image 'makeorg/comap'
+  command "-H #{server} -p 2376 -s #{ssl_opt} -n kitchen metrics:5678"
+  action :create
+end

metadata

@@ -0,0 +1,183 @@
+--- !ruby/object:Gem::Specification
+name: comap
+version: !ruby/object:Gem::Version
+  version: 1.0.0.pre.a
+platform: ruby
+authors:
+- Sylvain Arrambourg
+- Samuel Bernard
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-02-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.2'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.51'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.51'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.13'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.13'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: thin
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+description: Container Orchestrator Metrics Aggregator for Prometheus
+email:
+- sar@make.org
+executables:
+- comap
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".gitlab-ci.yml"
+- ".kitchen.yml"
+- ".rspec"
+- ".ruby-version"
+- ".travis.yml"
+- Berksfile
+- CONTRIBUTING.md
+- Dockerfile
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- comap.gemspec
+- exe/comap
+- lib/comap.rb
+- lib/comap/app.rb
+- lib/comap/docker_client.rb
+- lib/comap/version.rb
+- metadata.rb
+- recipes/build_comap.rb
+- recipes/launch_comap.rb
+homepage: https://gitlab.com/sre-gems/comap
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">"
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.3
+signing_key: 
+specification_version: 4
+summary: Container Orchestrator Metrics Aggregator for Prometheus
+test_files: []