hybrid_platforms_conductor 32.3.6

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: e16a6630a51ed9208fa4a3cecfce6604c65a39cb782d9efa426898ec980d5071
+  data.tar.gz: 33f38588a1b795e323f76e909ef301905cacf9d77c0a418ac0809713960659b6
+SHA512:
+  metadata.gz: 16a3b130741673e1ce4bfee15a5a4d8c45c66e5712e130a746f2d6be1e9940a1052a2c9195cce748831ca1652cc52255002226b31c0494516250b5359074c592
+  data.tar.gz: 920f8a21967bc69ecaf6072eec422e119e989415ddcd26d9e42e2d38779a35ddcfe58f5819cd3165c12799cf297dc5530da9a8d80ea6ae8c6a81c662d62c8fda

data/bin/check-node

@@ -0,0 +1,24 @@
+#!/usr/bin/env ruby
+require 'hybrid_platforms_conductor/executable'
+
+test_node = nil
+executable = HybridPlatformsConductor::Executable.new(
+  check_options: false,
+  nodes_selection_options: false,
+  parallel_options: false
+) do |opts|
+  opts.on('-n', '--node NODE', 'Run the command on a specific node.') do |node|
+    test_node = node
+  end
+end
+nodes_handler = executable.nodes_handler
+actions_executor = executable.actions_executor
+deployer = executable.deployer
+
+executable.parse_options!
+raise 'No node selected. Please use --node option to set at least one.' if test_node.nil? || test_node.empty?
+raise "Unknown node: #{test_node}" unless nodes_handler.known_nodes.include?(test_node)
+
+# Make sure we always use why-run
+deployer.use_why_run = true
+deployer.deploy_on(test_node)

data/bin/deploy

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+require 'hybrid_platforms_conductor/executable'
+
+executable = HybridPlatformsConductor::Executable.new
+actions_executor = executable.actions_executor
+deployer = executable.deployer
+
+executable.parse_options!
+raise 'No node selected. Please use --node option to set at least one.' if executable.selected_nodes.empty?
+
+# Deploy all
+deployer.deploy_on(executable.selected_nodes)

data/bin/dump_nodes_json

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+require 'hybrid_platforms_conductor/executable'
+
+executable = HybridPlatformsConductor::Executable.new(parallel_options: false)
+deployer = executable.deployer
+json_dumper = executable.json_dumper
+deployer.timeout = 30
+deployer.use_why_run = true
+
+executable.parse_options!
+
+json_dumper.dump_json_for(executable.selected_nodes)

data/bin/free_ips

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+require 'range_operators'
+require 'hybrid_platforms_conductor/executable'
+
+executable = HybridPlatformsConductor::Executable.new(nodes_selection_options: false)
+
+nodes_handler = executable.nodes_handler
+
+executable.parse_options!
+
+# Group IPs per first 3 numbers
+nodes_handler.prefetch_metadata_of nodes_handler.known_nodes, :private_ips
+nodes_handler.
+  known_nodes.
+  map { |node| nodes_handler.get_private_ips_of(node) || [] }.
+  flatten.
+  uniq.
+  group_by { |ip| ip.split('.')[0..2].map(&:to_i) }.
+  sort.
+  map { |ip_root, ips| [ip_root, ips.map { |ip| ip.split('.').last.to_i }] }.
+  each do |ip_root, ip_fourths|
+    executable.out "Free IPs for #{ip_root.join('.')}: #{([256] + ip_fourths).missing.rangify}"
+  end

data/bin/free_veids

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+require 'range_operators'
+require 'hybrid_platforms_conductor/executable'
+
+executable = HybridPlatformsConductor::Executable.new(nodes_selection_options: false)
+
+nodes_handler = executable.nodes_handler
+
+executable.parse_options!
+
+nodes_handler.prefetch_metadata_of nodes_handler.known_nodes, :veid
+veids = nodes_handler.
+  known_nodes.
+  map { |node| nodes_handler.get_veid_of(node) ? nodes_handler.get_veid_of(node).to_i : nil }.
+  compact
+
+executable.out "Free VEIDs: #{([10_000] + veids).missing.rangify}"

data/bin/get_impacted_nodes

@@ -0,0 +1,43 @@
+#!/usr/bin/env ruby
+require 'hybrid_platforms_conductor/executable'
+
+git_from = 'master'
+git_to = nil
+smallest_set = false
+platform = nil
+platforms_handler = nil
+executable = HybridPlatformsConductor::Executable.new(nodes_selection_options: false) do |opts|
+  opts.on('-f', '--from-commit COMMIT_ID', "Specify the GIT commit from which we look for diffs. Defaults to #{git_from}.") do |commit_id|
+    git_from = commit_id
+  end
+  opts.on('-p', '--platform PLATFORM_NAME', "Specify the repository on which to perform the diff. Possible values are #{platforms_handler.known_platforms.join(', ')}") do |platform_name|
+    platform = platform_name
+  end
+  opts.on('-s', '--smallest-test-sample', 'Display the minimal set of nodes to check that would validate all modifications.') do
+    smallest_set = true
+  end
+  opts.on('-t', '--to-commit COMMIT_ID', 'Specify the GIT commit to which we look for diffs. Defaults to current checked out files.') do |commit_id|
+    git_to = commit_id
+  end
+end
+nodes_handler = executable.nodes_handler
+platforms_handler = executable.platforms_handler
+cmd_runner = executable.cmd_runner
+executable.parse_options!
+raise 'No platform specified. Please use --platform option' if platform.nil?
+
+all_impacted_nodes, impacted_nodes, impacted_services, impact_global = nodes_handler.impacted_nodes_from_git_diff(
+  platform,
+  from_commit: git_from,
+  to_commit: git_to,
+  smallest_set: smallest_set
+)
+
+executable.out '* Potentially all nodes of this platform are impacted.' if impact_global
+executable.out ''
+executable.out "* #{impacted_services.size} impacted services:\n#{impacted_services.join("\n")}"
+executable.out ''
+executable.out "* #{impacted_nodes.size} impacted nodes (directly):\n#{impacted_nodes.join("\n")}"
+executable.out ''
+executable.out "* #{all_impacted_nodes.size} impacted nodes (total#{smallest_set ? ' smallest set' : ''}):\n#{all_impacted_nodes.join("\n")}"
+executable.out ''

data/bin/last_deploys

@@ -0,0 +1,56 @@
+#!/usr/bin/env ruby
+require 'time'
+require 'terminal-table'
+require 'hybrid_platforms_conductor/executable'
+
+possible_string_sorts = %i[
+  date
+  node
+  user
+]
+possible_sorts = Hash[possible_string_sorts.map do |property_name|
+  [
+    property_name,
+    proc { |node, deploy_info| [deploy_info.key?(:error) || !deploy_info.key?(property_name) ? '' : deploy_info[property_name], node] }
+  ]
+end]
+
+sort_by = :node
+sort_desc = false
+executable = HybridPlatformsConductor::Executable.new(deploy_options: false) do |opts|
+  opts.on('-r', '--sort-by SORT', "Specify a sort. Possible values are: #{possible_sorts.keys.sort.join(', ')}. Each value can append _desc to specify a reverse sorting. Defaults to #{sort_by}.") do |sort_name|
+    if sort_name =~ /^(.+)_desc$/
+      sort_desc = true
+      sort_name = $1
+    end
+    sort_by = sort_name.to_sym
+  end
+end
+nodes_handler = executable.nodes_handler
+deployer = executable.deployer
+
+executable.parse_options!
+raise "Unknown sort name: #{sort_by}. Should be one of #{possible_sorts.keys.join(', ')}." unless possible_sorts.key?(sort_by)
+
+sorted_deploy_info = Hash[
+  deployer.deployment_info_from(nodes_handler.select_nodes(executable.selected_nodes.empty? ? [{ all: true }] : executable.selected_nodes)).
+  map do |node, deploy_info|
+    decorated_deploy_info = deploy_info.merge(node: node)
+    decorated_deploy_info[:date] = deploy_info[:date].strftime('%F %T') if deploy_info.key?(:date)
+    decorated_deploy_info[:services] = deploy_info[:services].join(', ') if deploy_info.key?(:services)
+    [node, decorated_deploy_info]
+  end
+].sort_by(&possible_sorts[sort_by])
+sorted_deploy_info.reverse! if sort_desc
+info_displayed = {
+  node: 'Node',
+  date: 'Date',
+  user: 'Admin',
+  services: 'Services',
+  error: 'Error'
+}
+executable.out(Terminal::Table.new(headings: info_displayed.values) do |table|
+  sorted_deploy_info.each do |_node, deploy_info|
+    table << info_displayed.keys.map { |key| deploy_info[key] }
+  end
+end)

data/bin/nodes_to_deploy

@@ -0,0 +1,104 @@
+#!/usr/bin/env ruby
+require 'time'
+require 'hybrid_platforms_conductor/executable'
+
+ignore_schedule = false
+ignore_deploy_info = false
+deploy_time = Time.now.utc
+executable = HybridPlatformsConductor::Executable.new(deploy_options: false) do |opts|
+  opts.on('--deployment-time DATETIME', 'Set the deployment time to be considered while matching the schedules. Defaults to now.') do |date_time_str|
+    deploy_time = Time.parse("#{date_time_str} UTC")
+  end
+  opts.on('--ignore-deployed-info', 'Ignore the current deployed information.') do
+    ignore_deploy_info = true
+  end
+  opts.on('--ignore-schedule', 'Ignore the deployment schedules.') do
+    ignore_schedule = true
+  end
+end
+nodes_handler = executable.nodes_handler
+deployer = executable.deployer
+
+executable.parse_options!
+
+nodes = (nodes_handler.select_nodes(executable.selected_nodes.empty? ? [{ all: true }] : executable.selected_nodes))
+
+unless ignore_schedule
+  # Select nodes to be deployed, based first on deployment schedule
+  executable.log_debug "Consider deployment time for schedule: #{deploy_time.strftime('%F %T')}."
+  nodes.select! do |node|
+    # Select nodes whose schedule for deployment includes the deployment time
+    deployment_schedules = nodes_handler.select_confs_for_node(node, executable.config.deployment_schedules)
+    if deployment_schedules.empty?
+      # Consider we always deploy this node
+      executable.log_warn "[ #{node} ] - No deployment schedule defined."
+      true
+    else
+      executable.log_debug "[ #{node} ] - Deployment schedules: #{JSON.pretty_generate(deployment_schedules.map { |deployment_schedule_info| deployment_schedule_info[:schedule].to_h })}"
+      if deployment_schedules.any? { |deployment_schedule_info| deployment_schedule_info[:schedule].occurring_at?(deploy_time) }
+        executable.log_debug "[ #{node} ] - Node selected from the deployment schedule."
+        true
+      else
+        executable.log_debug "[ #{node} ] - Node NOT selected from the deployment schedule."
+        false
+      end
+    end
+  end
+end
+
+unless ignore_deploy_info
+  # Select nodes that have a git diff impact with master
+  # Get their current deployment info
+  deploy_info = deployer.deployment_info_from(nodes)
+  # Cache the impacted nodes, as very often the git diff can be lengthy and the same diff will be reused by several nodes
+  # Hash< String,    Hash< String,    Array< String > > >
+  # Hash< repo_name, Hash< commit_id, Array< node   > > >
+  cache_impacted_nodes = {}
+  nodes.select! do |node|
+    node_deploy_info = deploy_info[node]
+    executable.log_debug "[ #{node} ] - Deployment info: #{JSON.pretty_generate(node_deploy_info)}"
+    if node_deploy_info.key?(:error)
+      executable.log_warn "[ #{node} ] - Deployment in error: #{node_deploy_info[:error]}"
+      true
+    elsif !node_deploy_info.key?(:exit_status) || node_deploy_info[:exit_status] != '0'
+      executable.log_warn "[ #{node} ] - Last deployment did not end successfully: #{node_deploy_info[:exit_status]}"
+      true
+    elsif node_deploy_info.key?(:repo_name_0)
+      node_impacted = false
+      # Loop over all possible repositories concerned by this deployment
+      repo_idx = 0
+      while node_deploy_info.key?("repo_name_#{repo_idx}".to_sym) do
+        repo_name = node_deploy_info["repo_name_#{repo_idx}".to_sym]
+        commit_id = node_deploy_info["commit_id_#{repo_idx}".to_sym]
+        impacted_nodes = cache_impacted_nodes.dig(repo_name, commit_id)
+        if impacted_nodes.nil?
+          impacted_nodes, _single_impacted_nodes, _impacted_services, _impact_global = nodes_handler.impacted_nodes_from_git_diff(
+            repo_name,
+            from_commit: commit_id,
+            to_commit: 'master'
+          )
+          cache_impacted_nodes[repo_name] = {} unless cache_impacted_nodes.key?(repo_name)
+          cache_impacted_nodes[repo_name][commit_id] = impacted_nodes
+        end
+        if impacted_nodes.include?(node)
+          executable.log_debug "[ #{node} ] - Diffs on #{repo_name} between #{commit_id} and master are impacting this node."
+          node_impacted = true
+          break
+        else
+          executable.log_debug "[ #{node} ] - No impacting diffs on #{repo_name} between #{commit_id} and master for this node."
+        end
+        repo_idx += 1
+      end
+      node_impacted
+    else
+      # The deployment log has no repo name info.
+      # Consider it always has to be deployed.
+      true
+    end
+  end
+end
+
+executable.out '===== Nodes to deploy ====='
+nodes.sort.each do |node|
+  executable.out node
+end

data/bin/report

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+require 'hybrid_platforms_conductor/executable'
+
+executable = HybridPlatformsConductor::Executable.new
+reports_handler = executable.reports_handler
+
+executable.parse_options!
+
+selected_nodes = executable.selected_nodes
+reports_handler.produce_report_for(selected_nodes.empty? ? [{ all: true }] : selected_nodes)

data/bin/run

@@ -0,0 +1,39 @@
+#!/usr/bin/env ruby
+require 'hybrid_platforms_conductor/executable'
+
+commands = []
+timeout = nil
+concurrent_execution = false
+interactive = false
+executable = HybridPlatformsConductor::Executable.new do |opts|
+  opts.on('-c', '--command CMD', 'Command to execute (can\'t be used with --interactive) (can be used several times, commands will be executed sequentially)') do |cmd|
+    commands << cmd
+  end
+  opts.on('-f', '--commands-file FILE_NAME', 'Execute commands taken from a file (can\'t be used with --interactive) (can be used several times, commands will be executed sequentially)') do |file_name|
+    commands << { file: file_name }
+  end
+  opts.on('-i', '--interactive', 'Run an interactive SSH session instead of executing a command (can\'t be used with --command or --commands-file)') do
+    interactive = true
+  end
+  opts.on('-p', '--parallel', 'Execute the commands in parallel (put the standard output in files <hybrid-platforms-dir>/run_logs/*.stdout)') do
+    concurrent_execution = true
+  end
+  opts.on('-t', '--timeout SECS', 'Timeout in seconds to wait for each command (defaults to no timeout)') do |nbr_secs|
+    timeout = nbr_secs.to_i
+  end
+end
+actions_executor = executable.actions_executor
+
+executable.parse_options!
+raise '--interactive or --command options have to be present' if commands.empty? && !interactive
+raise 'No node selected. Please use --node option to set at least one.' if executable.selected_nodes.empty?
+
+actions = []
+actions << { remote_bash: commands } unless commands.empty?
+actions << { interactive: true } if interactive
+actions_executor.execute_actions(
+  { executable.selected_nodes => actions },
+  timeout: timeout,
+  concurrent: concurrent_execution,
+  log_to_stdout: executable.log_debug? || !concurrent_execution
+)

data/bin/setup

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+require 'hybrid_platforms_conductor/executable'
+
+executable = HybridPlatformsConductor::Executable.new(nodes_selection_options: false)
+platforms_handler = executable.platforms_handler
+
+executable.parse_options!
+
+platforms_handler.known_platforms.each do |platform|
+  platform.setup
+end

data/bin/ssh_config

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+require 'hybrid_platforms_conductor/executable'
+
+ssh_exec = 'ssh'
+executable = HybridPlatformsConductor::Executable.new(nodes_selection_options: false, parallel_options: false) do |opts|
+  opts.on('-x', '--ssh-exec FILE_PATH', "Path to the SSH executable to be used. Useful to give default options (especially with GIT_SSH). Defaults to #{ssh_exec}.") do |file_path|
+    ssh_exec = file_path
+  end
+end
+actions_executor = executable.actions_executor
+
+executable.parse_options!
+
+executable.out actions_executor.connector(:ssh).ssh_config(ssh_exec: ssh_exec)

data/bin/test

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+require 'hybrid_platforms_conductor/executable'
+
+executable = HybridPlatformsConductor::Executable.new(
+  check_options: false,
+  parallel_options: false,
+  timeout_options: false
+)
+tests_runner = executable.tests_runner
+
+executable.parse_options!
+
+exit(tests_runner.run_tests(executable.selected_nodes))

data/bin/topograph

@@ -0,0 +1,54 @@
+#!/usr/bin/env ruby
+require 'hybrid_platforms_conductor/executable'
+
+executable = HybridPlatformsConductor::Executable.new(
+  check_options: false,
+  nodes_selection_options: false,
+  parallel_options: false
+)
+nodes_handler = executable.nodes_handler
+deployer = executable.deployer
+deployer.timeout = 30
+deployer.concurrent_execution = true
+deployer.use_why_run = true
+topographer = executable.topographer
+
+executable.parse_options!
+
+# Initialize the topograph
+from_nodes, to_nodes = topographer.resolve_from_to
+topographer.get_json_files
+
+# The list of clusters to group nodes into 1 graphviz node to simplify the graph.
+known_nodes_lists = nodes_handler.known_nodes_lists
+
+executable.section 'Compute graph' do
+  topographer.graph_for(nodes_handler.known_nodes)
+end
+executable.section 'Add nodes lists clusters' do
+  topographer.graph_for_nodes_lists(known_nodes_lists)
+end
+executable.section 'Define IP 24 clusters' do
+  topographer.define_clusters_ip_24
+end
+nodes = nil
+executable.section 'Select path' do
+  from_nodes = topographer.children_nodes(from_nodes)
+  to_nodes = topographer.ancestor_nodes(to_nodes)
+  nodes = from_nodes & to_nodes
+end
+executable.section "Filter only nodes #{nodes.sort.join(', ')}" do
+  topographer.filter_in_nodes(nodes + topographer.cluster_nodes)
+end
+executable.section 'Collapse nodes lists' do
+  topographer.collapse_nodes(known_nodes_lists)
+end
+executable.section 'Remove self references' do
+  topographer.remove_self_connections
+end
+executable.section 'Remove empty clusters' do
+  topographer.remove_empty_clusters
+end
+executable.section 'Write outputs' do
+  topographer.dump_outputs
+end