vop-plugins 0.3.1 → 0.3.5

data/extended/rails_machines/commands/write_found_machines_into_rails_db.rb.not

@@ -0,0 +1,30 @@
+param! "machines", multi: true
+
+contribute to: "machines_found" do |machines|
+  stats = Hash.new { |h,k| h[k] = 0 }
+
+  machines.each do |machine_row|
+    name = machine_row["name"]
+
+    if name.nil?
+      $logger.error "a machine has been found without a name : #{machine_row.pretty_inspect}"
+    end
+    existing = Machine.find_by_name(name)
+    if existing
+      existing.seen_at = Time.now()
+      existing.save
+      stats[:existing] += 1
+      $logger.debug "updating #{name}"
+    else
+      new_machine = Machine.create!(name: name)
+      stats[:new] += 1
+      $logger.debug "new #{name}"
+    end
+  end
+
+  # TODO maybe invalidate only if there are new machines?
+  @op.rails_machines!
+  @op.redis_machines!
+
+  stats
+end

data/extended/rails_machines/rails_machines.plugin

@@ -0,0 +1 @@
+depends_on :machines

data/extended/redis_cache/commands/flush_cache.rb

@@ -0,0 +1,5 @@
+run do |plugin|
+  redis = plugin.state[:redis]
+
+  redis.flushall
+end

data/extended/redis_cache/commands/invalidate_cache.rb

@@ -0,0 +1,26 @@
+param! "command"
+param! "raw_params", default: {}
+
+run do |params, plugin|
+  redis = plugin.state[:redis]
+
+  request = Request.new(@op, params["command"], params["raw_params"])
+  $logger.debug "invalidating #{request.cache_key}"
+
+  count = redis.del request.cache_key
+  $logger.debug "removed #{count} key(s) from cache : #{request.cache_key}"
+
+  command = @op.commands[params["command"]]
+  raise "no such command #{params["command"]}" if command.nil?
+
+  if command.invalidation_block
+    $logger.debug "calling custom invalidation block for #{command.name}"
+
+    block_param_names = command.invalidation_block.parameters.map { |x| x.last }
+    payload = @op.executor.prepare_payload(request, {}, block_param_names)
+
+    command.invalidation_block.call(*payload)
+  end
+
+  count
+end

data/extended/redis_cache/filters/cache_filter.rb

@@ -0,0 +1,78 @@
+require_relative "../helpers/cache_wrapper"
+
+run do |command, request, plugin|
+  should_cache = command.read_only
+
+  if should_cache
+    # TODO we need this workaround (to get the plugin) because the above "plugin" is the one for the command that's executed
+    redis = @op.plugin("redis_cache").state[:redis]
+    raise "no redis connection" unless redis
+
+    cache_wrapper = CacheWrapper.from_json redis.get(request.cache_key)
+    cached = nil
+    if cache_wrapper
+      cache_state = "hit"
+      cached = cache_wrapper.data
+      origin_class = cache_wrapper.options["origin"]
+      (main, child) = origin_class.split("/")
+      if main == "Array" && child == "Vop::Entity"
+        cached.map! do |entity_data|
+          Entity.json_create(plugin.op, entity_data)
+        end
+      end
+    else
+      cache_state = "miss"
+    end
+    $logger.debug "cache #{cache_state} : #{request.cache_key}"
+
+    result = nil
+    response = nil
+    unless cached.nil?
+      result = cached
+
+      description = if result.respond_to? :size
+        size_text = " "
+        size_text += result.size.to_s
+        size_text += if result.is_a? Array
+          " lines"
+        else
+          " bytes (?)"
+        end
+        size_text
+      else
+        result
+      end
+
+      timestamp = nil
+      begin
+        timestamp = Time.at(cache_wrapper.timestamp)
+      rescue TypeError => detail
+        $logger.error("cannot parse timestamp #{cache_wrapper.timestamp}")
+      end
+      response = Response.new(result, {}, timestamp)
+    else
+      response = request.next_filter.execute(request)
+      fresh_result = response.result
+      context = response.context
+
+      origin_class = fresh_result.class.to_s
+      if fresh_result.is_a?(Array)
+        origin_class += "/#{fresh_result.first.class.to_s}"
+      end
+      for_cache = CacheWrapper.new(fresh_result, {"origin" => origin_class})
+      json = nil
+      begin
+        # TODO filter sensitive data
+        json = for_cache.to_json
+        $logger.debug "caching for #{request.cache_key} (#{origin_class}) : #{json}"
+        redis.set(request.cache_key, json)
+      rescue => detail
+        $logger.warn("could not cache #{request.cache_key} - response not serializable")
+      end
+      result = fresh_result
+    end
+    raise ::Vop::InterruptChain.new(response)
+  else
+    request.next_filter.execute(request)
+  end
+end

data/extended/redis_cache/helpers/cache_wrapper.rb

@@ -0,0 +1,35 @@
+module Vop
+
+  class CacheWrapper
+
+    attr_reader :data, :options, :timestamp
+
+    def initialize(data, options = {}, timestamp = nil)
+      @data = data
+      @options = options
+      @timestamp = timestamp || Time.now
+    end
+
+    def to_json
+      begin
+        JSON.generate({
+          "options" => options,
+          "data" => data,
+          "timestamp" => @timestamp.to_i
+        })
+      rescue => detail
+        $logger.warn("problem generating JSON : #{detail.message}")
+        $logger.debug("offending payload : #{data.pretty_inspect}")
+        raise detail
+      end
+    end
+
+    def self.from_json string
+      return if string.nil?
+      payload = JSON.load(string)
+      self.new(payload["data"], payload["options"], payload["timestamp"])
+    end
+
+  end
+
+end

data/extended/redis_cache/redis_cache.plugin

@@ -0,0 +1,8 @@
+require "redis"
+
+on :init do |plugin|
+  plugin.state[:redis] = Redis.new()
+  $logger.debug "redis_cache init : #{plugin.state[:redis]}"
+end
+
+#autoload false

data/extended/spares/commands/convert_spare.rb

@@ -0,0 +1,93 @@
+description "converts a pre-installed spare machine into a usable installation"
+
+param! :machine, "the spare to convert"
+param! "new_name", "name for the new machine"
+
+run do |machine, new_name|
+  host = machine.parent
+
+  # TODO sanity check that this is called on a spare vm, not a host?
+  # TODO sanity check that the spare is in a usable state?
+  host_name = machine.name
+
+  @op.track_installation_status(
+    host_name: host_name,
+    vm_name: new_name,
+    status: "converting"
+  )
+
+  vm_name = machine.name.split(".").first
+  $logger.info "converting #{vm_name} into #{new_name} ..."
+
+  begin
+    # rename and shutdown
+    machine.set_hostname("new_name" => new_name)
+    begin
+      machine.sudo "shutdown -h now"
+    rescue
+      $logger.info "caught exception during VM shutdown, probably normal"
+    end
+
+    @op.wait("timeout" => 120, "block" => lambda do
+      vm = host.list_vms!.select { |x| x["name"] == vm_name }.first
+      raise "vm #{vm_name} not found on #{host.name}" if vm.nil?
+
+      vm["state"] == "shut off"
+    end)
+
+    # libvirt rename
+    host.rename_vm("vm_name" => vm_name, "new_name" => new_name)
+
+    # scan again so that we have SSH connect metadata
+    scanned = host.list_vms_for_scan
+    @op.machines_found(scanned)
+
+    # TODO adjust memory, CPU and disk (see 0.2.x)
+
+    # start converted VM
+    host.start_vm("name" => new_name)
+
+    @op.wait("timeout" => 120, "block" => lambda do
+      new_vm_list = @op.list_vms!("machine" => host.name)
+      vm = new_vm_list.select { |x| x["name"] == new_name }.first
+      raise "vm #{new_name} not found on #{host.name}" if vm.nil?
+
+      vm["state"] == "running"
+    end)
+
+    full_name = "#{new_name}.#{host.name}"
+
+    # wait until vm_addresses returns a meaningful result
+    @op.wait(
+      "timeout" => 30,
+      "block" => lambda do
+        addresses = host.vm_addresses!("name" => new_name)
+        $logger.debug "addresses : #{addresses.pretty_inspect}"
+        ! addresses.nil? && ! addresses.empty?
+      end
+    )
+
+    @op.ssh_options!("machine" => full_name)
+
+    # clean previous SSH keys for VMs with the same name/IP
+    vm_address = host.vm_address(name: new_name)
+    @op.clean_known_host(ip: vm_address)
+
+    @op.track_installation_status(
+      host_name: host_name,
+      vm_name: new_name,
+      status: "converted"
+    )
+  rescue => detail
+    $logger.error "problem converting spare #{vm_name} into #{new_name} : #{detail.message}"
+    @op.track_installation_status(
+      host_name: host_name,
+      vm_name: new_name,
+      status: "converting_failed"
+    )
+  ensure
+    host.post_convert_spare("converted" => vm_name)
+  end
+
+  full_name
+end

data/extended/spares/commands/list_spares.rb

@@ -0,0 +1,9 @@
+param! :machine
+
+run do |machine|
+  machine.list_vms.map do |vm|
+    vm["name"] =~ /^spare/ &&
+    vm["state"] == "running" &&
+    @op.test_ssh("machine" => "#{vm['name']}.#{machine.name}") ? vm["name"] : nil
+  end.compact
+end

data/extended/spares/commands/new_vm_from_spare.rb

@@ -0,0 +1,24 @@
+param! "name"
+
+param! :machine, description: "a virtualization host on which the VM should be created"
+
+run do |machine, name, params|
+  spare = full_name = nil
+
+  spares = machine.list_spares
+  # TODO this should happen within a lock
+  # TODO also, we should only use spares that have been installed completely
+  if spares.size > 0
+    spare = spares.first
+    spare_full_name = "#{spare}.#{machine.name}"
+    p = { :machine => spare_full_name, "new_name" => name }
+    $logger.info "calling convert_spare : #{p.pretty_inspect}"
+    full_name = @op.convert_spare(p)
+  end
+
+  if spare.nil?
+    full_name = machine.new_vm_from_scratch(params)
+  end
+
+  full_name
+end

data/extended/spares/commands/post_convert_spare.rb

@@ -0,0 +1,13 @@
+description "hook that is called when a spare has been converted"
+
+param! :machine
+param "converted"
+
+run do |machine, converted|
+  $logger.info "post_convert_spare #{machine.name} (#{converted})"
+  # TODO cleanup
+  #PrepareSparesWorker.perform_async(machine.name)
+
+  request = ::Vop::Request.new(@op, "prepare_spares", {"machine" => machine.name})
+  @op.execute_async(request)
+end

data/extended/spares/commands/prepare_spares.rb

@@ -0,0 +1,17 @@
+param! :machine
+param "count", description: "number of spares that should be available", default: 3
+
+run do |machine, count|
+  existing = machine.list_spares
+
+  result = []
+  1.upto(count.to_i) do |idx|
+    vm_name = "spare#{sprintf('%02d', idx)}"
+
+    unless existing.include? vm_name
+      machine.new_vm_from_scratch("name" => vm_name)
+      result << vm_name
+    end
+  end
+  result
+end

data/extended/spares/commands/wait.rb

@@ -0,0 +1,29 @@
+description "waits until a condition has been met"
+
+param! "block", "the block that should be executed periodically. should return true if the condition is met, false otherwise"
+param! "timeout", description: "number of seconds to wait for the condition to be met", default: 60
+param! "interval", description: "seconds to wait between tries", default: 1
+param  "error_text", description: "text that is displayed when the specified timeout is reached", default: "condition not met"
+
+run do |timeout, interval, error_text, params|
+  block = params["block"]
+  seconds_waited = 0
+
+  current_result = false
+  while (seconds_waited < timeout.to_i) do
+    begin
+      current_result = block.call()
+    rescue Exception => e
+      $logger.warn("got an exception while waiting : #{e.message}")
+    end
+
+    $logger.debug("checked condition : #{current_result}")
+    break if current_result
+
+    sleep(interval.to_i)
+    seconds_waited += interval.to_i
+  end
+
+  raise Exception.new(error_text) unless current_result
+  current_result
+end

data/extended/spares/commands/wtf.rb

@@ -0,0 +1,9 @@
+param! :machine
+param! "name"
+
+run do |machine, name|
+  full_name = machine.convert_spare("new_name" => name)
+  vm = @op.machines[full_name]
+  host = vm.parent
+  host.vm_addresses!("name" => name)
+end

data/extended/spares/spares.plugin

@@ -0,0 +1 @@
+depends_on :rails_machines

data/standard/apt/commands/add_apt_repo.rb

@@ -0,0 +1,14 @@
+param! :machine
+param! "url"
+param! "dist", default: "stable"
+param! "alias"
+param! "key"
+param "component", default: ""
+
+run do |params, machine, component|
+  # TODO verify that repo[:alias] is a valid filename?
+  machine.ssh("request_pty" => true,
+    "command" => "echo 'deb #{params["url"]} #{params["dist"]} #{component}' | sudo tee /etc/apt/sources.list.d/#{params["alias"]}.list")
+  machine.ssh("request_pty" => true, "command" => "wget -qO - #{params["key"]} | sudo apt-key add")
+  machine.sudo("apt-get update")
+end

data/standard/git/commands/deploy.rb

@@ -0,0 +1,19 @@
+param! :machine
+param! "git_url", description: "URL to the github repository to deploy"
+param! "domain", multi: true, description: "the domain on which the project should be deployed"
+param "subfolder", description: "folder inside git checkout that should be published (as web root)"
+
+contribute to: "deploy" do |machine, git_url, domain, params, subfolder|
+  machine.install_service("service" => "apache.apache")
+
+  web_root = machine.git_clone_web(git_url)
+  if subfolder
+    web_root = File.join(web_root, subfolder)
+  end
+  machine.add_static_vhost("server_name" => domain, "web_root" => web_root)
+
+  machine.parent.reverse_proxy.add_reverse_proxy(
+    "server_name" => domain,
+    "target_url" => "http://#{machine.internal_ip}/"
+  )
+end

data/standard/git/commands/deploy_from_git.rb

@@ -0,0 +1,34 @@
+param! :machine
+param! "git_url", description: "URL to the github repository to deploy"
+param! "domain", multi: true, description: "the domain on which the project should be deployed"
+param "subfolder", description: "folder inside git checkout that should be published (as web root)"
+
+contribute to: "deploy" do |machine, git_url, domain, params, subfolder|
+  host_name = machine.parent.name
+  @op.track_installation_status(
+    host_name: host_name,
+    vm_name: machine.name.split(".").first,
+    status: "deploying"
+  )
+
+  machine.install_service("service" => "apache.apache")
+
+  web_root = machine.git_clone_web(git_url)
+  if subfolder
+    web_root = File.join(web_root, subfolder)
+  end
+  machine.add_static_vhost("server_name" => domain, "web_root" => web_root)
+
+  # TODO actually, this should maybe be machine.publish(machine.internal_ip => domain)
+  # (and the apache-specific add_reverse_proxy contributes to publish)
+  machine.parent.reverse_proxy.add_reverse_proxy(
+    "server_name" => domain,
+    "target_url" => "http://#{machine.internal_ip}/"
+  )
+
+  @op.track_installation_status(
+    host_name: host_name,
+    vm_name: machine.name.split(".").first,
+    status: "deployed"
+  )
+end