logical-construct 0.0.1.localtesting → 0.0.1

data/lib/logical-construct/ground-control/core.rb

@@ -1,4 +1,3 @@
-p :loading => __FILE__
 require 'mattock'
 
 module LogicalConstruct

data/lib/logical-construct/ground-control/provision.rb

@@ -1,69 +1,186 @@
 require 'mattock'
-require 'rake/packagetask'
+require 'json'
+require 'logical-construct/resolving-task'
 
 module LogicalConstruct
   module GroundControl
     class Provision < Mattock::Tasklib
       class WebConfigure < Mattock::Task
-        setting :target_ipaddr, nil
-        setting :web_port, 51076
+        include ResolutionProtocol
+
+        setting :target_protocol, "http"
+        setting :target_address, nil
+        setting :target_port, 51076
+        runtime_setting :target_url
         setting :resolutions, {}
+        runtime_setting :web_resolutions
 
-        def action
-          connection = Excon.new(target_address)
-          index = connection.get
-          body = Nokogiri::HTML(index.body)
-          resolution_needed = body.xpath('//a[@href]')
-          resolution_needed.each do |link|
-            href = link['href']
-            connection.post(href, {"data" => resolve(href)})
-          end
+        def resolve_runtime_configuration
+          super
+          self.target_url ||= "#{target_protocol}://#{target_address}:#{target_port}/"
+          self.web_resolutions = Hash[resolutions.map do |name, value|
+            [web_path(name), value]
+          end]
         end
 
         def resolve(path)
-          resolved = resolutions.fetch(path)
+          resolved = web_resolutions.fetch(path)
           if resolved.respond_to? :call
             resolved = resolved.call
           end
           return resolved
+        rescue KeyError
+          puts "Can't find a resolution for #{path} in #{web_resolutions.keys.inspect} (ex #{resolutions.keys})"
+          raise
+        end
+
+        def uri(options)
+          uri_class = URI.scheme_list[target_protocol.upcase]
+          uri_hash = {:host => target_address, :port => target_port}
+          return uri_class.build(uri_hash.merge(options)).to_s
+        end
+
+        def resolution_needed
+          index = RestClient.get(uri(:path => '/'))
+          body = Nokogiri::HTML(index.body)
+          return body.xpath("//a[@rel='requirement']")
+        end
+
+
+        #XXX I would like this to become an actual RESTful client at some
+        #point, but seems it would mean building it from scratch
+        def action
+          require 'uri'
+          require 'rest-client'
+          require 'nokogiri'
+
+          until (link = resolution_needed.first).nil?
+            href = link['href']
+            begin
+              response = RestClient.post(uri(:path => href), :data => resolve(href))
+            rescue RestClient::InternalServerError => ex
+              require 'tempfile'
+              file = Tempfile.open('provision-error.html')
+              path = file.path
+              file.close!
+
+              File::open(path, "w") do |file|
+                file.write ex.http_body
+              end
+              puts "Written error response to #{path}"
+              puts "Try: chromium #{path}"
+              fail "Unsuccessful response from server!"
+            end
+          end
         end
       end
 
       default_namespace :provision
 
       setting :valise
-      setting :web_port, 51076
-      setting :target_ipaddr, nil
+      setting :target_protocol, "http"
+      setting(:target_address, nil).isnt(:copiable)
+      setting :target_port, 51076
       setting :resolutions, {}
-      setting :marshalling_path
-      setting :cookbooks_path
+      setting :marshalling_path, "marshall"
+
+      setting(:secret_data, nested {
+        setting :path
+        setting :tarball_path
+        setting :file_list
+      })
+
+      setting(:normal_data, nested {
+        setting :path
+        setting :tarball_path
+        setting :file_list
+      })
+
+      setting(:cookbooks, nested {
+        setting :path
+        setting :tarball_path
+        setting :file_list
+      })
+
+      setting :json_attribs_path
+      setting :roles, {}
+      setting :node_attribs, { "run_list" => [] }
+      setting :json_attribs, ""
 
       def default_configuration(core)
         core.copy_settings_to(self)
         super
+        self.cookbooks.path = "cookbooks"
+        self.secret_data.path = "data-bags/secret"
+        self.normal_data.path = "data-bags"
+      end
+
+      def resolve_configuration
+        super
+        self.json_attribs_path ||= File::join(marshalling_path, "node.json")
+
+        self.cookbooks.file_list ||= Rake::FileList[cookbooks.path + "/**/*"].exclude(%r{/[.]git/}).exclude(%r{[.]sw[.]$})
+        self.secret_data.file_list ||= Rake::FileList[secret_data.path + "/**/*"].exclude(%r{[.]sw[.]$})
+        self.normal_data.file_list ||=
+          Rake::FileList[normal_data.path + "/**/*"].exclude(%r{^#{secret_data.path}}).exclude(%r{[.]sw[.]$})
+
+        self.cookbooks.tarball_path ||= File::join(marshalling_path, "cookbooks.tgz")
+        self.secret_data.tarball_path ||= File::join(marshalling_path, "secret_data_bags.tgz")
+        self.normal_data.tarball_path ||= File::join(marshalling_path, "normal_data_bags.tgz")
+
+        resolutions["chef_config:cookbook_tarball"] ||= proc do
+          File::open(cookbooks.tarball_path, "rb")
+        end
+
+        resolutions["chef_config:secret_data_tarball"] ||= proc do
+          File::open(secret_data.tarball_path, "rb")
+        end
+
+        resolutions["chef_config:normal_data_tarball"] ||= proc do
+          File::open(normal_data.tarball_path, "rb")
+        end
       end
 
+      include Mattock::CommandLineDSL
       def define
         in_namespace do
-          task :collect, [:ipaddr] do |task, args|
-            self.target_ipaddr = args[:ipaddr]
+          directory marshalling_path
+
+          task :collect, [:ipaddr, :role] do |task, args|
+            self.target_address = args[:ipaddr]
+            unless args[:role].nil?
+              self.node_attribs["run_list"] = roles[args[:role]]
+            end
+            self.json_attribs = JSON.pretty_generate(node_attribs)
+            resolutions["chef_config:json_attribs"] ||= json_attribs
           end
 
-          WebConfigure.new(:web_configure => :collect) do |task|
-            self.copy_settings_to(task)
+          file secret_data.tarball_path => [marshalling_path] + secret_data.file_list do
+            cmd("tar", "--exclude **/*.sw?", "-czf", secret_data.tarball_path, secret_data.path).must_succeed!
           end
 
-          namespace :cookbook do
-            Rake::PackageTask.new("cookbook", :noversion) do |task|
-              task.need_tar_gz = true
-              task.package_dir = marshalling_path
-              task.package_files.include(cookbooks_path + "/**/*")
-            end
+          file normal_data.tarball_path => [marshalling_path] + normal_data.file_list do
+            cmd("tar",
+                "--exclude **/*.sw?",
+                "--exclude #{secret_data.path}",
+                "-czf", normal_data.tarball_path, normal_data.path).must_succeed!
           end
-        end
 
-        task root_task, [:ipaddr] => :web_configure
+          file cookbooks.tarball_path => [marshalling_path] + cookbooks.file_list do
+            cmd("tar", "--exclude .git", "--exclude **/*.sw?", "-czf", cookbooks.tarball_path, cookbooks.path).must_succeed!
+          end
+
+          manifest = LogicalConstruct::GenerateManifest.new(self, :manifest => [cookbooks.tarball_path, :collect]) do |manifest|
+            manifest.receiving_name = "configuration:Manifest"
+          end
+
+          WebConfigure.new(:web_configure => [:collect, :manifest, cookbooks.tarball_path]) do |task|
+            self.proxy_settings_to(task)
+            task.target_address = proxy_value.target_address
+          end
+        end
 
+        task root_task, [:ipaddr] => self[:web_configure]
       end
     end
   end

data/lib/logical-construct/ground-control/run-on-target.rb

@@ -5,6 +5,9 @@ module LogicalConstruct
   class RunOnTarget < Mattock::TaskLib
     include Mattock::CommandLineDSL
 
+    SSH_OPTIONS = [ "ControlMaster=auto", "ControlPersist=3600",
+      "StrictHostKeyChecking=no", "UserKnownHostsFile=/dev/null" ]
+
     runtime_setting(:remote_server)
 
     def default_configuration(setup)
@@ -16,10 +19,7 @@ module LogicalConstruct
       in_namespace do
         desc comment unless comment.nil?
         Mattock::RemoteCommandTask.new(name) do |task|
-          task.ssh_options << "ControlMaster=auto"
-          task.ssh_options << "ControlPersist=3600"
-          task.ssh_options << "StrictHostKeyChecking=no"
-          task.ssh_options << "UserKnownHostsFile=/dev/null"
+          task.ssh_options += SSH_OPTIONS
 
           task.runtime_definition do |task|
             copy_settings_to(task)

data/lib/logical-construct/ground-control/setup.rb

@@ -12,6 +12,7 @@ module LogicalConstruct
       nil_fields :valise
 
       def default_configuration(core)
+        super
         core.copy_settings_to(self)
       end
 
@@ -19,17 +20,9 @@ module LogicalConstruct
         in_namespace do
           task :collect, [:address] do |t, args|
             remote_server.address = args[:address]
-            p :collect => remote_server
           end
 
-          task :local_setup => [:collect]
-
-          task :remote_groundwork => [:local_setup]
-
-          task :remote_config => [:remote_groundwork]
-
-          task :remote_setup => [:remote_config]
-
+          task_spine(:collect, :local_setup, :remote_groundwork, :remote_config, :remote_setup)
           task :complete => [:local_setup, :remote_setup]
         end
 
@@ -40,18 +33,16 @@ module LogicalConstruct
       def default_subtasks
         in_namespace do
           CreateConstructDirectory.new(self)
-          EnsureEnv.new(self)
-          BundleSetup.new(self)
           build_files = BuildFiles.new(self)
           CopyFiles.new(self, build_files)
+          InstallInit.new(self)
         end
       end
     end
   end
 end
 
-require 'logical-construct/ground-control/setup/bundle-setup'
+require 'logical-construct/ground-control/setup/build-files'
 require 'logical-construct/ground-control/setup/create-construct-directory'
-require 'logical-construct/ground-control/setup/ensure-env'
 require 'logical-construct/ground-control/setup/copy-files'
-require 'logical-construct/ground-control/setup/build-files'
+require 'logical-construct/ground-control/setup/install-init'

data/lib/logical-construct/ground-control/setup/build-files.rb

@@ -1,3 +1,6 @@
+require 'mattock'
+require 'mattock/bundle-command-task'
+
 module LogicalConstruct
   class ConfigBuilder < Mattock::TaskLib
     include Mattock::TemplateHost
@@ -28,36 +31,54 @@ module LogicalConstruct
           file.write render(source_path)
         end
       end
+      file target_path => target_dir
       task :local_setup => target_path
     end
   end
 
   class BuildFiles < Mattock::TaskLib
+    include Mattock::CommandLineDSL
+
     default_namespace :build_files
 
     setting(:target_dir, "target_configs")
-    setting(:valise)
+    required_fields :valise, :construct_dir
 
     def default_configuration(parent)
       super
-      self.valise = parent.valise
+      parent.copy_settings_to(self)
     end
 
-    attr_reader :built_files
-
     def define
-      file_tasks = []
+      rakefile = nil
       in_namespace do
         directory target_dir
 
-        file_tasks = ["Rakefile", "Gemfile"].map do |path|
-          ConfigBuilder.new(self) do |task|
-            task.base_name = path
-          end
+        gemfile = ConfigBuilder.new(self) do |task|
+          task.base_name = "Gemfile"
+        end
+
+        Mattock::BundleCommandTask.new(:standalone => gemfile.target_path) do |bundle_build|
+          bundle_build.command = (
+            cmd("cd", target_dir) &
+            cmd("pwd") &
+            cmd("bundle", "install"){|bundler|
+            bundler.options << "--standalone"
+            bundler.options << "--binstubs=bin"
+          })
+        end
+
+        rakefile = ConfigBuilder.new(self) do |task|
+          task.base_name = "Rakefile"
+        end
+
+        initd = ConfigBuilder.new(self) do |task|
+          task.base_name = "construct.init.d"
+          task.extra[:construct_dir] = construct_dir
         end
       end
       desc "Template files to be created on the remote server"
-      task root_task => file_tasks.map{|t| t.target_path}
+      task root_task => [rakefile.target_path] + in_namespace(:standalone)
       task :local_setup => root_task
     end
   end

data/lib/logical-construct/ground-control/setup/copy-files.rb

@@ -1,14 +1,16 @@
 require 'mattock/command-task'
 
 module LogicalConstruct
-  class SecureCopyFile < Mattock::CommandTask
+  class RemoteCopyFile < Mattock::CommandTask
     nil_fields :destination_address
     nil_fields :source_dir, :destination_dir, :basename
     required_fields :source_path, :destination_path
     runtime_required_field :remote_server
     runtime_required_field :command
+    setting :exclude, []
 
     def default_configuration(copy)
+      super
       self.remote_server = copy.proxy_value.remote_server
     end
 
@@ -18,14 +20,27 @@ module LogicalConstruct
       self.destination_path ||= File::join(destination_dir, basename)
     end
 
+    def secure_shell_command
+      escaped_command("ssh") do |ssh|
+        ssh.options += RunOnTarget::SSH_OPTIONS.map{|opt| "-o #{opt}"}
+        ssh.options << "-l #{remote_server.user}" unless remote_server.user.nil?
+      end
+    end
+
     def resolve_runtime_configuration
       self.destination_address ||= [remote_server.address, destination_path].join(":")
-      if remote_server.user
-        self.destination_address = "#{remote_server.user}@#{destination_address}"
+
+      self.command = cmd("rsync") do |rsync|
+        rsync.options << "-a"
+        rsync.options << "--copy-unsafe-links"
+        rsync.options << "-e #{secure_shell_command}"
+        exclude.each do |pattern|
+          rsync.options << "--exclude #{pattern}"
+        end
+        rsync.options << source_path
+        rsync.options << destination_address
       end
-      self.command = cmd("scp",
-                         "-o ControlMaster=auto",
-                         source_path, destination_address)
+      super
     end
   end
 
@@ -34,8 +49,6 @@ module LogicalConstruct
 
     required_fields :files_dir, :remote_server, :construct_dir
 
-    setting :files, ["Rakefile", "Gemfile"]
-
     def default_configuration(setup, build_files)
       super()
       self.files_dir = build_files.target_dir
@@ -44,22 +57,21 @@ module LogicalConstruct
     end
 
     def define
-      files.each do |file|
-        in_namespace do
-          SecureCopyFile.new(self, file) do |task|
-            task.runtime_definition do
-              task.remote_server = remote_server
-            end
-            task.source_dir = files_dir
-            task.destination_dir = construct_dir
-            task.basename = file
+      in_namespace do
+        RemoteCopyFile.new(self, :construct_dir) do |task|
+          task.runtime_definition do
+            task.remote_server = remote_server
           end
+          task.exclude << "*.so"
+          task.exclude << "*.dynlib"
+          task.source_path = File::join(files_dir, "*")
+          task.destination_path = construct_dir
         end
-        bracket_task(:local_setup, file, :remote_config)
       end
+      bracket_task(:remote_groundwork, :construct_dir, :remote_config)
 
       desc "Copy locally generated files to the remote server"
-      task root_task => in_namespace(*files)
+      task root_task => in_namespace(:construct_dir)
       task :remote_config => root_task
     end
   end