logical-construct 0.0.5 → 0.1.0

data/lib/logical-construct/ground-control/generate-manifest.rb

@@ -0,0 +1,67 @@
+require 'logical-construct/node-client'
+
+module LogicalConstruct
+  class GenerateManifest < Mattock::Tasklib
+    default_namespace :manifest
+
+    setting :plan_archives, []
+    setting :graph_format, :jsonld
+    setting :target_address, 'localhost'
+    setting :target_port, 51076
+
+    def default_configuration(provision)
+      super
+      self.plan_archives = provision.proxy_value.plan_archives
+      self.target_address = provision.proxy_value.target_address
+      self.target_port = provision.proxy_value.local_target_port
+    end
+
+    def node_url
+      "http://#{target_address}:#{target_port}"
+    end
+
+    def define
+      in_namespace do
+        desc "Dump manifest (mostly for debugging)"
+        task :dump, [:format] do |task, args|
+          require 'rdf/turtle'
+          format = args[:format] || graph_format
+          format = format.to_sym
+
+          base_url = "urn:manifest"
+
+          graph = ::RDF::Graph.new
+          focus = RoadForest::RDF::GraphFocus.new(base_url, graph)
+          builder = NodeClient::ManifestBuilder.new(focus)
+
+          plan_archives.each do |archive|
+            builder.add(archive)
+          end
+
+          puts(RDF::Writer.for(format).buffer(:base_uri => base_url) do |writer|
+            focus.relevant_prefixes.each do |prefix, uri|
+              writer.prefix(prefix, uri)
+            end
+            writer.insert(graph)
+          end)
+        end
+
+        task :deliver do |task|
+          client = NodeClient.new
+          client.node_url = node_url
+          client.plan_archives = plan_archives
+          client.deliver_manifest
+        end
+
+        task :fulfill do |task|
+          client = NodeClient.new
+          client.node_url = node_url
+          client.plan_archives = plan_archives
+          client.deliver_plans
+        end
+      end
+      task self[:dump] => root_task
+      task self[:deliver] => root_task
+    end
+  end
+end

data/lib/logical-construct/ground-control/provision.rb

@@ -1,204 +1,109 @@
 require 'mattock'
-require 'json'
-require 'logical-construct/resolving-task'
+
+require 'logical-construct/ground-control/generate-manifest'
+require 'logical-construct/ground-control/build-plan'
+require 'logical-construct/protocol/ssh-tunnel'
 
 module LogicalConstruct
   module GroundControl
     class Provision < Mattock::Tasklib
-      class WebConfigure < Mattock::Task
-        include ResolutionProtocol
-
-        setting :target_protocol, "http"
-        setting :target_address, nil
-        setting :target_port, 51076
-        runtime_setting :target_url
-        setting :resolutions
-        runtime_setting :web_resolutions
-
-        def resolve_runtime_configuration
-          super
-          self.target_url ||= "#{target_protocol}://#{target_address}:#{target_port}/"
-          self.web_resolutions = Hash[resolutions.map do |name, value|
-            [web_path(name), value]
-          end]
-        end
-
-        def resolve(path)
-          resolved = web_resolutions.fetch(path)
-          if resolved.respond_to? :call
-            resolved = resolved.call
-          end
-          return resolved
-        rescue KeyError
-          puts "Can't find a resolution for #{path} in #{web_resolutions.keys.inspect} (ex #{resolutions.keys})"
-          raise
-        end
-
-        def uri(options)
-          uri_class = URI.scheme_list[target_protocol.upcase]
-          uri_hash = {:host => target_address, :port => target_port}
-          return uri_class.build(uri_hash.merge(options)).to_s
-        end
-
-        def resolution_needed
-          index = RestClient.get(uri(:path => '/'))
-          body = Nokogiri::HTML(index.body)
-          return body.xpath("//a[@rel='requirement']")
-        end
-
-
-        #XXX I would like this to become an actual RESTful client at some
-        #point, but seems it would mean building it from scratch
-        def action
-          require 'uri'
-          require 'rest-client'
-          require 'nokogiri'
-
-          until (link = resolution_needed.first).nil?
-            href = link['href']
-            begin
-              response = RestClient.post(uri(:path => href), :data => resolve(href))
-            rescue RestClient::InternalServerError => ex
-              require 'tempfile'
-              file = Tempfile.open('provision-error.html')
-              path = file.path
-              file.close!
-
-              File::open(path, "w") do |file|
-                file.write ex.http_body
-              end
-              puts "Written error response to #{path}"
-              puts "Try: chromium #{path}"
-              fail "Unsuccessful response from server!"
-            end
-          end
-        end
-      end
-
       default_namespace :provision
 
-      setting :valise
       setting :target_protocol, "http"
       setting(:target_address, nil).isnt(:copiable)
-      setting :target_port, 51076
-      setting :resolutions
-      setting :marshalling_path, "marshall"
-
-      setting(:secret_data, nested {
-        setting :path
-        setting :tarball_path
-        setting :file_list
-      })
-
-      setting(:normal_data, nested {
-        setting :path
-        setting :tarball_path
-        setting :file_list
-      })
-
-      setting(:cookbooks, nested {
-        setting :path
-        setting :tarball_path
-        setting :file_list
-      })
-
-      setting :json_attribs_path
-      setting :roles
-      setting :node_attribs
-      setting :json_attribs, ""
-
-      def default_configuration(core)
-        core.copy_settings_to(self)
-        super
-        self.cookbooks.path = "cookbooks"
-        self.secret_data.path = "data-bags/secret"
-        self.normal_data.path = "data-bags"
-        self.resolutions = {}
-        self.roles = {}
-        self.node_attribs = { "run_list" => [] }
-      end
-
-      def resolve_configuration
-        super
-        self.json_attribs_path ||= File::join(marshalling_path, "node.json")
+      setting :local_target_port, 51076
+      setting :remote_target_port, 30712
 
-        self.cookbooks.file_list ||= Rake::FileList[cookbooks.path + "/**/*"].exclude(%r{/[.]git/}).exclude(%r{[.]sw[.]$})
-        self.secret_data.file_list ||= Rake::FileList[secret_data.path + "/**/*"].exclude(%r{[.]sw[.]$})
-        self.normal_data.file_list ||=
-          Rake::FileList[normal_data.path + "/**/*"].exclude(%r{^#{secret_data.path}}).exclude(%r{[.]sw[.]$})
+      setting :plan_archives, []
 
-        self.cookbooks.tarball_path ||= File::join(marshalling_path, "cookbooks.tgz")
-        self.secret_data.tarball_path ||= File::join(marshalling_path, "secret_data_bags.tgz")
-        self.normal_data.tarball_path ||= File::join(marshalling_path, "normal_data_bags.tgz")
+      dir(:marshalling, "marshall")
+      dir(:plan_source, "plans")
 
-        resolutions["chef_config:cookbook_tarball"] ||= proc do
-          File::open(cookbooks.tarball_path, "rb")
-        end
-
-        resolutions["chef_config:secret_data_tarball"] ||= proc do
-          File::open(secret_data.tarball_path, "rb")
-        end
-
-        resolutions["chef_config:normal_data_tarball"] ||= proc do
-          File::open(normal_data.tarball_path, "rb")
-        end
+      def resolve_configuration
+        resolve_paths
+        super
       end
 
-      include Mattock::CommandLineDSL
       def define
+        manifest = nil
         in_namespace do
-          directory marshalling_path
+          directory marshalling.absolute_path
 
           task :collect, [:ipaddr] do |task, args|
             self.target_address = args[:ipaddr]
           end
 
-          task :build_json_attribs, [:role] do |task, args|
-            unless args[:role].nil?
-              self.node_attribs["run_list"] = roles[args[:role]]
-            end
-            self.json_attribs = JSON.pretty_generate(node_attribs)
-            resolutions["chef_config:json_attribs"] ||= json_attribs
+          tunnel = LogicalConstruct::SSHTunnel.new do |tunnel|
+            tunnel.target_address = proxy_value.target_address
+            copy_settings_to(tunnel)
+            self.local_target_port = tunnel.proxy_value.local_target_port
           end
 
-          desc "Print attribs (optionally: for :role)"
-          task :inspect_attribs, [:role] => :build_json_attribs do
-            puts json_attribs
+          manifest = LogicalConstruct::GenerateManifest.new(self)
+
+          start_flight = Mattock::Rake::RemoteCommandTask.define_task(:start_flight => :collect) do |start_flight|
+            start_flight.remote_server.address = proxy_value.target_address
+            start_flight.command =
+              Mattock::CommandLine.new("nohup",
+                                       "/opt/logical-construct/bin/flight-deck",
+                                       "-C start_server")
+            start_flight.command.redirect_stdin("/dev/null")
+            start_flight.command.redirect_stdout("/opt/logical-construct/flight_deck_server.log")
+            start_flight.command.copy_stream_to(2, 1)
+            start_flight.command.redirections << "&"
           end
 
-          file secret_data.tarball_path => [marshalling_path] + secret_data.file_list do
-            cmd("tar", "--exclude **/*.sw?", "-czf", secret_data.tarball_path, secret_data.path).must_succeed!
+          start_resolution = Mattock::Rake::RemoteCommandTask.define_task(:start_resolution => :deliver_manifest) do |start_flight|
+            start_flight.remote_server.address = proxy_value.target_address
+            start_flight.verbose = 3
+            start_flight.command =
+              Mattock::CommandLine.new("nohup",
+                                       "/opt/logical-construct/bin/flight-deck")
+            #TODO: Mattock CommandLine needs a "background"
+            #TODO: RemoteCommandTask should wrap the whole
+            #nohup-redirect-background thing
+            start_flight.command.redirect_stdin("/dev/null")
+            start_flight.command.redirect_stdout("/opt/logical-construct/flight_deck.log")
+            start_flight.command.copy_stream_to(2, 1)
+            start_flight.command.redirections << "&"
           end
 
-          file normal_data.tarball_path => [marshalling_path] + normal_data.file_list do
-            cmd("tar",
-                "--exclude **/*.sw?",
-                "--exclude #{secret_data.path}",
-                "-czf", normal_data.tarball_path, normal_data.path).must_succeed!
-          end
+          task manifest.root_task => :collect
 
-          file cookbooks.tarball_path => [marshalling_path] + cookbooks.file_list do
-            cmd("tar", "--exclude .git", "--exclude **/*.sw?", "-czf", cookbooks.tarball_path, cookbooks.path).must_succeed!
-          end
+          task_spine(:start_flight, :deliver_manifest, :fulfill_manifest, :start_resolution, :complete_provision)
+          task :deliver_manifest => manifest[:deliver]
+          task :fulfill_manifest => manifest[:fulfill]
 
-          manifest = LogicalConstruct::GenerateManifest.new(self, :manifest =>
-                                                            [
-                                                              cookbooks.tarball_path,
-                                                              secret_data.tarball_path,
-                                                              normal_data.tarball_path,
-                                                              :collect
-                                                            ]) do |manifest|
-            manifest.receiving_name = "configuration:Manifest"
-          end
+          tunnel.wrap(manifest[:deliver])
+          tunnel.wrap(manifest[:fulfill])
+          task :complete_provision => tunnel[:close_tunnel]
+        end
+
+        desc "Provision :ipaddr with specified configs"
+        task root_task, [:ipaddr] => self[:complete_provision]
+      end
+
+      def plan_task(name, &block)
+        plan = BuildPlan.new(self) do |build|
+          build.basename = name
+          yield build if block_given?
+        end
+        task self[:manifest] => plan.archive_path
 
-          WebConfigure.new(:web_configure => [:collect, :build_json_attribs, :manifest, cookbooks.tarball_path]) do |task|
-            self.proxy_settings_to(task)
-            task.target_address = proxy_value.target_address
+        in_namespace do
+          namespace :package do
+            desc "Compile and archive plan #{name.inspect}"
+            task name => plan.archive_path
           end
         end
 
-        desc "Provision :ipaddr with specified configs (optionally: for :role)"
-        task root_task, [:ipaddr, :role] => self[:web_configure]
+        plan_archives << plan.archive_path
+      end
+
+      def plans(*names)
+        names.each do |name|
+          plan_task(name)
+        end
       end
     end
   end

data/lib/logical-construct/ground-control/run-on-target.rb

@@ -18,7 +18,7 @@ module LogicalConstruct
     def remote_task(name, comment = nil)
       in_namespace do
         desc comment unless comment.nil?
-        Mattock::RemoteCommandTask.new(name) do |task|
+        Mattock::Rake::RemoteCommandTask.define_task(name) do |task|
           task.ssh_options += SSH_OPTIONS
 
           task.runtime_definition do |task|

data/lib/logical-construct/ground-control/setup.rb

@@ -6,7 +6,7 @@ module LogicalConstruct
 
       settings(
         :remote_server => nested( :address => nil, :user => "root"),
-        :construct_dir => "/var/logical-construct"
+        :construct_dir => "/opt/logical-construct"
       )
       nil_fields :valise, :platform
 
@@ -47,7 +47,4 @@ module LogicalConstruct
   end
 end
 
-require 'logical-construct/ground-control/setup/build-files'
-require 'logical-construct/ground-control/setup/create-construct-directory'
 require 'logical-construct/ground-control/setup/copy-files'
-require 'logical-construct/ground-control/setup/install-init'

data/lib/logical-construct/ground-control/setup/copy-files.rb

@@ -1,7 +1,7 @@
 require 'mattock/command-task'
 
 module LogicalConstruct
-  class RemoteCopyFile < Mattock::CommandTask
+  class RemoteCopyFile < Mattock::Rake::CommandTask
     nil_fields :destination_address
     nil_fields :source_dir, :destination_dir, :basename
     required_fields :source_path, :destination_path
@@ -58,7 +58,7 @@ module LogicalConstruct
 
     def define
       in_namespace do
-        RemoteCopyFile.new(self, :construct_dir) do |task|
+        RemoteCopyFile.define_task(self, :construct_dir) do |task|
           task.runtime_definition do
             task.remote_server = remote_server
           end

data/lib/logical-construct/ground-control/tools.rb

@@ -0,0 +1,66 @@
+require 'mattock'
+
+module LogicalConstruct
+  module GroundControl
+    class Tools < ::Mattock::Tasklib
+      include Mattock::CommandLineDSL
+
+      default_namespace :tools
+
+      dir(:plans, "plans")
+
+      def resolve_configuration
+        resolve_paths
+        super
+      end
+
+      def plans_dir
+        plans.absolute_path.sub(%r{/$},'')
+      end
+
+      def define
+        in_namespace do
+          namespace :create_plan do
+            rule %r{\A#{plans_dir}/[^/]*\Z} do |task, args|
+              cmd("mkdir", "-p", task.name).must_succeed! #ok
+            end
+
+            rule(%r{\A#{plans_dir}/[^/]*/plan\.rake\Z}, [:name] => ['%d']) do |task, args|
+              require 'logical-construct/target/implementation'
+
+              File::open(task.name, "w") do |file|
+                indent = 16
+                file.write(<<-EOR.gsub(/^#{" "*indent}/,''))
+                require 'logical-construct/plan'
+                include LogicalConstruct::Plan
+
+                core = Core.new do |core|
+                  core.namespace_name = :#{args[:name]}
+                  core.plan_rakefile.absolute_path = __FILE__
+                end
+
+                core.in_namespace do
+                  #Plan tasks go here
+                  #
+                  #Important tasks to make dependencies to:
+
+                EOR
+
+                Target::Implementation.task_list.each do |task_name|
+                  file.puts("  #task :#{task_name}")
+                end
+
+                file.puts("end")
+              end
+            end
+          end
+
+          desc "Create a new plan to be part of a provisioning"
+          task :create_plan, [:name] do |task, args|
+            Rake::Task[File::join(plans_dir, args[:name], "plan.rake")].invoke(args[:name])
+          end
+        end
+      end
+    end
+  end
+end