conjure 0.0.0 → 0.0.1

data/History.md

@@ -1,3 +1,10 @@
+### Version 0.0.1
+2013-10-11
+
+* Deploy to cloud servers using Docker and DigitalOcean
+* Import and export the deployed app's database
+* Documentation improvements
+
 ### Version 0.0.0
 2013-10-07
 

data/README.md

@@ -1,5 +1,88 @@
-### Conjure
+## Conjure
 
 Magically powerful deployment for Rails applications.
 
-_More instructions forthcoming._
+### Requirements
+
+Deploying a Rails application with Conjure currently requires the
+following:
+
+  * A DigitalOcean account
+
+  * A Public/private SSH keypair to use for bootstrapping new cloud
+    servers. Generating a new keypair for this purpose is recommended.
+
+Also, your Rails application requires all of the following:
+
+  * It must have a `.ruby-version` file indicating which version of
+    Ruby to run
+
+  * It must be able to run in production mode with a single
+    Postgres database (any existing database.yml will be ignored)
+
+  * It must be checked out locally into a git repository with a valid
+    `origin` remote
+
+  * The public SSH key you're using must have permission to check out
+    the project from `origin`
+
+### Getting Started
+
+First, install the Conjure gem by either adding it to your Gemfile
+
+    group :development do
+      gem "conjure"
+    end
+
+and then running `bundle`, OR by installing it directly:
+
+    gem install conjure
+
+Then add a file to your Rails project called
+`config/conjure.yml`. This should be a YAML file with the following
+fields (all fields are required):
+
+  * `digitalocean_client_id` and `digitalocean_api_key`: These
+    credentials are available after logging in to your Digital Ocean
+    account.
+
+  * `digitalocean_region`: The geographic region for deploying new
+    cloud servers. If unsure, use "New York 1".
+
+  * `private_key_file` and `public_key_file`: Pathnames to local files
+    (relative to your project's `config` directory) that contain the
+    private and public SSH keys to use for deployment. It's
+    recommended to generate a new keypair rather than using your
+    personal SSH keys, since Conjure currently copies the specified
+    private key to the server during deployment.
+
+Here's an example conjure.yml file:
+
+    digitalocean_client_id: XXXXXXXX
+    digitalocean_api_key: XXXXXXXX
+    digitalocean_region: New York 1
+    private_key_file: conjure_key
+    public_key_file: conjure_key.pub
+
+Finally, tell Conjure to deploy your app:
+
+    conjure deploy
+
+The last line of the output will tell you the IP address of the
+deployed server. Repeating the command will reuse the existing server
+rather than deploying a new one.
+
+### Additional Commands
+
+These commands are available after you've deployed with `conjure
+deploy`.
+
+    conjure export FILE
+
+This will produce a Postgres SQL dump of the currently-deployed
+server's production database, and save it to the local file `FILE`.
+
+    conjure import FILE
+
+This will overwrite the production database on the currently-deployed
+server with a Postgres SQL dump from the local file `FILE`.

data/lib/conjure.rb

@@ -1,6 +1,6 @@
 module Conjure
 
-  VERSION = "0.0.0" unless defined?(VERSION)
+  VERSION = "0.0.1" unless defined?(VERSION)
   autoload :Command, "conjure/command"
   autoload :Service, "conjure/service"
 

data/lib/conjure/command.rb

@@ -1,10 +1,57 @@
 module Conjure
   class Command < Thor
     desc "deploy", "Deploys the app"
-    def deploy(source_path = Dir.pwd)
-      Service::RailsApplication.create source_path
+    def deploy
+      Service::RailsApplication.create github_url, app_name, rails_environment, config(Dir.pwd)
+    end
+
+    desc "import FILE", "Imports the production database from a postgres SQL dump"
+    def import(file)
+      Service::PostgresClient.create(docker_host, "#{app_name}_#{rails_environment}").import file
+      puts "[export] #{File.size file} bytes imported from #{file}"
+    end
+
+    desc "export FILE", "Exports the production database to a postgres SQL dump"
+    def export(file)
+      Service::PostgresClient.create(docker_host, "#{app_name}_#{rails_environment}").export file
+      puts "[export] #{File.size file} bytes exported to #{file}"
+    end
+
+    default_task :help
+
+    private
+
+    def rails_environment
+      "production"
+    end
+
+    def docker_host
+      Service::DockerHost.create "#{app_name}-#{rails_environment}", config(Dir.pwd)
+    end
+
+    def config(source_path)
+      require "ostruct"
+      config_path = File.join source_path, "config", "conjure.yml"
+      data = YAML.load_file config_path
+      data["config_path"] = File.dirname config_path
+      OpenStruct.new data
+    end
+
+    def app_name
+      name_from_github_url github_url
+    end
+
+    def github_url
+      git_origin_url Dir.pwd
+    end
+
+    def git_origin_url(source_path)
+      remote_info = `cd #{source_path}; git remote -v |grep origin`
+      remote_info.match(/(git@github.com[^ ]+)/)[1]
+    end
+
+    def name_from_github_url(github_url)
+      github_url.match(/\/([^.]+)\.git$/)[1]
     end
-    default_task :deploy
   end
 end
-

data/lib/conjure/service.rb

@@ -2,30 +2,16 @@ module Conjure
   module Service
     autoload :RailsApplication, "conjure/service/rails_application"
     autoload :RailsServer, "conjure/service/rails_server"
-    autoload :RvmShell, "conjure/service/rvm_shell"
     autoload :MachineInstance, "conjure/service/machine_instance"
-    autoload :SourceTree, "conjure/service/source_tree"
+    autoload :DockerHost, "conjure/service/docker_host"
+    autoload :CloudServer, "conjure/service/cloud_server"
+    autoload :PostgresServer, "conjure/service/postgres_server"
+    autoload :PostgresClient, "conjure/service/postgres_client"
   end
 
   class Basic
-    def dependencies
-      []
-    end
-
-    def started?
-      false
-    end
-
-    def start
-    end
-
-    def save
-      dependencies.each &:start
-      start unless started?
-    end
-
     def self.create(*args)
-      new(*args).save
+      new(*args)
     end
   end
 end

data/lib/conjure/service/cloud_server.rb

@@ -0,0 +1,108 @@
+module Conjure
+  module Service
+    class CloudServer < Basic
+      require "fog"
+
+      def initialize(name, config = {})
+        @name = name
+        @config = config
+      end
+
+      def run(command, options = {})
+        set_fog_credentials
+        upload_files options[:files].to_a
+        result = server.ssh(command).first
+        remove_files options[:files].to_a
+        result
+      end
+
+      def upload_files(files)
+        dir_names = files.map{|local_path, remote_path| File.dirname remote_path}.uniq
+        server.ssh "mkdir -p #{dir_names.join ' '}" if dir_names.any?
+        files.each{|local_path, remote_path| server.scp local_path, remote_path}
+      end
+
+      def remove_files(files)
+        files.each{|local_path, remote_path| server.ssh "rm -f #{remote_path}"}
+      end
+
+      def server
+        @server ||= existing_server
+        @server ||= new_server
+        @server.wait_for { ready? }
+        @server
+      end
+
+      def ip_address
+        @server.public_ip_address
+      end
+
+      def existing_server
+        server = connection.servers.find{|s| s.name == @name }
+        puts " [cloud] Using existing server #{@name}" if server
+        server
+      end
+
+      def new_server
+        puts " [cloud] Launching new server #{@name}"
+        connection.servers.bootstrap bootstrap_options.merge(fog_credentials)
+      end
+
+      def connection
+        @connection ||= Fog::Compute.new compute_options
+      end
+
+      def bootstrap_options
+        {
+          name: @name,
+          flavor_id: flavor_id,
+          region_id: region_id,
+          image_id: image_id,
+        }
+      end
+
+      def compute_options
+        {
+          provider: :digitalocean,
+          digitalocean_api_key: config.digitalocean_api_key,
+          digitalocean_client_id: config.digitalocean_client_id,
+        }
+      end
+
+      def flavor_id
+        @flavor_id ||= connection.flavors.find{|f| f.name == "512MB"}.id
+      end
+
+      def region_id
+        @region_id ||= connection.regions.find{|r| r.name == config.digitalocean_region}.id
+      end
+
+      def image_id
+        @image_id ||= connection.images.find{|r| r.name == "Ubuntu 13.04 x64"}.id
+      end
+
+      def config
+        @config
+      end
+
+      def set_fog_credentials
+        Fog.credentials.merge! fog_credentials
+      end
+
+      def private_key_file
+        Pathname.new(config.config_path).join config.private_key_file
+      end
+
+      def public_key_file
+        Pathname.new(config.config_path).join config.public_key_file
+      end
+
+      def fog_credentials
+        {
+          private_key_path: private_key_file,
+          public_key_path: public_key_file,
+        }
+      end
+    end
+  end
+end

data/lib/conjure/service/docker_host.rb

@@ -0,0 +1,171 @@
+module Conjure
+  module Service
+    class DockerHost < Basic
+      VERBOSE = false
+
+      def initialize(server_name, config = {})
+        @server_name = server_name
+        @config = config
+      end
+
+      def server
+        @server ||= Service::CloudServer.create @server_name, config
+      end
+
+      def config
+        @config
+      end
+
+      def ip_address
+        @server.ip_address
+      end
+
+      def new_docker_path
+        puts "[docker] Installing docker"
+        server.run "curl https://get.docker.io/gpg | apt-key add -"
+        server.run "echo 'deb https://get.docker.io/ubuntu docker main' >/etc/apt/sources.list.d/docker.list"
+        server.run "apt-get update"
+        server.run "DEBIAN_FRONTEND=noninteractive apt-get install -y linux-image-extra-`uname -r` lxc-docker"
+        existing_docker_path
+      end
+
+      def existing_docker_path
+        path = server.run("which docker").stdout.to_s.strip
+        path = nil if path == ""
+        puts "[docker] Using installed #{path}" if path
+        path
+      end
+
+      def docker_path
+        @docker_path ||= existing_docker_path
+        @docker_path ||= new_docker_path
+      end
+
+      def command(command, options = {})
+        full_command = "#{docker_path} #{command}"
+        full_command = "nohup #{full_command}" if options[:nohup]
+        full_command = "echo '#{shell_escape options[:stdin]}' | #{full_command}" if options[:stdin]
+        puts "   [scp] #{options[:files].inspect}" if VERBOSE and options[:files]
+        puts "   [ssh] #{full_command}" if VERBOSE
+        result = server.run full_command, files: options[:files]
+        raise "Docker error: #{result.stdout} #{result.stderr}" unless result.status == 0
+        result.stdout
+      end
+
+      def shell_escape(text)
+        text.gsub "'", "'\"'\"'"
+      end
+
+      def containers
+        ContainerSet.new self
+      end
+    end
+
+    class ContainerSet
+      def initialize(host)
+        @host = host
+      end
+
+      def create(options)
+        Container.new @host, options
+      end
+    end
+
+    class Container
+      def initialize(host, options)
+        @host = host
+        @label = options[:label]
+        @base_image = options[:base_image]
+        @ports = options[:ports].to_a
+        @volumes = options[:volumes].to_a
+        @setup_commands = options[:setup_commands].to_a
+        @daemon_command = options[:daemon_command]
+        @environment = options[:environment]
+        @files = options[:files]
+      end
+
+      def image_fingerprint
+        {base_image: @base_image, setup_commands: @setup_commands}
+      end
+
+      def image_name
+        hash = Digest::SHA1.hexdigest(image_fingerprint.to_yaml).first(12)
+        "#{@label}_#{hash}"
+      end
+
+      def run
+        unless id
+          build
+          puts "[docker] Starting #{@label} image"
+          container_id = @host.command("run -d #{@label}").strip
+          if(!id)
+            output = @host.command "logs #{container_id}"
+            raise "Docker: #{@label} daemon exited with: #{output}"
+          end
+        end
+        puts "[docker] #{@label} is running at #{ip_address}"
+      end
+
+      def raise_build_errors(build_output)
+        match = build_output.match(/Error build: The command \[([^\]]*)\] returned a non-zero code:/)
+        if match
+          failed_command = match[1]
+          last_section = build_output.split("--->").last
+          last_section.gsub!(/Running in [0-9a-f]+/, "")
+          last_section.gsub!(/Error build: The command.*/m, "")
+          raise "Docker: build step '#{failed_command}' failed: #{last_section.strip}"
+        end
+      end
+
+      def raise_run_errors(run_output)
+      end
+
+      def dockerfile
+        lines = ["FROM #{@base_image}"]
+        lines += @environment.map{|k, v| "ENV #{k} #{v}"} if @environment
+        lines += @setup_commands.map{|c| "RUN #{c}"}
+        lines << "EXPOSE #{@ports.map{|p| "#{p}:#{p}"}.join ' '}" if @ports.to_a.any?
+        lines << "VOLUME #{@volumes.inspect}" if @volumes.to_a.any?
+        lines << "ENTRYPOINT #{@daemon_command}" if @daemon_command
+        lines.join "\n"
+      end
+
+      def build
+        puts "[docker] Building #{@label} image"
+        raise_build_errors(@host.command "build -t #{@label} -", stdin: dockerfile)
+      end
+
+      def command(command, options = {})
+        build
+        puts "[docker] Executing #{@label} image"
+        file_options = options[:files] ? "-v /files:/files" : ""
+        @host.command "run #{file_options} #{@label} #{command}", files: files_hash(options[:files])
+      end
+
+      def files_hash(files_array)
+        files_array.to_a.inject({}) do |hash, local_file|
+          hash.merge local_file => "/files/#{File.basename local_file}"
+        end
+      end
+
+      def shell_command(command)
+        "bash -c '#{@host.shell_escape command}'"
+      end
+
+      def id
+        @id ||= @host.command("ps | grep #{@label}: ; true").strip.split("\n").first.to_s[0..11]
+        @id = nil if @id == ""
+        @id
+      end
+
+      def ip_address
+        status["NetworkSettings"]["IPAddress"]
+      end
+
+      def status
+        require "json"
+        JSON.parse(@host.command "inspect #{id}").first
+      end
+    end
+  end
+end

data/lib/conjure/service/postgres_client.rb

@@ -0,0 +1,32 @@
+module Conjure
+  module Service
+    class PostgresClient < Basic
+      def initialize(host, db_name)
+        server = PostgresServer.create host
+        server.run
+        @server_ip = server.ip_address
+        @db_name = db_name
+        @container = host.containers.create(
+          label: "pgclient",
+          base_image: "ubuntu",
+          setup_commands: [
+            "apt-get install -y python-software-properties software-properties-common",
+            "add-apt-repository -y ppa:pitti/postgresql",
+            "apt-get update",
+            "apt-get install -y postgresql-9.2 postgresql-client-9.2 postgresql-contrib-9.2",
+          ],
+        )
+      end
+
+      def export(file)
+        File.open file, "w" do |f|
+          f.write @container.command("/usr/lib/postgresql/9.2/bin/pg_dump -U root -h #{@server_ip} #{@db_name}")
+        end
+      end
+
+      def import(file)
+        @container.command "/usr/lib/postgresql/9.2/bin/psql -U root -h #{@server_ip} -d #{@db_name} -f /files/#{File.basename file}", files: [file]
+      end
+    end
+  end
+end

data/lib/conjure/service/postgres_server.rb

@@ -0,0 +1,31 @@
+module Conjure
+  module Service
+    class PostgresServer < Basic
+      def initialize(host)
+        @container = host.containers.create(
+          label: "postgres",
+          base_image: "ubuntu",
+          setup_commands: [
+            "apt-get install -y python-software-properties software-properties-common",
+            "add-apt-repository -y ppa:pitti/postgresql",
+            "apt-get update",
+            "apt-get install -y postgresql-9.2 postgresql-client-9.2 postgresql-contrib-9.2",
+            "service postgresql start; su postgres -c 'createuser -d -r -s root; createdb -O root root'; service postgresql stop",
+            "echo 'host all all 0.0.0.0/0 trust' >>/etc/postgresql/9.2/main/pg_hba.conf",
+            "echo \"listen_addresses='*'\" >>/etc/postgresql/9.2/main/postgresql.conf",
+          ],
+          daemon_command: "su postgres -c '/usr/lib/postgresql/9.2/bin/postgres -c config_file=/etc/postgresql/9.2/main/postgresql.conf'",
+          volumes: ["/var/lib/postgresql/9.2/main"],
+        )
+      end
+
+      def run
+        @container.run
+      end
+
+      def ip_address
+        @container.ip_address
+      end
+    end
+  end
+end

data/lib/conjure/service/rails_application.rb

@@ -1,15 +1,15 @@
 module Conjure
   module Service
     class RailsApplication < Basic
-      def initialize(source_path)
-        source_path = source_path
-        @instance = Service::MachineInstance.new
-        @source_tree = Service::SourceTree.new source_path, @instance
-        @server = Service::RailsServer.new @source_tree, @instance
-      end
-
-      def dependencies
-        [@instance, @source_tree, @server]
+      def initialize(github_url, name = "myapp", environment = "production", config = {})
+        @name = name
+        @environment = environment
+        docker = Service::DockerHost.create "#{name}-#{environment}", config
+        postgres = Service::PostgresServer.create docker
+        postgres.run
+        rails = Service::RailsServer.create docker, github_url, name, postgres.ip_address, environment
+        rails.run
+        puts "[deploy] Application deployed to #{docker.ip_address}"
       end
     end
   end

data/lib/conjure/service/rails_server.rb

@@ -1,35 +1,72 @@
 module Conjure
   module Service
     class RailsServer < Basic
-      def initialize(source_tree, instance)
-        @working_dir = "codebase"
-        @instance = instance
-        @source_tree = source_tree
-        @rvm_shell = Service::RvmShell.new instance, "1.9.3", "codebase"
+      def file_contents(config, file_path)
+        file_path = File.join config.config_path, file_path
+        `cat #{file_path}`
       end
 
-      def dependencies
-        [@instance, @source_tree, @rvm_shell]
+      def initialize(host, github_url, app_name, database_ip_address, rails_environment = "production")
+        config = host.config
+        ruby_version = file_contents(config, "../.ruby-version").strip
+        github_private_key = file_contents(config, config.private_key_file).gsub("\n", "\\n")
+        github_public_key = file_contents(config, config.public_key_file).gsub("\n", "\\n")
+        @container = host.containers.create(
+          label: "rails",
+          base_image: "ubuntu",
+          setup_commands: [
+           "apt-get install -y curl git",
+            "curl -L https://get.rvm.io | bash -s stable",
+            "rvm install #{ruby_version}",
+            "bash -c 'source /usr/local/rvm/scripts/rvm; rvm use #{ruby_version}@global --default'",
+            "mkdir -p /root/.ssh; echo '#{github_private_key}' > /root/.ssh/id_rsa",
+            "mkdir -p /root/.ssh; echo '#{github_public_key}' > /root/.ssh/id_rsa.pub",
+            "chmod -R go-rwx /root/.ssh",
+            "echo 'Host github.com\\n\\tStrictHostKeyChecking no\\n' >> /root/.ssh/config",
+            "git clone #{github_url}",
+            "apt-get install -y #{apt_packages_required_for_gems.join ' '}",
+            "cd #{app_name}; bundle --deployment",
+
+            "echo 'deb http://us.archive.ubuntu.com/ubuntu/ precise universe' >>/etc/apt/sources.list",
+            "apt-get install -y python-software-properties software-properties-common",
+            "add-apt-repository -y ppa:chris-lea/node.js-legacy",
+            "apt-get update",
+            "apt-get install -y nodejs",
+
+            "echo '#{database_yml database_ip_address, app_name, rails_environment}' >/#{app_name}/config/database.yml",
+            "cd #{app_name}; bundle exec rake db:setup",
+          ],
+          environment: {
+            PATH:"/usr/local/rvm/gems/ruby-1.9.3-p448@global/bin:/usr/local/rvm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+            RAILS_ENV: rails_environment,
+            GITHUB_TOKEN: ENV["GITHUB_TOKEN"],
+            FRECKLE_SUBDOMAIN: "neomind",
+          },
+          daemon_command: "cd #{app_name}; bundle exec rails server -p 80",
+          ports: [80],
+          volumes: ["/#{app_name}/log"],
+        )
       end
 
-      def started?
-        shell("bundle check").include? "dependencies are satisfied"
+      def run
+        @container.run
       end
 
-      def start
-        puts "Installing gems..."
-        execute "cd codebase; bundle"
-        puts "Starting rails server..."
-        if execute("rails server -d").include? "application starting"
-          puts "The app is running at http://localhost:4000/"
-        else
-          puts "The rails server failed to start."
-        end
+      def database_yml(database_ip_address, app_name, rails_environment)
+        {
+          rails_environment => {
+            "adapter" => "postgresql",
+            "database" => "#{app_name}_#{rails_environment}",
+            "encoding" => "utf8",
+            "host" => database_ip_address,
+            "username" => "root",
+            "template" => "template0",
+          }
+        }.to_yaml.gsub("\n", "\\n")
       end
 
-      def execute(command)
-        command = "cd #{@working_dir}; #{command}" if @working_dir
-        @rvm_shell.execute command
+      def apt_packages_required_for_gems
+        ["libpq-dev"]
       end
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: conjure
 version: !ruby/object:Gem::Version
-  version: 0.0.0
+  version: 0.0.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,8 +9,24 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-08 00:00:00.000000000 Z
+date: 2013-10-11 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: fog
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.15.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.15.0
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -52,11 +68,13 @@ extensions: []
 extra_rdoc_files: []
 files:
 - lib/conjure.rb
+- lib/conjure/service/cloud_server.rb
 - lib/conjure/service/rails_server.rb
+- lib/conjure/service/postgres_server.rb
 - lib/conjure/service/rails_application.rb
 - lib/conjure/service/machine_instance.rb
-- lib/conjure/service/source_tree.rb
-- lib/conjure/service/rvm_shell.rb
+- lib/conjure/service/postgres_client.rb
+- lib/conjure/service/docker_host.rb
 - lib/conjure/service.rb
 - lib/conjure/command.rb
 - README.md

data/lib/conjure/service/rvm_shell.rb

@@ -1,30 +0,0 @@
-module Conjure
-  module Service
-    class RvmShell < Basic
-      def initialize(machine_instance, ruby_version, gemset)
-        @instance = machine_instance
-        @ruby_version = ruby_version
-        @gemset = gemset
-      end
-
-      def dependences
-        [@instance]
-      end
-
-      def start
-        shell "sudo apt-get -y install curl libyaml-dev build-essential libsqlite3-dev nodejs"
-        shell "curl -L https://get.rvm.io | bash -s -- --ignore-dotfiles"
-        shell "echo \"source $HOME/.rvm/scripts/rvm\" >> ~/.bash_profile"
-        shell "source $HOME/.rvm/scripts/rvm"
-        shell "rvm install #{@ruby_version}"
-        shell "rvm use #{@ruby_version}@#{@gemset} --create --default"
-      end
-
-      def execute(command)
-        command.gsub! "'", "'\\\\''"
-        command = "bash --login -c '#{command}' 2>&1"
-        @instance.remote_command_output command
-      end
-    end
-  end
-end

data/lib/conjure/service/source_tree.rb

@@ -1,25 +0,0 @@
-module Conjure
-  module Service
-    class SourceTree < Basic
-      def initialize(source_path, machine_instance)
-        @source_path = source_path
-        @instance = machine_instance
-      end
-
-      def copy_to(instance, dest_path)
-        options = "-a -e \"ssh #{instance.ssh_options}\""
-        source = "#{@source_path}/"
-        dest = "#{instance.ssh_address}:#{dest_path}"
-        system "rsync #{options} #{source} #{dest}"
-      end
-
-      def start(dest_path="codebase")
-        puts "Transferring source code..."
-        copy_to @instance, dest_path
-        size = "#{@instance.remote_command_output "du -hs #{dest_path}"}"
-        size = size.split(/\s/).first
-        puts "Installed codebase size is #{size}."
-      end
-    end
-  end
-end