octopusci 0.0.1

data/LICENSE

@@ -0,0 +1 @@
+TODO: add license

data/README

@@ -0,0 +1,63 @@
+The purpose of this project is provide a simple CI server that will work with
+GitHub Post-Receive hook. It is also specifically designed to handle multiple
+build/job queues for each branch.
+
+This would basically allow you to every time code is pushed to the central
+repository enqueue a build/job for the specific branch. That way if you
+have topic branches that are being pushed to the central repository along
+side the mainline branch they will get queue properly as well.
+
+My idea for implementation at this point is that if I can detect the branch
+from the GitHub Post-Receive hook then I can identify branch. If I can
+identify branch then I can maintain individual queues for each branch.
+
+Then the execution would look at config values for predefined branches and
+priorities for order of running them so that the mainline branch running its
+jobs could take precedence over non specified topic branches.
+
+Install redis and get it starting up appropriately
+
+gem install octopusci
+sudo octopusci-skel
+octopusci-db-migrate
+
+Then update the /etc/octopusci/config.yml appropriately.
+
+Add any jobs you would like to the /etc/octopusci/jobs directory as rb files
+and octopusci will load them appropriately when started.
+
+Figure out what directory the gem is installed in by running the following
+command and stripping off the lib/octopusci.rb at the end.
+
+gem which octopusci
+
+Once you have the path we can use that path to setup Passenger with Apache
+or something else like nginx as well as setup the database. Note: You will
+need to setup a database user and a database for octopusci. The settings for
+these should be stored in /etc/octopusci/config.yml.
+
+rake -f /path/of/octpusci/we/got/before/Rakefile db:migrate
+
+<VirtualHost *:80>
+  ServerName octopusci.example.com
+  PassengerAppRoot /path/of/octpusci/we/got/before
+  DocumentRoot /path/of/octpusci/we/got/before/lib/octopusci/server/public
+  <Directory /path/of/octpusci/we/got/before/lib/octopusci/server/public>
+    Order allow,deny
+    Allow from all
+    AllowOverride all
+    Options -MultiViews
+  </Directory>
+</VirtualHost>
+
+The above will give us the web Octopusci web interface.
+
+If you are developing you can simply start this up by running
+rackup -p whatever_port while inside the octopusci directory where the
+config.ru file exists.
+
+I recommend you setup the second half of Octopusci (octopusci-tentacles) with
+God or some other monitoring system. However, for development you can simply
+run octopusci-tentacles directoly as follows:
+
+otopusci-tentacles

data/bin/octopusci

@@ -0,0 +1,56 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+
+require 'rubygems'
+require 'optparse'
+
+# This hash will hold all of the options
+# parsed from the command-line by
+# OptionParser.
+options = {}
+
+optparse = OptionParser.new do|opts|
+  # Set a banner, displayed at the top
+  # of the help screen.
+  opts.banner = "Usage: optparse1.rb [options] file1 file2 ..."
+
+  # Define the options, and what they do
+  options[:verbose] = false
+  opts.on( '-v', '--verbose', 'Output more information' ) do
+    options[:verbose] = true
+  end
+
+  options[:quick] = false
+  opts.on( '-q', '--quick', 'Perform the task quickly' ) do
+    options[:quick] = true
+  end
+
+  options[:logfile] = nil
+  opts.on( '-l', '--logfile FILE', 'Write log to FILE' ) do |file|
+    options[:logfile] = file
+  end
+
+  # This displays the help screen, all programs are
+  # assumed to have this option.
+  opts.on( '-h', '--help', 'Display this screen' ) do
+    puts opts
+    exit
+  end
+end
+
+# Parse the command-line. Remember there are two forms
+# of the parse method. The 'parse' method simply parses
+# ARGV, while the 'parse!' method parses ARGV and removes
+# any options found there, as well as any parameters for
+# the options. What's left is the list of files to resize.
+optparse.parse!
+
+puts "Being verbose" if options[:verbose]
+puts "Being quick" if options[:quick]
+puts "Logging to file #{options[:logfile]}" if options[:logfile]
+
+ARGV.each do|f|
+  puts "Resizing image #{f}..."
+  sleep 0.5
+end

data/bin/octopusci-db-migrate

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+
+require 'rubygems'
+require 'octopusci'
+
+ActiveRecord::Base.logger = Logger.new(STDOUT)
+ActiveRecord::Migration.verbose = true
+ActiveRecord::Migrator.migrate(File.expand_path(File.dirname(__FILE__) + "/../db/migrate"))

data/bin/octopusci-skel

@@ -0,0 +1,52 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+
+require 'rubygems'
+require 'fileutils'
+
+if Process.uid != 0
+  puts "Must run as root"
+  exit 1
+end
+
+JOBS_PATH = '/etc/octopusci/jobs'
+CONFIG_PATH = '/etc/octopusci/config.yml'
+
+FileUtils.mkdir_p(JOBS_PATH)
+
+if !File.exists?(CONFIG_PATH)
+  File.open(CONFIG_PATH, 'w') do |f|
+    f << "general:
+  jobs_path: \"/etc/octopusci/jobs\"
+
+smtp:
+  notification_from_email: somefrom@example.com
+  address: smtp.gmail.com
+  port: 587
+  authentication: plain
+  enable_starttls_auto: true
+  user_name: someuser@example.com
+  password: somepassword
+  raise_delivery_errors: true
+
+db:
+  adapter: mysql
+  host: localhost
+  database: octopusci
+  username: someusername
+  password: somepassword
+
+projects:
+  - { name: octopusci, owner: cyphactor, job_klass: SomeJobClass, repo_uri: 'git@github.com:cyphactor/octopusci.git', default_email: devs@example.com }
+
+stages:
+  - test_b
+    "
+  end
+  puts "Created example #{CONFIG_PATH}, please modify appropriately"
+else
+  puts "#{CONFIG_PATH} already exists, exiting to avoid modification."
+  puts "If you would like to generated the example config again please rename the existing #{CONFIG_PATH}."
+end
+

data/bin/octopusci-tentacles

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+
+require 'rubygems'
+require 'octopusci'
+
+Octopusci::WorkerLauncher.launch

data/bin/pusci-stage

@@ -0,0 +1,67 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+
+require 'rubygems'
+require 'optparse'
+require 'octopusci'
+
+# This hash will hold all of the options
+# parsed from the command-line by
+# OptionParser.
+options = {}
+
+optparse = OptionParser.new do|opts|
+  # Set a banner, displayed at the top
+  # of the help screen.
+  opts.banner = "Usage: pusci-stage [options] stage_name"
+
+  # Define the options, and what they do
+  options[:add] = false
+  opts.on('-a', '--add', 'Add a stage back into the pool') do
+    options[:add] = true
+  end
+
+  options[:rem] = false
+  opts.on( '-r', '--rem', 'Rem a stage from the pool' ) do
+    options[:rem] = true
+  end
+
+  options[:list] = false
+  opts.on( '-l', '--list', 'List all stages' ) do
+    options[:list] = true
+  end
+
+  options[:pool] = false
+  opts.on( '-p', '--pool', 'List all stages currently in the pool' ) do
+    options[:pool] = true
+  end
+
+  # This displays the help screen, all programs are
+  # assumed to have this option.
+  opts.on( '-h', '--help', 'Display the help screen' ) do
+    puts opts
+    exit
+  end
+end
+
+# Parse the command-line. Remember there are two forms
+# of the parse method. The 'parse' method simply parses
+# ARGV, while the 'parse!' method parses ARGV and removes
+# any options found there, as well as any parameters for
+# the options. What's left is the list of files to resize.
+optparse.parse!
+
+if options[:list] == true
+  Octopusci::StageLocker.stages.each do |s|
+    puts s
+  end
+elsif options[:pool] == true
+  puts Octopusci::StageLocker.pool
+elsif options[:add] == true
+  Octopusci::StageLocker.push(ARGV[0])
+elsif options[:rem] == true
+  Octopusci::StageLocker.rem(ARGV[0])
+end
+
+exit

data/config.ru

@@ -0,0 +1,5 @@
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/lib')
+require 'resque/server'
+require 'octopusci/server'
+
+run Rack::URLMap.new("/" => Octopusci::Server.new, "/resque" => Resque::Server.new)

data/db/migrate/0001_init.rb

@@ -0,0 +1,29 @@
+class Init < ActiveRecord::Migration
+  def self.up
+    create_table :jobs do |t|
+      t.string :ref
+      t.string :compare
+      t.string :repo_name
+      t.string :repo_owner_name
+      t.string :repo_owner_email
+      t.timestamp :repo_pushed_at
+      t.timestamp :repo_created_at
+      t.text :repo_desc
+      t.string :repo_url
+      t.string :before_commit
+      t.boolean :forced
+      t.string :after_commit
+      t.boolean :running
+      t.boolean :successful
+      t.text :output
+      t.timestamps
+      t.timestamp :started_at
+      t.timestamp :ended_at
+      t.text :payload
+    end
+  end
+
+  def self.down
+    drop_table :jobs
+  end
+end

data/lib/octopusci/config.rb

@@ -0,0 +1,95 @@
+require 'octopusci/notifier'
+require 'active_record'
+
+module Octopusci
+  class Config
+    class MissingConfigField < RuntimeError; end
+    class ConfigNotInitialized < RuntimeError; end
+    
+    def initialize()
+      @options = {}
+    end
+    
+    # read the configuration values into the object from a YML file.
+    def load_yaml(yaml_file)
+      @options = YAML.load_file(yaml_file)
+    end
+    
+    # allow options to be accessed as if this object is a Hash.
+    def [](key_name)
+      if @options.nil?
+        raise ConfigNotInitialized, "Can't access the '#{key_name}' field because the config hasn't been initialized."
+      end
+      if !@options.has_key?(key_name.to_s())
+        raise MissingConfigField, "'#{key_name}' is NOT defined as a config field."
+      end
+      return @options[key_name.to_s()]
+    end
+    
+    # allow options to be set as if this object is a Hash.
+    def []=(key_name, value)
+      @options[key_name.to_s()] = value
+    end
+    
+    def has_key?(key_name)
+      return @options.has_key?(key_name)
+    end
+    
+    # allow options to be read and set using method calls.  This capability is primarily for
+    # allowing the configuration to be defined through a block passed to the configure() function
+    # from an initializer or similar file.
+    def method_missing(key_name, *args)
+      key_name_str = key_name.to_s()
+      if key_name_str =~ /=$/ then
+        self[key_name_str.chop()] = args[0]
+      else
+        return self[key_name_str]
+      end
+    end
+  end
+  
+  # On evaluation of the module it defines a new singleton of Config.
+  if (!defined?(CONFIG))
+    CONFIG = Config.new()
+  end
+
+  def self.configure(yaml_file = nil, &block)    
+    CONFIG.load_yaml(yaml_file) if !yaml_file.nil?
+    yield CONFIG if block
+
+    Notifier.default :from => Octopusci::CONFIG['smtp']['notification_from_email']
+    Notifier.delivery_method = :smtp
+    Notifier.smtp_settings = {
+      :address => Octopusci::CONFIG['smtp']['address'],
+      :port => Octopusci::CONFIG['smtp']['port'].to_s,
+      :authentication => Octopusci::CONFIG['smtp']['authentication'],
+      :enable_starttls_auto => Octopusci::CONFIG['smtp']['enable_starttls_auto'],
+      :user_name => Octopusci::CONFIG['smtp']['user_name'],
+      :password => Octopusci::CONFIG['smtp']['password'],
+      :raise_delivery_errors => Octopusci::CONFIG['smtp']['raise_delivery_errors']
+    }
+    Notifier.logger = Logger.new(STDOUT)
+  end
+end
+
+# Load the actual config file
+Octopusci.configure("/etc/octopusci/config.yml")
+
+if Octopusci::CONFIG['stages'] == nil
+  raise "You have defined stages as an option but have no items in it."
+end
+
+ActiveRecord::Base.establish_connection(
+  :adapter => Octopusci::CONFIG['db']['adapter'],
+  :host => Octopusci::CONFIG['db']['host'],
+  :database => Octopusci::CONFIG['db']['database'],
+  :username => Octopusci::CONFIG['db']['username'],
+  :password => Octopusci::CONFIG['db']['password']
+)
+
+Dir.open(Octopusci::CONFIG['general']['jobs_path']) do |d|
+  job_file_names = d.entries.reject { |e| e == '..' || e == '.' }
+  job_file_names.each do |f_name|
+    require Octopusci::CONFIG['general']['jobs_path'] + "/#{f_name}"
+  end
+end

data/lib/octopusci/errors.rb

@@ -0,0 +1,4 @@
+module Octopusci
+  # Raised when a method that is meant to be pure virtual that is not overloaded is called
+  class PureVirtualMethod < RuntimeError; end
+end

data/lib/octopusci/helpers.rb

@@ -0,0 +1,60 @@
+module Octopusci
+  module Helpers
+    # Take the github payload hash and translate it to the Job model's attrs
+    # so that we can easily use the github payload hash to update_attributes
+    # on the Job mode.l
+    def self.gh_payload_to_job_attrs(gh_pl)
+      attrs = {}
+            
+      # ref
+      attrs[:ref] = gh_pl["ref"]
+      # compare
+      attrs[:compare] = gh_pl["compare"]
+      # repo_name
+      attrs[:repo_name] = gh_pl["repository"]["name"]
+      # repo_owner_name
+      attrs[:repo_owner_name] = gh_pl["repository"]["owner"]["name"]
+      # repo_owner_email
+      attrs[:repo_owner_email] = gh_pl["repository"]["owner"]["email"]
+      # repo_pushed_at
+      attrs[:repo_pushed_at] = gh_pl["repository"]["pushed_at"]
+      # repo_created_at
+      attrs[:repo_created_at] = gh_pl["repository"]["created_at"]
+      # repo_desc
+      attrs[:repo_desc] = gh_pl["repository"]["description"]
+      # repo_url
+      attrs[:repo_url] = gh_pl["repository"]["url"]
+      # before_commit
+      attrs[:before_commit] = gh_pl["before"]
+      # forced
+      attrs[:forced] = gh_pl["forced"]
+      # after_commit
+      attrs[:after_commit] = gh_pl["after"]
+      
+      attrs[:payload] = gh_pl
+      
+      return attrs
+    end
+    
+    # Get the information specified in the config about this project. If
+    # project info can't be found for the given project_name and project_owner
+    # this method returns nil. Otherwise, this project returns a hash of the
+    # project info that it found in the config.
+    def self.get_project_info(project_name, project_owner)
+      Octopusci::CONFIG["projects"].each do |proj|
+        if (proj['name'] == project_name) && (proj['owner'] == project_owner)
+          return proj
+        end
+      end
+      return nil
+    end
+    
+    def self.decode(str)
+      ::MultiJson.decode(str)
+    end
+    
+    def self.encode(str)
+      ::MultiJson.encode(str)
+    end
+  end
+end

data/lib/octopusci/job.rb

@@ -0,0 +1,43 @@
+module Octopusci
+  class Job
+    def self.run(github_payload, stage, job_id, job_conf)
+      raise PureVirtualMethod, "The self.commit_run method needs to be defined on your Octopusci::Job."
+    end
+
+    def self.perform(project_name, branch_name, job_id, job_conf)
+      if Octopusci::CONFIG.has_key?('stages')
+        # Get the next available stage from redis which locks it by removing it
+        # from the list of available
+        stage = Octopusci::StageLocker.pop
+      end
+      
+      begin
+        # Using redis to get the associated github_payload
+        github_payload = Octopusci::Queue.github_payload(project_name, branch_name)
+        
+        job = ::Job.where("jobs.repo_name = ? && jobs.ref = ?", github_payload['repository']['name'], github_payload['ref']).order('jobs.created_at DESC').first
+        if job
+          job.started_at = Time.new
+          job.running = true
+          job.save
+        end
+        
+        # Run the commit run and report about status and output
+        Bundler.with_clean_env {
+          self.run(github_payload, stage, job_id, job_conf)
+        }
+        
+        if job
+          job.ended_at = Time.new
+          job.running = false
+          job.save
+        end
+      ensure
+        if Octopusci::CONFIG.has_key?('stages')
+          # Unlock the stage by adding it back to the list of available stages
+          Octopusci::StageLocker.push(stage)
+        end
+      end
+    end    
+  end
+end

data/lib/octopusci/notifier/job_complete.html.erb

@@ -0,0 +1,16 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta content="text/html; charset=UTF-8" http-equiv="Content-Type" />
+  </head>
+  <body>
+    <h1>Octopusci Build (<%= @status_str %>) - <%= @job.repo_name %> / <%= @job.ref.gsub(/refs\/heads\//, '') %></h1>
+    <h2>Status: <%= @status_str %></h2>
+    <h2>Command Output:</h2>
+    <pre>
+      <code>
+        <%= @cmd_output %>
+      </code>
+    </pre>
+  </body>
+</html>

data/lib/octopusci/notifier/job_complete.text.erb

@@ -0,0 +1,9 @@
+# Octopusci Build
+  
+## Command Status
+
+    <%= @cmd_status %>
+
+## Command Output
+
+    <%= @cmd_output %>

data/lib/octopusci/notifier.rb

@@ -0,0 +1,37 @@
+require 'action_mailer'
+
+# Set the ActionMailer view_path to lib where this library is so that when
+# it searches for class_name/method for the templates it can find them when
+# we don't use the standard Rails action mailer view locations.
+ActionMailer::Base.view_paths = File.dirname(__FILE__) + '/../'
+
+module Octopusci
+  class Notifier < ActionMailer::Base    
+    def job_complete(recipient, cmd_output, cmd_status, github_payload, job_id)
+      @job = ::Job.find(job_id)
+      @job.output = cmd_output
+      @job.running = false
+      if cmd_status == 0
+        @job.successful = true
+      else
+        @job.successful = false
+      end
+      @job.save
+      
+      if recipient
+        @cmd_output = cmd_output
+        @cmd_status = cmd_status
+        @github_payload = github_payload
+        if @cmd_status == 0
+          @status_str = 'success'
+        else
+          @status_str = 'failed'
+        end
+        mail(:to => recipient, :subject => "Octopusci Build (#{@status_str}) - #{@job.repo_name} / #{@job.ref.gsub(/refs\/heads\//, '')}") do |format|
+          format.text
+          format.html
+        end
+      end
+    end
+  end
+end

data/lib/octopusci/queue.rb

@@ -0,0 +1,41 @@
+require 'resque'
+
+module Octopusci
+  module Queue
+    def self.enqueue(job_klass, proj_name, branch_name, github_payload, job_conf)
+      resque_opts = { "class" => job_klass, "args" => [proj_name, branch_name] }
+      gh_pl_key = github_payload_key(proj_name, branch_name)
+
+      if lismember('commit', resque_opts)
+        Resque.redis.set(gh_pl_key, Resque::encode(github_payload))
+        # Get the most recent job for this project and update it with the data
+        job = ::Job.where("jobs.repo_name = ? && jobs.ref = ?", proj_name, '/refs/heads/' + branch_name).order('jobs.created_at DESC').first
+        if job
+          job.update_attributes(Octopusci::Helpers.gh_payload_to_job_attrs(github_payload))
+        end
+      else
+        # Create a new job for this project with the appropriate data
+        job = ::Job.create(Octopusci::Helpers.gh_payload_to_job_attrs(github_payload).merge({ :running => false }))
+        resque_opts["args"] << job.id
+        resque_opts["args"] << job_conf
+        Resque.redis.set(gh_pl_key, Resque::encode(github_payload))
+        Resque.push('commit', resque_opts)
+      end
+    end
+
+    def self.lismember(queue, item)
+      size = Resque.size(queue)
+      [Resque.peek(queue, 0, size)].flatten.any? { |v|
+        v == item
+      }
+    end
+    
+    def self.github_payload(project_name, branch_name)
+      Resque::decode(Resque.redis.get(github_payload_key(project_name, branch_name)))
+    end
+    
+    def self.github_payload_key(proj_name, branch_name)
+      "octpusci:github_payload:#{proj_name}:#{branch_name}"
+    end
+  end
+end

data/lib/octopusci/schema.rb

@@ -0,0 +1,9 @@
+require 'active_record'
+
+class Job < ActiveRecord::Base
+  serialize :payload
+  
+  def branch_name
+    self.ref.gsub(/refs\/heads\//, '')
+  end
+end