polisher 0.1 → 0.2

data/README.rdoc

@@ -0,0 +1,25 @@
+== Rubygem Polisher
+Copyright (C) 2010 Red Hat, Inc.
+Written by Mohammed Morsi <mmorsi@redhat.com>
+
+=== Intro
+Polisher is a post-publishing processing webapp for rubygems. It allows for
+the configuration of events to be run on target gem/version updates. Polisher
+makes uses of the gemcutter webhook API in conjunction with the sinatra/rack
+web framework to subscribe to gem updates to run any specified/arbitrary event 
+handler callbacks.
+
+=== Running
+Simply 'gem install polisher'
+
+Also checkout the source from github via
+
+'git clone git://github.com/movitto/polisher.git'
+
+To run the spec suite simply run
+
+'rake spec'
+
+
+=== Using
+ See the generated API and specs

data/Rakefile

@@ -0,0 +1,92 @@
+# polisher project Rakefile
+
+require 'lib/common'
+require 'db/connection'
+require 'rake/rdoctask'
+require 'spec/rake/spectask'
+require 'rake/gempackagetask'
+
+#task :default => :test
+
+env = ENV['RACK_ENV']
+env ||= 'development'
+
+logger = Logger.new(STDOUT)
+
+GEM_NAME='polisher'
+PKG_VERSION='0.2'
+
+namespace :db do
+  task :migrate do
+    desc "Migrate the database"
+    Polisher::DB.connect Polisher::DB.load_config('./config/database.yml', env), logger
+    Polisher::DB.migrate './db/migrations'
+  end
+
+  task :rollback do
+    desc "Rollback the database"
+    Polisher::DB.connect Polisher::DB.load_config('./config/database.yml', env), logger
+    Polisher::DB.rollback './db/migrations'
+  end
+
+  task :drop_tables do
+    desc "Drop all tables in the database"
+    Polisher::DB.connect Polisher::DB.load_config('./config/database.yml', env), logger
+    Polisher::DB.drop_tables './db/migrations'
+  end
+end
+
+task 'test_env' do
+  env = 'test'
+  create_missing_polisher_dirs(:artifacts_dir => 'spec/artifacts', :db_data_dir => 'db/data', :log_dir => 'log')
+end
+
+desc "Run all specs"
+Spec::Rake::SpecTask.new('spec' => ['test_env', 'db:drop_tables', 'db:migrate']) do |t|
+  t.spec_files = FileList['spec/*_spec.rb']
+end
+
+task :rdoc do
+  desc "Create RDoc documentation"
+  system "rdoc --title 'Polisher documentation' lib/"
+end
+
+Rake::RDocTask.new do |rd|
+    rd.main = "README.rdoc"
+    rd.rdoc_dir = "doc/site/api"
+    rd.rdoc_files.include("README.rdoc", "polisher.rb", "db/**/*.rb", "lib/**/*.rb")
+end
+
+task :create_gem do
+  desc "Create a new gem"
+  system "gem build polisher.gemspec"
+end
+
+PKG_FILES = FileList['bin/*', 'config/*.yml', 'config.ru', 'COPYING', 
+'db/**/*.rb', 'lib/**/*.rb', 'LICENSE', 'polisher.rb', 'public/**/*', 'Rakefile', 
+'README.rdoc', 'spec/**/*.rb', 'TODO', 'views/**/*.haml']
+
+DIST_FILES = FileList[
+  "pkg/*.tgz", "pkg/*.gem"
+]
+
+SPEC = Gem::Specification.new do |s|
+    s.name = GEM_NAME
+    s.version = PKG_VERSION
+    s.files = PKG_FILES
+
+    s.required_ruby_version = '>= 1.8.1'
+    s.required_rubygems_version = Gem::Requirement.new(">= 1.3.3")
+
+    s.author = "Mohammed Morsi"
+    s.email = "mmorsi@redhat.com"
+    s.date = %q{2010-01-13}
+    s.description = "Ruby gem polisher"
+    s.summary = "Ruby gem polisher"
+    s.homepage = "http://github.com/movitto/polisher"
+end
+
+Rake::GemPackageTask.new(SPEC) do |pkg|
+    pkg.need_tar = true
+    pkg.need_zip = true
+end

data/TODO

@@ -0,0 +1,6 @@
+- Remaining TODO's, FIXME's, etc
+- Syncronization locks needed! (on all shared artifacts, db objects)
+- More event handlers, generic handler interface, 
+  workflow integration to automatically chain handlers together
+- multiple user support, RBAC and ACL mechanisms
+- simple command line utility / api interface

data/bin/server

@@ -0,0 +1,3 @@
+#!/bin/sh
+rake db:migrate
+thin start --debug -R config.ru 

data/config/database.yml

@@ -1,48 +1,13 @@
-# PostgresSQL.  Versions 8.2 recommended.
-#
-# Fedora Install Instructions:
-#   sudo yum install ruby-postgres postgresql-server
-#   sudo /sbin/service postgresql initdb
-#   sudo /sbin/service postgresql start
-#   sudo su - postgres
-#   $   createdb polisher
-#   $   psql polisher
-#   $-# CREATE USER polisher WITH PASSWORD 'polisher';
-#   $-# CREATE DATABASE polisher_development;
-#   $-# GRANT ALL PRIVILEGES ON DATABASE polisher_development to polisher;
-#   $-# CREATE DATABASE polisher_test;
-#   $-# GRANT ALL PRIVILEGES ON DATABASE polisher_test to polisher;
-#   $-# CREATE DATABASE polisher;
-#   $-# GRANT ALL PRIVILEGES ON DATABASE polisher to polisher;
-#   $-# \q
-#   $   exit
-#   rake db:migrate
-#
-#   On a local system you may need to edit this file to have only the following
-#   /var/lib/pgsql/data/pg_hba.conf
-#   local all all trust
-#   host all all 127.0.0.1 255.255.255.255 trust
+# polisher database configuration
 
+production:
+  adapter: sqlite3
+  database:  db/data/polisher.sqlite
 
 development:
-  adapter: postgresql
-  database: polisher
-  username: polisher
-  password: polisher
-  host: localhost
+  adapter: sqlite3
+  database:  db/data/polisher-development.sqlite
 
-# Warning: The database defined as 'test' will be erased and
-# re-generated from your development database when you run 'rake'.
-# Do not set this db to the same as development or production.
 test:
-  adapter: postgresql
-  database: polisher
-  username: polisher
-  host: localhost
-
-production:
-  adapter: postgresql
-  database: polisher
-  username: polisher
-  password: polisher
-  host: localhost
+  adapter: sqlite3
+  database:  db/data/polisher-test.sqlite

data/config/polisher.yml

@@ -1,10 +1,20 @@
 development:
-  gem_api_key:       "YOUR_GEMCUTTER_KEY_HERE"
-  smtp_server:       "localhost"
-  email_from:        "user@host.com"
-  email_to:          "user@host.com"
-  bugzilla_post_url: "https://bugzilla.redhat.com/post_bug.cgi"
+  gem_api_key:       "YOUR GEMCUTTER API KEY HERE"
+  email_from:        "foo@example.host"
+  email_server:      "localhost"
+  email_subject:     "gem #{gem_name} updated"
+  email_body:        "gem #{gem_name} updated"
 
 test:
+  gem_api_key:       "YOUR GEMCUTTER API KEY HERE"
+  email_from:        "foo@example.host"
+  email_server:      "localhost"
+  email_subject:     "gem #{gem_name} updated"
+  email_body:        "gem #{gem_name} updated"
 
 production:
+  gem_api_key:       "YOUR GEMCUTTER API KEY HERE"
+  email_from:        "foo@example.host"
+  email_server:      "localhost"
+  email_subject:     "gem #{gem_name} updated"
+  email_body:        "gem #{gem_name} updated"

data/config.ru

@@ -0,0 +1,25 @@
+require 'rubygems'
+require 'sinatra'
+
+dir = File.dirname(__FILE__) 
+
+set :artifacts_dir     => dir + '/artifacts'
+set :polisher_config   => dir + '/config/polisher.yml'
+set :db_config         => dir + '/config/database.yml'
+set :db_data_dir       => dir + '/db/data'
+set :log_dir          => dir + '/log/'
+set :environment       => :development
+set :run               => false
+set :logging, true
+
+#log = File.new(dir + "/log/sinatra.log", "a+")
+#$stdout.reopen(log)
+#$stderr.reopen(log)
+
+set :raise_errors => true
+use Rack::ShowExceptions
+
+$: << "lib/"
+
+require 'polisher.rb'
+run Sinatra::Application

data/db/connection.rb

@@ -0,0 +1,49 @@
+# ruby gem polisher common routines
+#
+# Copyright (C) 2010 Red Hat, Inc.
+# Written by Mohammed Morsi <mmorsi@redhat.com>
+#
+# This program is free software, you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation, either version 3
+# of the License, or (at your option) any later version.
+# 
+# You should have received a copy of the the GNU Affero
+# General Public License, along with Polisher. If not, see 
+# <http://www.gnu.org/licenses/>
+
+require 'active_record'
+
+Dir[File.dirname(__FILE__) + '/models/*.rb'].each { |model| require model }
+
+module Polisher
+  module DB
+  
+    def self.load_config(file_path, env = 'development')
+      YAML::load(File.open(file_path))[env.to_s]
+    end
+  
+    def self.connect(config, logger)
+      ActiveRecord::Base.logger = logger
+      ActiveRecord::Base.establish_connection config
+    end
+  
+    def self.migrate(migrations_dir)
+      ActiveRecord::Migration.verbose = true
+      ActiveRecord::Migrator.migrate(migrations_dir)
+    end 
+  
+    # perform a single polisher db rollback
+    def self.rollback(migrations_dir)
+      ActiveRecord::Migration.verbose = true
+      ActiveRecord::Migrator.rollback(migrations_dir)
+    end 
+
+    # drop all tables in the db
+    def self.drop_tables(migrations_dir)
+      ActiveRecord::Migration.verbose = true
+      ActiveRecord::Migrator.down(migrations_dir, 0)
+    end
+
+  end
+end

data/db/migrations/001_create_sources.rb

@@ -0,0 +1,27 @@
+# Copyright (C) 2010 Red Hat, Inc.
+# Written by Mohammed Morsi <mmorsi@redhat.com>
+#
+# This program is free software, you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation, either version 3
+# of the License, or (at your option) any later version.
+# 
+# You should have received a copy of the the GNU Affero
+# General Public License, along with Polisher. If not, see 
+# <http://www.gnu.org/licenses/>
+
+class CreateSources < ActiveRecord::Migration
+  def self.up
+    create_table :sources do |t|
+      t.string :name
+      t.string :uri
+    end
+
+    # create entry for the official gemcutter repo
+    Source.create :name => "gemcutter", :uri => "http://gemcutter.org"
+  end
+
+  def self.down
+    drop_table :sources
+  end
+end

data/db/migrations/002_create_managed_gems.rb

@@ -0,0 +1,24 @@
+# Copyright (C) 2010 Red Hat, Inc.
+# Written by Mohammed Morsi <mmorsi@redhat.com>
+#
+# This program is free software, you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation, either version 3
+# of the License, or (at your option) any later version.
+# 
+# You should have received a copy of the the GNU Affero
+# General Public License, along with Polisher. If not, see 
+# <http://www.gnu.org/licenses/>
+
+class CreateManagedGems < ActiveRecord::Migration
+  def self.up
+    create_table :managed_gems do |t|
+      t.string        :name
+      t.references    :source
+    end
+  end
+
+  def self.down
+    drop_table :managed_gems
+  end
+end

data/db/migrations/003_create_events.rb

@@ -0,0 +1,27 @@
+# Copyright (C) 2010 Red Hat, Inc.
+# Written by Mohammed Morsi <mmorsi@redhat.com>
+#
+# This program is free software, you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation, either version 3
+# of the License, or (at your option) any later version.
+# 
+# You should have received a copy of the the GNU Affero
+# General Public License, along with Polisher. If not, see 
+# <http://www.gnu.org/licenses/>
+
+class CreateEvents < ActiveRecord::Migration
+  def self.up
+    create_table :events do |t|
+      t.references :managed_gem
+      t.string     :process
+      t.string     :gem_version
+      t.string     :version_qualifier
+      t.string     :process_options
+    end
+  end
+
+  def self.down
+    drop_table :events
+  end
+end

data/db/models/event.rb

@@ -0,0 +1,61 @@
+# Copyright (C) 2010 Red Hat, Inc.
+# Written by Mohammed Morsi <mmorsi@redhat.com>
+#
+# This program is free software, you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation, either version 3
+# of the License, or (at your option) any later version.
+# 
+# You should have received a copy of the the GNU Affero
+# General Public License, along with Polisher. If not, see 
+# <http://www.gnu.org/licenses/>
+
+class Event < ActiveRecord::Base
+   belongs_to :managed_gem
+   alias :gem :managed_gem
+
+   validates_presence_of   :managed_gem_id, :process
+
+   # TODO right mow just returning a fixed list, at some point dynamically generate
+   def self.processes
+      ["create_package", "update_repo", "run_test_suite", "notify_subscribers"]
+   end
+
+   #  version qualifiers
+   VERSION_QUALIFIERS = ['', '=', '>', '<', '>=', '<=']
+
+   validates_inclusion_of :version_qualifier, :in => VERSION_QUALIFIERS, 
+                          :if => Proc.new { |e| !e.version_qualifier.nil? }
+
+   validates_presence_of :gem_version,
+                         :if => Proc.new { |e| !e.version_qualifier.nil? }
+
+   validates_presence_of :version_qualifier,
+                         :if => Proc.new { |e| !e.gem_version.nil? }
+
+   # determine if event applies to specified version
+   def applies_to_version?(version)
+     gv, ev = version.to_f, gem_version.to_f
+     return (["", nil].include? version_qualifier ) ||
+            (version_qualifier == "="  && gv == ev) ||
+            (version_qualifier == ">"  && gv >  ev) ||
+            (version_qualifier == "<"  && gv <  ev) ||
+            (version_qualifier == ">=" && gv >= ev) ||
+            (version_qualifier == "<=" && gv <= ev)
+   end
+
+   # run the event
+   def run
+      # covert process to method name
+      handler = method(process.intern)
+
+      # generate array of params from semi-colon seperated options
+      params  = process_options.nil? ? [] : process_options.split(';')
+
+      # first param is always the gem
+      params.unshift gem
+
+      # invoke
+      handler.call *params
+   end
+end

data/db/models/managed_gem.rb

@@ -0,0 +1,111 @@
+# Copyright (C) 2010 Red Hat, Inc.
+# Written by Mohammed Morsi <mmorsi@redhat.com>
+#
+# This program is free software, you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation, either version 3
+# of the License, or (at your option) any later version.
+# 
+# You should have received a copy of the the GNU Affero
+# General Public License, along with Polisher. If not, see 
+# <http://www.gnu.org/licenses/>
+
+require 'uri'
+require 'json'
+require 'net/http'
+
+# Gem representation in polisher, associated w/ rubygem being
+# managed, used to track updates and invoke handlers
+class ManagedGem < ActiveRecord::Base
+   belongs_to :source
+   has_many :events
+
+   validates_presence_of :name, :source_id
+   validates_uniqueness_of :name, :scope => :source_id
+
+   # TODO add validation to verify gem can be found in the associated gem source
+
+   # helper, extract source uri from specified gem uri
+   def self.uri_to_source_uri(gem_uri)
+      uri = URI.parse(gem_uri)
+      return uri.scheme + "://" + uri.host
+   end
+
+   # subscribe to updates to this gem from the associated gem source
+   def subscribe(args = {})
+      callback_url = args[:callback_url]
+      api_key = POLISHER_CONFIG["gem_api_key"]
+
+      subscribe_path = '/api/v1/web_hooks'
+      headers = { 'Authorization' => api_key }
+      data = "gem_name=#{name}&url=#{callback_url}"
+
+      http = Net::HTTP.new(URI.parse(source.uri).host, 80) 
+      res = http.post(subscribe_path, data, headers)
+      # TODO handle res = #<Net::HTTPNotFound:0x7f1df8319e40> This gem could not be found
+   end
+
+   # determine if we are subscribed to gem
+   def subscribed?
+      api_key = POLISHER_CONFIG["gem_api_key"]
+
+      subscribe_path = '/api/v1/web_hooks.json'
+      headers = { 'Authorization' => api_key }
+
+      http = Net::HTTP.new(URI.parse(source.uri).host, 80) 
+      res  = http.get(subscribe_path, headers).body
+      res  = JSON.parse(res)
+      return res.has_key?(name)
+   end
+
+   # unsubscribe to updates to this gem from associated gem source
+   def unsubscribe(args = {})
+      return unless subscribed?
+      callback_url = args[:callback_url]
+      api_key = POLISHER_CONFIG["gem_api_key"]
+
+      subscribe_path = "/api/v1/web_hooks/remove?gem_name=#{name}&url=#{callback_url}"
+      headers = { 'Authorization' => api_key }
+
+      http = Net::HTTP.new(URI.parse(source.uri).host, 80) 
+      res = http.delete(subscribe_path, headers)
+   end
+
+   # return hash of gem attributes/values retreived from remote source
+   def get_info
+       info = nil
+       source_uri = URI.parse(source.uri).host
+       get_path = "/api/v1/gems/#{name}.json"
+       Net::HTTP.start(source_uri, 80) { |http|
+          info = JSON.parse(http.get(get_path).body)
+       }
+       return info
+   end
+
+   def download_to(args = {})
+     path = args.has_key?(:path) ? args[:path] : nil
+     dir  = args.has_key?(:dir)  ? args[:dir]  : nil
+     version = args.has_key?(:version) ? args[:version]  : nil
+
+     info = get_info
+     gem_uri = info["gem_uri"]
+     version = info["version"] if version.nil?
+     path = dir + "/#{name}-#{version}.gem" if path.nil?
+
+     # handle redirects
+     found = false
+     until found  # TODO should impose a max tries
+       uri = URI.parse(gem_uri)
+       http = Net::HTTP.new(uri.host, 80) 
+       res =  http.get(uri.path)
+       if res.code == "200"
+          File.write path, res.body
+          found = true
+       else
+          gem_uri = res.header['location']
+       end 
+     end 
+
+     return path
+   end
+end

data/db/models/source.rb

@@ -0,0 +1,35 @@
+# Copyright (C) 2010 Red Hat, Inc.
+# Written by Mohammed Morsi <mmorsi@redhat.com>
+#
+# This program is free software, you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation, either version 3
+# of the License, or (at your option) any later version.
+# 
+# You should have received a copy of the the GNU Affero
+# General Public License, along with Polisher. If not, see 
+# <http://www.gnu.org/licenses/>
+
+# Source represents a remote endpoint which we will use
+# the gem API to get gems / subscribe to updates
+class Source < ActiveRecord::Base
+   has_many :managed_gems
+
+   alias :gems :managed_gems
+
+   validates_presence_of :name
+   validates_presence_of :uri
+   validates_uniqueness_of :name
+   validates_uniqueness_of :uri
+
+   # TODO validate format of uri
+
+   # TODO should have additional validation method that contacts gem source uri and 
+   # makes sure it satisfiest gem API requests
+   
+   # remove trailing slash in uri if present
+   before_save :clean_uri!
+   def clean_uri!
+     self.uri = self.uri[0...self.uri.size-1] if self.uri[-1].chr == '/'
+   end
+end

data/lib/common.rb

@@ -0,0 +1,39 @@
+# ruby gem polisher common routines
+#
+# Copyright (C) 2010 Red Hat, Inc.
+# Written by Mohammed Morsi <mmorsi@redhat.com>
+#
+# This program is free software, you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation, either version 3
+# of the License, or (at your option) any later version.
+# 
+# You should have received a copy of the the GNU Affero
+# General Public License, along with Polisher. If not, see 
+# <http://www.gnu.org/licenses/>
+
+# read entire file into string
+def File.read_all(path)
+  File.open(path, 'rb') {|file| return file.read }
+end
+
+# write contents of file from string
+def File.write(path, str)
+  File.open(path, 'wb') {|file| file.write str }
+end
+
+# create any missing directories
+def create_missing_polisher_dirs(args = {})
+  artifacts_dir = args[:artifacts_dir]
+  db_data_dir   = args[:db_data_dir]
+  log_dir       = args[:log_dir]
+
+  [artifacts_dir + '/gems', 
+   artifacts_dir + '/repos', 
+   artifacts_dir + '/SOURCES', 
+   artifacts_dir + '/SPECS', 
+   artifacts_dir + '/templates', 
+   log_dir, db_data_dir].each { |dir|
+     FileUtils.mkdir_p(dir) unless File.directory? dir
+   }
+end