polisher 0.3 → 0.4

data/db/models/project.rb

@@ -0,0 +1,110 @@
+# Copyright (C) 2010 Red Hat, Inc.
+# Written by Mohammed Morsi <mmorsi@redhat.com>
+#
+# This program is free software, you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation, either version 3
+# of the License, or (at your option) any later version.
+# 
+# You should have received a copy of the the GNU Affero
+# General Public License, along with Polisher. If not, see 
+# <http://www.gnu.org/licenses/>
+
+class Project < ActiveRecord::Base
+  # TODO on delete, destroy these
+  has_many :project_source_versions
+  has_many :sources, :through => :project_source_versions
+  has_many :events
+
+  has_many :project_dependencies
+  has_many :project_dependents, :class_name => "ProjectDependency", :foreign_key => "depends_on_project_id"
+
+  alias :dependencies :project_dependencies
+  alias :dependents :project_dependents
+
+  validates_presence_of   :name
+  validates_uniqueness_of :name
+
+  # Download all project sources to specified :dir
+  def download_to(args = {})
+    # If a version isn't specified we can't lookup project_source_versions entry for uri substitutions
+    # FIXME the latter case in this ternary operator should be something along the lines of sources_for_all_versions returning those only associated w/ project_version = nil (also being sure to do uri_params substition) (?)
+    srcs = args.has_key?(:version) ? sources_for_version(args[:version]) : sources
+    srcs.each { |source| source.download_to args }
+  end
+
+  # Return all events associated w/ particular version of the project
+  def events_for_version(version)
+    evnts = events
+    evnts.find_all { |event| event.applies_to_version?(version) }
+  end
+
+  # Return all dependencies associated w/ particular version of the project
+  def dependencies_for_version(version)
+    deps = project_dependencies
+    deps.find_all { |dep| dep.project_version == version || dep.project_version.nil? }
+  end
+
+  # Return all project_source_versions associated w/ particular version of the project
+  def project_source_versions_for_version(version)
+    psa = project_source_versions
+    psa.find_all { |ps| ps.project_version == version || ps.project_version.nil? }
+  end
+
+  # Return all sources associated w/ particular version of the project, each w/ uri formatted
+  # using project_source_versions source_uri_params
+  def sources_for_version(version)
+    project_source_versions_for_version(version).collect { |ps|
+      ps.source.format_uri!(ps.source_uri_params)
+      ps.source
+    }
+  end
+
+  # Get the project primary source
+  def primary_source
+    ps = project_source_versions.all.find { |ps| ps.primary_source }
+    # TODO special case if no sources are marked as primary, grab the first ? (also in primary_source_for_version below)
+    return ps.nil? ? nil : ps.source
+  end
+
+  # Set the project primary source
+  def primary_source=(source)
+    project_source_versions << ProjectSourceVersion.new(:project => self, :source => source, :primary_source => true)
+    #source.save! ; save!
+  end
+
+  # Return the primary source for the specified version
+  def primary_source_for_version(version)
+    ps = project_source_versions_for_version(version).find { |ps| ps.primary_source }
+    ps.source.format_uri!(ps.source_uri_params) unless ps.nil?
+    return ps.nil? ? nil : ps.source
+  end
+
+  # Return all versions which we have configured this project for
+  def versions
+    # TODO should we return configured project_depents.depends_on_project_version as well ?
+    (project_source_versions.collect { |ps| ps.project_version } +
+     events.collect { |e| e.version } +
+     project_dependencies.collect { |d| d.project_version }).uniq - [nil]
+  end
+
+  # Release specified project version
+  def released_version(version, args = {})
+    # process dependencies
+    dependencies_for_version(version).each { |dep|
+      dargs = {}
+      dargs = dep.depends_on_project_params.to_h.merge!(args) unless dep.depends_on_project_params.nil?
+
+      # if dep_version.nil? grab all configured depends_on_project versions
+      dep_versions = dep.depends_on_project_version
+      dep_versions = dep_versions.nil? ? dep.depends_on_project.versions : [dep_versions]
+
+      dep_versions.each { |dv| dep.depends_on_project.released_version(dv, dargs) }
+    }
+
+    # process events
+    args[:version] = version
+    events_for_version(version).each { |event| event.run(args) }
+  end
+
+end

data/db/models/project_dependency.rb

@@ -0,0 +1,27 @@
+# Copyright (C) 2010 Red Hat, Inc.
+# Written by Mohammed Morsi <mmorsi@redhat.com>
+#
+# This program is free software, you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation, either version 3
+# of the License, or (at your option) any later version.
+#
+# You should have received a copy of the the GNU Affero
+# General Public License, along with Polisher. If not, see
+# <http://www.gnu.org/licenses/>
+
+class ProjectDependency < ActiveRecord::Base
+  belongs_to :project
+  belongs_to :depends_on_project, :class_name => "Project", :foreign_key => "depends_on_project_id"
+
+  validates_presence_of :project_id
+  validates_presence_of :depends_on_project_id
+
+  validates_uniqueness_of :depends_on_project_version, :scope => [:project_id, :depends_on_project_id, :depends_on_project_version]
+
+  before_save :normalize_versions
+  def normalize_versions
+    self.project_version = nil if project_version == ""
+    self.depends_on_project_version = nil if depends_on_project_version == ""
+  end
+end

data/db/models/project_source_version.rb

@@ -0,0 +1,31 @@
+# Copyright (C) 2010 Red Hat, Inc.
+# Written by Mohammed Morsi <mmorsi@redhat.com>
+#
+# This program is free software, you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation, either version 3
+# of the License, or (at your option) any later version.
+#
+# You should have received a copy of the the GNU Affero
+# General Public License, along with Polisher. If not, see
+# <http://www.gnu.org/licenses/>
+
+class ProjectSourceVersion < ActiveRecord::Base
+  belongs_to :project
+  belongs_to :source
+
+  # FIXME destroy source on deletion only if no other project_source_versions sharing the source exist
+
+  validates_uniqueness_of :source_id, :scope => [:project_id, :project_version]
+
+  # validate only one primary_source set to 'true' in scope of (project_id, project_version)
+  validates_uniqueness_of :primary_source,
+                          :scope => [:project_id, :project_version],
+                          :if    => Proc.new { |ps| ps.primary_source }
+
+  before_save :normalize_versions
+  def normalize_versions
+    self.project_version = nil if project_version == ""
+    self.source_version = nil if source_version == ""
+  end
+end

data/db/models/source.rb

@@ -10,26 +10,92 @@
 # General Public License, along with Polisher. If not, see 
 # <http://www.gnu.org/licenses/>
 
-# Source represents a remote endpoint which we will use
-# the gem API to get gems / subscribe to updates
+require 'curl' # requires 'curb' package
+
 class Source < ActiveRecord::Base
-   has_many :managed_gems
+  # TODO on delete, destroy these
+  has_many :project_source_versions
+  has_many :projects, :through => :project_source_versions
+
+  validates_presence_of   :name
+  validates_uniqueness_of :name
+  validates_presence_of   :source_type
+  validates_presence_of   :uri
+  validates_uniqueness_of :uri
+
+  # TODO additional source types
+
+  SOURCE_TYPES = ['archive', 'patch', 'spec', 'gem', 'file', 'git_repo']
+
+  validates_inclusion_of :source_type, :in => SOURCE_TYPES
+
+  # Extract filename of this source from path
+  def filename
+    URI::parse(uri).path.split('/').last
+  end
+
+  # Return all project_source_versions associated w/ particular version of the source
+  def project_source_versions_for_version(version)
+    psa = project_source_versions
+    psa.find_all { |ps| ps.source_version == version || ps.source_version.nil? }
+  end
+
+  # Return all projects associated w/ particular version of the source
+  def projects_for_version(version)
+    project_source_versions_for_version(version).collect { |ps| ps.project }
+  end
+
+  # Return all versions which we have configured this project for
+  def versions
+    (project_source_versions.collect { |ps| ps.source_version }).uniq - [nil]
+  end
+
+  # Swap any occurence of the specified hash
+  # keys w/ their cooresponding values in the local source uri
+  def format_uri!(variables)
+    params = {}
+    if variables.class == String
+      params = variables.to_h
+    elsif variables.class == Hash
+      params = variables
+    else
+      return
+    end
+
+    turi = uri
+    params.each { |k,v| turi.gsub!("%{#{k}}", v.to_s) }
+    uri = turi
+  end
+
+  # Download source, args may contain any of the following
+  # * :path path to download source to
+  # * :dir  directory to download source to, filename will be generated from the last part of the uri
+   def download_to(args = {})
+     # TODO handle source_type == git_repo
+
+     path = args.has_key?(:path) ? args[:path] : nil
+     dir  = args.has_key?(:dir)  ? args[:dir]  : nil
+
+     # format the uri w/ any additional params
+     format_uri! args
 
-   alias :gems :managed_gems
+     begin
+       # generate path which to d/l the file to
+       fn = filename
+       path = "#{dir}/#{fn}" if path.nil?
+       dir  = File.dirname(path)
+       raise ArgumentError unless File.writable?(dir)
 
-   validates_presence_of :name
-   validates_presence_of :uri
-   validates_uniqueness_of :name
-   validates_uniqueness_of :uri
+       # d/l the file
+       curl = Curl::Easy.new(uri)
+       curl.follow_location = true # follow redirects
+       curl.perform
+       File.write path, curl.body_str
 
-   # TODO validate format of uri
+     rescue Exception => e
+       raise RuntimeError, "could not download project source from #{uri} to #{path}"
+     end
 
-   # TODO should have additional validation method that contacts gem source uri and 
-   # makes sure it satisfiest gem API requests
-   
-   # remove trailing slash in uri if present
-   before_save :clean_uri!
-   def clean_uri!
-     self.uri = self.uri[0...self.uri.size-1] if self.uri[-1].chr == '/'
+     return path
    end
 end

data/lib/common.rb

@@ -12,24 +12,23 @@
 # General Public License, along with Polisher. If not, see 
 # <http://www.gnu.org/licenses/>
 
-# read entire file into string
+# Read entire file into string
 def File.read_all(path)
   File.open(path, 'rb') {|file| return file.read }
 end
 
-# write contents of file from string
+# Write contents of file from string
 def File.write(path, str)
   File.open(path, 'wb') {|file| file.write str }
 end
 
-# create any missing directories
+# Create any missing directories
 def create_missing_polisher_dirs(args = {})
   artifacts_dir = args[:artifacts_dir]
   db_data_dir   = args[:db_data_dir]
   log_dir       = args[:log_dir]
 
-  [artifacts_dir + '/gems', 
-   artifacts_dir + '/repos', 
+  [artifacts_dir + '/repos',
    artifacts_dir + '/SOURCES', 
    artifacts_dir + '/SPECS', 
    artifacts_dir + '/templates', 
@@ -37,3 +36,36 @@ def create_missing_polisher_dirs(args = {})
      FileUtils.mkdir_p(dir) unless File.directory? dir
    }
 end
+
+# Set up and return polisher config from application
+def load_polisher_config(app)
+  config = {}
+  loaded_config =  YAML::load(File.open(app.polisher_config))[app.environment.to_s]
+  config.merge!(loaded_config) unless loaded_config.nil?
+
+  # Attempt to parse gem api key from ~/.gem/credentials if missing
+  if config["gem_api_key"].nil?
+    gcfile = File.expand_path("~/.gem/credentials")
+    if File.exists?(gcfile)
+      config["gem_api_key"] = File.read_all(gcfile).scan(/:rubygems_api_key:\s(.*)/).to_s
+    end
+  end
+  return config
+end
+
+class String
+  
+  # Parse/split string around element delimiters (;) and 
+  # key/value delimiters (=) and convert to hash.
+  def to_h
+    ret = {}
+    split(';').each { |p| u = p.split('='); ret[u[0]] = u[1] }
+    ret
+  end
+
+  # Convert hash into string
+  def self.from_h(hash)
+    hash.keys.collect { |k| k.to_s + "=" + hash[k].to_s }.join(";")
+  end
+
+end

data/lib/dsl.rb

@@ -0,0 +1,292 @@
+# Polisher dsl
+#
+# Copyright (C) 2010 Red Hat, Inc.
+# Written by Mohammed Morsi <mmorsi@redhat.com>
+#
+# This program is free software, you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation, either version 3
+# of the License, or (at your option) any later version.
+#
+# You should have received a copy of the the GNU Affero
+# General Public License, along with Polisher. If not, see
+# <http://www.gnu.org/licenses/>
+
+require 'libxml'
+require 'rest_client'
+
+module Polisher
+
+# Helper method to handle (print) xml status response
+def handle_response(operation, response, exception_on_fail = false)
+  rr = LibXML::XML::Document.string(response.body).root
+  success = rr.children.find { |c| c.name == "success" }.content.strip == "true"
+  msg  = rr.children.find { |c| c.name == "message" }.content.strip
+  puts "#{operation} returned w/ success = #{success} and the message: #{msg}"
+  raise RuntimeError, "#{operation} returned w/ failed status: #{msg}" if exception_on_fail && !success
+end
+module_function :handle_response
+
+# DSL representations of model classes (so as to not require db connection on client side)
+
+# DSL project
+class Project
+  # Project attributes
+  attr_accessor :id, :name
+
+  # Means to store project version and dependency params to be used when setting up project_source_versions and deps
+  attr_accessor :project_version, :dependency_params
+
+  def initialize(args = {})
+    args = {:id => nil, :name => nil}.merge(args)
+    @id = args[:id] ; @name = args[:name]
+    @dependency_params = {}
+  end
+
+  # Add a method to project for each source type, eg add_archive, add_path, add_repo, etc.
+  # TODO maintain as source_types are added (or put this this in a seperate module)
+  # XXX ran into this when doing it metaprogramatically http://coderrr.wordpress.com/2008/10/29/using-define_method-with-blocks-in-ruby-18/
+  def add_archive args = {}, &block; add_source('archive', args, &block); end
+  def add_patch   args = {}, &block; add_source('patch',   args, &block); end
+  def add_gem     args = {}, &block; add_source('gem',     args, &block); end
+  def add_file    args = {}, &block; add_source('file',    args, &block); end
+
+  def add_source  type, args = {}, &block
+    args[:source_type] = type # automatically set the source type
+    src = source(args)
+
+    # Dispatch source to caller blocker to register specific project/source versions.
+    if !block.nil?
+      block.call src
+
+    # If no block is given register default */* project/source version
+    # TODO should this execute regardless of whether there is a block or not, eg only if no project/source versions were created/exist (need to perform a new query to get that info)
+    else
+      version nil, :corresponds_to => src
+    end
+  end
+
+  # Create Project from xml and return
+  def self.from_xml(xml_str)
+    project = Polisher::Project.new
+
+    xml = LibXML::XML::Document.string(xml_str).root
+    project.id   = xml.children.find { |c| c.name == "id" }.content.to_i
+    project.name = xml.children.find { |c| c.name == "name" }.content.strip
+    # TODO associated versions, sources, events
+    #xml.children.find     { |c| c.name == "version" }.children.each  { |c|
+       #uri = c.children.find { |c| c.name == "uri"}
+       #project.sources  << uri.content.strip unless uri.nil?
+    #}
+
+    return project
+  end
+
+  # Retrieve and return all projects
+  def self.all
+    projects = []
+    RestClient.get("#{$polisher_uri}/projects") { |response|
+      xml = LibXML::XML::Document.string(response.body).root
+      xml.children.find_all { |c| c.name == "project" }.each { |s|
+        projects << Polisher::Project.from_xml(s.to_s)
+      }
+    }
+    return projects
+  end
+
+  # Create project
+  def create
+    RestClient.post("#{$polisher_uri}/projects/create", :name => name) { |response| Polisher.handle_response('create project', response, true) }
+  end
+
+  # Delete project
+  def delete
+    RestClient.delete("#{$polisher_uri}/projects/destroy/#{id}") { |response| Polisher.handle_response('delete project', response, true) }
+  end
+
+  # Create new Event w/ the specified version qualifier, version, process, and process options
+  def on_version(*args)
+    args.unshift nil unless ['', '=', '>', '<', '>=', '<='].include?(args[0]) # XXX don't like having to replicate entire event version qualifiers, but don't want the activerecord dependency
+    version_qualifier = args[0]
+    version           = args[1]
+    process           = args[2]
+    process_options   = args[3]
+    process_options   = "" if process_options.nil?
+    process_options   += ";" + String.from_h(args[4]) if args.size == 5
+
+    process.gsub!(/\s/, '_')
+
+    RestClient.post("#{$polisher_uri}/events/create",
+                   :project_id => id, :process => process, :version => version,
+                   :version_qualifier => version_qualifier, :process_options => process_options) { |response| Polisher.handle_response('create event', response) }
+  end
+
+  # Associate specified project version w/ corresponding source version if specified,
+  # else if not just return self
+  def version(version, args = {})
+    version = nil if version == "*"
+    @project_version = version
+    @dependency_params = args unless args.empty?
+
+    if args.has_key?(:corresponds_to)
+      # dispatch to source.version so we don't have to implement twice
+      source = args[:corresponds_to]
+      source.version source.source_version, :corresponds_to => self
+
+    elsif args.has_key?(:depends_on)
+      project = args.delete(:depends_on)
+      args = project.dependency_params.merge(args) unless project.dependency_params.empty?
+      depends_on_args = String.from_h(args) unless args.empty?
+      args = {:project_id => id, :project_version => version,
+              :depends_on_project_id  => project.id,  :depends_on_project_version  => project.project_version, 
+              :depends_on_project_params => depends_on_args}
+      RestClient.post("#{$polisher_uri}/project_dependencies/create", args) { |response| Polisher.handle_response('created project dependency', response) }
+
+    else
+      return self
+    end
+  end
+
+  # Test fire project released event for specified version
+  def released(version, params = {})
+     resource = RestClient::Resource.new("#{$polisher_uri}/projects/released", :timeout => 1000)  # give event handlers plenty of time to run
+
+     sparams = "name=#{name}&version=#{version}"
+     params.each { |k,v| sparams += "&#{k}=#{v}" }
+     resource.post sparams do |response|
+       Polisher.handle_response('released project', response)
+     end
+  end
+end
+
+# DSL Source
+class Source
+  # Source attributes
+  attr_accessor :id, :name, :uri, :source_type
+
+  # Means to store source version and optional uri params to be used when setting up project sources
+  attr_accessor :source_version, :uri_args
+
+  # Means to store primary_source value when creating projects sources
+  attr_accessor :primary_source
+
+  def initialize(args = {})
+    args = { :id => nil, :name => nil, :uri => nil, :source_type => nil, :primary_source => false}.merge(args)
+    @id = args[:id] ; @name = args[:name] ; @uri = args[:uri] ; @source_type = args[:source_type] ; @primary_sources = args[:primary_source]
+    @uri_args = '' ; @primary_source = false
+  end
+
+  # Create Source from xml and return
+  def self.from_xml(xml_str)
+    source = Polisher::Source.new
+
+    xml = LibXML::XML::Document.string(xml_str).root
+    source.id   = xml.children.find { |c| c.name == "id" }.content.to_i
+    source.name = xml.children.find { |c| c.name == "name" }.content.strip
+    source.uri  = xml.children.find { |c| c.name == "uri" }.content.strip
+    source.source_type = xml.children.find { |c| c.name == "source_type" }.content.strip
+    # TODO associated versions, projects, events
+
+    return source
+  end
+
+  # Retrieve and return all sources
+  def self.all
+    sources = []
+    RestClient.get("#{$polisher_uri}/sources") { |response|
+      xml = LibXML::XML::Document.string(response.body).root
+      xml.children.find_all { |c| c.name == "source" }.each { |s|
+        sources << Polisher::Source.from_xml(s.to_s)
+      }
+    }
+    return sources
+  end
+
+  # Create source
+  def create
+    RestClient.post("#{$polisher_uri}/sources/create", :name => name, :uri => uri, :source_type => source_type) { |response| Polisher.handle_response('create project', response, true) }
+  end
+
+  # Associate specified project source w/ corresponding source version if specified,
+  # else if not just return self
+  def version(version, args = {})
+    version = nil if version == "*"
+    project = args.delete(:corresponds_to)
+
+    @uri_args = String.from_h(args) unless args.empty?
+
+    if project.nil?
+      @source_version = version
+      return self
+    end
+
+    args = {:project_id => project.id, :project_version => project.project_version,
+            :source_id  => id,  :source_version  => version, :source_uri_params => @uri_args,
+            :primary_source => @primary_source }
+    RestClient.post("#{$polisher_uri}/project_source_versions/create", args) { |response| Polisher.handle_response('created project source', response) }
+  end
+
+  # Set source as primary in project/source associations
+  def is_the_primary_source
+    @primary_source = true
+  end
+end
+
+end # module Polisher
+
+# Set polisher uri for all connections
+def polisher(uri)
+  # XXX do this better
+  $polisher_uri = uri
+end
+
+# Retrieve list of all projects, invoking yield w/ each, before returning the list
+def projects
+  projects = Polisher::Project.all
+  projects.each { |project| yield project if block_given? }
+  return projects
+end
+
+# Find or create new project w/ specified args
+def project(args = {})
+  projects { |project|
+    project = nil if (args.has_key?(:name) && args[:name] != project.name) ||
+                     (args.has_key?(:id)   && args[:id]   != project.id)
+    unless project.nil?
+      yield project if block_given?
+      return project
+    end
+  }
+  proj = Polisher::Project.new args
+  proj.create
+  proj = project(args)
+  yield proj if block_given?
+  return proj
+end
+
+# Retrieve list of all sources, invoking yield w/ each, before returning the list
+def sources
+  sources = Polisher::Source.all
+  sources.each { |source| yield source if block_given? }
+  return sources
+end
+
+# Find or create new source w/ specified args
+def source(args = {})
+  sources { |src|
+    src = nil if (args.has_key?(:name) && args[:name] != src.name) ||
+                 (args.has_key?(:id)   && args[:id]   != src.id)   ||
+                 (args.has_key?(:source_type) && args[:source_type] != src.source_type) ||
+                 (args.has_key?(:uri)  && args[:uri]  != src.uri)
+    unless src.nil?
+      yield src if block_given?
+      return src
+    end
+  }
+  src = Polisher::Source.new args
+  src.create
+  src = source(args)
+  src.primary_source = args[:primary_source] if args.has_key?(:primary_source) # XXX ugly hack
+  yield src if block_given?
+  return src
+end