overlay 1.1.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: af4c5cc9e0f3df5433871de6e021c072fc3e5122
-  data.tar.gz: 5a769dbe4d754ec40db970a19bd2b536299d30c6
+  metadata.gz: 9e7cf4853cb51c3c998a8b0d89691dd7fe769b71
+  data.tar.gz: a3a51bcfb29b2d56ebd16b25547c609476506afd
 SHA512:
-  metadata.gz: 9566bcd6762c5035425aa5467fc1e5151366793335ad31c2de3668c5893f93f3e85d10b5a8a08d2924688c9e6c6660a8b4b38da490b338706ef1703bef328aab
-  data.tar.gz: b1981b111c43b6bab9b650ce86a07fab40fc77d83dc669982fda21f6be5ddecbd1ae67f79d1f8b25002a0d3a144cdf060194a468a36014eda30fa33facdade74
+  metadata.gz: 5dae53b5bd7073acba446d4a605341b473139bef5e763582946bf9173251474e29ce523cd65bb9e6e6cc074c6a58b0e966c8bc093ee44d4892b8717172ca43e6
+  data.tar.gz: 4e7f06ee20f890721b59bd548b1e1ffee3d9b75da166206518314ac52563157d94750878e1b1f8e0f043f7202129f711da8a86591c6a1dcb3e68a63cf5a6e0de

data/README.rdoc

@@ -1,15 +1,55 @@
-# Overlay
+Overlay
+====================
 
-Overlay a repository on to an existing Rails application.
+Rails engine that allows for overlaying external templates onto an existing Rails application.  Overlayed directories are prepended to the view path to allow overwriting of deployed templates.
 
-## Featrures
+Features
+====================
 
-## Installation
+GithubRepo Features
+---------------------
 
-## Configuration
+* Overlay separate directories in a single repo to specific places in your Rails application.
+* Update files in realtime utilizing self registering post commit webhooks on github.
+* Run code on file update via the GithubRepo #after_process_hook block.
+* Utilize an OverlayPublisher application and a redis server to centralize hook management and publish changes to a fleet of servers.
 
-## Usage
+Installation
+====================
 
-## Examples
+Add the gem to your Gemfile:
+
+    gem 'overlay'
+
+Configuration
+====================
+
+Add an initializer to your Rails `config/initializers` directory.  This file should configure your repositories and launch the initial overlay.  Here is a sample initializer:
+
+    require 'overlay'
+
+    Overlay.configure do |config|
+      config.relative_root_url  = Rails.application.config.relative_url_root
+
+      github_repo = Overlay::GithubRepo.new(
+          'repo_org',
+          'repo_name',
+          'repo_user_user:repo_password',
+          'source_root_directory',
+          'source_destination_path'
+        )
+      config.repositories << repo_config
+    end
+
+    # Overlay files after rails is initialized
+    #
+    Rails.application.config.after_initialize do
+      Overlay::Github.instance.process_overlays
+    end
+
+Usage
+====================
+
+Once Overlay is configured, on startup, a process will be forked to run the initial pull-down of files from the repository.  Overlay will update specific files on change in the repo through use of Github webhooks.
 
 This project rocks and uses MIT-LICENSE.

data/app/controllers/overlay/github_controller.rb

@@ -1,18 +1,15 @@
-require_dependency "overlay/application_controller"
-
 module Overlay
-  class GithubController < ApplicationController
+  class GithubController < Overlay::ApplicationController
     def update
       render nothing: true
 
       Overlay.configuration.repositories.each do |repo_config|
         next unless repo_config.class == GithubRepo
-        branch = repo_config[:branch] || 'master'
+        branch = repo_config.branch
 
         if (params[:repository] && params[:ref])
-          if (params[:repository][:name] == repo_config[:repo]) && (params[:ref] == "refs/heads/#{branch}")
-            logger.info "Enqueueing GithubJob for repo: #{repo_config[:repo]} and branch: #{repo_config[:branch] || 'master'}"
-            Overlay::Github.overlay_repo repo_config
+          if (params[:repository][:name] == repo_config.repo) && (params[:ref] == "refs/heads/#{branch}")
+            Overlay::Github.instance.process_hook(params, repo_config)
           end
         end
       end

data/lib/overlay/configuration.rb

@@ -1,16 +1,15 @@
+
 module Overlay
   VALID_OPTIONS_KEYS = [
-    :site,
-    :endpoint,
-    :repo,
-    :user,
-    :auth,
     :repositories,
     :host_name,
     :host_port,
     :relative_root_url
   ].freeze
 
+  # Exceptions
+  class RequiredParameterError < StandardError; end
+
   class << self
     attr_accessor :configuration
   end
@@ -28,11 +27,106 @@ module Overlay
     end
 
     def reset
-       @site = 'https://github.com'
-       @endpoint = 'https://api.github.com'
-       @repositories = Set.new
+      @repositories = Set.new
     end
   end
 
-  GithubRepo = Struct.new(:user, :repo, :branch, :root_source_path, :root_dest_path)
+  # Configure a github repository.  Required parameters:
+  # :org,
+  # :repo,
+  # :auth,
+  # :root_source_path,
+  # :root_dest_path
+
+  # Optional parameters:
+  # :branch,
+  # :use_publisher
+  # :registration_address
+  # :redis_server
+  # :redis_port
+  # :endpoint,
+  # :site,
+  class GithubRepo
+    attr_accessor :root_source_path, :root_dest_path, :branch
+    attr_accessor :use_publisher, :redis_server, :redis_port, :registration_server
+    attr_reader   :repo, :org, :auth, :endpoint, :site
+
+    # Internal repo api hook
+    attr_accessor :github_api
+
+    REQUIRED_PARAMS            = [:org, :repo, :auth, :root_source_path]
+    REQUIRED_PUBLISHER_PARAMS  = [:redis_server, :redis_port, :registration_server]
+
+    def initialize(org, repo, auth, root_source_path, root_dest_path)
+      @org              = org
+      @repo             = repo
+      @auth             = auth
+      @root_source_path = root_source_path
+      @root_dest_path   = root_dest_path
+      @branch           = 'master'
+      @use_publisher    = false
+
+      # Quick sanity check
+      validate
+
+      # Create a hook to the Github API
+      initialize_api
+    end
+
+    def endpoint=(endpoint_addr)
+      @endpoint = endpoint_addr
+
+      # re-initialize api
+      initialize_api
+    end
+
+    def site=(site_addr)
+      @site = site_addr
+
+      # re-initialize api
+      initialize_api
+    end
+
+    # Make sure that this configuration has all required parameters
+    def validate
+      REQUIRED_PARAMS.each do |param|
+        raise RequiredParameterError, "Overlay GithubRepo missing required paramater: #{param}" if (send(param).nil? || send(param).empty?)
+      end
+
+      # If we are using a publisher, check required publisher params
+      if @use_publisher
+        REQUIRED_PUBLISHER_PARAMS.each do |param|
+          raise RequiredParameterError, "Overlay GithubRepo missing required paramater: #{param}" if (send(param).nil?)
+        end
+      end
+    end
+
+    # Add code to be called after processing a hook
+    def after_process_hook(&hook)
+      raise ArgumentError, "No block given" unless block_given?
+      @post_hook = hook
+    end
+
+    # Run post_hook code
+    def post_hook
+      @post_hook.call unless @post_hook.nil?
+    end
+
+    # Retrieve API hook to repo
+    def initialize_api
+      ::Github.reset!
+      ::Github.configure do |github_config|
+        github_config.endpoint    = @endpoint if @endpoint
+        github_config.site        = @site if @site
+        github_config.basic_auth  = @auth
+        github_config.repo        = @repo
+        github_config.org         = @org
+        github_config.adapter     = :net_http
+        github_config.ssl         = {:verify => false}
+      end
+
+      @github_api = ::Github::Repos.new
+    end
+
+  end
 end

data/lib/overlay/engine.rb

@@ -6,7 +6,7 @@ module Overlay
       ActionController::Base.class_eval do
         before_filter do
           Overlay.configuration.repositories.each do |repo_config|
-            prepend_view_path repo_config[:root_dest_path]
+            prepend_view_path repo_config.root_dest_path
           end
         end
       end

data/lib/overlay/github.rb

@@ -1,40 +1,104 @@
 require 'github_api'
 require 'fileutils'
 require 'socket'
-
+require 'singleton'
+require 'redis'
+
+# The github class is responsible for managing overlaying
+# directories in a Github repo on the current application.
+# Call to action is based on either a webhook call to the github controller
+# or a publish event to a redis key.
+#
 module Overlay
   class Github
+    include Singleton
     include Overlay::Engine.routes.url_helpers
+
+    attr_accessor :subscribed_configs
+
+    def initialize
+      @subscribed_configs = []
+    end
+
     # Cycle through all configured repositories and overlay
-    #
-    def self.process_overlays
+    # This function should be called only at initialization time as it causes a
+    # full overlay to be run
+    def process_overlays
       # This can be called in an application initializer which will
       # load anytime the environment is loaded.  Make sure we are prepared to run
       # this.
       #
       return unless (config.host_port || ENV['SERVER_HOST_PORT'] || defined? Rails::Server)
 
-      # Configure github api
-      configure
-
       Overlay.configuration.repositories.each do |repo_config|
         next unless repo_config.class == GithubRepo
 
-        # Validate repository config
-        raise 'Respository config missing user' if (!repo_config[:user] || repo_config[:user].nil?)
-        raise 'Respository config missing repo' if (!repo_config[:repo] || repo_config[:repo].nil?)
+        # Register this server's endpoint as a webhook or subscribe
+        # to a redis pub/sub
+        if repo_config.use_publisher
+          publisher_subscribe(repo_config)
+        else
+          register_web_hook(repo_config)
+        end
 
-        repo_config[:branch] ||= 'master'
+        # Now that all the set-up is done, for a process
+        # to overlay the repo.
+        fork_it(:overlay_repo, repo_config)
+      end
+    end
 
-        register_web_hook(repo_config)
+    # Take a hook hash and process each changelist.
+    # For every file updated, clone it back to us.
+    def process_hook hook, repo_config
+      # Grab the commit array
+      commits = hook['commits']
+
+      # We don't care if there aren't commits
+      return if commits.nil?
+
+      commits.each do |commit|
+        # There will be three entries in each commit with file paths: added, removed, and modified.
+        added_files = commit['added']
+        added_files.each do |file|
+          # Do we care?
+          if my_file?(file, repo_config)
+            Rails.logger.info "Overlay found added file in hook: #{file}"
+
+            # Make sure that the directory is in place
+            FileUtils.mkdir_p(destination_path(File.dirname(file), repo_config))
+            clone_file(file, repo_config)
+          end
+        end
+
+        modified_files = commit['added']
+        modified_files.each do |file|
+          # Do we care?
+          if my_file?(file, repo_config)
+            Rails.logger.info "Overlay found modified file in hook: #{file}"
+            clone_file(file, repo_config)
+          end
+        end
 
-        overlay_repo repo_config
+        removed_files = commit['added']
+        removed_files.each do |file|
+          # Do we care?
+          if my_file?(file, repo_config)
+            Rails.logger.info "Overlay found deleted file in hook: #{file}"
+            File.delete(destination_path(file, repo_config))
+          end
+        end
       end
+
+      # Call post hook code
+      repo_config.post_hook
     end
 
+    private
+
     # Register our listener on the repo
     #
-    def self.register_web_hook repo_config
+    def register_web_hook(repo_config)
+      Rails.logger.info "Overlay register webhook for repo: org => #{repo_config.org}, repo => #{repo_config.repo}"
       # Make sure our routes are loaded
       Rails.application.reload_routes!
 
@@ -44,46 +108,78 @@ module Overlay
       path = Overlay::Engine.routes.url_for({:controller=>"overlay/github", :action=>"update", :only_path => true})
       uri  = ActionDispatch::Http::URL::url_for({:host => host, :port => port, :path => "#{config.relative_root_url}#{path}"})
 
+      github_api = repo_config.github_api
+
       # Retrieve current web hooks
-      current_hooks = github_repo.hooks.list(repo_config[:user], repo_config[:repo]).response.body
+      current_hooks = github_api.hooks.list(repo_config.org, repo_config.repo).response.body
       if current_hooks.find {|hook| hook.config.url == uri}.nil?
         # register hook
-        github_repo.hooks.create(repo_config[:user], repo_config[:repo], name: 'web', active: true, config: {:url => uri, :content_type => 'json'})
+        github_api.hooks.create(repo_config.org, repo_config.repo, name: 'web', active: true, config: {:url => uri, :content_type => 'json'})
       end
     end
 
-    def self.overlay_repo repo_config
-      Thread.new do
-        Rails.logger.info "Start processing repo with config #{repo_config.inspect}"
-        # Get our root entries
-        #
-        root = repo_config[:root_source_path] || '/'
-
-        # If we have a root defined, jump right into it
-        #
-        if (root != '/')
-          overlay_directory(root, repo_config)
-        else
-          root_entries = github_repo.contents.get(repo_config[:user], repo_config[:repo], root, ref: repo_config[:branch]).response.body
-
-          # We aren't pulling anything out of root.  Cycle through directories and overlay
-          #
-          root_entries.each do |entry|
-            if entry.type == 'dir'
-              overlay_directory(entry.path, repo_config)
-            end
+    # Retrieve Subscribe to a OverlayPublisher redis key
+    # Fork a process that subscribes to the redis key and processes updates.
+    def publisher_subscribe repo_config
+      return unless @subscribed_configs.find_index(repo_config).nil?
+
+      # Validate our settings
+      repo_config.validate
+
+      # Register this repo with the manager
+      uri = URI.parse(repo_config.registration_server)
+      http = Net::HTTP.new(uri.host, uri.port)
+      request = Net::HTTP::Post.new("/register")
+      request.add_field('Content-Type', 'application/json')
+      request.body = {
+        'organization' => repo_config.org,
+        'repo' => repo_config.repo,
+        'auth' => repo_config.auth,
+        'endpoint' => repo_config.endpoint,
+        'site' => repo_config.site
+      }.to_json
+
+      response = http.request(request)
+
+      # Retrieve publish key
+      publish_key = JSON.parse(response.read_body)['publish_key']
+
+      Rails.logger.info "Overlay subscribing to redis channel: #{publish_key}"
+
+      # Subscribe to redis channel
+      fork_it(:subscribe_to_channel, publish_key, repo_config)
+
+      @subscribed_configs << repo_config
+    end
+
+    # Overlay all the files specifiec by the repo_config.  This
+    # process can be long_running so we fork.  We should only be running
+    # this method in initialization of the application.
+    def overlay_repo repo_config
+      Rails.logger.info "Overlay started processing repo with config #{repo_config.inspect}"
+
+      # Get our root entries
+      root = repo_config.root_source_path || '/'
+
+      # If we have a root defined, jump right into it
+      if root != '/'
+        overlay_directory(root, repo_config)
+      else
+        root_entries = repo_config.github_api.contents.get(repo_config.org, repo_config.repo, root, ref: repo_config.branch).response.body
+
+        # We aren't pulling anything out of root.  Cycle through directories and overlay
+        root_entries.each do |entry|
+          if entry.type == 'dir'
+            overlay_directory(entry.path, repo_config)
           end
         end
-        Rails.logger.info "Finished processing repo with config #{repo_config.inspect}"
       end
+      Rails.logger.info "Overlay finished processing repo with config #{repo_config.inspect}"
     end
 
-    def self.overlay_directory path, repo_config
-      root_path = repo_config[:root_dest_path].empty? ? "#{Rails.application.root}" : "#{Rails.application.root}/#{repo_config[:root_dest_path]}"
-      dynamic_path = path.partition(repo_config[:root_source_path]).last
-
-      FileUtils.mkdir_p "#{root_path}/#{dynamic_path}"
-      directory_entries = github_repo.contents.get(repo_config[:user], repo_config[:repo], path, ref: repo_config[:branch]).response.body
+    def overlay_directory path, repo_config
+      FileUtils.mkdir_p destination_path(path, repo_config)
+      directory_entries = repo_config.github_api.contents.get(repo_config.org, repo_config.repo, path, ref: repo_config.branch).response.body
 
       directory_entries.each do |entry|
         if entry.type == 'dir'
@@ -94,36 +190,60 @@ module Overlay
       end
     end
 
-    def self.clone_file path, repo_config
-      root_path = repo_config[:root_dest_path].empty? ? "#{Rails.application.root}" : "#{Rails.application.root}/#{repo_config[:root_dest_path]}"
-      dynamic_path = path.partition(repo_config[:root_source_path]).last
+    def clone_file path, repo_config
+      file = repo_config.github_api.contents.get(repo_config.org, repo_config.repo, path, ref: repo_config.branch).response.body.content
+      File.open(destination_path(path, repo_config), "wb") { |f| f.write(Base64.decode64(file)) }
+      Rails.logger.info "Overlay cloned file: #{path}"
+    end
+
+    # Fork a new process and subscribe to a redis channel
+    def subscribe_to_channel key, repo_config
+      redis = Redis.new(:timeout => 30, :host => repo_config.redis_server, :port => repo_config.redis_port)
+
+      # This key is going to receive a publish event
+      # for any changes to the target repo.  We need to verify
+      # that the payload references our branch and our watch direstory
+      redis.subscribe(key) do |on|
+        on.message do |channel, msg|
+          Rails.logger.info "Overlay received publish event for channel #{publish_key} with payload: #{msg}"
+          hook = JSON.parse(msg)
+
+          # Make sure this is the branch we are watching
+          if (hook['ref'] == "refs/heads/#{repo_config.branch}")
+            Rails.logger.info "Overlay enqueueing Github hook processing job for repo: #{repo_config.repo} and branch: #{repo_config.branch}"
+            process_hook(hook, repo_config)
+            Rails.logger.info "Overlay done processing job for repo: #{repo_config.repo} and branch: #{repo_config.branch}"
+          end
+        end
+      end
+    end
 
-      file = github_repo.contents.get(repo_config[:user], repo_config[:repo], path, ref: repo_config[:branch]).response.body.content
-      File.open("#{root_path}/#{dynamic_path}", "wb") { |f| f.write(Base64.decode64(file)) }
+    def my_file? file_path, repo_config
+      return true if repo_config.root_dest_path.empty?
+      file_path.starts_with?(repo_config.root_source_path)
     end
 
-    private
+    def root_path repo_config
+      repo_config.root_dest_path.empty? ? "#{Rails.application.root}" : "#{Rails.application.root}/#{repo_config.root_dest_path}"
+    end
 
-    def self.github_repo
-      @@github ||= ::Github::Repos.new
+    def destination_path file_path, repo_config
+      raise "The file #{file_path} isn't handled by this repo with root path: #{repo_config.root_source_path}" unless my_file?(file_path, repo_config)
+      dynamic_path = file_path.partition(repo_config.root_source_path).last
+      return "#{root_path(repo_config)}#{dynamic_path}"
     end
 
-    def self.config
-      @@overlay_config ||= Overlay.configuration
+    def config
+      @overlay_config ||= Overlay.configuration
     end
 
-    # Configure the github api
-    def self.configure
-      # Validate required config
-      raise 'Configuration github_overlays.basic_auth not set' if (!config.auth || config.auth.nil?)
-
-      ::Github.configure do |github_config|
-        github_config.endpoint    = config.endpoint if config.endpoint
-        github_config.site        = config.site if config.site
-        github_config.basic_auth  = config.auth
-        github_config.adapter     = :net_http
-        github_config.ssl         = {:verify => false}
+    # Process the passed in function symbol and args in a fork
+    def fork_it method, *args
+      pid = Process.fork do
+        send(method, *args)
+        Process.exit
       end
+      Process.detach(pid)
     end
   end
 end