jackal-code-fetcher 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 256f04afa587e31c48aafa77aa2fc3261b725567
+  data.tar.gz: 2642b577fca02320ce3eef689d1e7e62037e5ea3
+SHA512:
+  metadata.gz: f2ddf0e3296657871dce882e18573cc0f4d1bdfa5be2189806a3448dbaa9f6a14f8eed3222c53e7d1bda0d6b40e431384b0af4e91cf6cdb513fbcbac65ce5fbf
+  data.tar.gz: 5dcd3979d0545c31933095c1cd4b2c246a29820a6dfc8085ced701e385cb6266e7eb5e30d0d86f014e8161aafe95054b94a34def399a613a4c824539f140a6fd

data/CHANGELOG.md

@@ -0,0 +1,2 @@
+# v0.1.0
+* Initial release

data/CONTRIBUTING.md

@@ -0,0 +1,25 @@
+# Contributing
+
+## Branches
+
+### `master` branch
+
+The master branch is the current stable released version.
+
+### `develop` branch
+
+The develop branch is the current edge of development.
+
+## Pull requests
+
+* https://github.com/carnivore-rb/jackal-code-fetcher/pulls
+
+Please base all pull requests of the `develop` branch. Merges to
+`master` only occur through the `develop` branch. Pull requests
+based on `master` will likely be cherry picked.
+
+## Issues
+
+Need to report an issue? Use the github issues:
+
+* https://github.com/carnivore-rb/jackal-code-fetcher/issues

data/LICENSE

@@ -0,0 +1,13 @@
+   Copyright 2015 Chris Roberts
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.md

@@ -0,0 +1,59 @@
+# Jackal Code Fetcher
+
+Fetch code from a repository and stash reference in the asset store.
+
+## Configuration
+
+The code fetcher uses the asset store to persist compressed code
+asset. Configure:
+
+```json
+{
+  "jackal": {
+    "assets": {
+      "provider": PROVIDER,
+      "credentials": {
+        CREDENTIALS
+      }
+    }
+  }
+}
+```
+
+## Supported Remotes
+
+### GitHub
+
+Access tokens are used for fetching private repositories. Token can
+be provided via direct configuration:
+
+```json
+{
+  "jackal": {
+    "code_fetcher": {
+      "config": {
+        "github": {
+          "access_token": TOKEN
+        }
+      }
+    }
+  }
+}
+```
+
+or it can be provided via application level configuration:
+
+```json
+{
+  "jackal": {
+    "github": {
+      "access_token": ACCESS_TOKEN
+    }
+  }
+}
+```
+
+# Info
+
+* Repository: https://github.com/carnivore-rb/jackal-code-fetcher
+* IRC: Freenode @ #carnivore

data/jackal-code-fetcher.gemspec

@@ -0,0 +1,16 @@
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__)) + '/lib/'
+require 'jackal-code-fetcher/version'
+Gem::Specification.new do |s|
+  s.name = 'jackal-code-fetcher'
+  s.version = Jackal::CodeFetcher::VERSION.version
+  s.summary = 'Message processing helper'
+  s.author = 'Chris Roberts'
+  s.email = 'code@chrisroberts.org'
+  s.homepage = 'https://github.com/carnivore-rb/jackal-code-fetcher'
+  s.description = 'Code fetching helper'
+  s.require_path = 'lib'
+  s.license = 'Apache 2.0'
+  s.add_dependency 'jackal'
+  s.add_dependency 'git'
+  s.files = Dir['lib/**/*'] + %w(jackal-code-fetcher.gemspec README.md CHANGELOG.md CONTRIBUTING.md LICENSE)
+end

data/lib/jackal-code-fetcher/git_hub.rb

@@ -0,0 +1,161 @@
+require 'jackal-code-fetcher'
+
+module Jackal
+  module CodeFetcher
+    # GitHub code fetcher
+    class GitHub < Callback
+
+      # Setup callback
+      def setup(*_)
+        require 'uri'
+        require 'git'
+        require 'fileutils'
+      end
+
+      # @return [String] working directory
+      def working_directory
+        memoize(:working_directory, :direct) do
+          FileUtils.mkdir_p(path = config.fetch(:working_directory, '/tmp/jackal-code-fetcher'))
+          path
+        end
+      end
+
+      # Determine validity of message
+      #
+      # @param message [Carnivore::Message]
+      # @return [Truthy, Falsey]
+      def valid?(message)
+        super do |payload|
+          payload.get(:data, :code_fetcher, :info, :commit_sha) &&
+            !payload.get(:data, :code_fetcher, :asset)
+        end
+      end
+
+      # Fetch code and push to asset store
+      #
+      # @param message [Carnivore::Message]
+      def execute(message)
+        failure_wrap(message) do |payload|
+          retried = false
+          begin
+            store_reference(payload)
+          rescue Git::GitExecuteError => e
+            unless(retried)
+              retried = true
+              delete_repository(payload)
+              error "Reference extraction from repository failed. Repository deleted and retrying. (Error: #{e.class} - #{e})"
+              retry
+            else
+              raise
+            end
+          end
+          job_completed(:code_fetcher, payload, message)
+        end
+      end
+
+      # Delete local repository path
+      #
+      # @param payload [Smash]
+      # @return [TrueClass, FalseClass]
+      def delete_repository(payload)
+        path = repository_path(payload)
+        if(File.exists?(path))
+          warn "Deleting repository directory: #{path}"
+          FileUtils.rm_rf(path)
+          true
+        else
+          false
+        end
+      end
+
+      # Fetch reference from GitHub repository and store compressed
+      # copy in the asset store
+      #
+      # @param payload [Smash]
+      # @return [TrueClass]
+      def store_reference(payload)
+        repo_dir = fetch_repository(payload)
+        pack_and_store(repo_dir, payload)
+      end
+
+      # Build github URL for fetching
+      #
+      # @param payload [Smash]
+      # @return [String]
+      def github_url(payload)
+        if(payload.get(:data, :code_fetcher, :info, :private))
+          uri = URI.parse(payload.get(:data, :code_fetcher, :info, :url))
+          uri.scheme = 'https'
+          uri.user = config.fetch(:github, :access_token,
+            app_config.get(:github, :access_token)
+          )
+          uri.to_s
+        else
+          payload.get(:data, :code_fetcher, :info, :url)
+        end
+      end
+
+      # Generate local path
+      #
+      # @return [String] path
+      def repository_path(payload)
+        File.join(
+          working_directory,
+          payload.get(:data, :code_fetcher, :info, :owner),
+          payload.get(:data, :code_fetcher, :info, :name)
+        )
+      end
+
+      # Fetch repository from GitHub
+      #
+      # @param payload [Smash]
+      # @return [String] path to repository directory
+      def fetch_repository(payload)
+        repo_path = repository_path(payload)
+        if(File.directory?(repo_path))
+          debug "Pulling changes to: #{repo_path}"
+          repo = Git.open(repo_path)
+          repo.checkout('master')
+          repo.pull
+          repo.fetch
+        else
+          debug "Initiating repository clone to: #{repo_path}"
+          Git.clone(github_url(payload), repo_path)
+        end
+        repo_path
+      end
+
+      # Store reference in asset store
+      #
+      # @param path [String] local path to repository
+      # @param payload [Smash]
+      # @return [TrueClass]
+      def pack_and_store(path, payload)
+        repo = Git.open(path)
+        repo.checkout(
+          payload.get(:data, :code_fetcher, :info, :commit_sha)
+        )
+        asset_key = [
+          payload.get(:data, :code_fetcher, :info, :owner),
+          payload.get(:data, :code_fetcher, :info, :name),
+          payload.get(:data, :code_fetcher, :info, :commit_sha)
+        ].join('-') + '.zip'
+        _path = File.join(working_directory, Celluloid.uuid)
+        begin
+          tmp_path = File.join(_path, asset_key)
+          FileUtils.mkdir_p(tmp_path)
+          FileUtils.cp_r(File.join(repository_path(payload), '.'), tmp_path)
+          FileUtils.rm_rf(File.join(tmp_path, '.git'))
+          tarball = asset_store.pack(tmp_path)
+          asset_store.put(asset_key, tarball)
+        ensure
+          FileUtils.rm_rf(_path)
+        end
+        payload.set(:data, :code_fetcher, :asset, asset_key)
+        true
+      end
+
+    end
+
+  end
+end

data/lib/jackal-code-fetcher/version.rb

@@ -0,0 +1,6 @@
+module Jackal
+  module CodeFetcher
+    # Current library version
+    VERSION = Gem::Version.new('0.1.0')
+  end
+end

data/lib/jackal-code-fetcher.rb

@@ -0,0 +1,9 @@
+require 'jackal'
+
+module Jackal
+  module CodeFetcher
+    autoload :GitHub, 'jackal-code-fetcher/git_hub'
+  end
+end
+
+require 'jackal-code-fetcher/version'

metadata

@@ -0,0 +1,80 @@
+--- !ruby/object:Gem::Specification
+name: jackal-code-fetcher
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Chris Roberts
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-03-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: jackal
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: git
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Code fetching helper
+email: code@chrisroberts.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- CONTRIBUTING.md
+- LICENSE
+- README.md
+- jackal-code-fetcher.gemspec
+- lib/jackal-code-fetcher.rb
+- lib/jackal-code-fetcher/git_hub.rb
+- lib/jackal-code-fetcher/version.rb
+homepage: https://github.com/carnivore-rb/jackal-code-fetcher
+licenses:
+- Apache 2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Message processing helper
+test_files: []
+has_rdoc: