jackal-code-fetcher 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +2 -0
- data/CONTRIBUTING.md +25 -0
- data/LICENSE +13 -0
- data/README.md +59 -0
- data/jackal-code-fetcher.gemspec +16 -0
- data/lib/jackal-code-fetcher/git_hub.rb +161 -0
- data/lib/jackal-code-fetcher/version.rb +6 -0
- data/lib/jackal-code-fetcher.rb +9 -0
- metadata +80 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 256f04afa587e31c48aafa77aa2fc3261b725567
|
4
|
+
data.tar.gz: 2642b577fca02320ce3eef689d1e7e62037e5ea3
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: f2ddf0e3296657871dce882e18573cc0f4d1bdfa5be2189806a3448dbaa9f6a14f8eed3222c53e7d1bda0d6b40e431384b0af4e91cf6cdb513fbcbac65ce5fbf
|
7
|
+
data.tar.gz: 5dcd3979d0545c31933095c1cd4b2c246a29820a6dfc8085ced701e385cb6266e7eb5e30d0d86f014e8161aafe95054b94a34def399a613a4c824539f140a6fd
|
data/CHANGELOG.md
ADDED
data/CONTRIBUTING.md
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
# Contributing
|
2
|
+
|
3
|
+
## Branches
|
4
|
+
|
5
|
+
### `master` branch
|
6
|
+
|
7
|
+
The master branch is the current stable released version.
|
8
|
+
|
9
|
+
### `develop` branch
|
10
|
+
|
11
|
+
The develop branch is the current edge of development.
|
12
|
+
|
13
|
+
## Pull requests
|
14
|
+
|
15
|
+
* https://github.com/carnivore-rb/jackal-code-fetcher/pulls
|
16
|
+
|
17
|
+
Please base all pull requests of the `develop` branch. Merges to
|
18
|
+
`master` only occur through the `develop` branch. Pull requests
|
19
|
+
based on `master` will likely be cherry picked.
|
20
|
+
|
21
|
+
## Issues
|
22
|
+
|
23
|
+
Need to report an issue? Use the github issues:
|
24
|
+
|
25
|
+
* https://github.com/carnivore-rb/jackal-code-fetcher/issues
|
data/LICENSE
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
Copyright 2015 Chris Roberts
|
2
|
+
|
3
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
you may not use this file except in compliance with the License.
|
5
|
+
You may obtain a copy of the License at
|
6
|
+
|
7
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
|
9
|
+
Unless required by applicable law or agreed to in writing, software
|
10
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
See the License for the specific language governing permissions and
|
13
|
+
limitations under the License.
|
data/README.md
ADDED
@@ -0,0 +1,59 @@
|
|
1
|
+
# Jackal Code Fetcher
|
2
|
+
|
3
|
+
Fetch code from a repository and stash reference in the asset store.
|
4
|
+
|
5
|
+
## Configuration
|
6
|
+
|
7
|
+
The code fetcher uses the asset store to persist compressed code
|
8
|
+
asset. Configure:
|
9
|
+
|
10
|
+
```json
|
11
|
+
{
|
12
|
+
"jackal": {
|
13
|
+
"assets": {
|
14
|
+
"provider": PROVIDER,
|
15
|
+
"credentials": {
|
16
|
+
CREDENTIALS
|
17
|
+
}
|
18
|
+
}
|
19
|
+
}
|
20
|
+
}
|
21
|
+
```
|
22
|
+
|
23
|
+
## Supported Remotes
|
24
|
+
|
25
|
+
### GitHub
|
26
|
+
|
27
|
+
Access tokens are used for fetching private repositories. Token can
|
28
|
+
be provided via direct configuration:
|
29
|
+
|
30
|
+
```json
|
31
|
+
{
|
32
|
+
"jackal": {
|
33
|
+
"code_fetcher": {
|
34
|
+
"config": {
|
35
|
+
"github": {
|
36
|
+
"access_token": TOKEN
|
37
|
+
}
|
38
|
+
}
|
39
|
+
}
|
40
|
+
}
|
41
|
+
}
|
42
|
+
```
|
43
|
+
|
44
|
+
or it can be provided via application level configuration:
|
45
|
+
|
46
|
+
```json
|
47
|
+
{
|
48
|
+
"jackal": {
|
49
|
+
"github": {
|
50
|
+
"access_token": ACCESS_TOKEN
|
51
|
+
}
|
52
|
+
}
|
53
|
+
}
|
54
|
+
```
|
55
|
+
|
56
|
+
# Info
|
57
|
+
|
58
|
+
* Repository: https://github.com/carnivore-rb/jackal-code-fetcher
|
59
|
+
* IRC: Freenode @ #carnivore
|
@@ -0,0 +1,16 @@
|
|
1
|
+
$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__)) + '/lib/'
|
2
|
+
require 'jackal-code-fetcher/version'
|
3
|
+
Gem::Specification.new do |s|
|
4
|
+
s.name = 'jackal-code-fetcher'
|
5
|
+
s.version = Jackal::CodeFetcher::VERSION.version
|
6
|
+
s.summary = 'Message processing helper'
|
7
|
+
s.author = 'Chris Roberts'
|
8
|
+
s.email = 'code@chrisroberts.org'
|
9
|
+
s.homepage = 'https://github.com/carnivore-rb/jackal-code-fetcher'
|
10
|
+
s.description = 'Code fetching helper'
|
11
|
+
s.require_path = 'lib'
|
12
|
+
s.license = 'Apache 2.0'
|
13
|
+
s.add_dependency 'jackal'
|
14
|
+
s.add_dependency 'git'
|
15
|
+
s.files = Dir['lib/**/*'] + %w(jackal-code-fetcher.gemspec README.md CHANGELOG.md CONTRIBUTING.md LICENSE)
|
16
|
+
end
|
@@ -0,0 +1,161 @@
|
|
1
|
+
require 'jackal-code-fetcher'
|
2
|
+
|
3
|
+
module Jackal
|
4
|
+
module CodeFetcher
|
5
|
+
# GitHub code fetcher
|
6
|
+
class GitHub < Callback
|
7
|
+
|
8
|
+
# Setup callback
|
9
|
+
def setup(*_)
|
10
|
+
require 'uri'
|
11
|
+
require 'git'
|
12
|
+
require 'fileutils'
|
13
|
+
end
|
14
|
+
|
15
|
+
# @return [String] working directory
|
16
|
+
def working_directory
|
17
|
+
memoize(:working_directory, :direct) do
|
18
|
+
FileUtils.mkdir_p(path = config.fetch(:working_directory, '/tmp/jackal-code-fetcher'))
|
19
|
+
path
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
# Determine validity of message
|
24
|
+
#
|
25
|
+
# @param message [Carnivore::Message]
|
26
|
+
# @return [Truthy, Falsey]
|
27
|
+
def valid?(message)
|
28
|
+
super do |payload|
|
29
|
+
payload.get(:data, :code_fetcher, :info, :commit_sha) &&
|
30
|
+
!payload.get(:data, :code_fetcher, :asset)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
# Fetch code and push to asset store
|
35
|
+
#
|
36
|
+
# @param message [Carnivore::Message]
|
37
|
+
def execute(message)
|
38
|
+
failure_wrap(message) do |payload|
|
39
|
+
retried = false
|
40
|
+
begin
|
41
|
+
store_reference(payload)
|
42
|
+
rescue Git::GitExecuteError => e
|
43
|
+
unless(retried)
|
44
|
+
retried = true
|
45
|
+
delete_repository(payload)
|
46
|
+
error "Reference extraction from repository failed. Repository deleted and retrying. (Error: #{e.class} - #{e})"
|
47
|
+
retry
|
48
|
+
else
|
49
|
+
raise
|
50
|
+
end
|
51
|
+
end
|
52
|
+
job_completed(:code_fetcher, payload, message)
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
# Delete local repository path
|
57
|
+
#
|
58
|
+
# @param payload [Smash]
|
59
|
+
# @return [TrueClass, FalseClass]
|
60
|
+
def delete_repository(payload)
|
61
|
+
path = repository_path(payload)
|
62
|
+
if(File.exists?(path))
|
63
|
+
warn "Deleting repository directory: #{path}"
|
64
|
+
FileUtils.rm_rf(path)
|
65
|
+
true
|
66
|
+
else
|
67
|
+
false
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
# Fetch reference from GitHub repository and store compressed
|
72
|
+
# copy in the asset store
|
73
|
+
#
|
74
|
+
# @param payload [Smash]
|
75
|
+
# @return [TrueClass]
|
76
|
+
def store_reference(payload)
|
77
|
+
repo_dir = fetch_repository(payload)
|
78
|
+
pack_and_store(repo_dir, payload)
|
79
|
+
end
|
80
|
+
|
81
|
+
# Build github URL for fetching
|
82
|
+
#
|
83
|
+
# @param payload [Smash]
|
84
|
+
# @return [String]
|
85
|
+
def github_url(payload)
|
86
|
+
if(payload.get(:data, :code_fetcher, :info, :private))
|
87
|
+
uri = URI.parse(payload.get(:data, :code_fetcher, :info, :url))
|
88
|
+
uri.scheme = 'https'
|
89
|
+
uri.user = config.fetch(:github, :access_token,
|
90
|
+
app_config.get(:github, :access_token)
|
91
|
+
)
|
92
|
+
uri.to_s
|
93
|
+
else
|
94
|
+
payload.get(:data, :code_fetcher, :info, :url)
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
# Generate local path
|
99
|
+
#
|
100
|
+
# @return [String] path
|
101
|
+
def repository_path(payload)
|
102
|
+
File.join(
|
103
|
+
working_directory,
|
104
|
+
payload.get(:data, :code_fetcher, :info, :owner),
|
105
|
+
payload.get(:data, :code_fetcher, :info, :name)
|
106
|
+
)
|
107
|
+
end
|
108
|
+
|
109
|
+
# Fetch repository from GitHub
|
110
|
+
#
|
111
|
+
# @param payload [Smash]
|
112
|
+
# @return [String] path to repository directory
|
113
|
+
def fetch_repository(payload)
|
114
|
+
repo_path = repository_path(payload)
|
115
|
+
if(File.directory?(repo_path))
|
116
|
+
debug "Pulling changes to: #{repo_path}"
|
117
|
+
repo = Git.open(repo_path)
|
118
|
+
repo.checkout('master')
|
119
|
+
repo.pull
|
120
|
+
repo.fetch
|
121
|
+
else
|
122
|
+
debug "Initiating repository clone to: #{repo_path}"
|
123
|
+
Git.clone(github_url(payload), repo_path)
|
124
|
+
end
|
125
|
+
repo_path
|
126
|
+
end
|
127
|
+
|
128
|
+
# Store reference in asset store
|
129
|
+
#
|
130
|
+
# @param path [String] local path to repository
|
131
|
+
# @param payload [Smash]
|
132
|
+
# @return [TrueClass]
|
133
|
+
def pack_and_store(path, payload)
|
134
|
+
repo = Git.open(path)
|
135
|
+
repo.checkout(
|
136
|
+
payload.get(:data, :code_fetcher, :info, :commit_sha)
|
137
|
+
)
|
138
|
+
asset_key = [
|
139
|
+
payload.get(:data, :code_fetcher, :info, :owner),
|
140
|
+
payload.get(:data, :code_fetcher, :info, :name),
|
141
|
+
payload.get(:data, :code_fetcher, :info, :commit_sha)
|
142
|
+
].join('-') + '.zip'
|
143
|
+
_path = File.join(working_directory, Celluloid.uuid)
|
144
|
+
begin
|
145
|
+
tmp_path = File.join(_path, asset_key)
|
146
|
+
FileUtils.mkdir_p(tmp_path)
|
147
|
+
FileUtils.cp_r(File.join(repository_path(payload), '.'), tmp_path)
|
148
|
+
FileUtils.rm_rf(File.join(tmp_path, '.git'))
|
149
|
+
tarball = asset_store.pack(tmp_path)
|
150
|
+
asset_store.put(asset_key, tarball)
|
151
|
+
ensure
|
152
|
+
FileUtils.rm_rf(_path)
|
153
|
+
end
|
154
|
+
payload.set(:data, :code_fetcher, :asset, asset_key)
|
155
|
+
true
|
156
|
+
end
|
157
|
+
|
158
|
+
end
|
159
|
+
|
160
|
+
end
|
161
|
+
end
|
metadata
ADDED
@@ -0,0 +1,80 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: jackal-code-fetcher
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Chris Roberts
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2015-03-04 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: jackal
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: git
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
description: Code fetching helper
|
42
|
+
email: code@chrisroberts.org
|
43
|
+
executables: []
|
44
|
+
extensions: []
|
45
|
+
extra_rdoc_files: []
|
46
|
+
files:
|
47
|
+
- CHANGELOG.md
|
48
|
+
- CONTRIBUTING.md
|
49
|
+
- LICENSE
|
50
|
+
- README.md
|
51
|
+
- jackal-code-fetcher.gemspec
|
52
|
+
- lib/jackal-code-fetcher.rb
|
53
|
+
- lib/jackal-code-fetcher/git_hub.rb
|
54
|
+
- lib/jackal-code-fetcher/version.rb
|
55
|
+
homepage: https://github.com/carnivore-rb/jackal-code-fetcher
|
56
|
+
licenses:
|
57
|
+
- Apache 2.0
|
58
|
+
metadata: {}
|
59
|
+
post_install_message:
|
60
|
+
rdoc_options: []
|
61
|
+
require_paths:
|
62
|
+
- lib
|
63
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
64
|
+
requirements:
|
65
|
+
- - ">="
|
66
|
+
- !ruby/object:Gem::Version
|
67
|
+
version: '0'
|
68
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
69
|
+
requirements:
|
70
|
+
- - ">="
|
71
|
+
- !ruby/object:Gem::Version
|
72
|
+
version: '0'
|
73
|
+
requirements: []
|
74
|
+
rubyforge_project:
|
75
|
+
rubygems_version: 2.2.2
|
76
|
+
signing_key:
|
77
|
+
specification_version: 4
|
78
|
+
summary: Message processing helper
|
79
|
+
test_files: []
|
80
|
+
has_rdoc:
|