github_snapshot 0.1.1 → 0.1.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +8 -8
- data/README.md +1 -0
- data/lib/github_snapshot/organization.rb +1 -0
- data/lib/github_snapshot/repository.rb +22 -10
- data/lib/github_snapshot/snapshot.rb +18 -17
- data/lib/github_snapshot/utilities.rb +14 -1
- data/lib/github_snapshot/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
NmFhNTVlMmRiZTMxNTg0Y2ZkODlhMjIyOTRlZmI2NWNmZTAyMTc0NQ==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
6
|
+
Mjk5ZDY1MzUzMDlkZmNmNmUxZWRiMzBkOTdkZDJkZjYxMTk0ZTllMg==
|
7
7
|
!binary "U0hBNTEy":
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
M2E1ODFiZjZhZTc2YTI4OTQ0MWZjYjEwNzI2NmIzYmY3ZTc5MDIyY2MzYzM0
|
10
|
+
OTI3OGM1M2Q0ZmVlYjc0MDVhMjBiMzNlYTZmY2MwZGM5MDQ1MDcyM2E4MTJm
|
11
|
+
YmUzNTZkNDI3ZGY1NjhhNmZjMjY1NTc4YWQ4NGYyOTIwNDk1NWM=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
NTBhMjg0OGJkYTFkZjAwOTZiMmViNDAyYjc2ODc4NjBjZWQzNTUyOWY0NGY4
|
14
|
+
YThjNTU4OTI4NDQyMWNhNDNlYzk3MTllNmFiMzJjZjAwMzA4NDM3MzA5ZTIw
|
15
|
+
OTVmNzg4M2RjZWZmMjZmODU5YjY2ODAzN2Q2ZDU2OGY0MzhiMGI=
|
data/README.md
CHANGED
@@ -47,6 +47,7 @@ organizations:
|
|
47
47
|
s3bucket: <s3 bucket to store the backups>
|
48
48
|
backup_folder: <backup folder were the repos will be cloned to>
|
49
49
|
releases_to_keep: <how many releases to keep>
|
50
|
+
git_clone_timeout: <timeout time for git clone>
|
50
51
|
```
|
51
52
|
|
52
53
|
Then, simply run the gem's binary:
|
@@ -4,6 +4,7 @@ require "timeout"
|
|
4
4
|
require_relative "utilities"
|
5
5
|
|
6
6
|
module GithubSnapshot
|
7
|
+
Error = Class.new(RuntimeError)
|
7
8
|
|
8
9
|
class Repository
|
9
10
|
|
@@ -35,11 +36,16 @@ module GithubSnapshot
|
|
35
36
|
return nil
|
36
37
|
end
|
37
38
|
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
39
|
+
begin
|
40
|
+
Dir.chdir "#{organization.name}"
|
41
|
+
clone
|
42
|
+
clone_wiki if self.has_wiki?
|
43
|
+
prune_old_backups
|
44
|
+
Dir.chdir ".."
|
45
|
+
rescue GithubSnapshot::Error
|
46
|
+
GithubSnapshot.logger.error "#{canonical_name} - skipping due to error"
|
47
|
+
return nil
|
48
|
+
end
|
43
49
|
|
44
50
|
GithubSnapshot.logger.info "#{canonical_name} - success"
|
45
51
|
end
|
@@ -48,21 +54,27 @@ module GithubSnapshot
|
|
48
54
|
|
49
55
|
def clone
|
50
56
|
GithubSnapshot.logger.info "#{canonical_name} - cloning"
|
57
|
+
|
58
|
+
# Cloning large repos is a very sensitive operation; here we rely on
|
59
|
+
# Timeout to make sure the script doesn't hang
|
51
60
|
begin
|
52
|
-
Timeout::timeout (
|
61
|
+
Timeout::timeout (GithubSnapshot.git_clone_timeout) {
|
53
62
|
GithubSnapshot.exec "#{GithubSnapshot.git_clone_cmd} #{ssh_url} #{folder}"
|
54
63
|
}
|
55
|
-
rescue Timeout::Error => e
|
56
|
-
|
64
|
+
rescue Timeout::Error, Utilities::ExecError => e
|
65
|
+
message = e.class == Timeout::Error ? 'timedout' : 'exec error'
|
66
|
+
GithubSnapshot.logger.error "Could not clone #{canonical_name}, #{message}"
|
67
|
+
raise GithubSnapshot::Error
|
57
68
|
end
|
58
|
-
|
69
|
+
|
70
|
+
Utilities.tar "#{folder}", GithubSnapshot.logger
|
59
71
|
GithubSnapshot.exec "rm -rf #{folder}"
|
60
72
|
end
|
61
73
|
|
62
74
|
def clone_wiki
|
63
75
|
GithubSnapshot.logger.info "#{canonical_name} - cloning wiki"
|
64
76
|
GithubSnapshot.exec "#{GithubSnapshot.git_clone_cmd} #{wiki_ssh_url} #{wiki_folder}"
|
65
|
-
|
77
|
+
Utilities.tar "#{wiki_folder}", GithubSnapshot.logger
|
66
78
|
GithubSnapshot.exec "rm -rf #{wiki_folder}"
|
67
79
|
end
|
68
80
|
|
@@ -10,27 +10,22 @@ module GithubSnapshot
|
|
10
10
|
@@git_clone_cmd = "git clone --quiet --mirror"
|
11
11
|
@@time_now = Time.now.getutc.strftime("%Y%m%d%H%M")
|
12
12
|
@@releases_to_keep
|
13
|
+
@@git_clone_timeout
|
13
14
|
|
14
|
-
def self.logger
|
15
|
-
|
16
|
-
end
|
17
|
-
|
18
|
-
def self.
|
19
|
-
@@git_clone_cmd
|
20
|
-
end
|
21
|
-
|
22
|
-
def self.time_now
|
23
|
-
@@time_now
|
24
|
-
end
|
25
|
-
|
26
|
-
def self.releases_to_keep
|
27
|
-
@@releases_to_keep
|
28
|
-
end
|
15
|
+
def self.logger; @@logger; end
|
16
|
+
def self.git_clone_cmd; @@git_clone_cmd; end
|
17
|
+
def self.time_now; @@time_now; end
|
18
|
+
def self.releases_to_keep; @@releases_to_keep; end
|
19
|
+
def self.git_clone_timeout; @@git_clone_timeout; end
|
29
20
|
|
30
21
|
def self.releases_to_keep=(releases_to_keep)
|
31
22
|
@@releases_to_keep = releases_to_keep
|
32
23
|
end
|
33
24
|
|
25
|
+
def self.git_clone_timeout=(git_clone_timeout)
|
26
|
+
@@git_clone_timeout = git_clone_timeout
|
27
|
+
end
|
28
|
+
|
34
29
|
def self.exec(cmd)
|
35
30
|
Utilities.exec cmd, @@logger
|
36
31
|
end
|
@@ -49,7 +44,8 @@ module GithubSnapshot
|
|
49
44
|
@organizations = config['organizations']
|
50
45
|
@s3_bucket = config['s3bucket']
|
51
46
|
@backup_folder = config['backup_folder']
|
52
|
-
GithubSnapshot.releases_to_keep
|
47
|
+
GithubSnapshot.releases_to_keep = config['releases_to_keep']
|
48
|
+
GithubSnapshot.git_clone_timeout = config['git_clone_timeout']
|
53
49
|
|
54
50
|
@github = Github.new do |config|
|
55
51
|
config.login = username
|
@@ -75,7 +71,12 @@ module GithubSnapshot
|
|
75
71
|
|
76
72
|
def download_from_s3
|
77
73
|
GithubSnapshot.logger.info "downloading fom s3"
|
78
|
-
|
74
|
+
begin
|
75
|
+
GithubSnapshot.exec "s3cmd sync --delete-removed s3://#{s3_bucket}/ #{backup_folder}/"
|
76
|
+
rescue Utilities::ExecError
|
77
|
+
GithubSnapshot.logger.info "s3cmd doesn't respect exit status\n"\
|
78
|
+
"there is a good chance that the sync was successful"
|
79
|
+
end
|
79
80
|
end
|
80
81
|
|
81
82
|
def backup_orgs
|
@@ -1,4 +1,7 @@
|
|
1
1
|
module Utilities
|
2
|
+
Error = Class.new(RuntimeError)
|
3
|
+
ExecError = Class.new(Error)
|
4
|
+
|
2
5
|
module_function
|
3
6
|
|
4
7
|
def exec(cmd, logger)
|
@@ -6,7 +9,17 @@ module Utilities
|
|
6
9
|
if err.empty?
|
7
10
|
logger.debug out unless out.empty?
|
8
11
|
else
|
9
|
-
logger.error
|
12
|
+
logger.error "Open3 error:\n#{'='*79}\n#{err}Command was:\n#{cmd}\n#{'='*79}\n"
|
13
|
+
raise Utilities::ExecError
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def tar(file, logger)
|
18
|
+
if File.exists? file
|
19
|
+
Utilities.exec "tar zcf #{file}.tar.gz #{file}", logger
|
20
|
+
else
|
21
|
+
logger.error "Unable to tar #{file}"
|
10
22
|
end
|
11
23
|
end
|
24
|
+
|
12
25
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: github_snapshot
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Artur Rodrigues
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-08-
|
12
|
+
date: 2013-08-29 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: bundler
|