github_snapshot 0.1.1 → 0.1.2

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    YjA2YjVjZjU1YTFmNWNiNTdiMjg5OTVhM2ZlNjRiNTk1ZjczYjkwZQ==
+    NmFhNTVlMmRiZTMxNTg0Y2ZkODlhMjIyOTRlZmI2NWNmZTAyMTc0NQ==
   data.tar.gz: !binary |-
-    Yjc0MTg5ZDFiNzcyYTg3MWFlYWFiZjA1ZTRkMzg0ZjU5Yjk4NzU1ZA==
+    Mjk5ZDY1MzUzMDlkZmNmNmUxZWRiMzBkOTdkZDJkZjYxMTk0ZTllMg==
 !binary "U0hBNTEy":
   metadata.gz: !binary |-
-    OGIxYTA0MTJiZTkxMWJkNzJmMmVjNjg4Y2ZiMGMxNTA1MmI5NDE0NTc4YWQ4
-    YWYyNGFiNGNhOWQyNjAyMTcxMjk1YWQ3NTg0N2IxNzIwNTU0NTEyNGRiNTcy
-    NDczOWZiOTg0OTYyMTY0MmVjM2QxZGZjNzExYTM1N2E4M2ZlN2U=
+    M2E1ODFiZjZhZTc2YTI4OTQ0MWZjYjEwNzI2NmIzYmY3ZTc5MDIyY2MzYzM0
+    OTI3OGM1M2Q0ZmVlYjc0MDVhMjBiMzNlYTZmY2MwZGM5MDQ1MDcyM2E4MTJm
+    YmUzNTZkNDI3ZGY1NjhhNmZjMjY1NTc4YWQ4NGYyOTIwNDk1NWM=
   data.tar.gz: !binary |-
-    OGRiMjViOTZiZmQ4ZTk3N2QxYWI1YjZjYWQ2YjNhNWQ4YWJjYzUzM2ViMTIw
-    OTEwNmIxMTJmMGE0ZjI4YmIwOGYzMmVjOTQwNTIzMzZjZTIxYzM4YTQxOTNl
-    Y2U5OWE0M2NjOWRlNmFiMGI5OWNhNTU0ZDk4YWJhYjRiZDlhZDg=
+    NTBhMjg0OGJkYTFkZjAwOTZiMmViNDAyYjc2ODc4NjBjZWQzNTUyOWY0NGY4
+    YThjNTU4OTI4NDQyMWNhNDNlYzk3MTllNmFiMzJjZjAwMzA4NDM3MzA5ZTIw
+    OTVmNzg4M2RjZWZmMjZmODU5YjY2ODAzN2Q2ZDU2OGY0MzhiMGI=

data/README.md

@@ -47,6 +47,7 @@ organizations:
 s3bucket: <s3 bucket to store the backups>
 backup_folder: <backup folder were the repos will be cloned to>
 releases_to_keep: <how many releases to keep>
+git_clone_timeout: <timeout time for git clone>
 ```
 
 Then, simply run the gem's binary:

data/lib/github_snapshot/organization.rb

@@ -13,6 +13,7 @@ module GithubSnapshot
 
     def backup
       repos.to_a.each do |repo|
+        GithubSnapshot.exec "mkdir -p #{name}"
         repository = Repository.new repo, self
         repository.backup
       end

data/lib/github_snapshot/repository.rb

@@ -4,6 +4,7 @@ require "timeout"
 require_relative "utilities"
 
 module GithubSnapshot
+  Error = Class.new(RuntimeError)
 
   class Repository
 
@@ -35,11 +36,16 @@ module GithubSnapshot
         return nil
       end
 
-      Dir.chdir "#{organization.name}"
-      clone
-      clone_wiki if self.has_wiki?
-      prune_old_backups
-      Dir.chdir ".."
+      begin
+        Dir.chdir "#{organization.name}"
+        clone
+        clone_wiki if self.has_wiki?
+        prune_old_backups
+        Dir.chdir ".."
+      rescue GithubSnapshot::Error
+        GithubSnapshot.logger.error "#{canonical_name} - skipping due to error"
+        return nil
+      end
 
       GithubSnapshot.logger.info "#{canonical_name} - success"
     end
@@ -48,21 +54,27 @@ module GithubSnapshot
 
     def clone
       GithubSnapshot.logger.info "#{canonical_name} - cloning"
+
+      # Cloning large repos is a very sensitive operation; here we rely on
+      # Timeout to make sure the script doesn't hang
       begin
-        Timeout::timeout (300) {
+        Timeout::timeout (GithubSnapshot.git_clone_timeout) {
           GithubSnapshot.exec "#{GithubSnapshot.git_clone_cmd} #{ssh_url} #{folder}"
         }
-      rescue Timeout::Error => e
-        logger.error "Could not clone #{canonical_name}, timedout"
+      rescue Timeout::Error, Utilities::ExecError => e
+        message = e.class == Timeout::Error ? 'timedout' : 'exec error'
+        GithubSnapshot.logger.error "Could not clone #{canonical_name}, #{message}"
+        raise GithubSnapshot::Error
       end
-      GithubSnapshot.exec "tar zcf #{folder}.tar.gz #{folder}"
+
+      Utilities.tar "#{folder}", GithubSnapshot.logger
       GithubSnapshot.exec "rm -rf #{folder}"
     end
 
     def clone_wiki
       GithubSnapshot.logger.info "#{canonical_name} - cloning wiki"
       GithubSnapshot.exec "#{GithubSnapshot.git_clone_cmd} #{wiki_ssh_url} #{wiki_folder}"
-      GithubSnapshot.exec "tar zcf #{wiki_folder}.tar.gz #{wiki_folder}"
+      Utilities.tar "#{wiki_folder}", GithubSnapshot.logger
       GithubSnapshot.exec "rm -rf #{wiki_folder}"
     end
 

data/lib/github_snapshot/snapshot.rb

@@ -10,27 +10,22 @@ module GithubSnapshot
   @@git_clone_cmd          = "git clone --quiet --mirror"
   @@time_now               = Time.now.getutc.strftime("%Y%m%d%H%M")
   @@releases_to_keep
+  @@git_clone_timeout
 
-  def self.logger
-    @@logger
-  end
-
-  def self.git_clone_cmd
-    @@git_clone_cmd
-  end
-
-  def self.time_now
-    @@time_now
-  end
-
-  def self.releases_to_keep
-    @@releases_to_keep
-  end
+  def self.logger;            @@logger;             end
+  def self.git_clone_cmd;     @@git_clone_cmd;      end
+  def self.time_now;          @@time_now;           end
+  def self.releases_to_keep;  @@releases_to_keep;   end
+  def self.git_clone_timeout; @@git_clone_timeout;  end
 
   def self.releases_to_keep=(releases_to_keep)
     @@releases_to_keep = releases_to_keep
   end
 
+  def self.git_clone_timeout=(git_clone_timeout)
+    @@git_clone_timeout = git_clone_timeout
+  end
+
   def self.exec(cmd)
     Utilities.exec cmd, @@logger
   end
@@ -49,7 +44,8 @@ module GithubSnapshot
       @organizations = config['organizations']
       @s3_bucket     = config['s3bucket']
       @backup_folder = config['backup_folder']
-      GithubSnapshot.releases_to_keep = config['releases_to_keep']
+      GithubSnapshot.releases_to_keep  = config['releases_to_keep']
+      GithubSnapshot.git_clone_timeout = config['git_clone_timeout']
 
       @github = Github.new do |config|
         config.login           = username
@@ -75,7 +71,12 @@ module GithubSnapshot
 
     def download_from_s3
       GithubSnapshot.logger.info "downloading fom s3"
-      GithubSnapshot.exec "s3cmd sync --delete-removed s3://#{s3_bucket}/ #{backup_folder}/"
+      begin
+        GithubSnapshot.exec "s3cmd sync --delete-removed s3://#{s3_bucket}/ #{backup_folder}/"
+      rescue Utilities::ExecError
+        GithubSnapshot.logger.info "s3cmd doesn't respect exit status\n"\
+                                   "there is a good chance that the sync was successful"
+      end
     end
 
     def backup_orgs

data/lib/github_snapshot/utilities.rb

@@ -1,4 +1,7 @@
 module Utilities
+  Error     = Class.new(RuntimeError)
+  ExecError = Class.new(Error)
+
   module_function
 
   def exec(cmd, logger)
@@ -6,7 +9,17 @@ module Utilities
     if err.empty?
       logger.debug out unless out.empty?
     else
-      logger.error(err + " command was: #{cmd}")
+      logger.error "Open3 error:\n#{'='*79}\n#{err}Command was:\n#{cmd}\n#{'='*79}\n"
+      raise Utilities::ExecError
+    end
+  end
+
+  def tar(file, logger)
+    if File.exists? file
+      Utilities.exec "tar zcf #{file}.tar.gz #{file}", logger
+    else
+      logger.error "Unable to tar #{file}"
     end
   end
+
 end

data/lib/github_snapshot/version.rb

@@ -1,3 +1,3 @@
 module GithubSnapshot
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: github_snapshot
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Artur Rodrigues
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-28 00:00:00.000000000 Z
+date: 2013-08-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler