github_snapshot 0.1.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    OGI2N2ZkYjNkY2I0MTZkNjFjYzcxYjMyOWU1NjE1NjU5ZjI3NGEwYQ==
+  data.tar.gz: !binary |-
+    YTFlNTIzZDBmNTVmOTUyYTQ1NmFiZjdjNjk0YzdjZDBiMmIxMjlkZA==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    YjMyMzIxOTg4MTI0MDkwYmU3YTUyY2E5MGFiNjI5YTMxZDM0YTcxN2RjNTVh
+    MTczZjZlMzkxOGJhNmExNTVmNTU0YWU2MTk1MmQxOTliYTYyZmQ0MDAyMGNm
+    MzRmMDM3NWFiOTNhMDcyOTVlOWFhM2JlNWQ0MjkxMWZhMDI0NjA=
+  data.tar.gz: !binary |-
+    ODZjYWZjY2IzM2I1NDcxOTMyY2RiZjQyYjUxMzE1ODg3YTAyMzViY2Q3ZjEx
+    NGY5N2YwNmEwYTc0Mzg0NTJjOTZmMTU0MjEzNTViMDY0MzAxMWQyYmMzOGQ3
+    NWFkNjg3Y2QxYzZkMGE4ZGVkYmViOGEwZjgzMjdmZTY5OGQ1ZjY=

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+config.yml

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in github_snapshot.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Innvent | Innovation & Venturing
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,46 @@
+# GithubSnapshot
+
+Snapshoting organization's repositories, including wikis
+
+## Installation
+
+`github_snapshot` should be run as a command line tool, so it makes sense installing it globally
+
+    $ gem install github_snapshot
+
+[`s3cmd`](https://github.com/s3tools/s3cmd) should also be installed on the system and properly configured.
+
+## Usage
+
+`github_snapshot` expects a `config.yml` file on the folder it is run, here is a sample:
+
+```yaml
+username: <github user with read access to the repositories>
+password: <github password>
+organizations:
+  - organization1
+  - organization2
+s3bucket: <s3 bucket to store the backups>
+backup_folder: <backup folder were the repos will be cloned to>
+releases_to_keep: <how many releases to keep>
+```
+
+Then, simply run the gem's binary:
+
+```bash
+$ github_snapshot
+```
+
+**You could be prompted to check the authenticity for `github.com` host**. You can either say yes during the first script execution or disable strict host key checking for it:
+
+```bash
+$ echo -e "Host github.com\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
+```
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/github_snapshot

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+begin
+  require 'github_snapshot'
+rescue LoadError
+  require 'rubygems'
+  require 'github_snapshot'
+end
+
+gs = GithubSnapshot::Snapshot.new
+gs.backup

data/github_snapshot.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'github_snapshot/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "github_snapshot"
+  spec.version       = GithubSnapshot::VERSION
+  spec.authors       = ["Artur Rodrigues" , "Joao Sa"]
+  spec.email         = ["arturhoo@gmail.com"]
+  spec.description   = %q{Snapshoting organization's repositories, including wikis}
+  spec.summary       = %q{Snapshoting organization's repositories}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "pry"
+  spec.add_development_dependency "awesome_print"
+
+  spec.add_dependency "github_api", "~> 0.10.2"
+  spec.add_dependency "rake"
+end

data/lib/github_snapshot/organization.rb

@@ -0,0 +1,24 @@
+require_relative "repository"
+
+module GithubSnapshot
+
+  class Organization
+
+    attr_reader :name, :folder, :repos
+
+    def initialize(name, folder, repos)
+      @name   = name
+      @folder = folder
+      @repos  = repos
+    end
+
+    def backup
+      repos.to_a.each do |repo|
+        repository = Repository.new repo, self
+        repository.backup
+      end
+    end
+
+  end
+
+end

data/lib/github_snapshot/repository.rb

@@ -0,0 +1,84 @@
+require "open3"
+require "rake"
+require "timeout"
+require_relative "utilities"
+
+module GithubSnapshot
+
+  class Repository
+
+    attr_reader :name, :folder, :wiki_folder, :ssh_url, :wiki_ssh_url,
+                :canonical_name, :pushed_at, :gh_has_wiki, :organization
+
+    def initialize(repo, organization)
+      @name           = repo['name']
+      @folder         = "#{organization.folder}/#{@name}-#{GithubSnapshot.time_now}.git"
+      @wiki_folder    = @folder.gsub('.git', '.wiki.git')
+      @ssh_url        = repo['ssh_url']
+      @wiki_ssh_url   = @ssh_url.gsub('.git', '.wiki.git')
+      @canonical_name = "#{organization.name}/#{@name}"
+      @pushed_at      = repo['pushed_at']
+      @gh_has_wiki    = repo['has_wiki']
+      @organization   = organization
+    end
+
+    def has_wiki?
+      Open3.capture3("git ls-remote #{wiki_ssh_url}")[1].empty? && gh_has_wiki
+    end
+
+    def backup
+      GithubSnapshot.logger.info "#{canonical_name} - backing up"
+
+      # Go to next repo if the repository is empty
+      unless pushed_at
+        GithubSnapshot.logger.info "#{canonical_name} is empty"
+        return nil
+      end
+
+      clone
+      clone_wiki if self.has_wiki?
+      prune_old_backups
+
+      GithubSnapshot.logger.info "#{canonical_name} - success"
+    end
+
+  private
+
+    def clone
+      GithubSnapshot.logger.info "#{canonical_name} - cloning"
+      begin
+        Timeout::timeout (300) {
+          GithubSnapshot.exec "#{GithubSnapshot.git_clone_cmd} #{ssh_url} #{folder}"
+        }
+      rescue Timeout::Error => e
+        logger.error "Could not clone #{canonical_name}, timedout"
+      end
+      GithubSnapshot.exec "tar zcf #{folder}.tar.gz #{folder}"
+      GithubSnapshot.exec "rm -rf #{folder}"
+    end
+
+    def clone_wiki
+      GithubSnapshot.logger.info "#{canonical_name} - cloning wiki"
+      GithubSnapshot.exec "#{GithubSnapshot.git_clone_cmd} #{wiki_ssh_url} #{wiki_folder}"
+      GithubSnapshot.exec "tar zcf #{wiki_folder}.tar.gz #{wiki_folder}"
+      GithubSnapshot.exec "rm -rf #{wiki_folder}"
+    end
+
+    def prune_old_backups
+      GithubSnapshot.logger.info "#{canonical_name} - pruning old backups"
+      file_regex  = "#{organization.folder}/#{name}*"
+      zipped_bkps = FileList[file_regex].exclude(/wiki/)
+      zipped_bkps.sort[0..-(GithubSnapshot.releases_to_keep + 1)].each do |file|
+        File.delete file
+      end if zipped_bkps.length > GithubSnapshot.releases_to_keep
+
+      wiki_regex   = "#{organization.name}/#{name}*wiki*"
+      zipped_wikis = FileList[wiki_regex]
+      zipped_wikis.sort[0..-(GithubSnapshot.releases_to_keep + 1)].each do |file|
+        File.delete file
+      end if zipped_wikis.length > GithubSnapshot.releases_to_keep
+    end
+
+  end
+
+end

data/lib/github_snapshot/snapshot.rb

@@ -0,0 +1,98 @@
+require "github_api"
+require "logger"
+require_relative "utilities"
+require_relative "organization"
+
+module GithubSnapshot
+  @@logger                 = Logger.new(STDOUT)
+  @@logger.level           = Logger::INFO
+  @@logger.datetime_format = '%Y-%m-%d %H:%M:%S '
+  @@git_clone_cmd          = "git clone --quiet --mirror"
+  @@time_now               = Time.now.getutc.strftime("%Y%m%d%H%M")
+  @@releases_to_keep
+
+  def self.logger
+    @@logger
+  end
+
+  def self.git_clone_cmd
+    @@git_clone_cmd
+  end
+
+  def self.time_now
+    @@time_now
+  end
+
+  def self.releases_to_keep
+    @@releases_to_keep
+  end
+
+  def self.releases_to_keep=(releases_to_keep)
+    @@releases_to_keep = releases_to_keep
+  end
+
+  def self.exec(cmd)
+    Utilities.exec cmd, @@logger
+  end
+
+  class Snapshot
+    require "yaml"
+
+    attr_reader :username, :password, :organizations, :s3_bucket, :backup_folder,
+                :releases_to_keep, :git_clone_cmd, :time_now, :github, :logger
+
+    def initialize(config_file="config.yml")
+      config = YAML.load(File.read(config_file))
+
+      @username      = config['username']
+      @password      = config['password']
+      @organizations = config['organizations']
+      @s3_bucket     = config['s3bucket']
+      @backup_folder = config['backup_folder']
+      GithubSnapshot.releases_to_keep = config['releases_to_keep']
+
+      @github = Github.new do |config|
+        config.login           = username
+        config.password        = password
+        config.auto_pagination = true
+      end
+    end
+
+    def backup
+      create_backup_folder
+      download_from_s3
+      backup_orgs
+      upload_to_s3
+
+      total_size = %x[ du -sh #{backup_folder} | cut -f1 ]
+      GithubSnapshot.logger.info "backup finished, total size is #{total_size}"
+    end
+
+  private
+    def create_backup_folder
+      GithubSnapshot.exec "mkdir -p #{backup_folder}"
+    end
+
+    def download_from_s3
+      GithubSnapshot.logger.info "downloading fom s3"
+      GithubSnapshot.exec "s3cmd sync --delete-removed s3://#{s3_bucket}/ #{backup_folder}/"
+    end
+
+    def backup_orgs
+      organizations.each do |org|
+        GithubSnapshot.logger.info "#{org} - initializing"
+        repos        = github.repos.list org: org
+        organization = Organization.new org, "#{backup_folder}/#{org}", repos
+        organization.backup
+        GithubSnapshot.logger.info "#{org} - ending"
+      end
+    end
+
+    def upload_to_s3
+      GithubSnapshot.logger.info "uploading to s3"
+      GithubSnapshot.exec "s3cmd sync --delete-removed #{backup_folder}/ s3://#{s3_bucket}/"
+    end
+
+  end
+
+end

data/lib/github_snapshot/utilities.rb

@@ -0,0 +1,12 @@
+module Utilities
+  module_function
+
+  def exec(cmd, logger)
+    out, err, status = Open3.capture3 cmd
+    if err.empty?
+      logger.debug out unless out.empty?
+    else
+      logger.error(err + " command was: #{cmd}")
+    end
+  end
+end

data/lib/github_snapshot/version.rb

@@ -0,0 +1,3 @@
+module GithubSnapshot
+  VERSION = "0.1.0"
+end

data/lib/github_snapshot.rb

@@ -0,0 +1,5 @@
+require "github_snapshot/snapshot"
+
+module GithubSnapshot
+  # Your code goes here...
+end

metadata

@@ -0,0 +1,143 @@
+--- !ruby/object:Gem::Specification
+name: github_snapshot
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Artur Rodrigues
+- Joao Sa
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-08-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: awesome_print
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: github_api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.10.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.10.2
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Snapshoting organization's repositories, including wikis
+email:
+- arturhoo@gmail.com
+executables:
+- github_snapshot
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/github_snapshot
+- github_snapshot.gemspec
+- lib/github_snapshot.rb
+- lib/github_snapshot/organization.rb
+- lib/github_snapshot/repository.rb
+- lib/github_snapshot/snapshot.rb
+- lib/github_snapshot/utilities.rb
+- lib/github_snapshot/version.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.7
+signing_key: 
+specification_version: 4
+summary: Snapshoting organization's repositories
+test_files: []