github_records_archiver 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 20d21627161e4f5e187d555f3a2f12b925a633c6c8c3c2ca3bf28b6aee26ed18
-  data.tar.gz: '0805b746a6b6495fddf921a9638245286ec6d89da41dfcf6c30117f514ea8fb7'
+  metadata.gz: 04613c74d230d25723b56ff88a3e34c840d8a8ae56bc7543a386f0bb91fc7ca4
+  data.tar.gz: 0eff9d4f24718e4db5107ccfb6ffd8c45f5783f31de16b51edea0324cda20803
 SHA512:
-  metadata.gz: 448ef59368b08c0ffb7e221ecad0b0674ed5cbfb85a1dceec9f2ba3bbd5103d00694032decbf896f6febd34fb7ddae2e6324e539e4da70482e2a6df0170e6b8a
-  data.tar.gz: 2eaec591f98b3359ee9f3769c2797f71db0b94c85480be4f5eb29b80903fa7eccfb357c12b5616347f09fb335982aa5305ae83cd69cfe17a6fa96feef665220d
+  metadata.gz: 0704f880ce05d5f0990f36a02bd7962452e072255d88f53d00f5878b035908a7171cf155bdb4284b6cdc8ca5d94ff89f8fbd98fd55bbe6fffb9f6b6226ff860a
+  data.tar.gz: 328a9351d1161903ffe67f0b3bbcacb8f1830baf87bd72a157c5306d7ab8af75c84d7df49d689d34c9745fe58c843b479573ba038ab726cd9febd4d89d824867

data/.coveralls.yml

@@ -0,0 +1 @@
+service_name: travis-ci

data/.gitignore

@@ -1,5 +1,6 @@
 /.env
 /archive
-Gemfile.lock
-spec/examples.txt
-*.gem
+/coverage/
+/Gemfile.lock
+/spec/examples.txt
+/*.gem

data/.rubocop.yml

@@ -2,7 +2,8 @@ AllCops:
   Exclude:
     - archive/**/*
     - vendor/**/*
-
+    - bin/github-records-archiver
+    
 Style/Documentation:
   Enabled: false
 

data/Gemfile

@@ -3,3 +3,5 @@ source 'https://rubygems.org'
 git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
 
 gemspec
+
+gem 'coveralls', require: false

data/README.md

@@ -1,6 +1,6 @@
 # GitHub Records Archiver
 
-[![Build Status](https://travis-ci.org/benbalter/github-records-archiver.svg?branch=master)](https://travis-ci.org/benbalter/github-records-archiver) [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat-square)](http://makeapullrequest.com)
+[![Build Status](https://travis-ci.org/benbalter/github_records_archiver.svg?branch=master)](https://travis-ci.org/benbalter/github_records_archiver) [![Gem Version](https://badge.fury.io/rb/github_records_archiver.svg)](http://badge.fury.io/rb/github_records_archiver) [![Coverage Status](https://coveralls.io/repos/github/benbalter/github_records_archiver/badge.svg)](https://coveralls.io/github/benbalter/github_records_archiver) [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat-square)](http://makeapullrequest.com)
 
 Backs up a GitHub organization's repositories and all their associated information for archival purposes.
 
@@ -14,38 +14,72 @@ Backs up a GitHub organization's repositories and all their associated informati
 ## Requirements
 
 1. Ruby
-2. A GitHub [personal access token](https://github.com/settings/tokens/new) with `public_repo` and `repo` scope.
+2. A GitHub [personal access token](https://github.com/settings/tokens/new) with `repo` scope.
 
 ## Setup
 
-1. `git clone https://github.com/benbalter/github-records-archiver`
-2. `cd github-records-archiver`
-3. `gem install bundler`
-4. `bundle install`
+If you have Ruby installed, simply run:
 
-## Usage
+```shell
+gem install github_records_archiver
+```
 
-`bin/archive [ORGANIZATION]`
+## Basic usage
 
-You'll want to set the following environmental variable:
+```shell
+$ github_records_archiver archive ORGANIZATION --token PERSONAL_ACCESS_TOKEN`
+```
+Alternatively, you could pass the personal access token as the `GITHUB_TOKEN` environmental variable:
 
-* `GITHUB_TOKEN` - Your personal access token
-
-You *may* set the following environmental variables:
-
-* `GITHUB_ARCHIVE_DIR` to specify the output directory. It will default to `./archive`.
-* `GITHUB_ORGANIZATION` - The organization to archive if none is passed as an argument.
-
-These can be passed as `GITHUB_TOKEN=123ABC GITHUB_ORGANIZATION=whitehouse bin/archive`.
-
-You can also add the values to a `.env` file in the project's root directory, which will be automatically set as environmental variables.
+```shell
+$ GITHUB_TOKEN=1234 github_records_archiver archive ORGANIZATION`
+```
 
 ## Output
 
-The script will create an `archive` directory, with one folder for each repository.
+The script will create an `archive` directory, with one folder for each organization.
+
+Within each organization folder, there will be one folder per repository.
 
-Within each folder will be the repository content as a git repository.
+Within each repository folder will be the repository content as a git repository.
 
 If the repository has a Wiki, the wiki will be cloned as a `wiki` subfolder, as a Git repository.
 
 If the repository has issues or pull requests, it will create an `issues` sub-folder with each issue and its associated comments stored as both markdown (human readable) and JSON (machine readable).
+
+Example output:
+
+```
+├─ archive
+├─── organization
+├──── repository
+├────── README.md
+├────── LICENSE.txt
+├──── wiki
+├────── wiki-page.md
+├──── issues
+├────── 1.md
+├────── 1.json
+├─── another organization
+├──── another-repository
+├────── README.md
+├────── LICENSE.txt
+├──── wiki
+├────── wiki-page.md
+├──── issues
+├────── 1.md
+├────── 1.json
+```
+
+## Advanced usage
+
+You may set the following flags:
+
+* `--dest-dir` - the destination archive directory, defaults to `./archive`
+* `--verbose` - verbose output while archiving
+
+Additionally, the following commands are also available:
+
+* `delete [ORGANIZATION]` - delete the entire archive directory or an organization's archive
+* `help` - display help information
+* `version` - display the GitHub Record Archiver version

data/bin/github-records-archiver

@@ -0,0 +1,77 @@
+#!/usr/bin/env ruby
+
+require 'thor'
+require 'parallel'
+require_relative '../lib/github_records_archiver'
+
+class GitHubRecordsArchiverCLI < Thor
+  package_name 'GitHub Records Archiver'
+  class_option :dest_dir, type: :string, required: false,
+    desc: "The destination directory for the archive", default: GitHubRecordsArchiver.dest_dir
+  class_option :token, type: :string, required: false, desc: "A GitHub personal access token with repo scope"
+  class_option :verbose, type: :boolean, desc: 'Display verbose output while archiving', default: false
+
+  desc 'version', 'Outputs the GitHubRecordsArchiver version'
+  def version
+    say "GitHub Records Archiver v#{GitHubRecordsArchiver::VERSION}"
+  end
+
+  desc "delete [ORGANIZATION]", "Deletes all archives, or the archive for an organization"
+  def delete(org_name = nil)
+    path = GitHubRecordsArchiver.dest_dir
+    path = File.join path, org_name if org_name
+    FileUtils.rm_rf(path) if yes? "Are you sure? Remove #{path}?"
+  end
+
+  desc 'archive ORGANIZATION', 'Create or update archive for the given organization'
+  def archive(org_name)
+    start_time # Memoize start time for comparison
+    @org_name = org_name
+
+    GitHubRecordsArchiver.shell = shell
+    %i(token dest_dir verbose).each do |option|
+      next unless options[option]
+      GitHubRecordsArchiver.public_send "#{option}=".to_sym, options[option]
+    end
+
+    say "Starting archive for @#{org.name} in #{org.archive_dir}"
+    shell.indent(2) do
+      archive_teams
+      archive_repos
+    end
+    say "Done in #{Time.now - start_time} seconds.", :green
+  end
+
+  private
+
+  def start_time
+    @start_time ||= Time.now
+  end
+
+  def organization
+    @organization ||= GitHubRecordsArchiver::Organization.new @org_name
+  end
+  alias_method :org, :organization
+
+  def archive_teams
+    say_status "Teams found:", org.teams.count, :white
+    Parallel.each(org.teams, progress: 'Archiving teams', &:archive)
+  end
+
+  def archive_repos
+    say_status "Repositories found:", org.repos.count, :white
+
+    Parallel.each(org.repos, progress: 'Archiving repos') do |repo|
+      begin
+        repo.clone
+        repo.wiki.clone if repo.has_wiki?
+        Parallel.each(repo.issues, &:archive)
+      rescue GitHubRecordsArchiver::GitError => e
+        say "Failed to archive #{repo.name}", :red
+        say e.message, :red
+      end
+    end
+  end
+end
+
+GitHubRecordsArchiverCLI.start(ARGV)

data/github_records_archiver.gemspec

@@ -28,6 +28,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'octokit', '~> 4.0'
   spec.add_dependency 'parallel', '~> 1.10'
   spec.add_dependency 'ruby-progressbar', '~> 1.0'
+  spec.add_dependency 'thor', '~> 0.19'
   spec.add_development_dependency 'addressable', '~> 2.5'
   spec.add_development_dependency 'bundler', '~> 1.16'
   spec.add_development_dependency 'pry', '~> 0.10'

data/lib/github_records_archiver.rb

@@ -1,18 +1,14 @@
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 
-# stdlib
 require 'yaml'
 require 'json'
 require 'logger'
 require 'fileutils'
 require 'open3'
-
-# gems
+require 'thor'
 require 'octokit'
-require 'dotenv'
+require 'dotenv/load'
 
-# Configuration
-Dotenv.load
 Octokit.auto_paginate = true
 
 module GitHubRecordsArchiver
@@ -28,8 +24,10 @@ module GitHubRecordsArchiver
   autoload :Wiki,          'github_records_archiver/wiki'
 
   class << self
+    attr_writer :token, :dest_dir, :verbose, :shell
+
     def token
-      ENV['GITHUB_TOKEN']
+      @token ||= ENV['GITHUB_TOKEN']
     end
 
     def client
@@ -37,7 +35,25 @@ module GitHubRecordsArchiver
     end
 
     def dest_dir
-      ENV['GITHUB_ARCHIVE_DIR'] || File.expand_path('./archive', Dir.pwd)
+      @dest_dir ||= File.expand_path('./archive', Dir.pwd)
+    end
+
+    def verbose
+      @verbose ||= false
+    end
+    alias verbose? verbose
+
+    def shell
+      @shell ||= Thor::Base.shell.new
+    end
+
+    def verbose_status(status, message, color = :white)
+      return unless verbose?
+      shell.say_status status, remove_token(message), color
+    end
+
+    def remove_token(string)
+      string.gsub(GitHubRecordsArchiver.token, '<GITHUB_TOKEN>')
     end
   end
 end

data/lib/github_records_archiver/git_repository.rb

@@ -3,11 +3,13 @@ module GitHubRecordsArchiver
 
   class GitRepository
     def clone
-      if Dir.exist? repo_dir # Repo already exists, just pull new objects
+      # Repo already exists, just pull new objects
+      if Dir.exist? File.join(repo_dir, '.git')
         Dir.chdir repo_dir do
           git 'pull'
         end
-      else # Clone Git content from scratch
+      else
+        # Clone Git content from scratch
         git 'clone', clone_url, repo_dir
       end
     end
@@ -43,8 +45,14 @@ module GitHubRecordsArchiver
     # Run a git command, piping output to stdout
     def git(*args)
       output, status = Open3.capture2e('git', *args)
+      cmd = "git #{args.join(' ')}"
+      cmd << " in #{Dir.pwd}" if args == ['pull']
+      GitHubRecordsArchiver.verbose_status 'Git command:', cmd
       return false if empty_repo?(output) || wiki_does_not_exist?(output)
-      raise GitError, output if status.exitstatus != 0
+      if status.exitstatus != 0
+        output = GitHubRecordsArchiver.remove_token(output)
+        GitHubRecordsArchiver.shell.say_status 'Git Error', output, :red
+      end
       output
     end
   end

data/lib/github_records_archiver/repository.rb

@@ -40,15 +40,13 @@ module GitHubRecordsArchiver
       end
     end
 
-    private
-
-    def repo_dir
-      @repo_dir ||= File.expand_path data[:name], GitHubRecordsArchiver.dest_dir
-    end
-
     def clone_url
       replacement = "https://#{GitHubRecordsArchiver.token}:x-oauth-basic@"
       data[:clone_url].gsub(%r{https?://}, replacement)
     end
+
+    def repo_dir
+      @repo_dir ||= File.expand_path full_name, GitHubRecordsArchiver.dest_dir
+    end
   end
 end

data/lib/github_records_archiver/version.rb

@@ -1,3 +1,3 @@
 module GitHubRecordsArchiver
-  VERSION = '0.1.0'.freeze
+  VERSION = '0.2.0'.freeze
 end

data/lib/github_records_archiver/wiki.rb

@@ -13,8 +13,7 @@ module GitHubRecordsArchiver
     end
 
     def repo_dir
-      name = "#{repository.full_name}/wiki"
-      @repo_dir ||= File.expand_path name, GitHubRecordsArchiver.dest_dir
+      @repo_dir ||= File.join repository.repo_dir, 'wiki'
     end
 
     private

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: github_records_archiver
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Ben Balter
@@ -66,6 +66,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.19'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.19'
 - !ruby/object:Gem::Dependency
   name: addressable
   requirement: !ruby/object:Gem::Requirement
@@ -168,10 +182,11 @@ description:
 email:
 - ben.balter@github.com
 executables:
-- archive
+- github-records-archiver
 extensions: []
 extra_rdoc_files: []
 files:
+- ".coveralls.yml"
 - ".github/CODEOWNERS"
 - ".github/config.yml"
 - ".github/no-response.yml"
@@ -185,7 +200,7 @@ files:
 - LICENSE.md
 - README.md
 - Rakefile
-- bin/archive
+- bin/github-records-archiver
 - docs/CODE_OF_CONDUCT.md
 - docs/CONTRIBUTING.md
 - github_records_archiver.gemspec

data/bin/archive

@@ -1,45 +0,0 @@
-#!/usr/bin/env ruby
-# Backs up a GitHub organization's repositories and
-# all their associated information for archival purposes.
-# Usage: ruby archive.rb
-
-require './lib/github_records_archiver'
-require 'parallel'
-
-def logger
-  @logger ||= Logger.new(STDOUT)
-end
-
-def log(msg)
-  logger.info(msg)
-end
-
-def error(msg)
-  logger.error(msg)
-end
-
-archiver = GitHubRecordsArchiver
-pwd      = Dir.pwd
-start    = Time.now
-org_name = ARGV[0] || ENV['GITHUB_ORGANIZATION']
-org      = archiver::Organization.new org_name
-
-log "Starting archive for @#{org.name} in #{org.archive_dir}"
-
-log "Found #{org.teams.count} teams"
-Parallel.each(org.teams, progress: 'Archiving teams', &:archive)
-
-log "Found #{org.repos.count} repos"
-Parallel.each(org.repos, progress: 'Archiving repos') do |repo|
-  begin
-    repo.clone
-    repo.wiki.clone if repo.has_wiki?
-    Parallel.each(repo.issues, &:archive)
-  rescue GitHubRecordsArchiver::GitError => e
-    error "Failed to archive #{repo.name}"
-    error e.message
-  end
-end
-
-Dir.chdir pwd
-log "Done in #{Time.now - start} seconds."