gitload 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: eae65fffc20fb78620aab8974a15a20daacee260
-  data.tar.gz: 022a121055c47e217b1fd85f5dd6c445db098843
+  metadata.gz: 42e39c1c4506254e1123f86afed50db8703bdd9e
+  data.tar.gz: fd301486eae7240f101a18346c3a340d259ce66f
 SHA512:
-  metadata.gz: 3f50d5f9f939f0946df260b6611a4a3f7f884fda61ecdddd0b336d393797a9eb57d9de1d53acfed46f11fd4b8a3afde3595ae2338a7beb0282b4a66d71cc1817
-  data.tar.gz: 312316480862831c155929f3272899e250818986fe2f0896eba4fa1d653123185dc56cd3d2e6e20ddc4ca7500bca01c754a73bb0fb1722870b8913f9d5bda048
+  metadata.gz: 4ebf0e8443d745ad42ead6a87aeb24cdce2a1f32e3d7830931bb68c9ceec1a5cf1adccd45e6c9ed5b60e6d8923dac42d7224a6cd9c4235833016fc66868ec1a0
+  data.tar.gz: 593ec1df8d4d5d2fbf68f713b4e514cb6e6625dab903a80b942d90b2361c8c286ba8e4bec9960cb34c83b1b6ad5a5422047071e86bf1bdc2316768dd4d8eca19

data/Gemfile

@@ -5,7 +5,10 @@ source "http://rubygems.org"
 
 gem 'paint', '~> 1.0'
 gem 'commander', '~> 4.2'
+
 gem 'octokit', '~> 4.6'
+gem 'bitbucket_rest_api', '~> 0.1.7'
+gem 'gitlab', '~> 3.7'
 
 # Add dependencies to develop your gem here.
 # Include everything needed to run rake, tests, features, etc.
@@ -14,6 +17,7 @@ group :development do
   gem 'rake', '~> 11.0'
   gem 'rspec', '~> 3.1'
   gem 'rspec-its', '~> 1.1'
+  gem 'rspec-collection_matchers', '~> 1.1'
   gem 'fakefs', '~> 0.9.0', :require => 'fakefs/safe'
   gem 'jeweler', '~> 2.0'
   gem 'rake-version', '~> 1.0'

data/VERSION

@@ -1 +1 @@
-0.1.0
+0.2.0

data/lib/gitload/command_line.rb

@@ -0,0 +1,29 @@
+require 'shellwords'
+
+module Gitload
+  module CommandLine
+    class << self
+      def execute *args
+        system escape_args(args.flatten)
+      end
+
+      def escape args
+        args.collect{ |arg| Shellwords.shellescape arg.to_s }.join(' ')
+      end
+
+      def print message, options = {}
+        if options[:color]
+          paint_args = options[:color].kind_of?(Array) ? options[:color] : [ options[:color] ]
+          paint_args.unshift message
+          message = Paint[*paint_args]
+        end
+
+        if options.fetch :new_line, true
+          puts message
+        else
+          print message
+        end
+      end
+    end
+  end
+end

data/lib/gitload/commands/load.rb

@@ -1,21 +1,14 @@
-require 'octokit'
+require 'forwardable'
 
 module Gitload
   module Commands
     class Load
       def initialize options = {}
+        @config = Config.new options
       end
 
       def run
-
-        Octokit.configure do |c|
-          c.access_token = ENV['GITLOAD_TOKEN']
-          c.auto_paginate = true
-        end
-
-        repos = Octokit.repositories ENV['GITLOAD_GITHUB_USER']
-
-        puts repos.collect{ |r| r.to_attrs[:name] }
+        @config.apply
       end
     end
   end

data/lib/gitload/config.rb

@@ -0,0 +1,127 @@
+require 'fileutils'
+
+module Gitload
+  class Config
+    attr_accessor :root
+    attr_accessor :cache
+    attr_accessor :clear_cache
+    attr_accessor :dry_run
+    attr_accessor :clone_url_type
+
+    def initialize options = {}
+      @root = options[:root]
+      @cache = options.cache
+      @clear_cache = options.clear_cache
+      @dry_run = options.dry_run
+      @clone_url_type = options[:clone_url_type]
+    end
+
+    def apply
+      filename = File.expand_path '~/.gitload.rb'
+      DSL.new(self).instance_eval File.read(filename), filename
+    end
+
+    def clone_options
+      {
+        dry_run: @dry_run,
+        clone_url_type: @clone_url_type
+      }
+    end
+
+    def load_or_cache_data relative_path
+      delete relative_path if @clear_cache
+
+      if @cache
+        data = self.load relative_path
+      end
+
+      unless data
+        data = yield
+      end
+
+      if @cache
+        save relative_path, data
+      end
+
+      data
+    end
+
+    def load relative_path
+      file = data_file relative_path
+      File.exist?(file) ? JSON.parse(File.read(file)) : nil
+    end
+
+    def save relative_path, contents
+      file = data_file relative_path
+      FileUtils.mkdir_p File.dirname(file)
+      File.open(file, 'w'){ |f| f.write JSON.dump(contents) }
+    end
+
+    def delete relative_path
+      FileUtils.rm_f data_file(relative_path)
+    end
+
+    class DSL
+      extend Forwardable
+
+      attr_reader :repos
+
+      def initialize config
+        @config = config
+        @repos = RepoChain.new @config
+      end
+
+      def chain repos = [], options = {}
+        RepoChain.new @config, repos, options
+      end
+
+      def cache
+        @config.cache = true
+      end
+
+      def root dest, &block
+
+        previous_root = @config.root
+        @config.root = dest
+
+        if block
+          instance_eval &block
+          @config.root = previous_root
+        end
+      end
+
+      def clone_url_type type, &block
+
+        previous_type = @config.clone_url_type
+        @config.clone_url_type = type
+
+        if block
+          instance_eval &block
+          @config.clone_url_type = previous_type
+        end
+      end
+
+      alias_method :into, :root
+
+      def_delegator :@repos, :add
+
+      def method_missing symbol, *args, &block
+        if Sources.sources.key? symbol
+          Sources.sources[symbol].source *args.unshift(@config), &block
+        else
+          super symbol, *args, &block
+        end
+      end
+    end
+
+    private
+
+    def data_file relative_path
+      File.join data_dir, "#{relative_path}.json"
+    end
+
+    def data_dir
+      File.join File.expand_path('~'), '.gitload'
+    end
+  end
+end

data/lib/gitload/repo.rb

@@ -0,0 +1,46 @@
+require 'shellwords'
+
+module Gitload
+  class Repo
+    attr_reader :clone_urls
+    attr_accessor :source, :api_data, :name, :fork, :owner, :owner_type, :cloned
+
+    def initialize source, api_data
+      @source = source
+      @api_data = api_data
+      @cloned = false
+      @clone_urls = {}
+    end
+
+    def cloned?
+      !!@cloned
+    end
+
+    def fork?
+      !!@fork
+    end
+
+    def clone_url type = nil
+      @clone_urls[type || :http]
+    end
+
+    def clone_to dest, options = {}
+      if @cloned && !options[:force]
+        return
+      elsif File.exists? dest
+        CommandLine.print "#{dest} already exists", color: :green
+        return
+      end
+
+      command = [ :git, :clone, clone_url(options[:clone_url_type]), dest ]
+
+      if options[:dry_run]
+        CommandLine.print CommandLine.escape(command), color: :yellow
+      else
+        CommandLine.execute command
+      end
+
+      @cloned = true
+    end
+  end
+end

data/lib/gitload/repo_chain.rb

@@ -0,0 +1,121 @@
+module Gitload
+  class RepoChain
+    extend Forwardable
+
+    attr_reader :config
+    attr_reader :repos
+
+    def initialize config, repos = [], options = {}
+      @config = config
+      @repos = repos
+      @rename = options[:rename]
+    end
+
+    def << repo
+      @repos << repo
+      self
+    end
+
+    def add repos
+      # TODO: remove
+      repos = repos.repos while repos.respond_to? :repos
+      @repos += repos
+      self
+    end
+
+    def by *owners
+      normalized_owners = owners.collect{ |owner| owner.to_s.downcase }
+      select{ |repo| normalized_owners.include? repo.owner.to_s.downcase }
+    end
+
+    def from *sources
+      select{ |repo| sources.include? repo.source }
+    end
+
+    def named *criteria
+      compiled_criteria = criteria.collect do |criterion|
+        if criterion.kind_of? Regexp
+          Proc.new{ |repo| repo.name.match criterion }
+        else
+          Proc.new{ |repo| repo.name.downcase == criterion.to_s.downcase }
+        end
+      end
+
+      select do |repo|
+        compiled_criteria.any? do |criterion|
+          criterion.call repo
+        end
+      end
+    end
+
+    def forks
+      select &:fork?
+    end
+
+    def select &block
+      dup @repos.select(&block)
+    end
+
+    def reject &block
+      dup @repos.reject(&block)
+    end
+
+    def rename pattern = nil, replacement = nil, &block
+      @rename_block = block || Proc.new{ |name| name.sub pattern, replacement }
+      self
+    end
+
+    def prefix prefix
+      @prefix = prefix
+      self
+    end
+
+    def clone
+      clone_into
+    end
+
+    def clone_to dest = nil
+      clone_repos :to, dest
+    end
+
+    def clone_into dest = nil
+      clone_repos :into, dest
+    end
+
+    def_delegator :@config, :root
+
+    private
+
+    def dup repos = nil
+      RepoChain.new @config, repos || @repos, rename: @rename
+    end
+
+    def clone_repos method, dest
+      dest = if !dest
+        File.expand_path(root.to_s)
+      elsif root
+        File.expand_path(dest.to_s, File.expand_path(root.to_s))
+      else
+        File.expand_path(dest.to_s)
+      end
+
+      puts dest
+      dir = File.dirname dest
+      #FileUtils.mkdir_p unless File.exists? dir
+
+      @repos.each do |repo|
+
+        repo_dest = if method == :into
+          name = repo.name
+          name = name.index(@prefix) == 0 ? name : "#{@prefix}#{name}" if @prefix
+          name = @rename_block.call name if @rename_block
+          File.join dest, name
+        else
+          dest
+        end
+
+        repo.clone_to repo_dest, @config.clone_options
+      end
+    end
+  end
+end

data/lib/gitload/sources/bitbucket.rb

@@ -0,0 +1,44 @@
+require 'bitbucket_rest_api'
+require 'json'
+
+module Gitload
+  module Sources
+    class Bitbucket
+      include Source
+
+      def initialize config, options = {}
+
+        @config = config
+
+        user = options.fetch :user, ENV['GITLOAD_BITBUCKET_USER']
+        password = options.fetch :password, ENV['GITLOAD_BITBUCKET_TOKEN']
+        @bitbucket_api = ::BitBucket.new basic_auth: "#{user}:#{password}"
+      end
+
+      def repos
+
+        data = @config.load_or_cache_data 'bitbucket' do
+          Utils.stringify_keys(@bitbucket_api.repos.list)
+        end
+
+        data = data.select{ |repo| repo['scm'] == 'git' }
+
+        data.collect{ |d| Repo.new d }
+      end
+
+      class Repo < Gitload::Repo
+        def initialize api_data
+          super :bitbucket, api_data
+
+          @name = api_data['slug']
+          @owner = api_data['owner']
+
+          @fork = api_data['is_fork']
+
+          @clone_urls[:ssh] = "git@bitbucket.org:#{@owner}/#{@name}.git"
+          @clone_urls[:http] = "https://AlphaHydrae@bitbucket.org/#{@owner}/#{@name}.git"
+        end
+      end
+    end
+  end
+end

data/lib/gitload/sources/github.rb

@@ -0,0 +1,44 @@
+require 'json'
+require 'octokit'
+
+module Gitload
+  module Sources
+    class GitHub
+      include Source
+
+      def initialize config, options = {}
+        @config = config
+
+        ::Octokit.configure do |c|
+          c.auto_paginate = true
+          c.access_token = options.fetch :access_token, ENV['GITLOAD_GITHUB_TOKEN']
+        end
+      end
+
+      def repos
+
+        data = @config.load_or_cache_data 'github' do
+          Utils.stringify_keys ::Octokit.repositories.collect(&:to_attrs)
+        end
+
+        data.collect{ |d| Repo.new d }
+      end
+
+      class Repo < Gitload::Repo
+        def initialize api_data
+          super :github, api_data
+
+          @name = api_data['name']
+          @owner = api_data['owner']['login']
+          @owner_type = api_data['owner']['type']
+
+          @fork = api_data['fork']
+
+          @clone_urls[:git] = api_data['git_url']
+          @clone_urls[:ssh] = api_data['ssh_url']
+          @clone_urls[:http] = api_data['clone_url']
+        end
+      end
+    end
+  end
+end

data/lib/gitload/sources/gitlab.rb

@@ -0,0 +1,40 @@
+require 'gitlab'
+require 'json'
+
+module Gitload
+  module Sources
+    class GitLab
+      include Source
+
+      def initialize config, options = {}
+        @config = config
+
+        ::Gitlab.configure do |c|
+          c.endpoint = 'https://gitlab.com/api/v3'
+          c.private_token = options.fetch :private_token, ENV['GITLOAD_GITLAB_TOKEN']
+        end
+      end
+
+      def repos
+
+        data = @config.load_or_cache_data 'gitlab' do
+          Utils.stringify_keys(::Gitlab.projects.auto_paginate.collect(&:to_h))
+        end
+
+        data.collect{ |d| Repo.new d }
+      end
+
+      class Repo < Gitload::Repo
+        def initialize api_data
+          super :gitlab, api_data
+
+          @name = api_data['path']
+          @owner = api_data['namespace']['path']
+
+          @clone_urls[:ssh] = api_data['ssh_url_to_repo']
+          @clone_urls[:http] = api_data['http_url_to_repo']
+        end
+      end
+    end
+  end
+end

data/lib/gitload/sources.rb

@@ -0,0 +1,28 @@
+module Gitload
+  module Sources
+    def self.sources
+      @sources ||= {}
+    end
+
+    module Source
+      def self.included base
+        base.extend ClassMethods
+      end
+
+      module ClassMethods
+        def source options = {}
+          new options
+        end
+      end
+    end
+  end
+end
+
+Dir[File.join File.dirname(__FILE__), File.basename(__FILE__, '.*'), '*.rb'].each{ |lib| require lib }
+
+module Gitload::Sources
+  constants.collect{ |c| const_get(c) }.select{ |c| Class === c && c.respond_to?(:source) }.each do |factory|
+    factory_name = factory.respond_to?(:source_name) ? factory.source_name : factory.name.to_s.downcase.sub(/.*::/, '').to_sym
+    sources[factory_name] = factory
+  end
+end

data/lib/gitload/utils.rb

@@ -0,0 +1,18 @@
+module Gitload
+  module Utils
+    def self.stringify_keys object
+      if object.kind_of? Hash
+        object.inject({}) do |memo,(key,value)|
+          memo[key.to_s] = stringify_keys value
+          memo
+        end
+      elsif object.kind_of? Array
+        object.collect do |value|
+          stringify_keys value
+        end
+      else
+        object
+      end
+    end
+  end
+end

data/lib/gitload.rb

@@ -2,7 +2,7 @@
 require 'paint'
 
 module Gitload
-  VERSION = '0.1.0'
+  VERSION = '0.2.0'
 end
 
 Dir[File.join File.dirname(__FILE__), File.basename(__FILE__, '.*'), '*.rb'].each{ |lib| require lib }

data/lib/program.rb

@@ -9,7 +9,10 @@ global_option '-c', '--config PATH', 'Use a custom configuration file (defaults
 
 command :load do |c|
   c.syntax = 'gitload load'
-  c.description = 'Downloads all configured repositories (default action)'
+  c.description = 'Clones all configured repositories (default action)'
+  c.option '--cache', 'Cache the downloaded repository data'
+  c.option '--clear-cache', 'Clears the latest cache data'
+  c.option '--dry-run', 'Show all repositories that will be cloned but without cloning them'
   c.action do |args,options|
     Gitload::Commands::Load.new(options).run
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitload
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Simon Oulevay (Alpha Hydrae)
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-28 00:00:00.000000000 Z
+date: 2017-02-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: paint
@@ -52,6 +52,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '4.6'
+- !ruby/object:Gem::Dependency
+  name: bitbucket_rest_api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.7
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.7
+- !ruby/object:Gem::Dependency
+  name: gitlab
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.7'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -94,6 +122,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: rspec-collection_matchers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: fakefs
   requirement: !ruby/object:Gem::Requirement
@@ -166,8 +208,17 @@ files:
 - VERSION
 - bin/gitload
 - lib/gitload.rb
+- lib/gitload/command_line.rb
 - lib/gitload/commands.rb
 - lib/gitload/commands/load.rb
+- lib/gitload/config.rb
+- lib/gitload/repo.rb
+- lib/gitload/repo_chain.rb
+- lib/gitload/sources.rb
+- lib/gitload/sources/bitbucket.rb
+- lib/gitload/sources/github.rb
+- lib/gitload/sources/gitlab.rb
+- lib/gitload/utils.rb
 - lib/program.rb
 homepage: https://github.com/AlphaHydrae/gitload
 licenses: