gem-src-srv 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: cac6ea3bf7198076c0055fa6f6d81da31836d94e8f81ee55b61f1545d691278b
+  data.tar.gz: dd0573345b33a24d185f9c3d63e9372eb119d4c0a184237ec83e757f22844b8c
+SHA512:
+  metadata.gz: efcecead3447218afdf071cfe9ab95c996ef42cd1f7111992f8b2949c973267aed547d342c0cad4ea2b482e3c73d7555fe24a04fe75a285099c16f7faffb8884
+  data.tar.gz: 4d3202c7c22157ae50b6491d9bce97bcc5e811a657c9130ad76886d6d09e5a6af64f5c598c5f3cc70a56bde99256c1d73306d86f882808fb321cd94848fdc8d2

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/.ruby-version

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in gem-src-srv.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,22 @@
+PATH
+  remote: .
+  specs:
+    gem-src-srv (0.1.1)
+      retryable (~> 3.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    rake (10.5.0)
+    retryable (3.0.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.16)
+  gem-src-srv!
+  rake (~> 10.0)
+
+BUNDLED WITH
+   1.16.1

data/LICENSE

@@ -0,0 +1,51 @@
+MIT License
+
+Copyright (c) 2018 Masataka Pocke Kuwabara
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+
+----
+
+
+This project copies some code from https://github.com/amatsuda/gem-src
+https://github.com/amatsuda/gem-src/blob/94565781c9c0941a02e1230d05f4e03b7587d187/LICENSE.txt
+
+Copyright (c) 2012 Akira Matsuda
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,49 @@
+# Gem::Src::Srv
+
+Run `git clone` after `gem install` concurrently. Inspired by [gem-src](https://github.com/amatsuda/gem-src)
+
+## Installation
+
+```bash
+$ gem install gem-src-srv
+```
+
+## Usage
+
+It clones a git repository after you install a gem with [ghq](https://github.com/motemen/ghq).
+
+```bash
+$ gem install rails
+$ ghq list | grep rails/rails
+github.com/rails/rails
+```
+
+Note: If you want to clone repositories with plain `git clone` command, you can send a pull-request!
+
+### Configuration
+
+You can set `GEM_SRC_SRV_PORT` and `GEM_SRC_SRV_CONCURRENCY` environment variables.
+
+See [lib/gem/src/srv/configuration.rb](https://github.com/pocke/gem-src-srv/blob/master/lib/gem/src/srv/configuration.rb).
+
+## Design
+
+gem-src is good, but it clones a repository synchronously. So it makes `gem install` slow.
+
+gem-src-srv clones a repository concurrently. So it does not block `gem install`, it's faster!
+
+### How concurrently?
+
+gem-src-srv works on Client-Server model.
+
+![img_20180225_124045652](https://user-images.githubusercontent.com/4361134/36637896-47fa1daa-1a29-11e8-828a-c37aaa36f6ac.jpg)
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/pocke/gem-src-srv.

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "gem/src/srv"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/gem-src-srv

@@ -0,0 +1,5 @@
+#!ruby
+
+require 'gem/src/srv'
+
+Gem::Src::Srv.start

data/gem-src-srv.gemspec

@@ -0,0 +1,26 @@
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "gem/src/srv/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "gem-src-srv"
+  spec.version       = Gem::Src::Srv::VERSION
+  spec.authors       = ["Masataka Pocke Kuwabara"]
+  spec.email         = ["kuwabara@pocke.me"]
+
+  spec.summary       = %q{Run `git clone` after `gem install` concurrently.}
+  spec.description   = %q{Run `git clone` after `gem install` concurrently.}
+  spec.homepage      = "https://github.com/pocke/gem-src-srv"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.16"
+  spec.add_development_dependency "rake", "~> 10.0"
+
+  spec.add_runtime_dependency "retryable", "~> 3.0"
+end

data/lib/gem/src/srv.rb

@@ -0,0 +1,24 @@
+require 'webrick'
+require 'net/http'
+require 'open-uri'
+require 'json'
+require 'etc'
+
+module Gem
+  module Src
+    module Srv
+      def self.start
+        queue = Queue.new
+        Worker.start(queue)
+        Server.start(queue)
+      end
+    end
+  end
+end
+
+require_relative "srv/version"
+require_relative "srv/configuration"
+require_relative "srv/irregular_repositories"
+require_relative "srv/fetcher"
+require_relative "srv/server"
+require_relative "srv/worker"

data/lib/gem/src/srv/configuration.rb

@@ -0,0 +1,14 @@
+module Gem
+  module Src
+    module Srv
+      module Configuration
+        def self.env(key)
+          ENV["GEM_SRC_SRV_#{key}"]
+        end
+
+        PORT = env('PORT')&.to_i || 64322
+        CONCURRENCY = env('CONCURRENCY')&.to_i || Etc.nprocessors
+      end
+    end
+  end
+end

data/lib/gem/src/srv/fetcher.rb

@@ -0,0 +1,86 @@
+module Gem::Src::Srv
+  class Fetcher
+    def self.tested_repositories
+      @tested_repositories ||= []
+    end
+
+    def tested_repositories
+      self.class.tested_repositories
+    end
+
+    def initialize(spec)
+      @spec = spec
+    end
+
+    def fetch
+      if IRREGULAR_REPOSITORIES.key? spec.name
+        return git_clone IRREGULAR_REPOSITORIES[spec.name]
+      end
+
+      git_clone(spec.homepage) ||
+        git_clone(github_url(spec.homepage)) ||
+        git_clone(source_code_uri) ||
+        git_clone(homepage_uri) ||
+        git_clone(github_url(homepage_uri)) ||
+        git_clone(github_organization_uri(spec.name))
+    end
+
+    private
+
+    attr_reader :spec
+
+    def git_clone(repository)
+      return if repository.nil? || repository.empty?
+      return if tested_repositories.include? repository
+      tested_repositories << repository
+      return if github?(repository) && !github_page_exists?(repository)
+
+      system 'ghq', 'get', repository
+    end
+
+    def github?(url)
+      URI.parse(url).host == 'github.com'
+    end
+
+    def github_page_exists?(url)
+      Net::HTTP.new('github.com', 443).tap {|h| h.use_ssl = true }.request_head(url).code != '404'
+    end
+
+    def github_url(url)
+      if url =~ /\Ahttps?:\/\/([^.]+)\.github\.(?:com|io)\/(.+)/
+        if $1 == 'www'
+          "https://github.com/#{$2}"
+        elsif $1 == 'wiki'
+          # https://wiki.github.com/foo/bar => https://github.com/foo/bar
+          "https://github.com/#{$2}"
+        else
+          # https://foo.github.com/bar => https://github.com/foo/bar
+          "https://github.com/#{$1}/#{$2}"
+        end
+      end
+    end
+
+    def api
+      @api ||= open("https://rubygems.org/api/v1/gems/#{spec.name}.yaml", &:read)
+    rescue OpenURI::HTTPError
+      ""
+    end
+
+    def source_code_uri
+      api_uri_for('source_code')
+    end
+
+    def homepage_uri
+      api_uri_for('homepage')
+    end
+
+    def github_organization_uri(name)
+      "https://github.com/#{name}/#{name}"
+    end
+
+    def api_uri_for(key)
+      uri = api[Regexp.new("^#{key}_uri: (.*)$"), 1]
+      uri =~ /\A(?:https?|git):\/\// ? uri : nil
+    end
+  end
+end

data/lib/gem/src/srv/irregular_repositories.rb

@@ -0,0 +1,66 @@
+# frozen-string-literal: true
+
+module Gem::Src::Srv
+  IRREGULAR_REPOSITORIES = {
+    'activesupport' => nil,
+    'actionview' => nil,
+    'actionpack' => nil,
+    'activemodel' => nil,
+    'activerecord' => nil,
+    'activejob' => nil,
+    'actionmailer' => nil,
+    'actioncable' => nil,
+    'railties' => nil,
+    'activeresource' => 'https://github.com/rails/activeresource.git',
+    'autoparse' => 'https://github.com/google/autoparse.git',
+    'aws-sdk-rails' => 'https://github.com/aws/aws-sdk-rails.git',
+    'bson' => 'https://github.com/mongodb/bson-ruby.git',
+    'compass-core' => 'https://github.com/Compass/compass.git',
+    'compass-import-once' => 'https://github.com/Compass/compass.git',
+    'cool.io' => 'https://github.com/tarcieri/cool.io.git',
+    'cucumber-core' => 'https://github.com/cucumber/cucumber-ruby-core.git',
+    'cucumber-wire' => 'https://github.com/cucumber/cucumber-ruby-wire.git',
+    'diff-lcs' => 'https://github.com/halostatue/diff-lcs.git',
+    'elasticsearch-api' => 'https://github.com/elastic/elasticsearch-ruby.git',
+    'elasticsearch-extensions' => 'https://github.com/elastic/elasticsearch-ruby.git',
+    'elasticsearch-transport' => 'https://github.com/elastic/elasticsearch-ruby.git',
+    'erubis' => 'https://github.com/kwatch/erubis.git',
+    'geocoder' => 'https://github.com/alexreisner/geocoder',
+    'hirb' => 'https://github.com/cldwalker/hirb',
+    'html2haml' => 'https://github.com/haml/html2haml',
+    'io-console' => nil,
+    'kaminari-actionview' => nil,
+    'kaminari-activerecord' => nil,
+    'kaminari-core' => nil,
+    'log4r' => 'https://github.com/colbygk/log4r',
+    'meta_request' => 'https://github.com/dejan/rails_panel',
+    'method_source' => 'https://github.com/banister/method_source',
+    'origin' => 'https://github.com/mongoid/origin',
+    'padrino' => 'https://github.com/padrino/padrino-framework',
+    'padrino-admin' => nil,
+    'padrino-cache' => nil,
+    'padrino-core' => nil,
+    'padrino-gen' => nil,
+    'padrino-helpers' => nil,
+    'padrino-mailer' => nil,
+    'padrino-performance' => nil,
+    'padrino-support' => nil,
+    'paranoia' => 'https://github.com/rubysherpas/paranoia',
+    'pdf-core' => 'https://github.com/prawnpdf/pdf-core',
+    'pg' => nil,
+    'rack-mini-profiler' => 'https://github.com/MiniProfiler/rack-mini-profiler',
+    'raindrops' => 'https://github.com/tmm1/raindrops',
+    'redis-actionpack' => 'https://github.com/redis-store/redis-actionpack',
+    'redis-activesupport' => 'https://github.com/redis-store/redis-activesupport',
+    'redis-rack' => 'https://github.com/redis-store/redis-rack',
+    'redis-rails' => 'https://github.com/redis-store/redis-rails',
+    'rom-mapper' => 'https://github.com/rom-rb/rom-mapper',
+    'rom-repository' => 'https://github.com/rom-rb/rom-repository',
+    'rom-sql' => 'https://github.com/rom-rb/rom-sql',
+    'rouge' => 'https://github.com/jneen/rouge',
+    'rubygems-update' => nil,
+    'spreadsheet' => 'https://github.com/zdavatz/spreadsheet',
+    'thin' => 'https://github.com/macournoyer/thin',
+    'uniform_notifier' => 'https://github.com/flyerhzm/uniform_notifier'
+  }.freeze
+end

data/lib/gem/src/srv/server.rb

@@ -0,0 +1,31 @@
+module Gem::Src::Srv
+  Spec = Struct.new(:name, :homepage)
+
+  class Server
+    def self.start(queue)
+      self.new(queue).start
+    end
+
+    def initialize(queue)
+      @queue = queue
+    end
+
+    def start
+      srv = WEBrick::HTTPServer.new(
+        BindAddress: '127.0.0.1',
+        Port: Configuration::PORT,
+      )
+      srv.mount_proc('/gem_install') do |req, resp|
+        if req.request_method != 'POST'
+          resp.status = 404
+          next
+        end
+
+        body = JSON.parse(req.body)
+        @queue << Spec.new(body['name'], body['homepage'])
+      end
+      trap('INT') { srv.shutdown }
+      srv.start
+    end
+  end
+end

data/lib/gem/src/srv/version.rb

@@ -0,0 +1,7 @@
+module Gem
+  module Src
+    module Srv
+      VERSION = "0.1.1"
+    end
+  end
+end

data/lib/gem/src/srv/worker.rb

@@ -0,0 +1,23 @@
+module Gem::Src::Srv
+  class Worker
+    def self.start(queue)
+      Configuration::CONCURRENCY.times do
+        self.new(queue).start
+      end
+    end
+
+    def initialize(queue)
+      @queue = queue
+    end
+
+    def start
+      Thread.new do
+        loop do
+          spec = @queue.pop
+          fetcher = Fetcher.new(spec)
+          fetcher.fetch
+        end
+      end
+    end
+  end
+end

data/lib/rubygems_plugin.rb

@@ -0,0 +1,25 @@
+require 'net/http'
+require 'uri'
+require 'retryable'
+require 'json'
+
+require 'gem/src/srv/configuration'
+
+
+Gem.post_install do |installer|
+  next if installer.spec.name == 'gem-src-srv'
+
+  spawn('gem-src-srv', [:out, :err] => '/dev/null')
+
+  Retryable.retryable(tries: 3, sleep: 0.1, on: Errno::ECONNREFUSED) do
+    port = Gem::Src::Srv::Configuration::PORT
+    uri = URI.parse("http://localhost:#{port}/gem_install")
+    req = Net::HTTP::Post.new(uri)
+    req.body = JSON.generate({
+      name: installer.spec.name,
+      homepage: installer.spec.homepage,
+    })
+    req.content_type = 'Application/json'
+    Net::HTTP.new(uri.host, uri.port).start {|http| http.request(req) }
+  end
+end

metadata

@@ -0,0 +1,104 @@
+--- !ruby/object:Gem::Specification
+name: gem-src-srv
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Masataka Pocke Kuwabara
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-02-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: retryable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: Run `git clone` after `gem install` concurrently.
+email:
+- kuwabara@pocke.me
+executables:
+- gem-src-srv
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- exe/gem-src-srv
+- gem-src-srv.gemspec
+- lib/gem/src/srv.rb
+- lib/gem/src/srv/configuration.rb
+- lib/gem/src/srv/fetcher.rb
+- lib/gem/src/srv/irregular_repositories.rb
+- lib/gem/src/srv/server.rb
+- lib/gem/src/srv/version.rb
+- lib/gem/src/srv/worker.rb
+- lib/rubygems_plugin.rb
+homepage: https://github.com/pocke/gem-src-srv
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.6
+signing_key: 
+specification_version: 4
+summary: Run `git clone` after `gem install` concurrently.
+test_files: []