npm-mirror 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 0b7ed4781e984fb77c715836c810d3c3aed5247c
+  data.tar.gz: 4063e539e0a1de7e343f8f3fd3ca4609fbce6a32
+SHA512:
+  metadata.gz: b7c5c7801ac4c8b286eef13a628f79761333a490bd53befeb0ede58b3138cd3724290b0f8783f452dcfb54422380dd57e7929e71d73c7592def78b7d8c0b013e
+  data.tar.gz: 6a884e45e6107b010440f424fef1a0da68c2cb57bb9ff1213c5710c6682c3b963f6e82c38ff8480c5ebb69de51b86f44739eb837fdab4cba5a15e43d752fe703

data/.gitignore

@@ -0,0 +1,22 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in npm-mirror.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Yue Du
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,65 @@
+# NPM Mirror 
+
+A NPM Mirror that doesn't need couchdb.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'npm-mirror'
+
+And then execute:
+
+    $ bundle
+
+## Usage
+
+    $ bundle exec bin/npm-mirror [path/to/config.yml]
+
+Or:
+
+    $ npm-mirror [path/to/config.yml]
+
+Here is an example of config.yml
+
+    - from: http://registry.npmjs.org/
+      to: /data/mirrors/npm
+      server: http://mymirrors.com/npm/
+      parallelism: 10
+      recheck: false
+
+Serving via Nginx
+
+    server {
+        listen 0.0.0.0:80;
+        server_name mymirrors.com;
+        root /data/mirrors/;
+        location /npm/ {
+            index index.json;
+
+            location ~ \.etag$ {
+                return 404;
+            }
+
+            location ~ /index\.json$ {
+                default_type application/json;
+            }
+
+            # for npm search
+            location = /npm/-/all/since {
+                rewrite ^ /npm/-/all/;
+            }
+        }
+    }
+
+npm install from your mirror
+
+    npm install -r http://mymirrors.com/npm/ package
+
+## Contributing
+
+1. Fork it ( https://github.com/ifduyue/npm-mirror/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/bin/npm-mirror

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+
+require 'yaml'
+require 'npm/mirror'
+
+if ARGV.empty?
+  Npm::Mirror::Mirror.new.run
+else
+  ARGV.each do |config_file|
+    configs = YAML.load_file config_file
+    fail "Invalid config file #{config_file}" unless configs.respond_to? :each
+
+    configs.each do |config|
+      from = config['from']
+      to = File.expand_path config['to']
+      server = config['server']
+      parallelism = config['parallelism']
+      recheck = config['recheck']
+      mirror = Npm::Mirror::Mirror.new(from, to, server, parallelism, recheck)
+      mirror.run
+    end
+  end
+end

data/bin/npm-mirror-correct-tarball-url

@@ -0,0 +1,22 @@
+#!/usr/bin/env ruby
+
+require 'yaml'
+require 'npm/mirror'
+
+if ARGV.empty?
+  Npm::Mirror::Mirror.new.correct_tarball_url
+else
+  ARGV.each do |config_file|
+    configs = YAML.load_file config_file
+    fail "Invalid config file #{config_file}" unless configs.respond_to? :each
+
+    configs.each do |config|
+      from = config['from']
+      to = File.expand_path config['to']
+      server = config['server']
+      parallelism = config['parallelism']
+      mirror = Npm::Mirror::Mirror.new(from, to, server, parallelism)
+      mirror.correct_tarball_url
+    end
+  end
+end

data/lib/npm/mirror.rb

@@ -0,0 +1,224 @@
+require 'net/http/persistent'
+require 'json'
+
+require 'npm/mirror/version'
+require 'npm/mirror/config'
+
+module Npm
+  module Mirror
+    autoload :Pool, 'npm/mirror/pool'
+
+    class Error < StandardError; end
+
+    class Mirror
+      attr_reader :from, :to, :server, :parallelism
+
+      def initialize(from = DEFAULT_FROM, to = DEFAULT_TO,
+                     server = DEFAULT_SERVER, parallelism = 10,
+                     recheck = false)
+        @from, @to, @server = from, to, server
+        parallelism ||= 10
+        recheck ||= false
+        @pool = Pool.new parallelism
+        @http = Net::HTTP::Persistent.new self.class.name
+        @recheck = recheck
+
+        puts "Mirroring  : #{from}"
+        puts "Datadir    : #{to}"
+        puts "Server     : #{server}"
+        puts "Parallelism: #{parallelism}"
+        puts "Recheck    : #{recheck}"
+      end
+
+      def from(*args)
+        File.join @from, *args
+      end
+
+      def to(*args)
+        File.join @to, *args
+      end
+
+      def link(*args)
+        File.join @server, *args
+      end
+
+      def id
+        l = @pool.size.to_s.size
+        if Thread.current[:id]
+          "Thread-#{Thread.current[:id].to_s.rjust(l, '0')}"
+        else
+          'Master'.ljust(%w(Thread- Master).map(&:size).max + l)
+        end
+      end
+
+      def fetch(url, path = nil)
+        uri = URI url
+        etag = etag_for path
+
+        req = Net::HTTP::Get.new uri.path
+        req.add_field 'If-None-Match', etag if etag
+
+        begin
+          resp = @http.request uri, req
+          fail Error, '503' if resp.code == '503'  # 503 backend read error
+        rescue => e
+          puts "[#{id}] Error fetching #{uri.path}: #{e.inspect}"
+          sleep 10
+          retry
+        end
+
+        puts "[#{id}] Fetched(#{resp.code}): #{uri}"
+        case resp.code.to_i
+        when 301  # Moved
+          return fetch resp['location'], path
+        when 302  # Found
+          return fetch resp['location'], path
+        when 200
+          return resp
+        when 304  # Not modified
+          return resp if @recheck
+          return nil
+        when 403
+          return nil
+        when 404  # couchdb returns json even it's 404
+          return resp
+        else
+          fail Error, "unexpected response #{resp.inspect}"
+        end
+      end
+
+      def fetch_index
+        url = from '-/all'
+        path = to '-/all/index.json'
+        resp = fetch url, path
+        fail Error, "Failed to fetch #{url}" if resp.nil?
+
+        if resp.code == '304'
+          if File.exist? path
+            json = JSON.load(File.open(path, 'rb').read)
+          else
+            json = {}
+          end
+        else
+          json = JSON.load resp.body
+        end
+
+        @pool.run
+
+        packages = json.keys
+        packages.each_slice(@pool.size * 10) do |slice|
+          slice.each do |package|
+            next if package.start_with? '_'
+            @pool.enqueue_job(package, &method(:fetch_package))
+          end
+          sleep 0.5
+        end
+
+        @pool.enqueue_job do
+          write_file path, resp.body, resp['etag']
+        end unless resp.code == '304'
+      end
+
+      def fetch_package(package)
+        url = from package
+        path = to package, 'index.json'
+        resp = fetch url, path
+        return if resp.nil?
+
+        if resp.code == '304'
+          json = JSON.load(File.open(path, 'rb').read)
+          tarball_links json
+        else
+          json = JSON.load resp.body
+          tarball_links json
+          write_file path, json.to_json, resp['etag']
+          write_package_versions(package, json)
+        end
+      end
+
+      def write_package_versions(package, json)
+        return unless json['versions']
+        json['versions'].each do |k, v|
+          path = to package, k, 'index.json'
+          write_file path, v.to_json, nil
+        end
+      end
+
+      def fetch_tarball(tarball_uri)
+        url = from tarball_uri
+        path = to tarball_uri
+        resp = fetch url, path
+        return if resp.nil? || resp.code == '304' || resp.body.size.zero?
+        write_file path, resp.body, resp['etag']
+      end
+
+      def write_file(path, bytes, etag = nil)
+        FileUtils.mkdir_p File.dirname(path)
+        File.open(path, 'wb') { |f| f << bytes }
+        File.open(path_for_etag(path), 'wb') { |f| f << etag } if etag
+      end
+
+      def tarball_links(json)
+        json.each do |_, v|
+          if v.is_a? Hash
+            if v['shasum'] && v['tarball']
+              if v['tarball'].start_with?(@from)
+                tarball = v['tarball'].split(/^#{@from}/, 2).last
+                v['tarball'] = link tarball
+              elsif v['tarball'].start_with?(@server)
+                tarball = v['tarball'].split(/^#{@server}/, 2).last
+              end
+              @pool.enqueue_job(tarball, &method(:fetch_tarball))
+            else
+              tarball_links v
+            end
+          end
+        end
+      end
+
+      def run
+        fetch_index
+        @pool.run_til_done
+      end
+
+      def path_for_etag(path)
+        if path.nil?
+          nil
+        elsif File.directory? path
+          File.join path, '.etag'
+        else
+          dirname, basename = File.split path
+          File.join dirname, ".#{basename}.etag"
+        end
+      end
+
+      def etag_for(path)
+        etag_path = path_for_etag path
+        if !etag_path
+          nil
+        elsif File.file?(etag_path) && File.readable?(etag_path)
+          File.open(etag_path, 'rb') { |f| f.readline.strip }
+        else
+          nil
+        end
+      end
+
+      def correct_tarball_url
+        Dir.glob("#{to}/*/index.json").each do |filename|
+          json = JSON.load(File.open(filename, 'rb').read)
+          next unless json['versions']
+          versions = json['versions'].keys
+          versions.each do |version|
+            next unless json['versions'][version]['dist']
+            tarball = URI json['versions'][version]['dist']['tarball']
+            tarball = link tarball.path
+            json['versions'][version]['dist']['tarball'] = tarball
+          end
+          File.open(filename, 'wb') { |f| f << json.to_json }
+          write_package_versions json['name'], json
+          puts filename
+        end
+      end
+    end
+  end
+end

data/lib/npm/mirror/config.rb

@@ -0,0 +1,9 @@
+require 'tmpdir'
+
+module Npm
+  module Mirror
+    DEFAULT_FROM = 'http://registry.npmjs.org/'
+    DEFAULT_TO = File.join(Dir.tmpdir, 'npm-mirror')
+    DEFAULT_SERVER = 'http://localhost/'
+  end
+end

data/lib/npm/mirror/pool.rb

@@ -0,0 +1,65 @@
+require 'thread'
+
+module Npm
+  module Mirror
+    class Pool
+      attr_accessor :size
+
+      def initialize(size)
+        @size = size
+        @queue = Queue.new
+        @running = false
+      end
+
+      def running?
+        @running
+      end
+
+      def run
+        return if running?
+        @running = true
+        @threads = Array.new(@size) do |i|
+          Thread.new do
+            Thread.current[:id] = i
+            catch(:exit) do
+              loop do
+                job, args = @queue.pop
+                job.call(*args)
+              end
+            end
+          end
+        end
+      end
+
+      def run_til_done
+        run
+
+        until @queue.empty? && @queue.num_waiting == @size
+          @threads.each { |t| t.join 0.2 }
+          l = @size.to_s.size
+          master = 'Master'.ljust(%w(Thread- Master).map(&:size).max + l)
+          puts "[#{master}] Queue size: #{@queue.size}, Waiting thread: " \
+               "#{@queue.num_waiting}"
+        end
+
+        terminate
+      end
+
+      def terminate
+        return unless running?
+
+        @running = false
+        @size.times do
+          enqueue_job { throw :exit }
+        end
+
+        @threads.each { |t| t.join 0.2 }
+        @threads.each { |t| t.kill }
+      end
+
+      def enqueue_job(*args, &block)
+        @queue << [block, args]
+      end
+    end  # class Pool
+  end  # Mirror
+end  # module Npm

data/lib/npm/mirror/version.rb

@@ -0,0 +1,5 @@
+module Npm
+  module Mirror
+    VERSION = '0.0.1'
+  end
+end

data/npm-mirror.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'npm/mirror/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "npm-mirror"
+  spec.version       = Npm::Mirror::VERSION
+  spec.authors       = ["Yue Du"]
+  spec.email         = ["ifduyue@gmail.com"]
+  spec.summary       = %q{NPM Mirror}
+  spec.homepage      = "https://github.com/ifduyue/npm-mirror"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "net-http-persistent", "~> 2.9.4"
+  spec.add_dependency "json"
+
+  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "pry"
+end

metadata

@@ -0,0 +1,128 @@
+--- !ruby/object:Gem::Specification
+name: npm-mirror
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Yue Du
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-05-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: net-http-persistent
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.9.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.9.4
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- ifduyue@gmail.com
+executables:
+- npm-mirror
+- npm-mirror-correct-tarball-url
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/npm-mirror
+- bin/npm-mirror-correct-tarball-url
+- lib/npm/mirror.rb
+- lib/npm/mirror/config.rb
+- lib/npm/mirror/pool.rb
+- lib/npm/mirror/version.rb
+- npm-mirror.gemspec
+homepage: https://github.com/ifduyue/npm-mirror
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: NPM Mirror
+test_files: []