gem-codesearch 0.1 → 0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3ba88fec2f6fc7cb946e0a1ba82b5296f41b8a14
-  data.tar.gz: 4ca8bd6a6b62f3da81b3f2ace20f5a13fe0d4c45
+  metadata.gz: 6e8c7bb53fd63dd6e17be73b4539e85b56b375cb
+  data.tar.gz: f13479a2283c8f7d9644cc32da9d94581ba36bd3
 SHA512:
-  metadata.gz: fd31437425bf9d8d8693af6784a2cbafe95f0caedc0a56f8ea095151144174f11e1ecf6ac2f35ba12aba3b45cc86e3a8c0d52eed760c35b8b6be2787d4ad081c
-  data.tar.gz: 304fb9e178cd06cc2b8279bf1872b20fa30789e53ab5dca747f1442ebf131610782dc758fecdda6a13a4df64c94ce14ea87031e8160019ba2e9a7de80bcec076
+  metadata.gz: a85f76d92147d3071d17e9a48b0f32f54fb7966bbc62d3d9c270367ce573fc821dc0904bbefbb08c1453a134103a60dcfc68614f454d1a975da2a720ef272e22
+  data.tar.gz: e8289838b82d9a148154f77e5ebf74a160f0d0942bdc8f1ee39b745bb7b8cfad02584a72223fe1f52fe0ec4e081eb1230a0d4b82f84fecc2ba6b0c349701a218

data/README.md

@@ -3,6 +3,10 @@
 gem-codesearch sets up a full text code search engine on mirror of rubygems.
 It use rubygems-mirror to mirror rubygems and milkode for search engine.
 
+300GB of free disk space is required to store the rubygems mirror,
+unpacked gems and milkode index at 2015-01.
+It will be larger in future.
+
 ## Usage
 
 This creates "latest-gem" package in milkode index.
@@ -14,7 +18,7 @@ This creates "latest-gem" package in milkode index.
     % gem-codesearch-setup all >& setup.log # It may take several days or more
 
 If "gem-codesearch-setup all" fails due to network or server errors,
-try again.
+try again to continue.
 
 After the index is created, enjoy code search.
 
@@ -29,6 +33,11 @@ After the index is created, enjoy code search.
     % cd gem-codesearch
     % rake all >& setup.log             # It may take several days or more
 
+## Links
+
+- https://github.com/akr/gem-codesearch
+- https://rubygems.org/gems/gem-codesearch
+
 ## Author
 
 Tanaka Akira

data/Rakefile

@@ -18,26 +18,29 @@ end
 
 task :all => [:mirror, :unpack, :index]
 
-MIRROR_DIR = "#{Dir.pwd}/mirror"
-LATEST_DIR = "#{Dir.pwd}/latest-gem"
-LOG_DIR = "#{Dir.pwd}/log"
+BASE_DIR = ENV['GEM_CODESEARCH_DIR'] || Dir.pwd
+MIRROR_URL = ENV['GEM_CODESEARCH_URL'] || 'http://rubygems.org/'
+
+MIRROR_DIR = "#{BASE_DIR}/mirror"
+LATEST_DIR = "#{BASE_DIR}/latest-gem"
+LOG_DIR = "#{BASE_DIR}/log"
 
 GEM_COMMAND = "#{RbConfig::CONFIG["bindir"]}/gem"
 MILK_COMMAND = "#{RbConfig::CONFIG["bindir"]}/milk"
 
-file ".gem/.mirrorrc" do |t|
+file "#{BASE_DIR}/.gem/.mirrorrc" do |t|
   FileUtils.mkpath File.dirname(t.name)
   File.write(t.name, <<"End")
 ---
-- from: http://production.s3.rubygems.org
+- from: #{MIRROR_URL}
   to: #{MIRROR_DIR}
 End
 end
 
-task :mirror => ".gem/.mirrorrc" do
+task :mirror => "#{BASE_DIR}/.gem/.mirrorrc" do
   FileUtils.mkpath MIRROR_DIR
   # HOME is set because gem mirror reads $HOME/.gem/.mirrorrc.
-  sh "HOME=#{Dir.pwd} #{GEM_COMMAND} mirror --verbose"
+  sh "HOME=#{BASE_DIR} #{GEM_COMMAND} mirror --verbose"
 end
 
 task :unpack do
@@ -61,18 +64,17 @@ task :unpack do
   #all_specs = all_specs.reject {|name,version,platform| /\Afoo-/ !~ name }
 
   latest_vnames = []
-  nonlatest_vnames = []
   h = all_specs.group_by {|name,version| name }
   h.each {|name, list|
     list = list.sort_by {|name,version| version }
     vnames = list.map {|name,version| "#{name}-#{version}" }
     latest_vnames << vnames.pop
-    nonlatest_vnames.concat list
   }
 
   already_unpacked = Dir.entries(LATEST_DIR)
-  (already_unpacked & nonlatest_vnames).each {|vname|
-    puts "remove: #{vname}
+  already_unpacked = already_unpacked - %w[. ..]
+  (already_unpacked - latest_vnames).each {|vname|
+    puts "remove: #{vname}"
     FileUtils.rmtree("#{LATEST_DIR}/#{vname}")
   }
 

data/gem-codesearch.gemspec

@@ -1,13 +1,15 @@
 Gem::Specification.new do |s|
   s.name = 'gem-codesearch'
-  s.version = '0.1'
-  s.date = '2015-01-09'
+  s.version = '0.2'
+  s.date = '2015-01-13'
   s.author = 'Tanaka Akira'
   s.email = 'akr@fsij.org'
   s.required_ruby_version = '>= 2.2.0'
   s.add_runtime_dependency 'rubygems-mirror', '~> 1.0', '>= 1.0.1'
   s.add_runtime_dependency 'milkode', '~> 1.8', '>= 1.8.6'
   s.add_runtime_dependency 'rake', '~> 10.4', '>= 10.4.2'
+  s.add_development_dependency 'builder', '~> 3.2', '>= 3.2.2'
+  s.add_development_dependency 'test-unit', '~> 3.0', '>= 3.0.8'
   s.executables << 'gem-codesearch-setup'
   s.files = %w[
     .gitignore
@@ -17,6 +19,10 @@ Gem::Specification.new do |s|
     bin/gem-codesearch-setup
     gem-codesearch.gemspec
   ]
+  s.test_files = %w[
+    test/test_all.rb
+    test/test_bin.rb
+  ]
   s.homepage = 'https://github.com/akr/gem-codesearch'
   s.license = 'MIT'
   s.summary = 'Set up a full text code search engine for rubygems mirror'

data/test/test_all.rb

@@ -0,0 +1,70 @@
+require 'test/unit'
+require 'tmpdir'
+require 'fileutils'
+require 'rbconfig'
+require 'socket'
+require 'pty'
+require 'io/console'
+
+GEM_COMMAND = "#{RbConfig::CONFIG["bindir"]}/gem"
+MILK_COMMAND = "#{RbConfig::CONFIG["bindir"]}/milk"
+RAKE_COMMAND = "#{RbConfig::CONFIG["bindir"]}/rake"
+
+topdir = File.dirname(File.dirname(File.realpath(__FILE__)))
+RAKEFILE = topdir + "/Rakefile"
+
+class TestGemCodesearch < Test::Unit::TestCase
+  def setup
+    @workdir = Dir.mktmpdir('gem-codesearch')
+    ENV['MILKODE_DEFAULT_DIR'] = "#{@workdir}/milkode"
+  end
+
+  def test_milk_init
+    assert(!File.exist?("#{@workdir}/milkode"))
+    system(MILK_COMMAND, 'init', '--default', :out => IO::NULL)
+    assert($?.success?)
+    assert(File.exist?("#{@workdir}/milkode"))
+  end
+
+  def start_gem_server
+    # "gem server" serves gem files already installed.
+    gem_server_command = [GEM_COMMAND, 'server', '-b', '127.0.0.1', '-p', '0']
+    pid = nil
+    PTY.open {|m, s|
+      s.raw!
+      pid = spawn(*gem_server_command, :out => s, :err => IO::NULL)
+      line = m.gets
+      unless %r{Server started at (http:\S+)} =~ line
+        flunk "unexpected 'gem server' message: #{line.inspect}"
+      end
+      url = $1
+      yield [pid, url]
+    }
+  ensure
+    if pid
+      Process.kill :TERM, pid
+      Process.wait pid
+    end
+  end
+
+  def test_gem_server
+    start_gem_server {|pid, url| }
+  end
+
+  def test_all
+    start_gem_server {|pid, url|
+      ENV['GEM_CODESEARCH_URL'] = url
+      ENV['GEM_CODESEARCH_DIR'] = @workdir
+      system(MILK_COMMAND, 'init', '--default', :out => IO::NULL)
+      system(RAKE_COMMAND, '-f', RAKEFILE, 'all', [:out, :err] => IO::NULL)
+      system(RAKE_COMMAND, '-f', RAKEFILE, 'all', [:out, :err] => IO::NULL) # update exisiting index.
+    }
+  end
+
+  def teardown
+    FileUtils.rmtree @workdir
+    ENV['MILKODE_DEFAULT_DIR'] = nil
+    ENV['GEM_CODESEARCH_URL'] = nil
+    ENV['GEM_CODESEARCH_DIR'] = nil
+  end
+end

data/test/test_bin.rb

@@ -0,0 +1,18 @@
+require 'test/unit'
+require 'rbconfig'
+
+RUBY_COMMAND = RbConfig.ruby
+
+topdir = File.dirname(File.dirname(File.realpath(__FILE__)))
+GEM_CODESEARCH_SETUP_COMMAND = topdir + "/bin/gem-codesearch-setup"
+
+class TestGemCodesearch < Test::Unit::TestCase
+  def test_setup_help
+    command = [RUBY_COMMAND, GEM_CODESEARCH_SETUP_COMMAND, 'help']
+    message = IO.popen(command) {|io|
+      io.read
+    }
+    assert_match(/rake help/, message)
+    assert_match(/rake all/, message)
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gem-codesearch
 version: !ruby/object:Gem::Version
-  version: '0.1'
+  version: '0.2'
 platform: ruby
 authors:
 - Tanaka Akira
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-09 00:00:00.000000000 Z
+date: 2015-01-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rubygems-mirror
@@ -70,6 +70,46 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 10.4.2
+- !ruby/object:Gem::Dependency
+  name: builder
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.2.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.2.2
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.8
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.8
 description: |
   gem-codesearch sets up a full text code search engine for rubygems mirror.
   It use rubygems-mirror to mirror rubygems and
@@ -86,6 +126,8 @@ files:
 - Rakefile
 - bin/gem-codesearch-setup
 - gem-codesearch.gemspec
+- test/test_all.rb
+- test/test_bin.rb
 homepage: https://github.com/akr/gem-codesearch
 licenses:
 - MIT
@@ -110,4 +152,6 @@ rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: Set up a full text code search engine for rubygems mirror
-test_files: []
+test_files:
+- test/test_all.rb
+- test/test_bin.rb