gem-codesearch 0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 3ba88fec2f6fc7cb946e0a1ba82b5296f41b8a14
+  data.tar.gz: 4ca8bd6a6b62f3da81b3f2ace20f5a13fe0d4c45
+SHA512:
+  metadata.gz: fd31437425bf9d8d8693af6784a2cbafe95f0caedc0a56f8ea095151144174f11e1ecf6ac2f35ba12aba3b45cc86e3a8c0d52eed760c35b8b6be2787d4ad081c
+  data.tar.gz: 304fb9e178cd06cc2b8279bf1872b20fa30789e53ab5dca747f1442ebf131610782dc758fecdda6a13a4df64c94ce14ea87031e8160019ba2e9a7de80bcec076

data/.gitignore

@@ -0,0 +1,34 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalisation:
+/.bundle/
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Tanaka Akira
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

data/README.md

@@ -0,0 +1,36 @@
+# gem-codesearch
+
+gem-codesearch sets up a full text code search engine on mirror of rubygems.
+It use rubygems-mirror to mirror rubygems and milkode for search engine.
+
+## Usage
+
+This creates "latest-gem" package in milkode index.
+
+    % gem install gem-codesearch
+    % milk init --default               # If you use milkode first time
+    % mkdir $HOME/gem-codesearch        # Make a some directory
+    % cd $HOME/gem-codesearch
+    % gem-codesearch-setup all >& setup.log # It may take several days or more
+
+If "gem-codesearch-setup all" fails due to network or server errors,
+try again.
+
+After the index is created, enjoy code search.
+
+    % gmilk -p latest-gem sort_by
+
+## Usage without install
+
+    % gem install rubygems-mirror milkode
+    % milk init --default               # If you use milkode first time
+    % cd $HOME
+    % git clone https://github.com/akr/gem-codesearch.git
+    % cd gem-codesearch
+    % rake all >& setup.log             # It may take several days or more
+
+## Author
+
+Tanaka Akira
+akr@fsij.org
+

data/Rakefile

@@ -0,0 +1,119 @@
+require 'rbconfig'
+require 'fileutils'
+require 'find'
+require 'pp'
+
+task :default => :help
+
+task :help do
+  puts <<'End'
+Usage:
+  rake help
+  rake all              # mirror, unpack, index
+  rake mirror
+  rake unpack
+  rake index
+End
+end
+
+task :all => [:mirror, :unpack, :index]
+
+MIRROR_DIR = "#{Dir.pwd}/mirror"
+LATEST_DIR = "#{Dir.pwd}/latest-gem"
+LOG_DIR = "#{Dir.pwd}/log"
+
+GEM_COMMAND = "#{RbConfig::CONFIG["bindir"]}/gem"
+MILK_COMMAND = "#{RbConfig::CONFIG["bindir"]}/milk"
+
+file ".gem/.mirrorrc" do |t|
+  FileUtils.mkpath File.dirname(t.name)
+  File.write(t.name, <<"End")
+---
+- from: http://production.s3.rubygems.org
+  to: #{MIRROR_DIR}
+End
+end
+
+task :mirror => ".gem/.mirrorrc" do
+  FileUtils.mkpath MIRROR_DIR
+  # HOME is set because gem mirror reads $HOME/.gem/.mirrorrc.
+  sh "HOME=#{Dir.pwd} #{GEM_COMMAND} mirror --verbose"
+end
+
+task :unpack do
+  FileUtils.mkpath LOG_DIR
+
+  available_gems = {}
+  Dir.foreach("#{MIRROR_DIR}/gems") {|filename|
+    next if /\.gem\z/ !~ filename
+    available_gems[filename] = true
+  }
+  FileUtils.mkpath LATEST_DIR
+  all_specs = File.open("#{MIRROR_DIR}/specs.4.8") {|f| Marshal.load(f) }
+  all_specs = all_specs.reject {|name,version,platform|
+    /\A\./ =~ name ||
+    /\A[0-9a-zA-Z._-]+\z/ !~ name ||
+    /\A[0-9a-zA-Z._-]+\z/ !~ version.to_s ||
+    platform != 'ruby' ||
+    !available_gems["#{name}-#{version}.gem"]
+  }
+
+  #all_specs = all_specs.reject {|name,version,platform| /\Afoo-/ !~ name }
+
+  latest_vnames = []
+  nonlatest_vnames = []
+  h = all_specs.group_by {|name,version| name }
+  h.each {|name, list|
+    list = list.sort_by {|name,version| version }
+    vnames = list.map {|name,version| "#{name}-#{version}" }
+    latest_vnames << vnames.pop
+    nonlatest_vnames.concat list
+  }
+
+  already_unpacked = Dir.entries(LATEST_DIR)
+  (already_unpacked & nonlatest_vnames).each {|vname|
+    puts "remove: #{vname}
+    FileUtils.rmtree("#{LATEST_DIR}/#{vname}")
+  }
+
+  File.open("#{LOG_DIR}/unpack.log.#{Time.now.strftime '%FT%T%:z'}", "a") {|log|
+    (latest_vnames - already_unpacked).each {|vname|
+      puts "unpack: #{vname}"
+      system GEM_COMMAND, 'unpack', "#{MIRROR_DIR}/gems/#{vname}.gem", :chdir => LATEST_DIR, :out => log
+      if !$?.success?
+        puts "failed to unpack #{vname}"
+      end
+      fix_permission("#{LATEST_DIR}/#{vname}")
+    }
+  }
+
+end
+
+task :index do
+  # Assume default database for milkode is already created.
+  # If not, do it as follows:
+  #   milk init --default
+  milkode_package_list = IO.popen([MILK_COMMAND, 'list']) {|f| f.read }
+  package_name = File.basename(LATEST_DIR)
+  if /^#{Regexp.escape package_name}$/ !~ milkode_package_list
+    system MILK_COMMAND, 'add', '--verbose', LATEST_DIR
+  else
+    system MILK_COMMAND, 'update', '--verbose', package_name
+  end
+end
+
+def fix_permission(dir)
+  return unless File.exist? dir
+  Find.find(dir) {|fn|
+    st = File.lstat(fn)
+    if st.file?
+      if !st.readable?
+        File.chmod(0644, fn)
+      end
+    elsif st.directory?
+      if !st.readable? || !st.executable?
+        File.chmod(0755, fn)
+      end
+    end
+  }
+end

data/bin/gem-codesearch-setup

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+require 'rbconfig'
+
+RAKE_COMMAND = "#{RbConfig::CONFIG["bindir"]}/rake"
+topdir = File.dirname(File.dirname(__FILE__))
+RAKEFILE = topdir + "/Rakefile"
+
+system RAKE_COMMAND, '-f', RAKEFILE, *ARGV

data/gem-codesearch.gemspec

@@ -0,0 +1,28 @@
+Gem::Specification.new do |s|
+  s.name = 'gem-codesearch'
+  s.version = '0.1'
+  s.date = '2015-01-09'
+  s.author = 'Tanaka Akira'
+  s.email = 'akr@fsij.org'
+  s.required_ruby_version = '>= 2.2.0'
+  s.add_runtime_dependency 'rubygems-mirror', '~> 1.0', '>= 1.0.1'
+  s.add_runtime_dependency 'milkode', '~> 1.8', '>= 1.8.6'
+  s.add_runtime_dependency 'rake', '~> 10.4', '>= 10.4.2'
+  s.executables << 'gem-codesearch-setup'
+  s.files = %w[
+    .gitignore
+    LICENSE
+    README.md
+    Rakefile
+    bin/gem-codesearch-setup
+    gem-codesearch.gemspec
+  ]
+  s.homepage = 'https://github.com/akr/gem-codesearch'
+  s.license = 'MIT'
+  s.summary = 'Set up a full text code search engine for rubygems mirror'
+  s.description = <<'End'
+gem-codesearch sets up a full text code search engine for rubygems mirror.
+It use rubygems-mirror to mirror rubygems and
+milkode for search engine.
+End
+end

metadata

@@ -0,0 +1,113 @@
+--- !ruby/object:Gem::Specification
+name: gem-codesearch
+version: !ruby/object:Gem::Version
+  version: '0.1'
+platform: ruby
+authors:
+- Tanaka Akira
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-01-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rubygems-mirror
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+- !ruby/object:Gem::Dependency
+  name: milkode
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.8.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.8.6
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.4'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 10.4.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.4'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 10.4.2
+description: |
+  gem-codesearch sets up a full text code search engine for rubygems mirror.
+  It use rubygems-mirror to mirror rubygems and
+  milkode for search engine.
+email: akr@fsij.org
+executables:
+- gem-codesearch-setup
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- LICENSE
+- README.md
+- Rakefile
+- bin/gem-codesearch-setup
+- gem-codesearch.gemspec
+homepage: https://github.com/akr/gem-codesearch
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.2.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Set up a full text code search engine for rubygems mirror
+test_files: []