solr_wrapper 0.7.0 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 76ae78f988ac44da700258433749f9af21e295c2
-  data.tar.gz: 17465fcf5dfddc6e79cb77e79daaefbe758285b9
+  metadata.gz: 7916cf546d675d2a3f008f4a271aa4fc2cf7b8ae
+  data.tar.gz: 3d2e1456ff74edb606a60cef2fa1731ae5a74351
 SHA512:
-  metadata.gz: acd8180341d4acee240a201be0d85919b5ff5725388b43c8a24f0d9753147f495a094515d7a7b356215d1e1f84d1fdd667c9f27aab107169cbc548cd7a473d7b
-  data.tar.gz: 672f9d286d56d2ec8a2b77076b998a385b8de8e27ec85575d2f15d709ada04c14f862217ed79bfee1a74f6f023a8e3c9706da4c16c573d9a0b1033234d452afb
+  metadata.gz: ff2b8b312c13c8b68c233f0799c68f36761109beea56fe2648485a89be28980b53d16d5646479ce58c0575cdde11025ff9ed74b81c923eb923953dcfbcf862cf
+  data.tar.gz: 58a248017fd9ec7330af89ca300d686710a6c71c36d96d9af5cd1967e8c1a703be7618835d30d3f5f7066958e1046fef686f3fa2b0d2d308e3c8f4990c3963fe

data/.rubocop.yml

@@ -10,4 +10,7 @@ Metrics/LineLength:
 
 # Offense count: 61
 Metrics/AbcSize:
-  Max: 59
+  Max: 59
+
+Style/RescueModifier:
+  Enabled: false

data/.travis.yml

@@ -1,8 +1,12 @@
 language: ruby
 sudo: false
 rvm:
-  - 2.2.0
+  - 2.2.4
+  - 2.3.0
   - jruby-9.0.1.0
 cache:
   directories:
-  - /tmp/solr-5.1.0
+  - /tmp/solr-5.4.1
+
+jdk:
+  - oraclejdk8

data/exe/solr_wrapper

@@ -5,14 +5,14 @@ require 'optparse'
 
 options = {}
 collection_options = {}
-OptionParser.new do |opts|
+args = OptionParser.new do |opts|
   opts.banner = "Usage: solr_wrapper [options]"
 
   opts.on("-v", "--[no-]verbose", "Run verbosely") do |v|
     options[:verbose] = v
   end
 
-  opts.on("--config", "Load configuration from a file") do |v|
+  opts.on("--config FILE", "Load configuration from a file") do |v|
     options[:config] = v
   end
 
@@ -20,7 +20,7 @@ OptionParser.new do |opts|
     options[:version] = v
   end
 
-  opts.on("-pPORT", "--port PORT", "Specify the port Solr should run at (default: 8983)") do |p|
+  opts.on("-pPORT", "--port PORT", "Specify the port Solr should run at (default: #{SolrWrapper.default_solr_port})") do |p|
     if p == 'random'
       options[:port] = nil
     else
@@ -55,20 +55,26 @@ OptionParser.new do |opts|
   opts.on("--no-checksum", "Skip running checksum validation on the downloaded file") do |d|
     options[:ignore_md5sum] = true
   end
-end.parse!
+end
+
+begin
+  args.parse!
+rescue => error
+  $stderr.puts "ERROR: #{error}\n"
+  $stderr.puts args.help
+  exit 1
+end
 
 # default to verbose
 options[:verbose] = true if options[:verbose].nil?
 
-instance = SolrWrapper.default_instance(options)
+instance = SolrWrapper.instance(options)
 $stderr.print "Starting Solr #{instance.version} on port #{instance.port} ... "
 instance.wrap do |conn|
   conn.with_collection(collection_options) do
     $stderr.puts "http://#{instance.host}:#{instance.port}/solr/"
     begin
-      while conn.status
-        sleep 1
-      end
+      conn.wait
     rescue Interrupt
       $stderr.puts "Solr is shutting down."
     end

data/lib/solr_wrapper/configuration.rb

@@ -3,8 +3,8 @@ module SolrWrapper
     attr_reader :options
 
     def initialize(options)
-      @options = options
-      read_config
+      @options = read_config(options[:config], options[:verbose])
+             .merge options
     end
 
     def solr_xml
@@ -35,7 +35,15 @@ module SolrWrapper
       # Check if the port option has been explicitly set to nil.
       # this means to start solr wrapper on a random open port
       return nil if options.key?(:port) && !options[:port]
-      options[:port] || SolrWrapper.default_instance_options[:port]
+      options.fetch(:port) { SolrWrapper.default_instance_options[:port] }.to_s
+    end
+
+    def zookeeper_host
+      options[:zookeeper_host]
+    end
+
+    def zookeeper_port
+      options[:zookeeper_port]
     end
 
     def download_path
@@ -43,7 +51,15 @@ module SolrWrapper
     end
 
     def download_dir
-      options[:download_dir]
+      options[:download_dir] || default_download_dir
+    end
+
+    def default_download_dir
+      if defined?(Rails) && Rails.root
+        File.join(Rails.root, 'tmp')
+      else
+        Dir.tmpdir
+      end
     end
 
     def solr_options
@@ -78,27 +94,52 @@ module SolrWrapper
       options[:version_file]
     end
 
+    def collection_options
+      hash = options.fetch(:collection, {})
+      Configuration.slice(convert_keys(hash), :name, :dir)
+    end
+
+    def configsets
+      configsets = options[:configsets] || []
+      configsets.map { |x| convert_keys(x) }
+    end
+
+    def poll_interval
+      options.fetch(:poll_interval, 1)
+    end
+
     private
 
-      def read_config
-        default_configuration_paths = ['.solr_wrapper', '~/.solr_wrapper']
+      def self.slice(source, *keys)
+        keys.each_with_object({}) { |k, hash| hash[k] = source[k] if source.has_key?(k) }
+      end
+
+      def read_config(config_file, verbose)
         default_configuration_paths.each do |p|
           path = File.expand_path(p)
-          options[:config] ||= path if File.exist? path
+          config_file ||= path if File.exist? path
+        end
+
+        unless config_file
+          $stdout.puts "No config specified" if verbose
+          return {}
         end
 
-        if options[:config]
-          $stdout.puts "Loading configuration from #{options[:config]}" if verbose?
-          config = YAML.load(ERB.new(IO.read(options[:config])).result(binding))
-          unless config
-            $stderr.puts "Unable to parse config #{options[:config]}" if verbose?
-            return
-          end
-          collection_config = config.delete(:collection) || {}
-          @options = config.merge(options)
-        elsif verbose?
-          $stdout.puts "No config specified"
+        $stdout.puts "Loading configuration from #{config_file}" if verbose
+        config = YAML.load(ERB.new(IO.read(config_file)).result(binding))
+        unless config
+          $stderr.puts "Unable to parse config #{config_file}" if verbose
+          return {}
         end
+        convert_keys(config)
+      end
+
+      def convert_keys(hash)
+        hash.each_with_object({}) { |(k, v), h| h[k.to_sym] = v }
+      end
+
+      def default_configuration_paths
+        ['.solr_wrapper', '~/.solr_wrapper']
       end
   end
 end

data/lib/solr_wrapper/downloader.rb

@@ -1,17 +1,23 @@
+require 'ruby-progressbar'
+
 module SolrWrapper
   class Downloader
     def self.fetch_with_progressbar(url, output)
-      pbar = ProgressBar.create(title: File.basename(url), total: nil, format: "%t: |%B| %p%% (%e )")
-      open(url, content_length_proc: lambda do|t|
-        if t && 0 < t
-          pbar.total = t
-        end
-      end,
-                progress_proc: lambda do|s|
-                  pbar.progress = s
-                end) do |io|
+      pbar = SafeProgressBar.new(title: File.basename(url), total: nil, format: '%t: |%B| %p%% (%e )')
+      open(url, content_length_proc: ->(bytes) { pbar.total = bytes }, progress_proc: ->(bytes) { pbar.progress = bytes }) do |io|
         IO.copy_stream(io, output)
       end
     end
+
+    class SafeProgressBar < ProgressBar::Base
+      def progress=(new_progress)
+        self.total = new_progress if total <= new_progress
+        super
+      end
+
+      def total=(new_total)
+        super if new_total && new_total > 0
+      end
+    end
   end
 end

data/lib/solr_wrapper/instance.rb

@@ -2,7 +2,6 @@ require 'digest'
 require 'fileutils'
 require 'json'
 require 'open-uri'
-require 'ruby-progressbar'
 require 'securerandom'
 require 'socket'
 require 'stringio'
@@ -13,7 +12,7 @@ require 'yaml'
 
 module SolrWrapper
   class Instance
-    attr_reader :config, :md5, :pid
+    attr_reader :config, :md5
 
     ##
     # @param [Hash] options
@@ -51,6 +50,10 @@ module SolrWrapper
       config.url
     end
 
+    def instance_dir
+      config.instance_dir
+    end
+
     def version
       config.version
     end
@@ -72,8 +75,10 @@ module SolrWrapper
 
         # Wait for solr to start
         unless status
-          sleep 1
+          sleep config.poll_interval
         end
+
+        after_start
       end
     end
 
@@ -81,15 +86,9 @@ module SolrWrapper
     # Stop Solr and wait for it to finish exiting
     def stop
       if config.managed? && started?
-
         exec('stop', p: port)
-        # Wait for solr to stop
-        while status
-          sleep 1
-        end
+        wait
       end
-
-      @pid = nil
     end
 
     ##
@@ -111,12 +110,31 @@ module SolrWrapper
       false
     end
 
+    def pid
+      return unless config.managed?
+
+      @pid ||= begin
+        out = exec('status').read
+        out.match(/process (?<pid>\d+) running on port #{port}/) do |m|
+          m[:pid].to_i
+        end
+      end
+    rescue
+      nil
+    end
+
     ##
     # Is Solr running?
     def started?
       !!status
     end
 
+    def wait
+      while (Process.getpgid(pid) rescue status)
+        sleep config.poll_interval
+      end
+    end
+
     ##
     # Create a new collection in solr
     # @param [Hash] options
@@ -127,12 +145,49 @@ module SolrWrapper
 
       create_options = { p: port }
       create_options[:c] = options[:name] if options[:name]
+      create_options[:n] = options[:config_name] if options[:config_name]
       create_options[:d] = options[:dir] if options[:dir]
       exec("create", create_options)
 
       options[:name]
     end
 
+    ##
+    # Update the collection configuration in zookeeper
+    # @param [Hash] options
+    # @option options [String] :config_name
+    # @option options [String] :dir
+    def upconfig(options = {})
+      options[:name] ||= SecureRandom.hex
+      options[:zkhost] ||= zkhost
+
+      upconfig_options = { upconfig: true, n: options[:name] }
+      upconfig_options[:d] = options[:dir] if options[:dir]
+      upconfig_options[:z] = options[:zkhost] if options[:zkhost]
+
+      exec 'zk', upconfig_options
+
+      options[:name]
+    end
+
+    ##
+    # Copy the collection configuration from zookeeper to a local directory
+    # @param [Hash] options
+    # @option options [String] :config_name
+    # @option options [String] :dir
+    def downconfig(options = {})
+      options[:name] ||= SecureRandom.hex
+      options[:zkhost] ||= zkhost
+
+      downconfig_options = { downconfig: true, n: options[:name] }
+      downconfig_options[:d] = options[:dir] if options[:dir]
+      downconfig_options[:z] = options[:zkhost] if options[:zkhost]
+
+      exec 'zk', downconfig_options
+
+      options[:name]
+    end
+
     ##
     # Create a new collection in solr
     # @param [String] name collection name
@@ -146,6 +201,7 @@ module SolrWrapper
     # @option options [String] :name
     # @option options [String] :dir
     def with_collection(options = {})
+      options = config.collection_options.merge(options)
       return yield if options.empty?
 
       name = create(options)
@@ -264,7 +320,6 @@ module SolrWrapper
         # JRuby
         env_str = config.env.map { |k, v| "#{Shellwords.escape(k)}=#{Shellwords.escape(v)}" }.join(" ")
         pid, input, output, error = IO.popen4(env_str + " " + args.join(" "))
-        @pid = pid
         stringio = StringIO.new
         if config.verbose? && !silence_output
           IO.copy_stream(output, $stderr)
@@ -277,7 +332,7 @@ module SolrWrapper
         input.close
         output.close
         error.close
-        exit_status = Process.waitpid2(@pid).last
+        exit_status = Process.waitpid2(pid).last
       else
         IO.popen(config.env, args + [err: [:child, :out]]) do |io|
           stringio = StringIO.new
@@ -288,8 +343,6 @@ module SolrWrapper
             IO.copy_stream(io, stringio)
           end
 
-          @pid = io.pid
-
           _, exit_status = Process.wait2(io.pid)
         end
       end
@@ -304,6 +357,16 @@ module SolrWrapper
 
     private
 
+    def after_start
+      create_configsets if config.cloud
+    end
+
+    def create_configsets
+      config.configsets.each do |configset|
+        upconfig(configset)
+      end
+    end
+
     def extracted_version
       File.read(config.version_file).strip if File.exists? config.version_file
     end
@@ -314,6 +377,10 @@ module SolrWrapper
       end
     end
 
+    def zkhost
+      "#{config.zookeeper_host}:#{config.zookeeper_port}" if config.cloud
+    end
+
     def raise_error_unless_extracted
       raise RuntimeError, "there is no solr instance at #{config.instance_dir}.  Run SolrWrapper.extract first." unless extracted?
     end

data/lib/solr_wrapper/settings.rb

@@ -1,3 +1,5 @@
+require 'delegate'
+
 module SolrWrapper
   # Configuraton that comes from static and dynamic sources.
   class Settings < Delegator
@@ -23,6 +25,11 @@ module SolrWrapper
       '127.0.0.1'
     end
 
+    def zookeeper_host
+      @zookeeper_host ||= static_config.zookeeper_port
+      @zookeeper_host ||= host
+    end
+
     ##
     # Get the port this Solr instance is running at
     def port
@@ -30,6 +37,12 @@ module SolrWrapper
       @port ||= random_open_port.to_s
     end
 
+    def zookeeper_port
+      @zookeeper_port ||= static_config.zookeeper_port
+      @zookeeper_port ||= "#{port.to_i + 1000}"
+    end
+
+
     ##
     # Get a (likely) URL to the solr instance
     def url
@@ -83,7 +96,6 @@ module SolrWrapper
 
       def download_dir
         @download_dir ||= static_config.download_dir
-        @download_dir ||= Dir.tmpdir
         FileUtils.mkdir_p @download_dir
         @download_dir
       end

data/lib/solr_wrapper/tasks/solr_wrapper.rake

@@ -4,8 +4,7 @@ require 'solr_wrapper'
 namespace :solr do
   desc "Load the solr options and solr instance"
   task :environment do
-    SolrWrapper.default_instance_options[:download_dir] ||= Rails.root.to_s + '/tmp' if defined? Rails
-    @solr_instance = SolrWrapper.default_instance
+    @solr_instance = SolrWrapper.instance
   end
 
   desc 'Install a clean version of solr. Replaces the existing copy if there is one.'

data/lib/solr_wrapper/version.rb

@@ -1,3 +1,3 @@
 module SolrWrapper
-  VERSION = '0.7.0'
+  VERSION = '0.10.0'
 end

data/lib/solr_wrapper.rb

@@ -7,12 +7,16 @@ require 'solr_wrapper/instance'
 
 module SolrWrapper
   def self.default_solr_version
-    '5.4.1'
+    '6.0.0'
+  end
+
+  def self.default_solr_port
+    '8983'
   end
 
   def self.default_instance_options
     @default_instance_options ||= {
-      port: '8983',
+      port: SolrWrapper.default_solr_port,
       version: SolrWrapper.default_solr_version
     }
   end
@@ -22,12 +26,16 @@ module SolrWrapper
   end
 
   def self.default_instance(options = {})
-    @default_instance ||= SolrWrapper::Instance.new options
+    @default_instance ||= instance(default_instance_options)
+  end
+
+  def self.instance(options = {})
+    SolrWrapper::Instance.new(options)
   end
 
   ##
   # Ensures a Solr service is running before executing the block
   def self.wrap(options = {}, &block)
-    default_instance(options).wrap &block
+    instance(options).wrap &block
   end
 end

data/spec/fixtures/sample_config.yml

@@ -0,0 +1,7 @@
+port: 9999
+collection:
+  dir: solr/config/
+  name: project-development
+configsets:
+  - dir: solr/config/
+    name: project-development-configset

data/spec/lib/solr_wrapper/configuration_spec.rb

@@ -21,4 +21,35 @@ describe SolrWrapper::Configuration do
       it { is_expected.to eq '8888' }
     end
   end
+
+  describe "#read_config" do
+    before do
+      allow(config).to receive(:default_configuration_paths).and_return([])
+    end
+    let(:options) { { config: 'spec/fixtures/sample_config.yml' } }
+    it "uses values from the config file" do
+      expect(config.port).to eq '9999'
+    end
+  end
+
+  describe "#collection_options" do
+    before do
+      allow(config).to receive(:default_configuration_paths).and_return([])
+    end
+    let(:options) { { config: 'spec/fixtures/sample_config.yml' } }
+    it "uses values from the config file" do
+      expect(config.collection_options).to eq(name: 'project-development', dir: 'solr/config/')
+    end
+  end
+
+  describe '#configsets' do
+    before do
+      allow(config).to receive(:default_configuration_paths).and_return([])
+    end
+    let(:options) { { config: 'spec/fixtures/sample_config.yml' } }
+
+    it 'uses values from the config file' do
+      expect(config.configsets).to include(name: 'project-development-configset', dir: 'solr/config/')
+    end
+  end
 end

data/spec/lib/solr_wrapper/instance_spec.rb

@@ -1,22 +1,73 @@
 require 'spec_helper'
 
 describe SolrWrapper::Instance do
-  let(:solr_instance) { SolrWrapper::Instance.new }
+  let(:options) { {} }
+  let(:solr_instance) { SolrWrapper::Instance.new(options) }
   subject { solr_instance }
   let(:client) { SimpleSolrClient::Client.new(subject.url) }
 
   describe "#with_collection" do
-    it "creates a new anonymous collection" do
+    let(:options) { { cloud: false } }
+    context "without a name" do
+      it "creates a new anonymous collection" do
+        subject.wrap do |solr|
+          solr.with_collection(dir: File.join(FIXTURES_DIR, "basic_configs")) do |collection_name|
+            core = client.core(collection_name)
+            unless defined? JRUBY_VERSION
+              expect(core.schema.field('id').name).to eq 'id'
+              expect(core.schema.field('id').stored).to eq true
+            end
+          end
+        end
+      end
+    end
+    context "with a config file" do
+      before do
+        allow(solr_instance.config).to receive(:collection_options)
+          .and_return(name: 'project-development', dir: 'solr/config/')
+        allow(solr_instance).to receive(:delete)
+      end
+
+      it "creates a new collection with options from the config" do
+        expect(solr_instance).to receive(:create).with(
+          hash_including(name: "project-development", dir: anything))
+        solr_instance.with_collection(dir: File.join(FIXTURES_DIR, "basic_configs")) {}
+      end
+    end
+  end
+
+  context 'with a SolrCloud instance' do
+    let(:options) { { cloud: true } }
+    it 'can upload configurations' do
       subject.wrap do |solr|
-        solr.with_collection(dir: File.join(FIXTURES_DIR, "basic_configs")) do |collection_name|
+        config_name = solr.upconfig dir: File.join(FIXTURES_DIR, 'basic_configs')
+        Dir.mktmpdir do |dir|
+          solr.downconfig name: config_name, dir: dir
+        end
+        solr.with_collection(config_name: config_name) do |collection_name|
           core = client.core(collection_name)
           unless defined? JRUBY_VERSION
-            expect(core.schema.field('id').name).to eq 'id'
-            expect(core.schema.field('id').stored).to eq true
+            expect(core.all.size).to eq 0
           end
         end
       end
     end
+
+    context 'with a config file' do
+      before do
+        allow(solr_instance.config).to receive(:configsets)
+          .and_return([name: 'project-development', dir: 'solr/config/'])
+      end
+
+      it 'creates a new configsets with options from the config' do
+        expect(subject).to receive(:upconfig).with(
+          hash_including(name: 'project-development', dir: anything))
+
+        subject.wrap do
+          # no-op
+        end
+      end
+    end
   end
 
   describe 'exec' do
@@ -56,9 +107,14 @@ describe SolrWrapper::Instance do
     it { is_expected.to eq 'http://127.0.0.1:8983/solr/' }
   end
 
+  describe "#instance_dir" do
+    subject { solr_instance.instance_dir }
+    it { is_expected.to start_with Dir.tmpdir }
+  end
+
   describe "#version" do
     subject { solr_instance.version }
-    it { is_expected.to eq '5.4.1' }
+    it { is_expected.to eq '6.0.0' }
   end
 
   describe "#md5" do

data/spec/lib/solr_wrapper_spec.rb

@@ -13,6 +13,14 @@ describe SolrWrapper do
     end
   end
 
+  describe '.instance' do
+    context 'without arguments' do
+      it 'inherits the defaults' do
+        expect(SolrWrapper.instance.port).to eq '8983'
+      end
+    end
+  end
+
   describe ".default_instance_options=" do
     it "sets default options" do
       SolrWrapper.default_instance_options = { port: '1234' }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: solr_wrapper
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.10.0
 platform: ruby
 authors:
 - Chris Beer
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-03-10 00:00:00.000000000 Z
+date: 2016-04-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rubyzip
@@ -146,6 +146,7 @@ files:
 - spec/fixtures/basic_configs/solrconfig.xml
 - spec/fixtures/basic_configs/stopwords.txt
 - spec/fixtures/basic_configs/synonyms.txt
+- spec/fixtures/sample_config.yml
 - spec/lib/solr_wrapper/configuration_spec.rb
 - spec/lib/solr_wrapper/instance_spec.rb
 - spec/lib/solr_wrapper_spec.rb
@@ -170,7 +171,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
 summary: Solr 5 service wrapper
@@ -183,8 +184,8 @@ test_files:
 - spec/fixtures/basic_configs/solrconfig.xml
 - spec/fixtures/basic_configs/stopwords.txt
 - spec/fixtures/basic_configs/synonyms.txt
+- spec/fixtures/sample_config.yml
 - spec/lib/solr_wrapper/configuration_spec.rb
 - spec/lib/solr_wrapper/instance_spec.rb
 - spec/lib/solr_wrapper_spec.rb
 - spec/spec_helper.rb
-has_rdoc: