solr_wrapper 0.0.7 → 1.0.0

data/lib/solr_wrapper/instance.rb

@@ -2,33 +2,64 @@ require 'digest'
 require 'fileutils'
 require 'json'
 require 'open-uri'
-require 'ruby-progressbar'
 require 'securerandom'
+require 'socket'
 require 'stringio'
 require 'tmpdir'
 require 'zip'
+require 'erb'
+require 'yaml'
 
 module SolrWrapper
   class Instance
-    attr_reader :options
+    attr_reader :config, :md5
 
     ##
     # @param [Hash] options
     # @option options [String] :url
-    # @option options [String] :version
-    # @option options [String] :port
-    # @option options [String] :version_file
-    # @option options [String] :instance_dir
-    # @option options [String] :download_path
-    # @option options [String] :md5sum
-    # @option options [Boolean] :verbose
-    # @option options [Boolean] :managed
+    # @option options [String] :instance_dir Directory to store the solr index files
+    # @option options [String] :version Solr version to download and install
+    # @option options [String] :port port to run Solr on
+    # @option options [Boolean] :cloud Run solr in cloud mode
+    # @option options [String] :version_file Local path to store the currently installed version
+    # @option options [String] :download_dir Local directory to store the downloaded Solr zip and its md5 file in (overridden by :solr_zip_path)
+    # @option options [String] :solr_zip_path Local path for storing the downloaded Solr zip file
+    # @option options [Boolean] :validate Should solr_wrapper download a new md5 and (re-)validate the zip file? (default: trueF)
+    # @option options [String] :md5sum Path/URL to MD5 checksum
+    # @option options [String] :solr_xml Path to Solr configuration
+    # @option options [String] :extra_lib_dir Path to directory containing extra libraries to copy into instance_dir/lib
+    # @option options [Boolean] :verbose return verbose info when running solr commands
     # @option options [Boolean] :ignore_md5sum
-    def initialize options = {}
-      @options = options
+    # @option options [Hash] :solr_options
+    # @option options [Hash] :env
+    # @option options [String] :config
+    def initialize(options = {})
+      @config = Settings.new(Configuration.new(options))
+      @md5 = MD5.new(@config)
     end
 
-    def wrap &block
+    def host
+      config.host
+    end
+
+    def port
+      config.port
+    end
+
+    def url
+      config.url
+    end
+
+    def instance_dir
+      config.instance_dir
+    end
+
+    def version
+      config.version
+    end
+
+    def wrap(&_block)
+      extract_and_configure
       start
       yield self
     ensure
@@ -38,37 +69,58 @@ module SolrWrapper
     ##
     # Start Solr and wait for it to become available
     def start
-      extract
-      if managed?
-        exec('start', p: port)
+      extract_and_configure
+      if config.managed?
+        exec('start', p: port, c: config.cloud)
 
         # Wait for solr to start
         unless status
-          sleep 1
+          sleep config.poll_interval
         end
+
+        after_start
       end
     end
 
     ##
     # Stop Solr and wait for it to finish exiting
     def stop
-      if managed? and started?
-
+      if config.managed? && started?
         exec('stop', p: port)
-        # Wait for solr to stop
-        while status
-          sleep 1
-        end
+        wait
+      end
+    end
+
+    ##
+    # Stop Solr and wait for it to finish exiting
+    def restart
+      if config.managed? && started?
+        exec('restart', p: port, c: config.cloud)
       end
     end
 
     ##
     # Check the status of a managed Solr service
     def status
-      return true unless managed?
+      return true unless config.managed?
 
-      out = exec('status', output: true).read
+      out = exec('status').read
       out =~ /running on port #{port}/
+    rescue
+      false
+    end
+
+    def pid
+      return unless config.managed?
+
+      @pid ||= begin
+        out = exec('status').read
+        out.match(/process (?<pid>\d+) running on port #{port}/) do |m|
+          m[:pid].to_i
+        end
+      end
+    rescue
+      nil
     end
 
     ##
@@ -77,23 +129,73 @@ module SolrWrapper
       !!status
     end
 
+    def wait
+      while (Process.getpgid(pid) rescue status)
+        sleep config.poll_interval
+      end
+    end
+
     ##
     # Create a new collection in solr
     # @param [Hash] options
     # @option options [String] :name
     # @option options [String] :dir
-    def create options = {}
+    def create(options = {})
       options[:name] ||= SecureRandom.hex
 
-      exec("create", c: options[:name], d: options[:dir], p: port)
+      create_options = { p: port }
+      create_options[:c] = options[:name] if options[:name]
+      create_options[:n] = options[:config_name] if options[:config_name]
+      create_options[:d] = options[:dir] if options[:dir]
+
+      # short-circuit if we're using persisted data with an existing core/collection
+      return if options[:persist] && create_options[:c] && client.exists?(create_options[:c])
+
+      exec("create", create_options)
 
       options[:name]
     end
-    
+
+    ##
+    # Update the collection configuration in zookeeper
+    # @param [Hash] options
+    # @option options [String] :config_name
+    # @option options [String] :dir
+    def upconfig(options = {})
+      options[:name] ||= SecureRandom.hex
+      options[:zkhost] ||= zkhost
+
+      upconfig_options = { upconfig: true, n: options[:name] }
+      upconfig_options[:d] = options[:dir] if options[:dir]
+      upconfig_options[:z] = options[:zkhost] if options[:zkhost]
+
+      exec 'zk', upconfig_options
+
+      options[:name]
+    end
+
+    ##
+    # Copy the collection configuration from zookeeper to a local directory
+    # @param [Hash] options
+    # @option options [String] :config_name
+    # @option options [String] :dir
+    def downconfig(options = {})
+      options[:name] ||= SecureRandom.hex
+      options[:zkhost] ||= zkhost
+
+      downconfig_options = { downconfig: true, n: options[:name] }
+      downconfig_options[:d] = options[:dir] if options[:dir]
+      downconfig_options[:z] = options[:zkhost] if options[:zkhost]
+
+      exec 'zk', downconfig_options
+
+      options[:name]
+    end
+
     ##
     # Create a new collection in solr
     # @param [String] name collection name
-    def delete name, options = {}
+    def delete(name, _options = {})
       exec("delete", c: name, p: port)
     end
 
@@ -102,35 +204,52 @@ module SolrWrapper
     # @param [Hash] options
     # @option options [String] :name
     # @option options [String] :dir
-    def with_collection options = {}
+    def with_collection(options = {})
+      options = config.collection_options.merge(options)
+      return yield if options.empty?
+
       name = create(options)
       begin
         yield name
       ensure
-        delete name
+        delete name unless options[:persist]
       end
     end
 
-    ##
-    # Get the port this Solr instance is running at 
-    def port
-      options.fetch(:port, "8983").to_s
-    end
-
     ##
     # Clean up any files solr_wrapper may have downloaded
     def clean!
       stop
-      FileUtils.remove_entry(download_path) if File.exists? download_path
-      FileUtils.remove_entry(tmp_save_dir, true) if File.exists? tmp_save_dir
-      FileUtils.remove_entry(instance_dir, true) if File.exists? instance_dir
-      FileUtils.remove_entry(md5sum_path) if File.exists? md5sum_path
-      FileUtils.remove_entry(version_file) if File.exists? version_file
+      remove_instance_dir!
+      FileUtils.remove_entry(config.download_dir, true) if File.exist?(config.download_dir)
+      FileUtils.remove_entry(config.tmp_save_dir, true) if File.exist? config.tmp_save_dir
+      md5.clean!
+      FileUtils.remove_entry(config.version_file) if File.exist? config.version_file
     end
 
-    protected
+    ##
+    # Clean up any files in the Solr instance dir
+    def remove_instance_dir!
+      FileUtils.remove_entry(instance_dir, true) if File.exist? instance_dir
+    end
+
+    def configure
+      raise_error_unless_extracted
+      FileUtils.cp config.solr_xml, File.join(config.instance_dir, 'server', 'solr', 'solr.xml') if config.solr_xml
+      FileUtils.cp_r File.join(config.extra_lib_dir, '.'), File.join(config.instance_dir, 'server', 'solr', 'lib') if config.extra_lib_dir
+    end
+
+    def extract_and_configure
+      extract.tap { configure }
+    end
+
+    # rubocop:disable Lint/RescueException
+
+    # extract a copy of solr to instance_dir
+    # Does noting if solr already exists at instance_dir
+    # @return [String] instance_dir Directory where solr has been installed
     def extract
-      return solr_dir if File.exists?(solr_binary) and extracted_version == version
+      return config.instance_dir if extracted?
 
       zip_path = download
 
@@ -138,196 +257,140 @@ module SolrWrapper
         Zip::File.open(zip_path) do |zip_file|
           # Handle entries one by one
           zip_file.each do |entry|
-            dest_file = File.join(tmp_save_dir,entry.name)
-            FileUtils.remove_entry(dest_file,true)
+            dest_file = File.join(config.tmp_save_dir, entry.name)
+            FileUtils.remove_entry(dest_file, true)
             entry.extract(dest_file)
           end
         end
 
       rescue Exception => e
-        abort "Unable to unzip #{zip_path} into #{tmp_save_dir}: #{e.message}"
+        abort "Unable to unzip #{zip_path} into #{config.tmp_save_dir}: #{e.message}"
       end
 
-      begin  
-        FileUtils.remove_dir(solr_dir,true)
-        FileUtils.cp_r File.join(tmp_save_dir, File.basename(default_url, ".zip")), solr_dir
-        self.extracted_version = version
-        FileUtils.chmod 0755, solr_binary
+      begin
+        FileUtils.remove_dir(config.instance_dir, true)
+        FileUtils.cp_r File.join(config.tmp_save_dir, File.basename(config.download_url, ".zip")), config.instance_dir
+        self.extracted_version = config.version
+        FileUtils.chmod 0755, config.solr_binary
       rescue Exception => e
-        abort "Unable to copy #{tmp_save_dir} to #{solr_dir}: #{e.message}"
+        abort "Unable to copy #{config.tmp_save_dir} to #{config.instance_dir}: #{e.message}"
       end
 
-      solr_dir
+      config.instance_dir
     ensure
-      FileUtils.remove_entry tmp_save_dir if File.exists? tmp_save_dir
+      FileUtils.remove_entry config.tmp_save_dir if File.exist? config.tmp_save_dir
     end
+    # rubocop:enable Lint/RescueException
 
-    def download
-      unless File.exists? download_path and validate? download_path
-        fetch_with_progressbar url, download_path
-        validate! download_path
-      end
-
-      download_path
-    end
+    protected
 
-    def validate? file
-      Digest::MD5.file(file).hexdigest == expected_md5sum
+    def extracted?
+      File.exist?(config.solr_binary) && extracted_version == config.version
     end
 
-    def validate! file
-      unless validate? file
-        raise "MD5 mismatch" unless options[:ignore_md5sum]
+    def download
+      unless File.exist?(config.solr_zip_path) && md5.validate?(config.solr_zip_path)
+        Downloader.fetch_with_progressbar config.download_url, config.solr_zip_path
+        md5.validate! config.solr_zip_path
       end
+      config.solr_zip_path
     end
 
     ##
     # Run a bin/solr command
+    # @param [String] cmd command to run
+    # @param [Hash] options key-value pairs to transform into command line arguments
+    # @return [StringIO] an IO object for the executed shell command
     # @see https://github.com/apache/lucene-solr/blob/trunk/solr/bin/solr
-    def exec cmd, options = {}
-      output = !!options.delete(:output)
-      args = [solr_binary, cmd] + options.map { |k,v| ["-#{k}", "#{v}"] }.flatten
+    # If you want to pass a boolean flag, include it in the +options+ hash with its value set to +true+
+    # the key will be converted into a boolean flag for you.
+    # @example start solr in cloud mode on port 8983
+    #   exec('start', {p: '8983', c: true})
+    def exec(cmd, options = {})
+      silence_output = !options.delete(:output)
+
+      args = [config.solr_binary, cmd] + config.solr_options.merge(options).map do |k, v|
+        case v
+        when true
+          "-#{k}"
+        when false, nil
+          # don't return anything
+        else
+          ["-#{k}", "#{v}"]
+        end
+      end.flatten.compact
 
       if IO.respond_to? :popen4
         # JRuby
-        pid, input, output, error = IO.popen4(args.join(" "))
-        
+        env_str = config.env.map { |k, v| "#{Shellwords.escape(k)}=#{Shellwords.escape(v)}" }.join(" ")
+        pid, input, output, error = IO.popen4(env_str + " " + args.join(" "))
         stringio = StringIO.new
-        if verbose? and !output
-          IO.copy_stream(output,$stderr)
-          IO.copy_stream(error,$stderr)
+        if config.verbose? && !silence_output
+          IO.copy_stream(output, $stderr)
+          IO.copy_stream(error, $stderr)
         else
           IO.copy_stream(output, stringio)
           IO.copy_stream(error, stringio)
         end
+
         input.close
         output.close
         error.close
-        
-        stringio.rewind
-
-        if $? != 0
-          raise "Failed to execute solr #{cmd}: #{stringio.read}"
-        end
-
-        stringio
+        exit_status = Process.waitpid2(pid).last
       else
-        IO.popen(args + [err: [:child, :out]]) do |io|
+        IO.popen(config.env, args + [err: [:child, :out]]) do |io|
           stringio = StringIO.new
-          if verbose? and !output
-            IO.copy_stream(io,$stderr)
+
+          if config.verbose? && !silence_output
+            IO.copy_stream(io, $stderr)
           else
             IO.copy_stream(io, stringio)
           end
-          _, exit_status = Process.wait2(io.pid)
-
-          stringio.rewind
-
-          if exit_status != 0
-            raise "Failed to execute solr #{cmd}: #{stringio.read}"
-          end
 
-          stringio
+          _, exit_status = Process.wait2(io.pid)
         end
       end
-    end
-
-    private  
-    def url
-      @download_url ||= options.fetch(:url, default_url)
-    end
 
-    def default_url
-      @default_url ||= begin
-        mirror_url = "http://www.apache.org/dyn/closer.cgi/lucene/solr/#{version}/solr-#{version}.zip?asjson=true"
-        json = open(mirror_url).read
-        doc = JSON.parse(json)
-        doc['preferred'] + doc['path_info']
+      stringio.rewind
+      if exit_status != 0 && cmd != 'status'
+        raise "Failed to execute solr #{cmd}: #{stringio.read}. Further information may be available in #{instance_dir}/logs"
       end
-    end
-
-    def md5url
-      "http://www.us.apache.org/dist/lucene/solr/#{version}/solr-#{version}.zip.md5"
-    end
-
-    def version
-      @version ||= options.fetch(:version, default_solr_version)
-    end
 
-    def default_solr_version
-      SolrWrapper.default_solr_version
+      stringio
     end
 
-    def download_path
-      @download_path ||= options.fetch(:download_path, default_download_path)  
-    end
-
-    def default_download_path
-      File.join(Dir.tmpdir, File.basename(default_url))
-    end
-
-    def solr_dir
-      @solr_dir ||= options.fetch(:instance_dir, File.join(Dir.tmpdir, File.basename(default_url, ".zip")))
-    end
-
-    def verbose?
-      !!options.fetch(:verbose, false)
-    end
+    private
 
-    def managed?
-      !!options.fetch(:managed, true)
+    def after_start
+      create_configsets if config.cloud
     end
 
-    def version_file
-      options.fetch(:version_file, File.join(solr_dir, "VERSION"))
-    end
-
-    def expected_md5sum
-      @md5sum ||= options.fetch(:md5sum, open(md5file).read.split(" ").first)
-    end
-    
-    def solr_binary
-      File.join(solr_dir, "bin", "solr")
-    end
-
-    def md5sum_path
-      File.join(Dir.tmpdir, File.basename(md5url))
+    def create_configsets
+      config.configsets.each do |configset|
+        upconfig(configset)
+      end
     end
 
-    def tmp_save_dir
-      @tmp_save_dir ||= Dir.mktmpdir
+    def extracted_version
+      File.read(config.version_file).strip if File.exist? config.version_file
     end
 
-    def fetch_with_progressbar url, output
-      pbar = ProgressBar.create(title: File.basename(url), total: nil, format: "%t: |%B| %p%% (%e )")
-      open(url, content_length_proc: lambda {|t|
-        if t && 0 < t
-          pbar.total = t
-        end
-        },
-        progress_proc: lambda {|s|
-          pbar.progress = s
-        }) do |io|
-        IO.copy_stream(io,output)
+    def extracted_version=(version)
+      File.open(config.version_file, "w") do |f|
+        f.puts version
       end
     end
 
-    def md5file
-      unless File.exists? md5sum_path
-        fetch_with_progressbar md5url, md5sum_path
-      end
-
-      md5sum_path
+    def zkhost
+      "#{config.zookeeper_host}:#{config.zookeeper_port}" if config.cloud
     end
 
-    def extracted_version
-      File.read(version_file).strip if File.exists? version_file
+    def client
+      SolrWrapper::Client.new(url)
     end
 
-    def extracted_version= version
-      File.open(version_file, "w") do |f|
-        f.puts version
-      end
+    def raise_error_unless_extracted
+      raise RuntimeError, "there is no solr instance at #{config.instance_dir}.  Run SolrWrapper.extract first." unless extracted?
     end
   end
-end
+end

data/lib/solr_wrapper/md5.rb

@@ -0,0 +1,43 @@
+module SolrWrapper
+  class MD5
+    attr_reader :config
+    def initialize(config)
+      @config = config
+    end
+
+    def clean!
+      FileUtils.remove_entry(config.md5sum_path) if File.exist? config.md5sum_path
+    end
+
+    def validate?(file)
+      return true if config.validate == false
+
+      Digest::MD5.file(file).hexdigest == expected_sum
+    end
+
+    def validate!(file)
+      unless validate? file
+        raise "MD5 mismatch" unless config.ignore_md5sum
+      end
+    end
+
+    private
+
+      def expected_sum
+        @md5sum ||= config.md5sum
+        @md5sum ||= read_file
+      end
+
+      def read_file
+        open(md5file).read.split(" ").first
+      end
+
+      def md5file
+        unless File.exist? config.md5sum_path
+          Downloader.fetch_with_progressbar config.md5url, config.md5sum_path
+        end
+
+        config.md5sum_path
+      end
+  end
+end

data/lib/solr_wrapper/rake_task.rb

@@ -0,0 +1,5 @@
+module SolrWrapper
+  module RakeTask
+    Dir[File.expand_path(File.join(File.dirname(__FILE__),"tasks/*.rake"))].each { |ext| load ext } if defined?(Rake)
+  end
+end