solr_wrapper 3.1.3 → 4.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e135734d52040fa5877fd21ad5149afc55ba8ffbc282a900ba448a9b2b230c28
-  data.tar.gz: 5363eb0bcad04e78b36267dbf35aa03a1fec56ee1487023622d3219a54501f87
+  metadata.gz: d0ec6a2ca3c8e571ae16e5ded77d9786ce333094a0fc8773d61b46d2bf1ba211
+  data.tar.gz: 3636273764eb165f5dc132bb2a558fd13b974c91f0f6e3a6d1cfaa318fb1b327
 SHA512:
-  metadata.gz: e0c3fca3bc208c4f9d71c6c14fcb0e99094d737e705ac12620c9ba725cc925b004a2dfa9bdb5210b05863df6ff94f4a3282a23b3e8f6aa4e2ef5de60a78eea2a
-  data.tar.gz: 3cd79dcfa0e2b3fd26bb1ce499712e09d019ade55799d9fbbdbdc6e832748df4f0bbc54689a5ba73f7e7d122db55751eff6a75bac37f6c41d19e6df55396b760
+  metadata.gz: da1cfd3a2b98fc21125970c6357c2763394e6620f9b22df2ee1d08594d0948c0cb99f0051f4c79c76cd4f89c170db00b078f80687163a119fa6a5882a3c56c52
+  data.tar.gz: 5ebff3d507a4ff72b3fc711756c2f9bfb1e443c617b4a55d12da8f64ecbe655da417c5b3063f81dead29cf94acfc0d3baa1cb439af5936933c5b73e68beafe61

data/README.md

@@ -31,13 +31,13 @@ SolrWrapper.wrap port: 8983,
 |Option         |                                         |
 |---------------|-----------------------------------------|
 | instance_dir  | Directory to store the solr index files |
-| url           | URL of the Zip file to download |
+| url           | URL of the TGZ artifact to download |
 | mirror_url    | Mirror to download the solr artifacts from (e.g. http://lib-solr-mirror.princeton.edu/dist/)|
 | version       | Solr version to download and install |
 | port          | port to run Solr on |
 | version_file  | Local path to store the currently installed version |
-| download_dir  | Local path for storing the downloaded Solr zip file |
-| solr_zip_path | Local path to the Solr zip file |
+| download_dir  | Local path for storing the downloaded Solr tgz file |
+| artifact_path | Local path to the Solr tgz file |
 | checksum      | Path/URL to checksum |
 | solr_xml      | Path to Solr configuration |
 | verbose       | (Boolean) |
@@ -46,6 +46,7 @@ SolrWrapper.wrap port: 8983,
 | solr_options  | (Hash) |
 | env           | (Hash) |
 | persist      | (Boolean) Preserves the data in you collection between startups |
+| contrib | (Array<Hash>) |
 
 ```ruby
 solr.with_collection(name: 'collection_name', dir: 'path_to_solr_configs')

data/exe/solr_wrapper

@@ -39,8 +39,8 @@ args = OptionParser.new do |opts|
     options[:cloud] = c
   end
 
-  opts.on("--solr_zip_path PATH", "Download/use solr at the given path") do |d|
-    options[:solr_zip_path] = d
+  opts.on("--artifact_path PATH", "Download/use solr at the given path") do |d|
+    options[:artifact_path] = d
   end
 
   opts.on("-iDIR", "--instance_directory DIR", "Install/use solr at the given directory") do |d|

data/lib/solr_wrapper/checksum_validator.rb

@@ -28,7 +28,7 @@ module SolrWrapper
         if config.default_download_url == config.static_config.archive_download_url
           "#{config.default_download_url}.#{suffix}"
         else
-          "https://archive.apache.org/dist/lucene/solr/#{config.static_config.version}/solr-#{config.static_config.version}.zip.#{suffix}"
+          "https://archive.apache.org/dist/#{config.mirror_artifact_path}.#{suffix}"
         end
       end
 

data/lib/solr_wrapper/configuration.rb

@@ -54,8 +54,8 @@ module SolrWrapper
       options[:zookeeper_port]
     end
 
-    def solr_zip_path
-      options[:solr_zip_path]
+    def artifact_path
+      options[:artifact_path]
     end
 
     def download_dir
@@ -93,13 +93,21 @@ module SolrWrapper
       end
     end
 
+    def mirror_artifact_path
+      if version > '9'
+        "solr/solr/#{version}/solr-#{version}.tgz"
+      else
+        "lucene/solr/#{version}/solr-#{version}.tgz"
+      end
+    end
+
     def closest_mirror_url
-      "https://www.apache.org/dyn/closer.lua/lucene/solr/#{version}/solr-#{version}.zip?asjson=true"
+      "https://www.apache.org/dyn/closer.lua/#{mirror_artifact_path}?asjson=true"
     end
 
     def mirror_url
       @mirror_url ||= if options[:mirror_url]
-        options[:mirror_url] + "lucene/solr/#{version}/solr-#{version}.zip"
+        options[:mirror_url] + mirror_artifact_path
       else
         begin
           json = HTTP.follow.get(closest_mirror_url).body
@@ -120,7 +128,7 @@ module SolrWrapper
     end
 
     def archive_download_url
-      "https://archive.apache.org/dist/lucene/solr/#{version}/solr-#{version}.zip"
+      "https://archive.apache.org/dist/#{mirror_artifact_path}"
     end
 
     def cloud
@@ -137,18 +145,22 @@ module SolrWrapper
 
     def collection_options
       hash = options.fetch(:collection, {})
-      Configuration.slice(convert_keys(hash), :name, :dir, :persist)
+      Configuration.slice(hash.transform_keys(&:to_sym), :name, :dir, :persist)
     end
 
     def configsets
       configsets = options[:configsets] || []
-      configsets.map { |x| convert_keys(x) }
+      configsets.map { |x| x.transform_keys(&:to_sym) }
     end
 
     def poll_interval
       options.fetch(:poll_interval, 1)
     end
 
+    def contrib
+      options.fetch(:contrib, [])
+    end
+
     private
 
       def self.slice(source, *keys)
@@ -174,11 +186,23 @@ module SolrWrapper
           $stderr.puts "Unable to parse config #{config_file}" if verbose?
           return {}
         end
-        convert_keys(config)
+
+        config.transform_keys!(&:to_sym)
+        absoluteize_paths(config, root: File.dirname(config_file))
       end
 
-      def convert_keys(hash)
-        hash.each_with_object({}) { |(k, v), h| h[k.to_sym] = v }
+      def absoluteize_paths(config, root: Dir.pwd)
+        return config unless config[:contrib]
+
+        config = config.dup
+
+        config[:contrib] = config[:contrib].map do |mapping|
+          mapping = mapping.transform_keys(&:to_sym)
+          mapping[:from] = File.expand_path(mapping[:from], root)
+          mapping[:to] ||= 'contrib/'
+        end
+
+        config
       end
 
       def default_configuration_paths

data/lib/solr_wrapper/instance.rb

@@ -6,10 +6,10 @@ require 'securerandom'
 require 'socket'
 require 'stringio'
 require 'tmpdir'
-require 'zip'
 require 'erb'
 require 'yaml'
 require 'retriable'
+require 'solr_wrapper/tgz_extractor'
 
 module SolrWrapper
   class Instance
@@ -23,9 +23,9 @@ module SolrWrapper
     # @option options [String] :port port to run Solr on
     # @option options [Boolean] :cloud Run solr in cloud mode
     # @option options [String] :version_file Local path to store the currently installed version
-    # @option options [String] :download_dir Local directory to store the downloaded Solr zip and its checksum file in (overridden by :solr_zip_path)
-    # @option options [String] :solr_zip_path Local path for storing the downloaded Solr zip file
-    # @option options [Boolean] :validate Should solr_wrapper download a new checksum and (re-)validate the zip file? (default: trueF)
+    # @option options [String] :download_dir Local directory to store the downloaded Solr artifact and its checksum file in (overridden by :artifact_path)
+    # @option options [String] :artifact_path Local path for storing the downloaded Solr artifact file
+    # @option options [Boolean] :validate Should solr_wrapper download a new checksum and (re-)validate the tgz file? (default: true)
     # @option options [String] :checksum Path/URL to checksum
     # @option options [String] :solr_xml Path to Solr configuration
     # @option options [String] :extra_lib_dir Path to directory containing extra libraries to copy into instance_dir/lib
@@ -36,6 +36,7 @@ module SolrWrapper
     # @option options [String] :config
     def initialize(options = {})
       @config = Settings.new(Configuration.new(options))
+      @started = false
     end
 
     def host
@@ -63,7 +64,7 @@ module SolrWrapper
       start
       yield self
     ensure
-      stop
+      stop if @started
     end
 
     ##
@@ -73,6 +74,8 @@ module SolrWrapper
       if managed?
         exec('start', p: port, c: config.cloud)
 
+        @started = true
+
         # Wait for solr to start
         unless status
           sleep config.poll_interval
@@ -241,6 +244,14 @@ module SolrWrapper
       raise_error_unless_extracted
       FileUtils.cp config.solr_xml, File.join(config.instance_dir, 'server', 'solr', 'solr.xml') if config.solr_xml
       FileUtils.cp_r File.join(config.extra_lib_dir, '.'), File.join(config.instance_dir, 'server', 'solr', 'lib') if config.extra_lib_dir
+
+      config.contrib.each do |mapping|
+        if File.directory? mapping[:from]
+          FileUtils.cp_r mapping[:from], File.join(config.instance_dir, mapping[:to])
+        else
+          FileUtils.cp mapping[:from], File.join(config.instance_dir, mapping[:to])
+        end
+      end
     end
 
     def extract_and_configure
@@ -255,25 +266,13 @@ module SolrWrapper
     def extract
       return config.instance_dir if extracted?
 
-      zip_path = download
+      downloaded_artifact_path = download
 
-      begin
-        Zip::File.open(zip_path) do |zip_file|
-          # Handle entries one by one
-          zip_file.each do |entry|
-            dest_file = File.join(config.tmp_save_dir, entry.name)
-            FileUtils.remove_entry(dest_file, true)
-            entry.extract(dest_file)
-          end
-        end
-
-      rescue Exception => e
-        abort "Unable to unzip #{zip_path} into #{config.tmp_save_dir}: #{e.message}"
-      end
+      SolrWrapper::TgzExtractor.new(downloaded_artifact_path, destination: config.tmp_save_dir).extract!
 
       begin
         FileUtils.remove_dir(config.instance_dir, true)
-        FileUtils.cp_r File.join(config.tmp_save_dir, File.basename(config.download_url, ".zip")), config.instance_dir
+        FileUtils.cp_r File.join(config.tmp_save_dir, File.basename(config.download_url, '.tgz')), config.instance_dir
         self.extracted_version = config.version
         FileUtils.chmod 0755, config.solr_binary
       rescue Exception => e
@@ -297,11 +296,11 @@ module SolrWrapper
     end
 
     def download
-      unless File.exist?(config.solr_zip_path) && checksum_validator.validate?(config.solr_zip_path)
-        Downloader.fetch_with_progressbar config.download_url, config.solr_zip_path
-        checksum_validator.validate! config.solr_zip_path
+      unless File.exist?(config.downloaded_artifact_path) && checksum_validator.validate?(config.downloaded_artifact_path)
+        Downloader.fetch_with_progressbar config.download_url, config.downloaded_artifact_path
+        checksum_validator.validate! config.downloaded_artifact_path
       end
-      config.solr_zip_path
+      config.downloaded_artifact_path
     end
 
     ##

data/lib/solr_wrapper/settings.rb

@@ -50,7 +50,7 @@ module SolrWrapper
 
     def instance_dir
       @instance_dir ||= static_config.instance_dir
-      @instance_dir ||= File.join(tmpdir, File.basename(download_url, ".zip"))
+      @instance_dir ||= File.join(tmpdir, File.basename(download_url, ".tgz"))
     end
 
     def managed?
@@ -62,9 +62,9 @@ module SolrWrapper
       @download_url ||= default_download_url
     end
 
-    def solr_zip_path
-      @solr_zip_path ||= static_config.solr_zip_path
-      @solr_zip_path ||= default_solr_zip_path
+    def downloaded_artifact_path
+      @downloaded_artifact_path ||= static_config.artifact_path
+      @downloaded_artifact_path ||= default_solr_artifact_path
     end
 
     def version_file
@@ -93,7 +93,7 @@ module SolrWrapper
         end
       end
 
-      def default_solr_zip_path
+      def default_solr_artifact_path
         File.join(download_dir, File.basename(download_url))
       end
 

data/lib/solr_wrapper/tgz_extractor.rb

@@ -0,0 +1,21 @@
+require 'minitar'
+require 'zlib'
+
+module SolrWrapper
+  class TgzExtractor
+    attr_reader :file, :destination
+
+    TAR_LONGLINK = '././@LongLink'
+
+    def initialize(file, destination: nil)
+      @file = file
+      @destination = destination || Dir.mktmpdir
+    end
+
+    def extract!
+      Minitar.unpack(Zlib::GzipReader.open(file), destination)
+    rescue StandardError => e
+      abort "Unable to extract #{file} into #{destination}: #{e.message}"
+    end
+  end
+end

data/lib/solr_wrapper/version.rb

@@ -1,3 +1,3 @@
 module SolrWrapper
-  VERSION = '3.1.3'
+  VERSION = '4.0.2'
 end

data/solr_wrapper.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "http"
-  spec.add_dependency "rubyzip"
+  spec.add_dependency "minitar"
   spec.add_dependency "ruby-progressbar"
   spec.add_dependency "retriable"
 

data/spec/fixtures/basic_configs/schema.xml

@@ -16,10 +16,10 @@
  limitations under the License.
 -->
 
-<!--  
+<!--
  This is the Solr schema file. This file should be named "schema.xml" and
  should be in the conf directory under the solr home
- (i.e. ./solr/conf/schema.xml by default) 
+ (i.e. ./solr/conf/schema.xml by default)
  or located where the classloader for the Solr webapp can find it.
 
  This example schema is the recommended starting point for users.
@@ -31,26 +31,26 @@
 
 <schema name="example" version="1.5">
   <!-- attribute "name" is the name of this schema and is only used for display purposes.
-       version="x.y" is Solr's version number for the schema syntax and 
+       version="x.y" is Solr's version number for the schema syntax and
        semantics.  It should not normally be changed by applications.
 
-       1.0: multiValued attribute did not exist, all fields are multiValued 
+       1.0: multiValued attribute did not exist, all fields are multiValued
             by nature
-       1.1: multiValued attribute introduced, false by default 
-       1.2: omitTermFreqAndPositions attribute introduced, true by default 
+       1.1: multiValued attribute introduced, false by default
+       1.2: omitTermFreqAndPositions attribute introduced, true by default
             except for text fields.
        1.3: removed optional field compress feature
        1.4: autoGeneratePhraseQueries attribute introduced to drive QueryParser
-            behavior when a single string produces multiple tokens.  Defaults 
+            behavior when a single string produces multiple tokens.  Defaults
             to off for version >= 1.4
-       1.5: omitNorms defaults to true for primitive field types 
+       1.5: omitNorms defaults to true for primitive field types
             (int, float, boolean, string...)
      -->
 
 
    <!-- Valid attributes for fields:
      name: mandatory - the name for the field
-     type: mandatory - the name of a field type from the 
+     type: mandatory - the name of a field type from the
        <types> fieldType section
      indexed: true if this field should be indexed (searchable or sortable)
      stored: true if this field should be retrievable
@@ -73,9 +73,9 @@
        given field.
        When using MoreLikeThis, fields used for similarity should be
        stored for best performance.
-     termPositions: Store position information with the term vector.  
+     termPositions: Store position information with the term vector.
        This will increase storage costs.
-     termOffsets: Store offset information with the term vector. This 
+     termOffsets: Store offset information with the term vector. This
        will increase storage costs.
      required: The field is required.  It will throw an error if the
        value does not exist
@@ -92,26 +92,26 @@
 
    <!-- If you remove this field, you must _also_ disable the update log in solrconfig.xml
       or Solr won't start. _version_ and update log are required for SolrCloud
-   --> 
+   -->
    <field name="_version_" type="long" indexed="true" stored="true"/>
-   
+
    <!-- points to the root document of a block of nested documents. Required for nested
       document support, may be removed otherwise
    -->
    <field name="_root_" type="string" indexed="true" stored="false"/>
 
    <!-- Only remove the "id" field if you have a very good reason to. While not strictly
-     required, it is highly recommended. A <uniqueKey> is present in almost all Solr 
+     required, it is highly recommended. A <uniqueKey> is present in almost all Solr
      installations. See the <uniqueKey> declaration below where <uniqueKey> is set to "id".
-   -->   
-   <field name="id" type="string" indexed="true" stored="true" required="true" multiValued="false" /> 
+   -->
+   <field name="id" type="string" indexed="true" stored="true" required="true" multiValued="false" />
 
    <!-- Dynamic field definitions allow using convention over configuration
-       for fields via the specification of patterns to match field names. 
+       for fields via the specification of patterns to match field names.
        EXAMPLE:  name="*_i" will match any field ending in _i (like myid_i, z_i)
        RESTRICTION: the glob-like pattern in the name attribute must have
        a "*" only at the start or the end.  -->
-   
+
    <dynamicField name="*_i"  type="int"    indexed="true"  stored="true"/>
    <dynamicField name="*_is" type="int"    indexed="true"  stored="true"  multiValued="true"/>
    <dynamicField name="*_s"  type="string"  indexed="true"  stored="true" />
@@ -133,7 +133,6 @@
 
    <dynamicField name="*_dt"  type="date"    indexed="true"  stored="true"/>
    <dynamicField name="*_dts" type="date"    indexed="true"  stored="true" multiValued="true"/>
-   <dynamicField name="*_p"  type="location" indexed="true" stored="true"/>
 
    <!-- some trie-coded dynamic fields for faster range queries -->
    <dynamicField name="*_ti" type="tint"    indexed="true"  stored="true"/>
@@ -149,13 +148,13 @@
 
    <dynamicField name="random_*" type="random" />
 
-   <!-- uncomment the following to ignore any fields that don't already match an existing 
-        field name or dynamic field, rather than reporting them as an error. 
-        alternately, change the type="ignored" to some other type e.g. "text" if you want 
-        unknown fields indexed and/or stored by default --> 
+   <!-- uncomment the following to ignore any fields that don't already match an existing
+        field name or dynamic field, rather than reporting them as an error.
+        alternately, change the type="ignored" to some other type e.g. "text" if you want
+        unknown fields indexed and/or stored by default -->
    <!--dynamicField name="*" type="ignored" multiValued="true" /-->
 
- <!-- Field to use to determine and enforce document uniqueness. 
+ <!-- Field to use to determine and enforce document uniqueness.
       Unless this field is marked with required="false", it will be a required field
    -->
  <uniqueKey>id</uniqueKey>
@@ -168,7 +167,7 @@
    <copyField source="title" dest="text"/>
    <copyField source="body" dest="text"/>
   -->
-  
+
     <!-- field type definitions. The "name" attribute is
        just a label to be used by field definitions.  The "class"
        attribute and any other attributes determine the real
@@ -200,7 +199,7 @@
        - If sortMissingLast="false" and sortMissingFirst="false" (the default),
          then default lucene sorting will be used which places docs without the
          field first in an ascending sort and last in a descending sort.
-    -->    
+    -->
 
     <!--
       Default numeric field types. For faster range queries, consider the tint/tfloat/tlong/tdouble types.
@@ -230,7 +229,7 @@
 
     <!-- The format for this date field is of the form 1995-12-31T23:59:59Z, and
          is a more restricted form of the canonical representation of dateTime
-         http://www.w3.org/TR/xmlschema-2/#dateTime    
+         http://www.w3.org/TR/xmlschema-2/#dateTime
          The trailing "Z" designates UTC time and is mandatory.
          Optional fractional seconds are allowed: 1995-12-31T23:59:59.999Z
          All other components are mandatory.
@@ -245,7 +244,7 @@
                NOW/DAY+6MONTHS+3DAYS
                   ... 6 months and 3 days in the future from the start of
                       the current day
-                      
+
          Consult the TrieDateField javadocs for more information.
 
          Note: For faster range queries, consider the tdate type
@@ -261,11 +260,11 @@
 
     <!-- The "RandomSortField" is not used to store or search any
          data.  You can declare fields of this type it in your schema
-         to generate pseudo-random orderings of your docs for sorting 
+         to generate pseudo-random orderings of your docs for sorting
          or function purposes.  The ordering is generated based on the field
          name and the version of the index. As long as the index version
          remains unchanged, and the same field name is reused,
-         the ordering of the docs will be consistent.  
+         the ordering of the docs will be consistent.
          If you want different psuedo-random orderings of documents,
          for the same version of the index, use a dynamicField and
          change the field name in the request.
@@ -456,13 +455,13 @@
         <filter class="solr.TrimFilterFactory" />
         <!-- The PatternReplaceFilter gives you the flexibility to use
              Java Regular expression to replace any sequence of characters
-             matching a pattern with an arbitrary replacement string, 
+             matching a pattern with an arbitrary replacement string,
              which may include back references to portions of the original
              string matched by the pattern.
-             
+
              See the Java Regular Expression documentation for more
              information on pattern and replacement string syntax.
-             
+
              http://docs.oracle.com/javase/7/docs/api/java/util/regex/package-summary.html
           -->
         <filter class="solr.PatternReplaceFilterFactory"
@@ -480,12 +479,12 @@
     </fieldType>
 
     <!-- since fields of this type are by default not stored or indexed,
-         any data added to them will be ignored outright.  --> 
+         any data added to them will be ignored outright.  -->
     <fieldType name="ignored" stored="false" indexed="false" multiValued="true" class="solr.StrField" />
 
     <!-- This point type indexes the coordinates as separate fields (subFields)
       If subFieldType is defined, it references a type, and a dynamic field
-      definition is created matching *___<typename>.  Alternately, if 
+      definition is created matching *___<typename>.  Alternately, if
       subFieldSuffix is defined, that is used to create the subFields.
       Example: if subFieldType="double", then the coordinates would be
         indexed in fields myloc_0___double,myloc_1___double.
@@ -496,9 +495,6 @@
      -->
     <fieldType name="point" class="solr.PointType" dimension="2" subFieldSuffix="_d"/>
 
-    <!-- A specialized field for geospatial search. If indexed, this fieldType must not be multivalued. -->
-    <fieldType name="location" class="solr.LatLonType" subFieldSuffix="_coordinate"/>
-
     <!-- An alternative geospatial field type new to Solr 4.  It supports multiValued and polygon shapes.
       For more information about this and other Spatial fields new to Solr 4, see:
       http://wiki.apache.org/solr/SolrAdaptersForLuceneSpatial4

data/spec/lib/solr_wrapper/configuration_spec.rb

@@ -101,7 +101,7 @@ describe SolrWrapper::Configuration do
       let(:options) { { mirror_url: 'http://lib-solr-mirror.princeton.edu/dist/', version: '7.2.1' } }
 
       it 'is the URL to the artifact on the preferred mirror' do
-        expect(config.mirror_url).to eq 'http://lib-solr-mirror.princeton.edu/dist/lucene/solr/7.2.1/solr-7.2.1.zip'
+        expect(config.mirror_url).to eq 'http://lib-solr-mirror.princeton.edu/dist/lucene/solr/7.2.1/solr-7.2.1.tgz'
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: solr_wrapper
 version: !ruby/object:Gem::Version
-  version: 3.1.3
+  version: 4.0.2
 platform: ruby
 authors:
 - Chris Beer
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-03-22 00:00:00.000000000 Z
+date: 2022-05-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: http
@@ -25,7 +25,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rubyzip
+  name: minitar
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -193,6 +193,7 @@ files:
 - lib/solr_wrapper/runner.rb
 - lib/solr_wrapper/settings.rb
 - lib/solr_wrapper/tasks/solr_wrapper.rake
+- lib/solr_wrapper/tgz_extractor.rb
 - lib/solr_wrapper/version.rb
 - solr_wrapper.gemspec
 - spec/fixtures/another_sample_config.yml