gi_cat_driver 0.1.0 → 0.1.1

data/README.md

@@ -44,6 +44,9 @@ Rubydoc API documentation is available at http://rubydoc.info/gems/gi_cat_driver
 
 ## Version History
 
+* 0.1.1
+  * Added parameter to ESIP OpenSearch query builder
+  * Added log output for GI-Cat harvest procedures
 * 0.1.0
   * Release stable version of gem features that invoke GI-Cat
 * 0.0.8
@@ -68,15 +71,13 @@ This gem follows the principles of [Semantic Versioning 2.0.0](http://semver.org
 ## Releasing
 
 1. Make sure to increment the version number (See the section about versioning above) and append a description of your changes to the Version History section above.
-2. Commit changes into the master branch of the repo on sourcecontrol.nsidc.org
-3. The commit will trigger a Jenkins job to run the tests
-4. When the changes are ready to be released, you can either push the gem manually or run a job in Jenkins
-4a. To push manually: You must have permissions on the GitHub repository through the NSIDC organization and you must be an owner of the 'gi_cat_driver'a gem on RubyGems.
-Once you have the credentials, run the command 'rake release' to tag the version number in the Git revision history and publish on RubyGems.
-To push to GitHub make sure you have the remotes configured (run 'git remote -v' to list available remotes. To add the GitHub repository to your remotes, run 'git remote add github https://github.com/nsidc/gi_cat_driver.git')
-4b: Using Jenkins: Run the job named 'GI-Cat_(gem)_Deploy_gi_cat_driver'
-
-NOTE: RubyGems will not allow you to release the same version of a gem more than once.
+2. Commit changes into the master branch of the repo on 'sourcecontrol.nsidc.org'. This will trigger a Jenkins job to run the tests.
+4. Assuming the change is merged with the master branch and you are ready to release them to GitHub run 'git push https://github.com/nsidc/gi_cat_driver.git master'
+      * There is a Jenkins job that can push master to GitHub if you are confident you wont have merge conflicts.
+      * Note: You must have permissions on the GitHub repository through the NSIDC organization.
+5. Make sure to release the new gem version to RubyGems by running 'rake release'
+      * Note: You must be configured as an owner of the 'gi_cat_driver'a gem on RubyGems.
+      * Also Note: RubyGems will not allow you to release the same version of a gem more than once.
 
 ## Contributing
 

data/lib/gi_cat_driver/esip_opensearch_query_builder.rb

@@ -9,17 +9,18 @@ module EsipOpensearchQueryBuilder
   end
 
   def self.get_query_string( query_params={} )
-    all_params = { 
-      :si => '', 
-      :ct => '', 
-      :st => '', 
-      :bbox => '', 
-      :rel => '', 
-      :loc => '', 
-      :ts => '', 
-      :te => '', 
-      :lac => '', 
-      :luc => '', 
+    all_params = {
+      :si => '',
+      :ct => '',
+      :st => '',
+      :bbox => '',
+      :rel => '',
+      :loc => '',
+      :ts => '',
+      :te => '',
+      :lac => '',
+      :luc => '',
+      :gdc => '',
       :outputFormat => ''
     }.merge(query_params)
 

data/lib/gi_cat_driver/version.rb

@@ -1,3 +1,3 @@
 module GiCatDriver
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

data/lib/gi_cat_driver.rb

@@ -12,13 +12,14 @@ module GiCatDriver
 
     ATOM_NAMESPACE = { "atom" => "http://www.w3.org/2005/Atom" }
     RELEVANCE_NAMESPACE = { "relevance" => "http://a9.com/-/opensearch/extensions/relevance/1.0/" }
-    attr_accessor :base_url, :harvestersid_array
-    
+    attr_accessor :base_url, :harvestersid_array, :harvestersinfo_array
+
     def initialize( url, username, password )
       @base_url = url.sub(/\/+$/, '')
       @admin_username = username
       @admin_password = password
       @harvestersid_array = []
+      @harvestersinfo_array = {}
     end
 
     def basic_auth_string
@@ -51,12 +52,6 @@ module GiCatDriver
       return (profile.empty? ? nil : profile.attr('id').value)
     end
 
-    def parse_profile_element( profile_name, xml_doc )
-      configs = Nokogiri.XML(xml_doc)
-
-      return configs.css("brokerConfiguration[name=#{profile_name}]")
-    end
-
     # Enable a profile with the specified name
     def enable_profile( profile_name )
       profile_id = find_profile_id(profile_name)
@@ -66,6 +61,17 @@ module GiCatDriver
       RestClient.get(activate_profile_request, standard_headers)
     end
 
+    # Given a profile id, put all the associated resource id and title into harvest info array
+    def get_harvest_resources(profile_id)
+      id = get_active_profile_distributor_id(profile_id)
+      harvest_resource_request = "#{@base_url}/services/conf/brokerConfigurations/#{profile_id}/distributors/#{id}"
+      response = RestClient.get(harvest_resource_request, standard_headers)
+      doc = Nokogiri::XML(response) 
+      doc.css("component").each do |component|
+        @harvestersinfo_array[component.css("id").text.to_sym] = component.css("title").text
+      end
+    end
+
     # Retrieve the ID for the active profile
     # Returns an integer ID reference to the active profile
     def get_active_profile_id
@@ -78,23 +84,12 @@ module GiCatDriver
     def enable_lucene
       set_lucene_enabled true
     end
-    
+
     # Disable Lucene indexes for GI-Cat search results
     def disable_lucene
       set_lucene_enabled false
     end
 
-    def set_lucene_enabled( enabled )
-      enable_lucene_request = "#{@base_url}/services/conf/brokerConfigurations/#{get_active_profile_id}/luceneEnabled"
-      RestClient.put(enable_lucene_request,
-        enabled.to_s,
-        standard_headers)
-
-      activate_profile_request = "#{@base_url}/services/conf/brokerConfigurations/#{get_active_profile_id}?opts=active"
-      RestClient.get(activate_profile_request,
-        standard_headers)
-    end
-
     # Find out whether Lucene indexing is turned on for the current profile
     # It is desirable to use REST to query GI-Cat for the value of this setting but GI-Cat does not yet support this.  
     # Instead, run a query and check that a 'relevance:score' element is present.
@@ -108,59 +103,119 @@ module GiCatDriver
 
       return result_scores.count > 0
     end
-    
-    # Build the harvester resource id array
+
+    # Build the harvester resource id 
     def add_harvester_resource_id(resource_id)
       @harvestersid_array.push(resource_id)
+      @harvestersinfo_array[resource_id.to_sym] = "default"
+    end
+
+    # Remove the harvester resourceinfo array
+    def clear_resource
+      @harvestersinfo_array.clear
+    end
+
+    # Harvest all resource in the active profile
+    def harvest_all_resources_for_active_configuration
+      get_harvest_resources(get_active_profile_id)
+      @harvestersinfo_array.each do |harvester_id, harvester_title|
+        harvest_resource_for_active_configuration(harvester_id.to_s, harvester_title)
+      end
+    end
+
+    # Run till harvest all the resources are completed or time out
+    # The default timeout is 300 seconds (5 minutes)
+    def confirm_harvest_done(waitmax=300)
+      begin
+        puts "Info: Max wait time (timeout) for current profile is set to #{waitmax} seconds"
+        Timeout::timeout(waitmax) do
+          @harvestersinfo_array.each  do |harvester_id, harvester_title|
+            havest_request_is_done(harvester_id.to_s, harvester_title)
+          end
+        end
+      rescue Timeout::Error
+        puts "Warning: re-harvest is time out(#{waitmax} seconds, we are going to reuse the previous harvest results"
+      end
+    end
+
+    private
+
+    # Retrieve the profile element using the name
+    def parse_profile_element( profile_name, xml_doc )
+      configs = Nokogiri.XML(xml_doc)
+
+      return configs.css("brokerConfiguration[name=#{profile_name}]")
+    end
+
+    # Retrive the distributor id given a profile id
+    def get_active_profile_distributor_id(id)
+      active_profile_request = "#{@base_url}/services/conf/brokerConfigurations/#{id}"
+      response = RestClient.get(active_profile_request, standard_headers)
+      id = Nokogiri::XML(response).css("component id").text
+      return id
+    end
+
+    # Toggle lucene indexes on when enabled is true, off when false
+    def set_lucene_enabled( enabled )
+      enable_lucene_request = "#{@base_url}/services/conf/brokerConfigurations/#{get_active_profile_id}/luceneEnabled"
+      RestClient.put(enable_lucene_request,
+        enabled.to_s,
+        standard_headers)
+
+      activate_profile_request = "#{@base_url}/services/conf/brokerConfigurations/#{get_active_profile_id}?opts=active"
+      RestClient.get(activate_profile_request,
+        standard_headers)
     end
 
     # Harvest from specified resource
-    def harvest_resource_for_active_configuration(harvesterid)
+    def harvest_resource_for_active_configuration(harvesterid, harvestername = "n/a")
       RestClient.get(
         "#{@base_url}/services/conf/brokerConfigurations/#{self.get_active_profile_id}/harvesters/#{harvesterid}/start",
         standard_headers){ |response, request, result, &block|
           case response.code
             when 200
-              p "Harvest initiated.  Please wait a couple minutes for the process to complete."
+              puts "#{Time.now}: Initiate harvesting GI-Cat resource #{harvestername}. Please wait a couple minutes for the process to complete."
             else
-              raise "Failed to start GI-Cat resource harvest."
+              raise "Failed to initiate harvesting GI-Cat resource #{harvestername}."
               response.return!(request, result, &block)
             end
         }
     end
 
-    # Harvest all resource in the active profile
-    def harvest_all_resources_for_active_configuration
-      harvestersid_array.each do |harvesterid|
-        harvest_resource_for_active_configuration(harvesterid)
-      end
-    end
-
+    # Parsing and handle the harvest status
+     def handle_harvest_status(harvest_status, harvestername="default")
+       timestamp = Time.now
+       puts "#{Time.now}: Harvest #{harvestername} status: #{harvest_status}"
+       case harvest_status
+       when /completed/
+         return :completed
+       when /error/
+         return :error
+       else 
+         return :pending
+       end
+     end
+     
     # Run till the harvest of a resource is completed
-    def havest_request_is_done(harvesterid)
+    def havest_request_is_done(harvesterid, harvestername="n/a")
       while(1) do
         rnum=rand
         request = @base_url + "/services/conf/giconf/status?id=#{harvesterid}&rand=#{rnum}"
         response = RestClient.get request
-        teststring = String.new(response.body)
-        if teststring.include?("Harvesting completed")
-          break
-        end
-      end
-    end
     
-    # Run till harvest all the resource is completed or time out
-    def confirm_harvest_done
-      begin
-        Timeout::timeout(10) do
-          harvestersid_array.each  do |harvesterid|
-            havest_request_is_done(harvesterid)
+        responsexml = Nokogiri::XML::Reader(response)
+        responsexml.each do |node|
+          if node.name == "status" && !node.inner_xml.empty? 
+            case handle_harvest_status(node.inner_xml, harvestername) 
+            when :error
+              fail "Error harvesting the resource #{harvestername}: #{harvest_status}"
+            when :completed
+              puts "Harvest succeed"
+              return
+            end
           end
         end
-      rescue Timeout::Error
-        puts "Warning: reharvest is time out, we are going to reuse the previous harvest results"
       end
     end
-    
   end
 end

data/spec/esip_opensearch_query_builder_spec.rb

@@ -6,17 +6,17 @@ describe EsipOpensearchQueryBuilder do
   describe "get_query_string returns ESIP OpenSearch URLs" do
     it "Returns a URL with empty parameters when called without arguments" do
       query = EsipOpensearchQueryBuilder::get_query_string()
-      query.should eq "?si=&ct=&st=&bbox=&rel=&loc=&ts=&te=&lac=&luc=&outputFormat="
+      query.should eq "?si=&ct=&st=&bbox=&rel=&loc=&ts=&te=&lac=&luc=&gdc=&outputFormat="
     end
 
     it "Returns a URL with a bounding box when called with a bbox argument" do
       query = EsipOpensearchQueryBuilder::get_query_string( :bbox => 'abcd')
-      query.should eq "?si=&ct=&st=&bbox=abcd&rel=&loc=&ts=&te=&lac=&luc=&outputFormat="
+      query.should eq "?si=&ct=&st=&bbox=abcd&rel=&loc=&ts=&te=&lac=&luc=&gdc=&outputFormat="
     end
 
     it "Returns a URL with a search term when called with a st argument" do
       query = EsipOpensearchQueryBuilder::get_query_string( :st => 'snow')
-      query.should eq "?si=&ct=&st=snow&bbox=&rel=&loc=&ts=&te=&lac=&luc=&outputFormat="
+      query.should eq "?si=&ct=&st=snow&bbox=&rel=&loc=&ts=&te=&lac=&luc=&gdc=&outputFormat="
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gi_cat_driver
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
   prerelease: 
 platform: ruby
 authors:
@@ -10,11 +10,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-20 00:00:00.000000000 Z
+date: 2013-04-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &70253266626500 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -22,15 +22,10 @@ dependencies:
         version: 1.6.7
   type: :runtime
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 1.6.7
+  version_requirements: *70253266626500
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &70253266625340 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -38,15 +33,10 @@ dependencies:
         version: 1.5.6
   type: :runtime
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 1.5.6
+  version_requirements: *70253266625340
 - !ruby/object:Gem::Dependency
   name: bundler
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &70253266624460 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,15 +44,10 @@ dependencies:
         version: '1.3'
   type: :development
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '1.3'
+  version_requirements: *70253266624460
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &70253266623780 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -70,15 +55,10 @@ dependencies:
         version: 10.0.3
   type: :development
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 10.0.3
+  version_requirements: *70253266623780
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &70253266623000 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -86,12 +66,7 @@ dependencies:
         version: 2.13.0
   type: :development
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 2.13.0
+  version_requirements: *70253266623000
 description: Configure and control deployed instances of GI-Cat.
 email:
 - stuart.reed@nsidc.org
@@ -135,7 +110,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 1.8.15
 signing_key: 
 specification_version: 3
 summary: Configure and control deployed instances of GI-Cat.