federal_register 0.7.6 → 0.7.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 63643d75cc0539e9244e7bcb4715569cdaabaa946467a527b14c3b1babc3f40c
-  data.tar.gz: 1a0a8b76e5e2deb3dc215cf2109e117989cc983d3caf7189575b25a7ab01e480
+  metadata.gz: 29108ec013bfbdf370e3ac4b2c5f2de8312ee5ed0d42fa4314a4ec67349ddaa7
+  data.tar.gz: 81773b6cedce1060dd60b0bc811f81e47db875e9e2feb0eefd39b8ce707687ce
 SHA512:
-  metadata.gz: 611b9c78a73332729c6fff76b56cc5d29e7ef5ecb1bae8357aaa8c7c9e1bef153c461235ec30b190c42a9ad83539572e198933ea59a611fd926173934b08799e
-  data.tar.gz: d0e720e9af49fcb7655b201b09fea1550f2b4f9fd2f0eeffbad4cbb8bee1601e36bb017009c8edb2147a69bbb645777f4d358485e829f11ac244713916b84b56
+  metadata.gz: cfaafa77066a18b25c3fb8168e677b9d5cf332da4cde50112a2e945adc8d0eedbaca0045d96617b20fc0e28eacbf5e3d26cf0425fc9629af1f257236aa260db2
+  data.tar.gz: 71f6cab320067badb01768221dec911e9bf57d777594a28b2f658489ac2bcf4e1162b74ceeb5dd3a549a116748d07a953903be776c20f9212342770672ca04b7

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    federal_register (0.6.7)
-      httparty (~> 0.14.0)
+    federal_register (0.7.7)
+      httparty (>= 0.14.0)
 
 GEM
   remote: https://rubygems.org/
@@ -14,10 +14,14 @@ GEM
     concurrent-ruby (1.0.5)
     diff-lcs (1.2.5)
     fakeweb (1.3.0)
-    httparty (0.14.0)
+    httparty (0.18.1)
+      mime-types (~> 3.0)
       multi_xml (>= 0.5.2)
     i18n (0.9.1)
       concurrent-ruby (~> 1.0)
+    mime-types (3.3.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2020.1104)
     multi_json (1.12.2)
     multi_xml (0.6.0)
     parallel (1.12.0)
@@ -35,6 +39,8 @@ GEM
     rspec-expectations (2.99.2)
       diff-lcs (>= 1.1.3, < 2.0)
     rspec-mocks (2.99.3)
+    rspec_junit_formatter (0.4.1)
+      rspec-core (>= 2, < 4, != 2.12.0)
     rubocop (0.51.0)
       parallel (~> 1.10)
       parser (>= 2.3.3.1, < 3.0)
@@ -60,8 +66,9 @@ DEPENDENCIES
   fakeweb (~> 1.3.0)
   federal_register!
   rspec (~> 2.6)
+  rspec_junit_formatter
   rubocop
   shoulda
 
 BUNDLED WITH
-   1.16.0
+   2.1.4

data/VERSION

@@ -1 +1 @@
-0.7.6
+0.7.7

data/federal_register.gemspec

@@ -5,7 +5,7 @@
 
 Gem::Specification.new do |s|
   s.name = "federal_register"
-  s.version = "0.7.6"
+  s.version = "0.7.7"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Andrew Carpenter", "Bob Burbach"]
@@ -35,6 +35,7 @@ Gem::Specification.new do |s|
     "lib/federal_register/document.rb",
     "lib/federal_register/document_image.rb",
     "lib/federal_register/document_search_details.rb",
+    "lib/federal_register/document_utilities.rb",
     "lib/federal_register/facet.rb",
     "lib/federal_register/facet/agency.rb",
     "lib/federal_register/facet/document.rb",
@@ -94,6 +95,7 @@ Gem::Specification.new do |s|
       s.add_development_dependency(%q<activesupport>, ["~> 3"])
       s.add_development_dependency(%q<fakeweb>, ["~> 1.3.0"])
       s.add_development_dependency(%q<rubocop>, [">= 0"])
+      s.add_development_dependency(%q<rspec_junit_formatter>, [">= 0"])
     else
       s.add_dependency(%q<httparty>, [">= 0.7.0"])
       s.add_dependency(%q<shoulda>, [">= 0"])
@@ -102,6 +104,7 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<activesupport>, ["~> 3"])
       s.add_dependency(%q<fakeweb>, ["~> 1.3.0"])
       s.add_dependency(%q<rubocop>, [">= 0"])
+      s.add_dependency(%q<rspec_junit_formatter>, [">= 0"])
     end
   else
     s.add_dependency(%q<httparty>, [">= 0.14.0"])
@@ -111,5 +114,6 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<activesupport>, ["~> 3"])
     s.add_dependency(%q<fakeweb>, ["~> 1.3.0"])
     s.add_dependency(%q<rubocop>, [">= 0"])
+    s.add_dependency(%q<rspec_junit_formatter>, [">= 0"])
   end
 end

data/lib/federal_register.rb

@@ -12,6 +12,7 @@ require "federal_register/result_set.rb"
 require "federal_register/public_inspection_issue_result_set.rb"
 require "federal_register/facet_result_set.rb"
 
+require "federal_register/document_utilities.rb"
 require "federal_register/agency.rb"
 require "federal_register/document.rb"
 require "federal_register/article.rb"

data/lib/federal_register/document.rb

@@ -1,5 +1,6 @@
 class FederalRegister::Document < FederalRegister::Base
   extend FederalRegister::Utilities
+  extend FederalRegister::DocumentUtilities
 
   add_attribute :abstract,
                 :abstract_html_url,
@@ -77,23 +78,8 @@ class FederalRegister::Document < FederalRegister::Base
     new(attributes, :full => true)
   end
 
-  # supports values like: '2016-26522', '2016-26522,2016-26594', '81 FR 76496', '81 FR 76496,81 FR 76685'
-  # note: no space after comma
-  def self.find_all(*args)
-    options, document_numbers_or_citations = extract_options(args)
-    document_numbers_or_citations.flatten!
-
-    fetch_options = {:result_class => self}
-    fetch_options.merge!(:query => {:fields => options[:fields]}) if options[:fields]
-
-    #TODO: fix this gross hack to ensure that find_all with a single document number
-    # is returned in the same way multiple document numbers are
-    if document_numbers_or_citations.size == 1
-      document_numbers_or_citations << " "
-    end
-
-    params = URI.encode(document_numbers_or_citations.compact.join(',').strip)
-    result_set = FederalRegister::ResultSet.fetch("/documents/#{params}.json", fetch_options)
+  def self.find_all_base_path
+    '/documents'
   end
 
   def agencies

data/lib/federal_register/document_utilities.rb

@@ -0,0 +1,57 @@
+module FederalRegister::DocumentUtilities
+  URL_CHARACTER_LIMIT = 2000
+
+  def find_all(*args)
+    options, document_numbers_or_citations = extract_options(args)
+    document_numbers_or_citations.flatten!
+
+    fetch_options = {:result_class => self}
+    fetch_options.merge!(:query => {:fields => options[:fields]}) if options[:fields]
+
+    document_numbers_or_citations.uniq!
+
+    if document_numbers_or_citations.empty?
+      raise "No documents or citation numbers were provided"
+    end
+
+    #TODO: fix this gross hack to ensure that find_all with a single document number
+    # is returned in the same way multiple document numbers are
+    if document_numbers_or_citations.size == 1
+      document_numbers_or_citations << " "
+    end
+
+    http_request_batches = calculate_request_batches(document_numbers_or_citations, fetch_options)
+
+    slice_size = (document_numbers_or_citations.count.to_f / http_request_batches).ceil
+    results    = []
+    document_numbers_or_citations.each_slice(slice_size).each do |slice|
+
+      params   = URI.encode(slice.join(',').strip)
+      url      = "#{find_all_base_path}/#{params}.json"
+      response = get(url, fetch_options).parsed_response
+      results += response['results']
+    end
+
+    FederalRegister::ResultSet.new(
+      {
+        'count'   => results.count,
+        'results' => results,
+      },
+      self
+    )
+  end
+
+  private
+
+  def calculate_request_batches(document_numbers_or_citations, fetch_options)
+    fetch_option_url_character_count = URI.encode_www_form(fetch_options).length # HTTPParty uses Net::HTTP
+    characters_available             = URL_CHARACTER_LIMIT - fetch_option_url_character_count
+    doc_number_character_count       = URI.encode(document_numbers_or_citations.compact.join(',').strip).length
+
+    if characters_available > doc_number_character_count
+      1
+    else
+      (doc_number_character_count.to_f / characters_available).ceil
+    end
+  end
+end

data/lib/federal_register/public_inspection_document.rb

@@ -1,5 +1,6 @@
 class FederalRegister::PublicInspectionDocument < FederalRegister::Base
   extend FederalRegister::Utilities
+  extend FederalRegister::DocumentUtilities
 
   add_attribute :agencies,
                 :docket_numbers,
@@ -43,21 +44,8 @@ class FederalRegister::PublicInspectionDocument < FederalRegister::Base
     FederalRegister::PublicInspectionIssueResultSet.fetch("/public-inspection-documents/current.json", :result_class => self)
   end
 
-  def self.find_all(*args)
-    options, document_numbers = extract_options(args)
-    document_numbers.flatten!
-
-    fetch_options = {:result_class => self}
-    fetch_options.merge!(:query => {:fields => options[:fields]}) if options[:fields]
-
-    #TODO: fix this gross hack to ensure that find_all with a single document number
-    # is returned in the same way multiple document numbers are
-    if document_numbers.size == 1
-      document_numbers << " "
-    end
-    
-    params = URI.encode(document_numbers.compact.join(',').strip)
-    result_set = FederalRegister::ResultSet.fetch("/public-inspection-documents/#{params}.json", fetch_options)
+  def self.find_all_base_path
+    '/public-inspection-documents'
   end
 
   def agencies

data/spec/document_spec.rb

@@ -16,9 +16,11 @@ describe FederalRegister::Document do
 
     it "fetches the document with only requested fields (when present)" do
       document_number = "2010-213"
+      params = URI.encode_www_form([["fields[]","title"],["fields[]", "start_page"]])
+
       FakeWeb.register_uri(
         :get,
-        "https://www.federalregister.gov/api/v1/documents/#{document_number}.json?fields[]=title&fields[]=start_page",
+        "https://www.federalregister.gov/api/v1/documents/#{document_number}.json?#{params}",
         :content_type => "text/json",
         :body => {:title => "Important Notice", :start_page => 12345}.to_json
       )
@@ -32,9 +34,11 @@ describe FederalRegister::Document do
     it "fetches the document with the provided publication date (when present)" do
       document_number = "2010-213"
       publication_date = "2010-02-02"
+      params = URI.encode_www_form([["publication_date", "#{publication_date}"]])
+      
       FakeWeb.register_uri(
         :get,
-        "https://www.federalregister.gov/api/v1/documents/#{document_number}.json?publication_date=#{publication_date}",
+        "https://www.federalregister.gov/api/v1/documents/#{document_number}.json?#{params}",
         :content_type => "text/json",
         :body => {:title => "Important Notice", :publication_date => publication_date}.to_json
       )
@@ -69,9 +73,11 @@ describe FederalRegister::Document do
     end
 
     it "fetches multiple matching documents with only requested fields (when present)" do
+      params = URI.encode_www_form([["fields[]","document_number"],["fields[]", "title"]])
+      
       FakeWeb.register_uri(
         :get,
-        "https://www.federalregister.gov/api/v1/documents/abc,def.json?fields[]=document_number&fields[]=title",
+        "https://www.federalregister.gov/api/v1/documents/abc,def.json?#{params}",
         :content_type =>"text/json",
         :body => {:results => [{:document_number => "abc", :title => "Important Notice"},
                                {:document_number => "def", :title => "Important Rule"}]}.to_json
@@ -81,13 +87,87 @@ describe FederalRegister::Document do
       result_set.results.map(&:title).sort.should === ['Important Notice','Important Rule']
       result_set.results.map(&:start_page).should === [nil, nil]
     end
+
+    it "handles results when no documents are returned" do
+      params = URI.encode_www_form([["fields[]","document_number"],["fields[]", "title"]])
+      FakeWeb.register_uri(
+        :get,
+        "https://www.federalregister.gov/api/v1/documents/bad_document_number,.json?#{params}",
+        :content_type =>"text/json",
+        :body => {"count":0,"results":[],"errors":{"not_found":["bad_docuemnt_number"]}}.to_json
+      )
+
+      result_set = FederalRegister::Document.find_all('bad_document_number', :fields => ["document_number", "title"])
+      result_set.count.should == 0
+    end
+
+    it "appends a trailing comma when a single document is provided to ensure API interface doesn't return a single document" do
+      params = URI.encode_www_form([["fields[]","document_number"],["fields[]", "title"]])
+      FakeWeb.register_uri(
+        :get,
+        "https://www.federalregister.gov/api/v1/documents/2016-26522,.json?#{params}",
+        :content_type =>"text/json",
+        :body => {:results => [{:document_number => "2016-26522", :title => "Important Notice"}],
+         }.to_json
+      )
+
+      FederalRegister::Document.find_all('2016-26522', :fields => ["document_number", "title"])
+    end
+
+    it "ensures document or citation numbers provided are unique before making requests to API Core." do
+      # Without calling unique, duplicate results could be returned since we're now batching up larger requests.
+      params = URI.encode_www_form([["fields[]","document_number"],["fields[]", "title"]])
+      FakeWeb.register_uri(
+        :get,
+        "https://www.federalregister.gov/api/v1/documents/2016-26522,.json?fields[]=#{params}",
+        :content_type =>"text/json",
+        :body => {:results => [{:document_number => "2016-26522", :title => "Important Notice"}],
+         }.to_json
+      )
+      FederalRegister::Document.find_all('2016-26522', '2016-26522', :fields => ["document_number", "title"])
+    end
+
+    #TODO: Add specs aroudn API response
+    it "batches up large requests when CGI parameters exceed the URL limit and returns the correct" do
+      params = URI.encode_www_form([["fields[]","document_number"],["fields[]", "title"]])
+      FakeWeb.register_uri(
+        :get,
+        "https://www.federalregister.gov/api/v1/documents/2016-26000,2016-26001,2016-26002,2016-26003,2016-26004,2016-26005,2016-26006,2016-26007,2016-26008,2016-26009,2016-26010,2016-26011,2016-26012,2016-26013,2016-26014,2016-26015,2016-26016,2016-26017,2016-26018,2016-26019,2016-26020,2016-26021,2016-26022,2016-26023,2016-26024,2016-26025,2016-26026,2016-26027,2016-26028,2016-26029,2016-26030,2016-26031,2016-26032,2016-26033,2016-26034,2016-26035,2016-26036,2016-26037,2016-26038,2016-26039,2016-26040,2016-26041,2016-26042,2016-26043,2016-26044,2016-26045,2016-26046,2016-26047,2016-26048,2016-26049,2016-26050,2016-26051,2016-26052,2016-26053,2016-26054,2016-26055,2016-26056,2016-26057,2016-26058,2016-26059,2016-26060,2016-26061,2016-26062,2016-26063,2016-26064,2016-26065,2016-26066,2016-26067,2016-26068,2016-26069,2016-26070,2016-26071,2016-26072,2016-26073,2016-26074,2016-26075,2016-26076,2016-26077,2016-26078,2016-26079,2016-26080,2016-26081,2016-26082,2016-26083,2016-26084,2016-26085,2016-26086,2016-26087,2016-26088,2016-26089,2016-26090,2016-26091,2016-26092,2016-26093,2016-26094,2016-26095,2016-26096,2016-26097,2016-26098,2016-26099,2016-26100.json?#{params}",
+        :content_type =>"text/json",
+        :body => {:results => [
+          {:document_number => "2016-26000", :title => "Important Notice"},
+          #...
+        ]}.to_json
+      )
+      FakeWeb.register_uri(
+        :get,
+        "https://www.federalregister.gov/api/v1/documents/2016-26101,2016-26102,2016-26103,2016-26104,2016-26105,2016-26106,2016-26107,2016-26108,2016-26109,2016-26110,2016-26111,2016-26112,2016-26113,2016-26114,2016-26115,2016-26116,2016-26117,2016-26118,2016-26119,2016-26120,2016-26121,2016-26122,2016-26123,2016-26124,2016-26125,2016-26126,2016-26127,2016-26128,2016-26129,2016-26130,2016-26131,2016-26132,2016-26133,2016-26134,2016-26135,2016-26136,2016-26137,2016-26138,2016-26139,2016-26140,2016-26141,2016-26142,2016-26143,2016-26144,2016-26145,2016-26146,2016-26147,2016-26148,2016-26149,2016-26150,2016-26151,2016-26152,2016-26153,2016-26154,2016-26155,2016-26156,2016-26157,2016-26158,2016-26159,2016-26160,2016-26161,2016-26162,2016-26163,2016-26164,2016-26165,2016-26166,2016-26167,2016-26168,2016-26169,2016-26170,2016-26171,2016-26172,2016-26173,2016-26174,2016-26175,2016-26176,2016-26177,2016-26178,2016-26179,2016-26180,2016-26181,2016-26182,2016-26183,2016-26184,2016-26185,2016-26186,2016-26187,2016-26188,2016-26189,2016-26190,2016-26191,2016-26192,2016-26193,2016-26194,2016-26195,2016-26196,2016-26197,2016-26198,2016-26199,2016-26200.json?#{params}",
+        :content_type =>"text/json",
+        :body => {:results => [
+          {:document_number => "2016-26000", :title => "Important Notice"},
+          #...
+        ]}.to_json
+      )
+
+      document_numbers = []
+      doc_suffix = 26000
+      201.times { document_numbers << "2016-#{doc_suffix}"; doc_suffix += 1 }
+
+      result_set = FederalRegister::Document.find_all(*document_numbers, :fields => ["document_number", "title"])
+
+      result_set.count.should       == 2
+      result_set.previous.should    == nil
+      result_set.next.should        == nil
+      result_set.total_pages.should == nil
+    end
   end
 
   describe ".search" do
     before(:each) do
+      params = URI.encode_www_form([["conditions[term]","Fish"]])
       FakeWeb.register_uri(
         :get,
-        "https://www.federalregister.gov/api/v1/documents.json?conditions[term]=Fish",
+        "https://www.federalregister.gov/api/v1/documents.json?#{params}",
         :content_type =>"text/json",
         :body => {:count => 3}.to_json
       )

data/spec/public_inspection_document_spec.rb

@@ -41,9 +41,10 @@ describe FederalRegister::PublicInspectionDocument do
 
   describe ".available_on" do
     it "fetches the document on PI on a given date" do
+      params = URI.encode_www_form([["conditions[available_on]", "2011-10-15"]])
       FakeWeb.register_uri(
         :get,
-        "https://www.federalregister.gov/api/v1/public-inspection-documents.json?conditions[available_on]=2011-10-15",
+        "https://www.federalregister.gov/api/v1/public-inspection-documents.json?#{params}",
         :content_type =>"text/json",
         :body => {:results => [{:document_number => "abc"}, {:document_number => "def"}]}.to_json
       )

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: federal_register
 version: !ruby/object:Gem::Version
-  version: 0.7.6
+  version: 0.7.7
 platform: ruby
 authors:
 - Andrew Carpenter
@@ -109,6 +109,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec_junit_formatter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Ruby API Client for FederalRegister.gov that handles searching documents
   and getting information about agencies
 email: andrew@criticaljuncture.org
@@ -136,6 +150,7 @@ files:
 - lib/federal_register/document.rb
 - lib/federal_register/document_image.rb
 - lib/federal_register/document_search_details.rb
+- lib/federal_register/document_utilities.rb
 - lib/federal_register/facet.rb
 - lib/federal_register/facet/agency.rb
 - lib/federal_register/facet/document.rb
@@ -195,8 +210,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6.2
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 3
 summary: Ruby API Client for FederalRegister.gov