propublica-nonprofits 0.4.1 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9f22696c280743b8ad727b26a7e42d245a6860df751e93de02417ccec08f886d
-  data.tar.gz: 633d7e23680142bb516fd2d5b5299fcd0c6db79a0162cfe5d4fb0d72f9961c77
+  metadata.gz: c05c2a3977841806bac525c3c642db9a94bf0755c3d098c4852276b092b49168
+  data.tar.gz: ecbbd8799e9d43592ea52d8e2ab4754f74fef539dec2a3eab0d7a415513acf1b
 SHA512:
-  metadata.gz: c7f1e635ba864223d2a962745d0643d67a77bc722dc1c07e8bd217e24ff80875a06ab9f9dbb12c03b5ca8ba6a2930ff94cdbb71773bd63b43bdf6dae993060fb
-  data.tar.gz: 9a94138157fb0f95608cc181e1cb5fe071e42c1df9cec12004b6c0740b6f09b1f900689cade98616c88f41244613b258d17e0e03b462b38e38c898c341063fdb
+  metadata.gz: 1e30af0a7b67f51505022bf3a6e8b63f91fff5bb702c5a531dbed24585194d8e9d0f966b4097d128c67d1a98bc26e44ca533acd64e4681fa6d7ab2ef17da55c3
+  data.tar.gz: 9c34f58bc0c29ccabd62af0d62974f71f3557a907551e0f83521e657b20e5534f4afa9c78d1203dd83270be48cd3f7420059dba90069e263eb885c1e58628c30

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    propublica-nonprofits (0.4.1)
+    propublica-nonprofits (1.0.0)
       faraday
       json
 

data/README.md

@@ -40,8 +40,11 @@ To search the API and return an array of Propublica::Nonprofits::Objects, use th
     result.details.deductibility_code
     result.filings_with_data.first.totassetsend
 
+    Want to be as memory and API efficient as possible? Be lazy!
+    `Propublica::Nonprofits.lazy_search("rural").first(10)` will only fetch first 10 items
+
 *Note: This will currently only return the first 100 organizations*
-*Note: This is returned as a Enumerator::Lazy, to be as memory efficient as possible, if you do something like `Propublica::Nonprofits.search("rural").first(10)`*
+*Note: This is returned as an Array*
 
 
 ### EIN Lookup - As Organization
@@ -80,11 +83,6 @@ consider using the `find_attributes` method.
     attributes.dig("organization", "name")
     attributes.dig("filings_without_data").first.dig("pdf_url")
 
-## TODO
-
-- Add more robust searching parameters https://projects.propublica.org/nonprofits/api/#endpoint-search-example
-- Handle paginated search results (with Enumerator::Lazy)
-
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/lib/propublica/nonprofits.rb

@@ -5,44 +5,74 @@ require "faraday"
 require "json"
 
 API_BASE_URL = %(https://projects.propublica.org)
-API_SEARCH_PATH = %(/nonprofits/api/v2/search.json)
+API_SEARCH_PATH = %(nonprofits/api/v2/search.json)
+
+def self.results(column_names)
+  columns = column_names.each
+
+  Enumerator.new do |yielder|
+    loop do
+      yielder << Event.where(columns.next => query)
+    end
+
+    yielder << Event.fuzzy_search(query)
+  end
+end
 
 module Propublica
   module Nonprofits
-    def self.search(term, state: nil, page: nil, fetch_all: false)
-      organizations = []
-      more_pages = true
-
-      page =
-        case
-        when page
-          page
-        when fetch_all && page
-          raise "Page is set but we are fetching all, chose one or the other"
-        else
-          0
-        end
+    class DataNotFetched < StandardError
+    end
 
-      while (more_pages)
-        response = connection.get do |request|
-          request.url(API_SEARCH_PATH)
+    def self.search(term, state: nil, ntee: nil, page: nil, fetch_all: false)
+      search_results(term, state: state, ntee: ntee, page: page, fetch_all: fetch_all)
+        .flat_map(&:itself)
+    end
 
-          request.params["q"] = term
-          request.params["state[id]"] = state if state
-          request.params["page"] = page if page
-        end
-        
+    def self.lazy_search(term, state: nil, ntee: nil, page: nil, fetch_all: false)
+      search_results(term, state: state, ntee: ntee, page: page, fetch_all: fetch_all)
+        .lazy
+        .flat_map(&:itself)
+    end
 
-        attributes = JSON.parse(response.body)
-        new_organizations = attributes.fetch("organizations", [])
-        organizations.push(*new_organizations)
+    def self.search_results(term, state: nil, ntee: nil, page: nil, fetch_all: false)
+      raise ArgumentError.new("`page` and `fetch_all` are both: choose one or the other") if fetch_all && page
+      page ||= 0
+      max_pages = nil
 
-        more_pages = fetch_all && new_organizations.any?
-        page += 1
-      end
+      Enumerator.new do |yielder|
+        loop do
+          params = {}
+          params["q"] = term
+          params["state[id]"] = state if state
+          params["ntee[id]"] = ntee if ntee
+          params["page"] = page if page
+
+          response = Faraday.default_connection.get("#{API_BASE_URL}/#{API_SEARCH_PATH}", params)
+          parsed_response =
+            if response.body.is_a?(Hash)
+              response.body
+            else
+              begin
+                JSON.parse(response.body)
+              rescue JSON::ParserError => e
+                raise JSON::ParserError.new("Propublica API Parsing Error: #{e.message}")
+              end
+            end
+
+          max_pages = parsed_response.dig("num_pages") || max_pages
 
-      organizations.lazy.map do |basic_attrs|
-        Propublica::Nonprofits::Organization.new("basic" => basic_attrs)
+          yielder <<
+            parsed_response
+              .fetch("organizations", [])
+              .map { |basic_attrs| Propublica::Nonprofits::Organization.new("basic" => basic_attrs) }
+
+          if fetch_all && page + 1 < max_pages
+            page += 1
+          else
+            raise(StopIteration)
+          end
+        end
       end
     end
 
@@ -52,15 +82,16 @@ module Propublica
     end
 
     def self.find_attributes(ein)
-      response = connection.get("/nonprofits/api/v2/organizations/#{ein}.json")
-      JSON.parse(response.body)
-    end
-
-    def self.connection
-      @connection ||= Faraday.new(url: API_BASE_URL)
+      response = Faraday.get("#{API_BASE_URL}/nonprofits/api/v2/organizations/#{ein}.json")
+      if response.body.is_a?(Hash)
+        response.body
+      else
+        begin
+          JSON.parse(response.body)
+        rescue JSON::ParserError => e
+          raise JSON::ParserError.new("Propublica API Parsing Error: #{e.message}")
+        end
+      end
     end
-
-    class Error < StandardError; end
-    class DataNotFetched < Error; end
   end
 end

data/lib/propublica/nonprofits/version.rb

@@ -1,5 +1,5 @@
 module Propublica
   module Nonprofits
-    VERSION = "0.4.1"
+    VERSION = "1.0.0"
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: propublica-nonprofits
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 1.0.0
 platform: ruby
 authors:
 - Ricky Chilcott