propublica-nonprofits 0.4.1 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9f22696c280743b8ad727b26a7e42d245a6860df751e93de02417ccec08f886d
-  data.tar.gz: 633d7e23680142bb516fd2d5b5299fcd0c6db79a0162cfe5d4fb0d72f9961c77
+  metadata.gz: baea7efcfc907f93a2765cfff5c3343fdf17e817a09ac15160f1025990ceee5a
+  data.tar.gz: 1da616c76aa6cc85b49176018db027e75f5a7a39f9518abcc3a114190883fefe
 SHA512:
-  metadata.gz: c7f1e635ba864223d2a962745d0643d67a77bc722dc1c07e8bd217e24ff80875a06ab9f9dbb12c03b5ca8ba6a2930ff94cdbb71773bd63b43bdf6dae993060fb
-  data.tar.gz: 9a94138157fb0f95608cc181e1cb5fe071e42c1df9cec12004b6c0740b6f09b1f900689cade98616c88f41244613b258d17e0e03b462b38e38c898c341063fdb
+  metadata.gz: bf924b498d17cbea10dcf305ae1099359ebba0971d7f09cb6542d7c958be3e5309babd862be78e45674d93ec5d0d2c060374799f7daf9961c89b8aa441fd03fc
+  data.tar.gz: 93196885e1d02886c94e6b7e407b3ea552042dd348c3855fda889ad2a8c14992f681c115da932148934c731c9354844966f28c379c8f51a6135aad0d3811ed44

data/Gemfile.lock

@@ -1,52 +1,38 @@
 PATH
   remote: .
   specs:
-    propublica-nonprofits (0.4.1)
-      faraday
+    propublica-nonprofits (1.1.0)
+      faraday (> 1.0, < 3.0)
       json
 
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.8.0)
-      public_suffix (>= 2.0.2, < 5.0)
+    addressable (2.8.1)
+      public_suffix (>= 2.0.2, < 6.0)
     coderay (1.1.2)
-    crack (0.4.3)
-      safe_yaml (~> 1.0.0)
-    faraday (1.5.1)
-      faraday-em_http (~> 1.0)
-      faraday-em_synchrony (~> 1.0)
-      faraday-excon (~> 1.1)
-      faraday-httpclient (~> 1.0.1)
-      faraday-net_http (~> 1.0)
-      faraday-net_http_persistent (~> 1.1)
-      faraday-patron (~> 1.0)
-      multipart-post (>= 1.2, < 3)
+    crack (0.4.5)
+      rexml
+    faraday (2.7.2)
+      faraday-net_http (>= 2.0, < 3.1)
       ruby2_keywords (>= 0.0.4)
-    faraday-em_http (1.0.0)
-    faraday-em_synchrony (1.0.0)
-    faraday-excon (1.1.0)
-    faraday-httpclient (1.0.1)
-    faraday-net_http (1.0.1)
-    faraday-net_http_persistent (1.2.0)
-    faraday-patron (1.0.0)
-    hashdiff (1.0.0)
+    faraday-net_http (3.0.2)
+    hashdiff (1.0.1)
     json (2.5.1)
     method_source (0.9.0)
     minitest (5.11.3)
     minitest-line (0.6.5)
       minitest (~> 5.0)
-    multipart-post (2.1.1)
     pry (0.11.3)
       coderay (~> 1.1.0)
       method_source (~> 0.9.0)
-    public_suffix (4.0.6)
+    public_suffix (5.0.1)
     rake (13.0.1)
-    ruby2_keywords (0.0.4)
-    safe_yaml (1.0.5)
-    vcr (4.0.0)
-    webmock (3.7.6)
-      addressable (>= 2.3.6)
+    rexml (3.2.5)
+    ruby2_keywords (0.0.5)
+    vcr (6.1.0)
+    webmock (3.18.1)
+      addressable (>= 2.8.0)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
 
@@ -60,8 +46,9 @@ DEPENDENCIES
   propublica-nonprofits!
   pry
   rake (~> 13.0)
+  rexml
   vcr
   webmock
 
 BUNDLED WITH
-   2.2.23
+   2.3.10

data/README.md

@@ -40,8 +40,11 @@ To search the API and return an array of Propublica::Nonprofits::Objects, use th
     result.details.deductibility_code
     result.filings_with_data.first.totassetsend
 
+    Want to be as memory and API efficient as possible? Be lazy!
+    `Propublica::Nonprofits.lazy_search("rural").first(10)` will only fetch first 10 items
+
 *Note: This will currently only return the first 100 organizations*
-*Note: This is returned as a Enumerator::Lazy, to be as memory efficient as possible, if you do something like `Propublica::Nonprofits.search("rural").first(10)`*
+*Note: This is returned as an Array*
 
 
 ### EIN Lookup - As Organization
@@ -80,11 +83,6 @@ consider using the `find_attributes` method.
     attributes.dig("organization", "name")
     attributes.dig("filings_without_data").first.dig("pdf_url")
 
-## TODO
-
-- Add more robust searching parameters https://projects.propublica.org/nonprofits/api/#endpoint-search-example
-- Handle paginated search results (with Enumerator::Lazy)
-
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/lib/propublica/nonprofits/version.rb

@@ -1,5 +1,5 @@
 module Propublica
   module Nonprofits
-    VERSION = "0.4.1"
+    VERSION = "1.1.0"
   end
 end

data/lib/propublica/nonprofits.rb

@@ -5,62 +5,90 @@ require "faraday"
 require "json"
 
 API_BASE_URL = %(https://projects.propublica.org)
-API_SEARCH_PATH = %(/nonprofits/api/v2/search.json)
+API_SEARCH_PATH = %(nonprofits/api/v2/search.json)
+
+def self.results(column_names)
+  columns = column_names.each
+
+  Enumerator.new do |yielder|
+    loop do
+      yielder << Event.where(columns.next => query)
+    end
+
+    yielder << Event.fuzzy_search(query)
+  end
+end
 
 module Propublica
   module Nonprofits
-    def self.search(term, state: nil, page: nil, fetch_all: false)
-      organizations = []
-      more_pages = true
-
-      page =
-        case
-        when page
-          page
-        when fetch_all && page
-          raise "Page is set but we are fetching all, chose one or the other"
-        else
-          0
-        end
+    class DataNotFetched < StandardError
+    end
 
-      while (more_pages)
-        response = connection.get do |request|
-          request.url(API_SEARCH_PATH)
+    def self.search(term, state: nil, ntee: nil, page: nil, fetch_all: false)
+      search_results(term, state: state, ntee: ntee, page: page, fetch_all: fetch_all)
+        .flat_map(&:itself)
+    end
 
-          request.params["q"] = term
-          request.params["state[id]"] = state if state
-          request.params["page"] = page if page
-        end
-        
+    def self.lazy_search(term, state: nil, ntee: nil, page: nil, fetch_all: false)
+      search_results(term, state: state, ntee: ntee, page: page, fetch_all: fetch_all)
+        .lazy
+        .flat_map(&:itself)
+    end
 
-        attributes = JSON.parse(response.body)
-        new_organizations = attributes.fetch("organizations", [])
-        organizations.push(*new_organizations)
+    def self.search_results(term, state: nil, ntee: nil, page: nil, fetch_all: false)
+      raise ArgumentError.new("`page` and `fetch_all` are both: choose one or the other") if fetch_all && page
+      page ||= 0
+      max_pages = nil
 
-        more_pages = fetch_all && new_organizations.any?
-        page += 1
-      end
+      Enumerator.new do |yielder|
+        loop do
+          params = {}
+          params["q"] = term
+          params["state[id]"] = state if state
+          params["ntee[id]"] = ntee if ntee
+          params["page"] = page if page
 
-      organizations.lazy.map do |basic_attrs|
-        Propublica::Nonprofits::Organization.new("basic" => basic_attrs)
+          response = connection.get("/#{API_SEARCH_PATH}", params)
+          max_pages = response.body.dig("num_pages") || max_pages
+
+          yielder <<
+            response
+              .body
+              .fetch("organizations", [])
+              .map { |basic_attrs| Propublica::Nonprofits::Organization.new("basic" => basic_attrs) }
+
+          if fetch_all && page + 1 < max_pages
+            page += 1
+          else
+            raise(StopIteration)
+          end
+        end
       end
-    end
 
-    def self.find(ein)
-      attributes = self.find_attributes(ein)
-      Propublica::Nonprofits::Organization.new(attributes)
+    rescue JSON::ParserError => e
+      raise JSON::ParserError.new("Propublica API Parsing Error: #{e.message}")
     end
 
-    def self.find_attributes(ein)
-      response = connection.get("/nonprofits/api/v2/organizations/#{ein}.json")
-      JSON.parse(response.body)
+    def self.find(ein)
+      Propublica::Nonprofits::Organization.new(
+        self.find_attributes(ein)
+      )
     end
 
     def self.connection
-      @connection ||= Faraday.new(url: API_BASE_URL)
+      Faraday
+        .new(API_BASE_URL) do |f|
+          f.response :json
+        end
     end
 
-    class Error < StandardError; end
-    class DataNotFetched < Error; end
+    def self.find_attributes(ein)
+      connection
+        .get("/nonprofits/api/v2/organizations/#{ein}.json")
+        .body
+
+    rescue JSON::ParserError => e
+      raise JSON::ParserError.new("Propublica API Parsing Error: #{e.message}")
+    end
   end
 end

data/propublica-nonprofits.gemspec

@@ -29,7 +29,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "vcr"
   spec.add_development_dependency "webmock"
+  spec.add_development_dependency "rexml"
 
-  spec.add_dependency "faraday"
+  spec.add_dependency "faraday", "> 1.0", "< 3.0"
   spec.add_dependency "json"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: propublica-nonprofits
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 1.1.0
 platform: ruby
 authors:
 - Ricky Chilcott
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-07-13 00:00:00.000000000 Z
+date: 2023-01-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -109,19 +109,39 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: faraday
+  name: rexml
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
@@ -183,7 +203,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.2.32
 signing_key:
 specification_version: 4
 summary: Ruby wrapper for the Propublica Nonprofits API https://projects.propublica.org/nonprofits/api/v2