propublica-nonprofits 0.1.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6f1548a6ab99d0cc5bc651916c1591c0339b0644531533e507f9a1395fd21aa1
-  data.tar.gz: f991cc23ad0289a3ff680f7be2bae80930b6e1777ca990917b4302e815674c73
+  metadata.gz: c05c2a3977841806bac525c3c642db9a94bf0755c3d098c4852276b092b49168
+  data.tar.gz: ecbbd8799e9d43592ea52d8e2ab4754f74fef539dec2a3eab0d7a415513acf1b
 SHA512:
-  metadata.gz: 5fa50cbfdd4a595c2772a97080be6cf71e465e1263e7b902916b142f849e8cd38522c9ce42acce04ac28d20597a1b6196189ec4c8e3f27280b5703dab3954038
-  data.tar.gz: 6292726cce32f0e3a403c29dd5e7fca49accfa43c45e8d2a5bc2e978fc0774ffd3bc880d75d7580ca24c0e5838f34a75ddf4b638a1e4040b0e9053781a94ae12
+  metadata.gz: 1e30af0a7b67f51505022bf3a6e8b63f91fff5bb702c5a531dbed24585194d8e9d0f966b4097d128c67d1a98bc26e44ca533acd64e4681fa6d7ab2ef17da55c3
+  data.tar.gz: 9c34f58bc0c29ccabd62af0d62974f71f3557a907551e0f83521e657b20e5534f4afa9c78d1203dd83270be48cd3f7420059dba90069e263eb885c1e58628c30

data/.gitignore

@@ -6,3 +6,6 @@
 /pkg/
 /spec/reports/
 /tmp/
+/*.gem
+/old-versions/*.gem
+.DS_Store

data/Gemfile.lock

@@ -1,48 +1,67 @@
 PATH
   remote: .
   specs:
-    propublica-nonprofits (0.1.0)
+    propublica-nonprofits (1.0.0)
       faraday
       json
 
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.5.2)
-      public_suffix (>= 2.0.2, < 4.0)
+    addressable (2.8.0)
+      public_suffix (>= 2.0.2, < 5.0)
     coderay (1.1.2)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
-    faraday (0.15.2)
+    faraday (1.5.1)
+      faraday-em_http (~> 1.0)
+      faraday-em_synchrony (~> 1.0)
+      faraday-excon (~> 1.1)
+      faraday-httpclient (~> 1.0.1)
+      faraday-net_http (~> 1.0)
+      faraday-net_http_persistent (~> 1.1)
+      faraday-patron (~> 1.0)
       multipart-post (>= 1.2, < 3)
-    hashdiff (0.3.7)
-    json (2.1.0)
+      ruby2_keywords (>= 0.0.4)
+    faraday-em_http (1.0.0)
+    faraday-em_synchrony (1.0.0)
+    faraday-excon (1.1.0)
+    faraday-httpclient (1.0.1)
+    faraday-net_http (1.0.1)
+    faraday-net_http_persistent (1.2.0)
+    faraday-patron (1.0.0)
+    hashdiff (1.0.0)
+    json (2.5.1)
     method_source (0.9.0)
     minitest (5.11.3)
-    multipart-post (2.0.0)
+    minitest-line (0.6.5)
+      minitest (~> 5.0)
+    multipart-post (2.1.1)
     pry (0.11.3)
       coderay (~> 1.1.0)
       method_source (~> 0.9.0)
-    public_suffix (3.0.2)
-    rake (10.5.0)
-    safe_yaml (1.0.4)
+    public_suffix (4.0.6)
+    rake (13.0.1)
+    ruby2_keywords (0.0.4)
+    safe_yaml (1.0.5)
     vcr (4.0.0)
-    webmock (3.4.2)
+    webmock (3.7.6)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
-      hashdiff
+      hashdiff (>= 0.4.0, < 2.0.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  bundler (~> 1.16)
+  bundler (>= 2.2.10)
   minitest (~> 5.0)
+  minitest-line
   propublica-nonprofits!
   pry
-  rake (~> 10.0)
+  rake (~> 13.0)
   vcr
   webmock
 
 BUNDLED WITH
-   1.16.2
+   2.2.23

data/README.md

@@ -40,8 +40,11 @@ To search the API and return an array of Propublica::Nonprofits::Objects, use th
     result.details.deductibility_code
     result.filings_with_data.first.totassetsend
 
+    Want to be as memory and API efficient as possible? Be lazy!
+    `Propublica::Nonprofits.lazy_search("rural").first(10)` will only fetch first 10 items
+
 *Note: This will currently only return the first 100 organizations*
-*Note: This is returned as a Enumerator::Lazy, to be as memory efficient as possible, if you do something like `Propublica::Nonprofits.search("rural").first(10)`*
+*Note: This is returned as an Array*
 
 
 ### EIN Lookup - As Organization
@@ -80,11 +83,6 @@ consider using the `find_attributes` method.
     attributes.dig("organization", "name")
     attributes.dig("filings_without_data").first.dig("pdf_url")
 
-## TODO
-
-- Add more robust searching parameters https://projects.propublica.org/nonprofits/api/#endpoint-search-example
-- Handle paginated search results (with Enumerator::Lazy)
-
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/bin/build-and-publish-gem

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+mkdir -p old-versions
+mv *.gem old-versions
+
+bundle exec gem build *.gemspec
+bundle exec gem push *.gem

data/bin/console

@@ -1,6 +1,5 @@
 #!/usr/bin/env ruby
 
-require "bundler/setup"
 require "propublica/nonprofits"
 
 # You can add fixtures and/or initialization code here to make experimenting

data/lib/propublica/nonprofits.rb

@@ -1,19 +1,78 @@
 require "propublica/nonprofits/version"
 require "propublica/nonprofits/organization"
 
-require 'faraday'
-require 'json'
+require "faraday"
+require "json"
 
-API_BASE_URL = %(https://projects.propublica.org/nonprofits/api/v2)
+API_BASE_URL = %(https://projects.propublica.org)
+API_SEARCH_PATH = %(nonprofits/api/v2/search.json)
+
+def self.results(column_names)
+  columns = column_names.each
+
+  Enumerator.new do |yielder|
+    loop do
+      yielder << Event.where(columns.next => query)
+    end
+
+    yielder << Event.fuzzy_search(query)
+  end
+end
 
 module Propublica
   module Nonprofits
-    def self.search(term)
-      response = Faraday.get("#{API_BASE_URL}/search.json?q='#{term}'")
-      attributes = JSON.parse(response.body)
-      organizations = attributes.dig("organizations")
-      organizations.lazy.map do |basic_attrs|
-        Propublica::Nonprofits::Organization.new("basic" => basic_attrs)
+    class DataNotFetched < StandardError
+    end
+
+    def self.search(term, state: nil, ntee: nil, page: nil, fetch_all: false)
+      search_results(term, state: state, ntee: ntee, page: page, fetch_all: fetch_all)
+        .flat_map(&:itself)
+    end
+
+    def self.lazy_search(term, state: nil, ntee: nil, page: nil, fetch_all: false)
+      search_results(term, state: state, ntee: ntee, page: page, fetch_all: fetch_all)
+        .lazy
+        .flat_map(&:itself)
+    end
+
+    def self.search_results(term, state: nil, ntee: nil, page: nil, fetch_all: false)
+      raise ArgumentError.new("`page` and `fetch_all` are both: choose one or the other") if fetch_all && page
+      page ||= 0
+      max_pages = nil
+
+      Enumerator.new do |yielder|
+        loop do
+          params = {}
+          params["q"] = term
+          params["state[id]"] = state if state
+          params["ntee[id]"] = ntee if ntee
+          params["page"] = page if page
+
+          response = Faraday.default_connection.get("#{API_BASE_URL}/#{API_SEARCH_PATH}", params)
+          parsed_response =
+            if response.body.is_a?(Hash)
+              response.body
+            else
+              begin
+                JSON.parse(response.body)
+              rescue JSON::ParserError => e
+                raise JSON::ParserError.new("Propublica API Parsing Error: #{e.message}")
+              end
+            end
+
+          max_pages = parsed_response.dig("num_pages") || max_pages
+
+          yielder <<
+            parsed_response
+              .fetch("organizations", [])
+              .map { |basic_attrs| Propublica::Nonprofits::Organization.new("basic" => basic_attrs) }
+
+          if fetch_all && page + 1 < max_pages
+            page += 1
+          else
+            raise(StopIteration)
+          end
+        end
       end
     end
 
@@ -23,11 +82,16 @@ module Propublica
     end
 
     def self.find_attributes(ein)
-      response = Faraday.get("#{API_BASE_URL}/organizations/#{ein}.json")
-      JSON.parse(response.body)
+      response = Faraday.get("#{API_BASE_URL}/nonprofits/api/v2/organizations/#{ein}.json")
+      if response.body.is_a?(Hash)
+        response.body
+      else
+        begin
+          JSON.parse(response.body)
+        rescue JSON::ParserError => e
+          raise JSON::ParserError.new("Propublica API Parsing Error: #{e.message}")
+        end
+      end
     end
-
-    class Error < StandardError; end
-    class DataNotFetched < Error; end
   end
 end

data/lib/propublica/nonprofits/organization.rb

@@ -1,8 +1,6 @@
 module Propublica
   module Nonprofits
     class Organization
-      attr_reader :basic
-
       def initialize(attributes)
         @attributes = attributes
       end
@@ -38,28 +36,44 @@ module Propublica
 
       def error
         ensure_full_request!
-        @error ||= attributes["error"]
+        @error ||= attributes["error"] || ""
+      end
+
+      def error?
+        !error.empty?
       end
 
       private
 
       attr_reader :attributes
+      attr_accessor :full_request_made
+
+      def full_request_made?
+        !!full_request_made
+      end
 
       def ensure_full_request!
-        return if @full_request == true
+        return if full_request_made?
 
-        required_keys = ["organization", "filings_with_data", "filings_without_data", "data_source", "api_version", "error"].freeze
-        @full_request = (attributes.keys & required_keys) == required_keys
-        return if @full_request == true
+        self.full_request_made = (attributes.keys & required_keys) == required_keys
+        return if full_request_made?
 
         fetch_full_request!
       end
 
+      def required_keys
+        ["organization", "filings_with_data", "filings_without_data", "data_source", "api_version", "error"].freeze
+      end
+
+      def ein
+        @attributes.dig("organization", "ein") || self.basic.ein
+      end
+
       def fetch_full_request!
         # Fetch all attributes and merge with what we have now
-        new_attrs = Propublica::Nonprofits.find_attributes(self.basic.ein)
+        new_attrs = Propublica::Nonprofits.find_attributes(ein)
         attributes.merge!(new_attrs)
-        @full_request = true
+        self.full_request_made = true
       end
     end
   end

data/lib/propublica/nonprofits/organization/dynamic_parser.rb

@@ -7,11 +7,7 @@ module Propublica
           fields.each do |field|
             define_method field do
               vars = self.instance_variable_get("@attributes")
-              vars.fetch(field.to_s) do
-                class_name = self.class.to_s.split("::").last
-                raise Propublica::Nonprofits::DataNotFetched,
-                  "#{class_name}##{field} not fetched from API. This may be due to an API error or because you tried to access a Basic property on the full results"
-              end
+              vars.fetch(field.to_s) { raise_field_fetch_error(field) }
             end
           end
         end
@@ -27,6 +23,12 @@ module Propublica
         private
 
         attr_reader :attributes
+
+        def raise_field_fetch_error(field)
+          class_name = self.class.to_s.split("::").last
+          raise Propublica::Nonprofits::DataNotFetched,
+            "#{class_name}##{field} not fetched from API. This may be due to an API error or because you tried to access a Basic property on the full results"
+        end
       end
     end
   end

data/lib/propublica/nonprofits/version.rb

@@ -1,5 +1,5 @@
 module Propublica
   module Nonprofits
-    VERSION = "0.1.0"
+    VERSION = "1.0.0"
   end
 end

data/propublica-nonprofits.gemspec

@@ -1,32 +1,32 @@
-
 lib = File.expand_path("../lib", __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require "propublica/nonprofits/version"
 
 Gem::Specification.new do |spec|
-  spec.name          = "propublica-nonprofits"
-  spec.version       = Propublica::Nonprofits::VERSION
-  spec.authors       = ["Ricky Chilcott"]
-  spec.email         = ["ricky@rakefire.io"]
+  spec.name = "propublica-nonprofits"
+  spec.version = Propublica::Nonprofits::VERSION
+  spec.authors = ["Ricky Chilcott"]
+  spec.email = ["ricky@rakefire.io"]
 
-  spec.summary       = %q{Ruby wrapper for the Propublica Nonprofits API https://projects.propublica.org/nonprofits/api/v2}
-  spec.description   = %q{Ruby wrapper for the Propublica Nonprofits API https://projects.propublica.org/nonprofits/api/v2}
-  spec.homepage      = "https://github.com/Rakefire/propublica-nonprofits"
-  spec.license       = "MIT"
+  spec.summary = %q{Ruby wrapper for the Propublica Nonprofits API https://projects.propublica.org/nonprofits/api/v2}
+  spec.description = %q{Ruby wrapper for the Propublica Nonprofits API https://projects.propublica.org/nonprofits/api/v2}
+  spec.homepage = "https://github.com/Rakefire/propublica-nonprofits"
+  spec.license = "MIT"
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
-  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+  spec.files = Dir.chdir(File.expand_path("..", __FILE__)) do
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   end
-  spec.bindir        = "exe"
-  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.bindir = "exe"
+  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.16"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "bundler", ">= 2.2.10"
+  spec.add_development_dependency "minitest-line"
   spec.add_development_dependency "minitest", "~> 5.0"
   spec.add_development_dependency "pry"
+  spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "vcr"
   spec.add_development_dependency "webmock"
 

metadata

@@ -1,43 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: propublica-nonprofits
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Ricky Chilcott
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2018-07-11 00:00:00.000000000 Z
+date: 2021-07-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: 2.2.10
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: 2.2.10
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: minitest-line
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -66,6 +66,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: vcr
   requirement: !ruby/object:Gem::Requirement
@@ -137,6 +151,7 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- bin/build-and-publish-gem
 - bin/console
 - bin/setup
 - bin/test
@@ -153,7 +168,7 @@ homepage: https://github.com/Rakefire/propublica-nonprofits
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -168,9 +183,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
-signing_key: 
+rubygems_version: 3.1.2
+signing_key:
 specification_version: 4
 summary: Ruby wrapper for the Propublica Nonprofits API https://projects.propublica.org/nonprofits/api/v2
 test_files: []