gov_scooper 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 5b86b619b7752b3318c28f910474c06fc0f15d8a
+  data.tar.gz: a0f45a04ca8a897e634fe956a426ce7dc1a1f49c
+SHA512:
+  metadata.gz: 4cfee6dc74f3563ec7443025855fb548738f1bd379f889cfb396fef171606a10a77ceb7b77f4fcb7145d8250904506e0bed1cd2685e192d96bafee6a5203d831
+  data.tar.gz: 10662831c4924da06a8ecf1c4d0feedd1059918cac3bad3209f4acc77f93cc297e0c2662f3fa766f99532d78f0448bdea0fb639154e2d1a3a2c3151f9b91f504

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.3.3
+before_install: gem install bundler -v 1.13.6

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in gov_scooper.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Jack Reed
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,46 @@
+# GovScooper
+
+Scoopin' up all of the metadata from Data.gov. GovScooper is a paginated harvester of the Data.gov CKAN API. It also enables you to save that metadata in a pairtree.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'gov_scooper'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install gov_scooper
+
+## Usage
+
+```ruby
+metadata_enumberable = DataGov::API.new.search
+metadata_enumberable.length #=> 10
+metadata_enumberable.map { |md| DataGov::Dataset.new(md).save_ckan_metadata }
+# metadata is now saved in a pairtree directory structure based off of id
+
+# Download the resources
+DataGov::Dataset.from_id('8a311c18-3060-438e-a8c7-9e37bcde6529').resources.map { |r| r.download }
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/mejackreed/GovScooper.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,8 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+Dir.glob('lib/tasks/*.rake').each { |r| load r}
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "gov_scooper"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/gov_scooper.gemspec

@@ -0,0 +1,34 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'gov_scooper/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'gov_scooper'
+  spec.version       = GovScooper::VERSION
+  spec.authors       = ['Jack Reed']
+  spec.email         = ['phillipjreed@gmail.com']
+
+  spec.summary       = 'Scooper of data.gov metadata'
+  spec.description   = 'Scooper of data.gov metadata'
+  spec.homepage      = 'https://github.com/mejackreed/gov_scooper'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'faraday'
+  spec.add_dependency 'faraday_middleware'
+  spec.add_dependency 'pairtree'
+  spec.add_dependency 'mime-types'
+  spec.add_dependency 'ruby-progressbar'
+  spec.add_dependency 'open_uri_redirections'
+
+  spec.add_development_dependency 'bundler', '~> 1.13'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+end

data/lib/data_gov/API.rb

@@ -0,0 +1,17 @@
+module DataGov
+  class API
+    ##
+    # @return [Enumerator]
+    def search(params = {})
+      client.paginated_get('/api/3/action/package_search', 'results', params)
+    end
+
+    def harvest_object(id)
+      client.get("/harvest/object/#{id}")
+    end
+
+    def client
+      @client ||= DataGov::Client.new
+    end
+  end
+end

data/lib/data_gov/client.rb

@@ -0,0 +1,54 @@
+require 'faraday'
+require 'faraday_middleware'
+
+module DataGov
+  class Client
+    ##
+    # @return [Hash] a parsed JSON hash
+    def get(path, params = {})
+      response = connection.get(path) do |req|
+        req.params = params
+      end
+      puts "#{response.status} #{response.env.url}"
+      response.body
+    end
+
+    def paginated_get(path, accessor, options = {})
+      Enumerator.new do |yielder|
+        params   = options.dup
+        rows     = params.delete(:rows) { 5 }
+        start    = params.delete(:start) { 1 }
+        max      = params.delete(:max) { 10 }
+        total    = 0
+
+        loop do
+          data = get(path, { rows: rows, start: start }.merge(params))
+
+          total += data['result'][accessor].length
+
+          data['result'][accessor].each do |element|
+            yielder.yield element
+          end
+
+          start += rows
+          puts "total: #{total}, max: #{max}, count: #{data['result']['count']}"
+          break if total >= data['result']['count'] || total >= max
+        end
+      end
+    end
+
+    private
+
+    def connection
+      @connection ||= begin
+        conn = Faraday.new(url: 'https://catalog.data.gov') do |faraday|
+          faraday.adapter Faraday.default_adapter
+          faraday.response :json
+        end
+        conn.options.timeout = 60
+        conn.options.open_timeout = 10
+        conn
+      end
+    end
+  end
+end

data/lib/data_gov/dataset.rb

@@ -0,0 +1,47 @@
+require 'pairtree'
+require 'JSON'
+
+module DataGov
+  class Dataset
+    attr_accessor :id, :ckan_metadata
+    attr_reader :resources
+
+    def initialize(ckan_metadata)
+      @ckan_metadata = ckan_metadata
+      @id = ckan_metadata['id']
+    end
+
+    def save_ckan_metadata
+      pairtree.open('ckan.json', 'w') do |io|
+        io.write(JSON.pretty_generate(ckan_metadata))
+      end
+    end
+
+    def resources
+      @resources ||= ckan_metadata['resources'].map do |resource|
+        DataGov::Resource.new(resource, self)
+      end
+    end
+
+    def download_resources
+      puts "Downloading resources for #{id}"
+      resources.map { |resource| resource.download }
+    end
+
+    def pairtree
+      @pairtree ||= Pairtree.at(pairtree_location, create: true)
+                            .mk(id.delete('-'))
+    end
+
+    def pairtree_location
+      ENV.fetch('DATA_DIR')
+    end
+
+    def self.from_id(id)
+      instance = new('')
+      instance.id = id
+      instance.ckan_metadata = JSON.parse(instance.pairtree.read('ckan.json'))
+      instance
+    end
+  end
+end

data/lib/data_gov/resource.rb

@@ -0,0 +1,54 @@
+require 'mime/types'
+require 'open-uri'
+require 'open_uri_redirections'
+require 'ruby-progressbar'
+
+module DataGov
+  class Resource
+    attr_reader :metadata, :dataset
+
+    def initialize(metadata, dataset)
+      @metadata = metadata
+      @dataset = dataset
+    end
+
+    def download
+      if dataset.pairtree.exists?(file_name)
+        puts "#{file_name} already exists, skipping download"
+        return
+      end
+      pbar = ProgressBar.create(title: file_name, total: nil)
+      begin
+        download = open(metadata['url'],
+                        allow_redirections: :safe,
+                        content_length_proc: lambda do |content_length|
+                          if content_length && 0 < content_length
+                            pbar.total = content_length
+                          end
+                        end,
+                        progress_proc: lambda do |s|
+                          if pbar.total
+                            pbar.progress += s
+                          else
+                            pbar.increment
+                          end
+                        end)
+        dataset.pairtree.open(file_name, 'w') { |io| IO.copy_stream(download, io) }
+      rescue StandardError => e
+        puts e
+      end
+    end
+
+    def file_name
+      "#{metadata['id']}.#{extension}"
+    end
+
+    def extension
+      mimetype.preferred_extension
+    end
+
+    def mimetype
+      MIME::Types[metadata['mimetype']].first || MIME::Types['text/plain'].first
+    end
+  end
+end

data/lib/data_gov.rb

@@ -0,0 +1,4 @@
+require 'data_gov/API'
+require 'data_gov/client'
+require 'data_gov/dataset'
+require 'data_gov/resource'

data/lib/gov_scooper/version.rb

@@ -0,0 +1,3 @@
+module GovScooper
+  VERSION = '0.1.0'
+end

data/lib/gov_scooper.rb

@@ -0,0 +1,6 @@
+require 'gov_scooper/version'
+require 'data_gov'
+
+module GovScooper
+  # Your code goes here...
+end

data/lib/tasks/gov_scooper.rake

@@ -0,0 +1,76 @@
+require 'json'
+require 'gov_scooper'
+require 'fileutils'
+# require 'pry'
+
+namespace :gov_scooper do
+  desc 'Sample OGM data and move it to a new directory'
+  task :sample do
+    number = Integer(ENV['GS_SAMPLE']) || 1
+    ogm_loc = ENV.fetch('DATA_DIR')
+    sample_output = ENV.fetch('GS_OUTPUT')
+    if ogm_loc.nil?
+      raise 'Please provide environment variable DATA_DIR'\
+            ' for opengeometdata directory location'
+    end
+    raise 'Please provide output directory GS_OUTPUT' if sample_output.nil?
+
+    layers = JSON.parse(File.read(File.join(ogm_loc, 'pairtree_root', 'layers.json')))
+    puts "#{layers.length} layers found"
+    random_layers = layers.to_a.sample(number).to_h
+    puts "Sampling #{random_layers.length} layers"
+    random_layers.values.each do |value|
+      output = File.join(sample_output, 'pairtree_root', value)
+      FileUtils.mkdir_p output
+      Dir[File.join(ogm_loc, 'pairtree_root', value, '*')].each do |file_name|
+        next if File.directory? file_name
+        puts "Copying #{file_name}"
+        FileUtils.cp file_name, output
+      end
+    end
+    ENV['DATA_DIR'] = sample_output
+    Rake::Task['gov_scooper:create_layers_json'].invoke
+    Rake::Task['gov_scooper:download_data'].invoke
+    ENV['DATA_DIR'] = ogm_loc
+  end
+  desc 'Download data for layers in a given directory - Be careful with this'
+  task :download_data do
+    ogm_loc = ENV.fetch('DATA_DIR')
+    if ogm_loc.nil?
+      raise 'Please provide environment variable DATA_DIR'\
+            ' for opengeometdata directory location'
+    end
+
+    layers = JSON.parse(File.read(File.join(ogm_loc, 'pairtree_root', 'layers.json')))
+    puts "#{layers.length} layers found"
+    resource_count = 0
+    layers.each do |layer|
+      dataset = DataGov::Dataset.from_id(layer[0])
+      puts "Downloading from dataset #{dataset.id}"
+      resources = dataset.resources
+      resource_count += resources.length
+      resources.map(&:download)
+    end
+    puts "#{resource_count} total resources"
+  end
+  desc 'Create layers.json'
+  task :create_layers_json do
+    ogm_loc = ENV.fetch('DATA_DIR')
+    if ogm_loc.nil?
+      raise 'Please provide environment variable DATA_DIR'\
+            ' for opengeometdata directory location'
+    end
+    layers = Dir[File.join(ogm_loc, 'pairtree_root', '**', 'ckan.json')]
+    h = layers.map do |f|
+      d = DataGov::Dataset.new(JSON.parse(File.read(f)))
+      {
+        d.id => f.sub(/.*pairtree_root\//, '').sub('ckan.json', '')
+      }
+    end
+    v = h.inject(:merge!)
+    puts "layers.json created for #{layers.count} files"
+    File.open(File.join(ogm_loc, 'pairtree_root', 'layers.json'), 'w') do |io|
+      io.write(JSON.pretty_generate(v))
+    end
+  end
+end

metadata

@@ -0,0 +1,188 @@
+--- !ruby/object:Gem::Specification
+name: gov_scooper
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Jack Reed
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-01-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: faraday_middleware
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pairtree
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mime-types
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby-progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: open_uri_redirections
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: Scooper of data.gov metadata
+email:
+- phillipjreed@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- gov_scooper.gemspec
+- lib/data_gov.rb
+- lib/data_gov/API.rb
+- lib/data_gov/client.rb
+- lib/data_gov/dataset.rb
+- lib/data_gov/resource.rb
+- lib/gov_scooper.rb
+- lib/gov_scooper/version.rb
+- lib/tasks/gov_scooper.rake
+homepage: https://github.com/mejackreed/gov_scooper
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2
+signing_key: 
+specification_version: 4
+summary: Scooper of data.gov metadata
+test_files: []