scraperwiki-api 0.0.1

data/.gitignore

@@ -0,0 +1,6 @@
+*.gem
+.bundle
+.yardoc
+Gemfile.lock
+doc/*
+pkg/*

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in unbreakable.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2011 Open North Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,37 @@
+# The ScraperWiki API Ruby Gem
+
+A Ruby wrapper for the ScraperWiki API.
+
+## Installation
+
+    gem install scraperwiki-api
+
+## Examples
+
+    >> require 'scraperwiki-api'
+
+    >> api = ScraperWiki::API.new 'my-api-key' # API key is optional
+
+    >> api.datastore_sqlite 'example-scraper', 'SELECT * FROM swdata LIMIT 10'
+    => [{"fieldA"=>"valueA", "fieldB"=>"valueB", "fieldC"=>"valueC"}, ...]
+
+    >> api.scraper_getinfo 'example-scraper'
+    => [{"code"=>"require 'nokogiri'\n...", "datasummary"=>...}]
+
+    >> api.scraper_getruninfo 'example-scraper'
+    => [{"run_ended"=>"1970-01-01T00:00:00", "first_url_scraped"=>...}]
+
+    >> api.scraper_getuserinfo 'johndoe'
+    => [{"username"=>"johndoe", "profilename"=>"John Doe", "coderoles"=>...}]
+
+    >> api.scraper_search searchquery: 'search terms'
+    => [{"description"=>"Scrapes websites for data.", "language"=>"ruby", ...]
+
+    >> api.scraper_usersearch searchquery: 'search terms'
+    => [{"username"=>"johndoe", "profilename"=>"John Doe", "date_joined"=>...}]
+
+## Bugs? Questions?
+
+Unbreakable's main repository is on GitHub: [http://github.com/opennorth/scraperwiki-api-ruby](http://github.com/opennorth/scraperwiki-api-ruby), where your contributions, forks, bug reports, feature requests, and feedback are greatly welcomed.
+
+Copyright (c) 2011 Open North Inc., released under the MIT license

data/Rakefile

@@ -0,0 +1,16 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec
+
+begin
+  require 'yard'
+  YARD::Rake::YardocTask.new
+rescue LoadError
+  task :yard do
+    abort 'YARD is not available. In order to run yard, you must: gem install yard'
+  end
+end

data/USAGE

@@ -0,0 +1 @@
+See README.md for full usage details.

data/lib/scraperwiki-api.rb

@@ -0,0 +1,306 @@
+require 'httparty'
+require 'scraperwiki-api/version'
+
+module ScraperWiki
+  # A Ruby wrapper for the ScraperWiki API.
+  # @see https://scraperwiki.com/docs/api
+  class API
+    include HTTParty
+    base_uri 'api.scraperwiki.com/api/1.0'
+
+    class Error < StandardError; end
+    class ScraperNotFound < Error; end
+
+    # Initializes a ScraperWiki API object.
+    def initialize(apikey = nil)
+      @apikey = apikey
+    end
+
+    # Queries and extracts data via a general purpose SQL interface.
+    #
+    # To make an RSS feed you need to use SQL's +AS+ keyword (e.g. "SELECT name
+    # AS description") to make columns called +title+, +link+, +description+,
+    # +guid+ (optional, uses link if not available) and +pubDate+ or +date+.
+    #
+    # +jsondict+ example output:
+    #
+    #     [
+    #       {
+    #         "fieldA": "valueA",
+    #         "fieldB": "valueB",
+    #         "fieldC": "valueC",
+    #       },
+    #       ...
+    #     ]
+    #
+    # +jsonlist+ example output:
+    #
+    #     {
+    #       "keys": ["fieldA", "fieldB", "fieldC"],
+    #       "data": [
+    #         ["valueA", "valueB", "valueC"],
+    #         ...
+    #       ]
+    #     }
+    #
+    # +csv+ example output:
+    #
+    #     fieldA,fieldB,fieldC
+    #     valueA,valueB,valueC
+    #     ...
+    #
+    # @param [String] shortname the scraper's shortname (as it appears in the URL)
+    # @param [String] query a SQL query
+    # @param [Hash] opts optional arguments
+    # @option opts [String] :format one of "jsondict", "jsonlist", "csv",
+    #   "htmltable" or "rss2"
+    # @option opts [String] :attach ";"-delimited list of shortnames of other
+    #   scrapers whose data you need to access
+    # @see https://scraperwiki.com/docs/ruby/ruby_help_documentation/
+    #
+    # @note The query string parameter is +name+, not +shortname+
+    #   {https://scraperwiki.com/docs/api#sqlite as documented}
+    def datastore_sqlite(shortname, query, opts = {})
+      if Array === opts[:attach]
+        opts[:attach] = opts[:attach].join ';'
+      end
+      request_with_apikey '/datastore/sqlite', {name: shortname, query: query}.merge(opts)
+    end
+
+    # Extracts data about a scraper's code, owner, history, etc.
+    #
+    # Example output:
+    # * The +runid+ is a Unix timestamp with microseconds and a UUID.
+    # * The value of +records+ is the same as that of +total_rows+ under +datasummary+.
+    # * +run_interval+ is the number of seconds between runs.
+    #
+    #     [
+    #       {
+    #         "code": "require 'nokogiri'\n...",
+    #         "datasummary": {
+    #           "tables": {
+    #             "swdata": {
+    #               "keys": [
+    #                 "fieldA",
+    #                 ...
+    #               ],
+    #               "count": 42,
+    #               "sql": "CREATE TABLE `swdata` (...)"
+    #             },
+    #             "swvariables": {
+    #               "keys": [
+    #                 "value_blob",
+    #                 "type",
+    #                 "name"
+    #               ],
+    #               "count": 2,
+    #               "sql": "CREATE TABLE `swvariables` (`value_blob` blob, `type` text, `name` text)"
+    #             },
+    #             ...
+    #           },
+    #           "total_rows": 44,
+    #           "filesize": 1000000
+    #         },
+    #         "description": "Scrapes websites for data.",
+    #         "language": "ruby",
+    #         "title": "Example scraper",
+    #         "tags": [],
+    #         "short_name": "example-scraper",
+    #         "userroles": {
+    #           "owner": [
+    #             "johndoe"
+    #           ],
+    #           "editor": [
+    #             "janedoe",
+    #             ...
+    #           ]
+    #         },
+    #         "last_run": "1970-01-01T00:00:00",
+    #         "created": "1970-01-01T00:00:00",
+    #         "runevents": [
+    #           {
+    #             "still_running": false,
+    #             "pages_scraped": 5,
+    #             "run_started": "1970-01-01T00:00:00",
+    #             "last_update": "1970-01-01T00:00:00",
+    #             "runid": "1325394000.000000_xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx",
+    #             "records_produced": 42
+    #           },
+    #           ...
+    #         ],
+    #         "records": 44,
+    #         "wiki_type": "scraper",
+    #         "privacy_status": "visible",
+    #         "run_interval": 604800,
+    #         "attachable_here": [],
+    #         "attachables": [],
+    #         "history": [
+    #           ...,
+    #           {
+    #             "date": "1970-01-01T00:00:00",
+    #             "version": 0,
+    #             "user": "johndoe",
+    #             "session": "Thu, 1 Jan 1970 00:00:08 GMT"
+    #           }
+    #         ]
+    #       }
+    #     ]
+    #
+    # @param [String] shortname the scraper's shortname (as it appears in the URL)
+    # @param [Hash] opts optional arguments
+    # @option opts [String] :version version number (-1 for most recent) [default -1]
+    # @option opts [String] :history_start_date history and runevents are
+    #   restricted to this date or after, enter as YYYY-MM-DD
+    # @option opts [String] :quietfields "|"-delimited list of fields to exclude
+    #   from the output. Must be a subset of 'code|runevents|datasummary|userroles|history'
+    #
+    # @note Returns an array although the array seems to always have only one item
+    # @note The +tags+ field seems to always be an empty array
+    # @note The query string parameter is +name+, not +shortname+
+    #   {https://scraperwiki.com/docs/api#getinfo as documented}
+    def scraper_getinfo(shortname, opts = {})
+      if Array === opts[:quietfields]
+        opts[:quietfields] = opts[:quietfields].join '|'
+      end
+      request_with_apikey '/scraper/getinfo', {name: shortname}.merge(opts)
+    end
+
+    # See what the scraper did during each run.
+    #
+    # Example output:
+    #
+    #     [
+    #       {
+    #         "run_ended": "1970-01-01T00:00:00",
+    #         "first_url_scraped": "http://www.iana.org/domains/example/",
+    #         "pages_scraped": 5,
+    #         "run_started": "1970-01-01T00:00:00",
+    #         "runid": "1325394000.000000_xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx",
+    #         "domainsscraped": [
+    #           {
+    #             "domain": "http://example.com",
+    #             "bytes": 1000000,
+    #             "pages": 5
+    #           }
+    #           ...
+    #         ],
+    #         "output": "...",
+    #         "records_produced": 42
+    #       }
+    #     ]
+    #
+    # @param [String] shortname the scraper's shortname (as it appears in the URL)
+    # @param [Hash] opts optional arguments
+    # @option opts [String] runid a run ID
+    #
+    # @note Returns an array although the array seems to always have only one item
+    # @note The query string parameter is +name+, not +shortname+
+    #   {https://scraperwiki.com/docs/api#getinfo as documented}
+    def scraper_getruninfo(shortname, opts = {})
+      request_with_apikey '/scraper/getruninfo', {name: shortname}.merge(opts)
+    end
+
+    # Find out information about a user.
+    #
+    # Example output:
+    #
+    #     [
+    #       {
+    #         "username": "johndoe",
+    #         "profilename": "John Doe",
+    #         "coderoles": {
+    #           "owner": [
+    #             "johndoe.emailer",
+    #             "example-scraper",
+    #             ...
+    #           ],
+    #           "email": [
+    #             "johndoe.emailer"
+    #           ],
+    #           "editor": [
+    #             "yet-another-scraper",
+    #             ...
+    #           ]
+    #         },
+    #         "datejoined": "1970-01-01T00:00:00"
+    #       }
+    #     ]
+    #
+    # @param [String] username a username
+    #
+    # @note Returns an array although the array seems to always have only one item
+    # @note The date joined field is +date_joined+ (with underscore) on
+    #   {#scraper_usersearch}
+    def scraper_getuserinfo(username)
+      request_with_apikey '/scraper/getuserinfo', username: username
+    end
+
+    # Search the titles and descriptions of all the scrapers.
+    #
+    # Example output:
+    #
+    #     [
+    #       {
+    #         "description": "Scrapes websites for data.",
+    #         "language": "ruby",
+    #         "created": "1970-01-01T00:00:00",
+    #         "title": "Example scraper",
+    #         "short_name": "example-scraper",
+    #         "privacy_status": "public"
+    #       },
+    #       ...
+    #     ]
+    #
+    # @param [Hash] opts optional arguments
+    # @option opts [String] :searchquery search terms
+    # @option opts [Integer] :maxrows number of results to return [default 5]
+    # @option opts [String] :requestinguser the name of the user making the
+    #   search, which changes the order of the matches
+    def scraper_search(opts = {})
+      request_with_apikey '/scraper/search', opts
+    end
+
+    # Search for a user by name.
+    #
+    # Example output:
+    #
+    #     [
+    #       {
+    #         "username": "johndoe",
+    #         "profilename": "John Doe",
+    #         "date_joined": "1970-01-01T00:00:00"
+    #       },
+    #       ...
+    #     ]
+    #
+    # @param [Hash] opts optional arguments
+    # @option opts [String] :searchquery search terms
+    # @option opts [Integer] :maxrows number of results to return [default 5]
+    # @option opts [String] :nolist space-separated list of usernames to exclude
+    #   from the output
+    # @option opts [String] :requestinguser the name of the user making the
+    #   search, which changes the order of the matches
+    #
+    # @note The date joined field is +datejoined+ (without underscore) on
+    #   {#scraper_getuserinfo}
+    def scraper_usersearch(opts = {})
+      if Array === opts[:nolist]
+        opts[:nolist] = opts[:nolist].join ' '
+      end
+      request '/scraper/usersearch', opts
+    end
+
+  private
+
+    def request_with_apikey(path, opts = {})
+      if @apikey
+        opts[:apikey] = @apikey
+      end
+      request path, opts
+    end
+
+    def request(path, opts)
+      self.class.get(path, query: opts).parsed_response
+    end
+  end
+end

data/lib/scraperwiki-api/version.rb

@@ -0,0 +1,5 @@
+module ScraperWiki
+  class API
+    VERSION = "0.0.1"
+  end
+end

data/scraperwiki-api.gemspec

@@ -0,0 +1,22 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "scraperwiki-api/version"
+
+Gem::Specification.new do |s|
+  s.name        = "scraperwiki-api"
+  s.version     = ScraperWiki::API::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Open North"]
+  s.email       = ["info@opennorth.ca"]
+  s.homepage    = "http://github.com/opennorth/scraperwiki-api-ruby"
+  s.summary     = %q{The ScraperWiki API Ruby Gem}
+  s.description = %q{A Ruby wrapper for the ScraperWiki API}
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_runtime_dependency('httparty', '~> 0.7.8')
+  s.add_development_dependency('rspec', '~> 2.6.0')
+end

data/spec/scraperwiki-api_spec.rb

@@ -0,0 +1,138 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+require 'time'
+
+class ScraperWiki::API
+  # We don't want to test the ScraperWiki API. We just want to check that the
+  # wrapper works.
+  describe ScraperWiki::API do
+    EXAMPLE_SHORTNAME = 'frabcus.emailer'
+    EXAMPLE_USERNAME = 'frabcus'
+    QUIETFIELDS = %w(code runevents datasummary userroles history)
+
+    before :all do
+      @api = ScraperWiki::API.new
+    end
+
+    describe '#datastore_sqlite' do
+      # @todo
+    end
+
+    describe '#scraper_getinfo' do
+      it 'should return a non-empty array containing a single hash' do
+        response = @api.scraper_getinfo EXAMPLE_SHORTNAME
+        response.should be_an(Array)
+        response.size.should == 1
+        response.first.should be_a(Hash)
+      end
+
+      it 'should respect the :version argument' do
+        bare = @api.scraper_getinfo(EXAMPLE_SHORTNAME).first
+        bare.should_not have_key('currcommit')
+        result = @api.scraper_getinfo(EXAMPLE_SHORTNAME, version: 1).first
+        result.should have_key('currcommit')
+        result['code'].should_not == bare['code']
+      end
+
+      it 'should respect the :history_start_date argument' do
+        bare = @api.scraper_getinfo(EXAMPLE_SHORTNAME).first
+        bare['history'].size.should be > 1
+        history_start_date = bare['history'][0]['date'][0..9]
+        result = @api.scraper_getinfo(EXAMPLE_SHORTNAME, history_start_date: history_start_date).first
+        result['history'].size.should == 1
+      end
+
+      it 'should respect the :quietfields argument (as an array)' do
+        result = @api.scraper_getinfo(EXAMPLE_SHORTNAME, quietfields: QUIETFIELDS).first
+        QUIETFIELDS.each do |key|
+          result.should_not have_key(key)
+        end
+      end
+
+      it 'should respect the :quietfields argument (as an string)' do
+        result = @api.scraper_getinfo(EXAMPLE_SHORTNAME, quietfields: QUIETFIELDS.join('|')).first
+        QUIETFIELDS.each do |key|
+          result.should_not have_key(key)
+        end
+      end
+    end
+
+    describe '#scraper_getruninfo' do
+      it 'should return a non-empty array containing a single hash' do
+        response = @api.scraper_getruninfo EXAMPLE_SHORTNAME
+        response.should be_an(Array)
+        response.size.should == 1
+        response.first.should be_a(Hash)
+      end
+
+      it 'should respect the :runid argument' do
+        runevents = @api.scraper_getinfo(EXAMPLE_SHORTNAME).first['runevents']
+        bare = @api.scraper_getruninfo(EXAMPLE_SHORTNAME).first
+        bare['runid'].should == runevents.first['runid']
+        response = @api.scraper_getruninfo(EXAMPLE_SHORTNAME, runid: runevents.last['runid']).first
+        response['runid'].should_not == bare['runid']
+      end
+    end
+
+    describe '#scraper_getuserinfo' do
+      it 'should return a non-empty array containing a single hash' do
+        response = @api.scraper_getuserinfo EXAMPLE_USERNAME
+        response.should be_an(Array)
+        response.size.should == 1
+        response.first.should be_a(Hash)
+      end
+    end
+
+    describe '#scraper_search' do
+      it 'should return a non-empty array of hashes' do
+        response = @api.scraper_search
+        response.should be_an(Array)
+        response.size.should_not be_zero
+        response.first.should be_a(Hash)
+      end
+
+      it 'should respect the :searchquery argument' do
+        @api.scraper_search(searchquery: EXAMPLE_SHORTNAME).find{|result|
+          result['short_name'] == EXAMPLE_SHORTNAME
+        }.should_not be_nil
+      end
+
+      it 'should respect the :maxrows argument' do
+        @api.scraper_search(maxrows: 1).size.should == 1
+      end
+    end
+
+    describe '#scraper_usersearch' do
+      it 'should return a non-empty array of hashes' do
+        response = @api.scraper_usersearch
+        response.should be_an(Array)
+        response.size.should_not be_zero
+        response.first.should be_a(Hash)
+      end
+
+      it 'should respect the :searchquery argument' do
+        @api.scraper_usersearch(searchquery: EXAMPLE_USERNAME).find{|result|
+          result['username'] == EXAMPLE_USERNAME
+        }.should_not be_nil
+      end
+
+      it 'should respect the :maxrows argument' do
+        @api.scraper_usersearch(maxrows: 1).size.should == 1
+      end
+
+      it 'should respect the :nolist argument (as an array)' do
+        usernames = @api.scraper_usersearch.map{|result| result['username']}
+        @api.scraper_usersearch(nolist: usernames).find{|result|
+          usernames.include? result['username']
+        }.should be_nil
+      end
+
+      it 'should respect the :nolist argument (as an string)' do
+        usernames = @api.scraper_usersearch.map{|result| result['username']}
+        @api.scraper_usersearch(nolist: usernames.join(' ')).find{|result|
+          usernames.include? result['username']
+        }.should be_nil
+      end
+    end
+  end
+end

data/spec/spec.opts

@@ -0,0 +1,5 @@
+--colour
+--format nested
+--loadby mtime
+--reverse
+--backtrace

data/spec/spec_helper.rb

@@ -0,0 +1,3 @@
+require 'rubygems'
+require 'rspec'
+require File.dirname(__FILE__) + '/../lib/scraperwiki-api'

metadata

@@ -0,0 +1,82 @@
+--- !ruby/object:Gem::Specification
+name: scraperwiki-api
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Open North
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-05-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: &70190599887060 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.7.8
+  type: :runtime
+  prerelease: false
+  version_requirements: *70190599887060
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &70190599883440 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.6.0
+  type: :development
+  prerelease: false
+  version_requirements: *70190599883440
+description: A Ruby wrapper for the ScraperWiki API
+email:
+- info@opennorth.ca
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- USAGE
+- lib/scraperwiki-api.rb
+- lib/scraperwiki-api/version.rb
+- scraperwiki-api.gemspec
+- spec/scraperwiki-api_spec.rb
+- spec/spec.opts
+- spec/spec_helper.rb
+homepage: http://github.com/opennorth/scraperwiki-api-ruby
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.12
+signing_key: 
+specification_version: 3
+summary: The ScraperWiki API Ruby Gem
+test_files:
+- spec/scraperwiki-api_spec.rb
+- spec/spec.opts
+- spec/spec_helper.rb