kickscraper 0.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 87aea1accb9c5d850a4e99e590e430eec8cce9c0
+  data.tar.gz: 72f2f4a49a7a6a102c0e850912e43e62549255a4
+SHA512:
+  metadata.gz: e992d00b4be9af29e8b3ad4c836cafb30265d5a003370df363c3516e692061c4be2f93d5d8b06804019b57c7d2a6c38b19f32492a0cb04648c8a1be5005b9615
+  data.tar.gz: 4f50b09cd04a405e4b80e43867590e7c3add968b68bf8fca74f42106a6c02066554b2f7636dd14958f2c0fce253bde677c670b4df5895c78c006f837806545a5

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+.DS_Store

data/.rspec

@@ -0,0 +1,3 @@
+--require spec_helper 
+-fd
+--color

data/CHANGELOG.md

@@ -0,0 +1,74 @@
+### 0.2.0
+
+* Changing the minor version because we've rewritten several things to accommodate Kickstarter's change to public search
+  * This encapsulates the changes starting at v 0.1.2
+
+### 0.1.4
+
+* changes
+  * Using ssl (https) for all public www.kickstarter.com searches
+
+### 0.1.3
+
+* changes
+  * Using the public url for categories instead of the api url
+  * Fixing project comments and updates so that they are now pulled in when a project didn't already have them
+  * Adding more tests to cover the new scenarios for pulling public info from Kickstarter without API credentials
+
+### 0.1.2
+
+* changes
+  * Using the public search instead of the authenticated API for project searches
+  * Allowing configuration with no email/pass (which now works for some features)
+  * NOTE: These changes were made in response to a change in Kickstarter's API, and there are currently some broken features
+
+### 0.1.1
+
+* enhancements
+  * Adding a created_projects method to user
+
+* bug fixes
+  * Fixing the backed_projects and starred_projects methods for user
+  * Fixes an error when searching for projects with spaces in the name
+
+### 0.1.0
+
+* enhancements
+  * adding support for Kickstarter categories!
+  * basic pagination when searching projects
+
+* optimize
+  * huge refactor of tests to DRY them up
+
+* deprecate
+  * client.newest_projects (replaced by client.recently_launched_projects)
+  * client.can_load_more_projects (replaced by client.more_projects_available?)
+
+* bug fixes
+  * a little bit more error handling
+
+
+### 0.0.3
+
+* enhancements
+  * added tests!
+  * a little bit of basic error checking
+
+* optimize
+  * DRYing up the API calls
+
+* bug fixes
+  * updating the search for "newest" projects, which was renamed by Kickstarter
+  * handling special characters in searches
+
+
+### 0.0.2
+
+* enhancements
+  * Project searches (popular, recent, ending soon)
+  * User biography
+
+
+### 0.0.1
+
+* Basic working API (getting projects and users)

data/CONTRIBUTORS.md

@@ -0,0 +1,14 @@
+CONTRIBUTORS
+------------
+
+
+#### Originally created by:
+
+* Mark Olson - github @markolson
+
+
+#### Contributions by:
+
+* Ben Rugg - github @benrugg
+* James Allen - github @jamlen
+* Mark Anderson - github @NolaMark

data/Gemfile

@@ -0,0 +1,9 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in kickscraper.gemspec
+gemspec
+
+group :test, :development do
+  gem 'guard-rspec'
+  gem 'terminal-notifier-guard'
+end

data/Guardfile

@@ -0,0 +1,6 @@
+guard :rspec do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{kickscraper/(?:client/)?(.+)\.rb$})  { |m| "spec/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+  watch(%r{spec/support/.*\.rb$})  { "spec" }
+end

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Mark Olson
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,144 @@
+# Kickscraper
+
+Kickscraper is a library for interfacing with Kickstarter's undocumented/unannounced API. With it, you can get a lot of data about projects, whether you've backed them or not, including reward levels, amount pledged, links to the videos, updates, if it's been successful or not, and details about the creator.
+
+Much of the of data available is documented in the [wiki for this project](https://github.com/markolson/kickscraper/wiki/_pages).
+
+
+## Status Update
+
+This version now uses Kickstarter's public search for all project searches. It should be a drop-in replacement for the
+previous version.
+
+
+## Installation
+
+    $ gem install kickscraper
+
+Or for use in another app, add it to your Gemfile
+
+    # use the gem
+    gem 'kickscraper'
+    
+    # or stay up to date with the repo (which should be stable)
+    gem 'kickscraper', :git => 'git://github.com/markolson/kickscraper.git' 
+
+
+## Quick way to get started in the console
+    
+    1. Put your real Kickstarter user credentials in spec/test_constants.rb (or leave blank, which will still allow some kickscraper features)
+    2. Enter the console with "rake console"
+    3. Start using kickscraper with any of the examples below (starting with "c = Kickscraper.client")
+
+
+## Sample Usage
+
+Provided with your user credentials this will list the first 20 or so projects you've backed, along with if the project is still active and if it has met it's funding goal.
+
+    require 'kickscraper'
+
+    Kickscraper.configure do |config|
+        config.email = 'your-kickstarter-email-address@domain.com'
+        config.password = 'This is not my real password. Seriously.'
+    end
+
+    client = Kickscraper.client
+    puts " A | C |"
+    puts "------------------------"
+    client.user.backed_projects.each {|x| 
+        print (x.active? ? ' X |' : '   |')
+        print (x.successful? ? ' X | ' : '   | ')
+        puts x.name
+    }
+
+
+## API Examples
+    c = Kickscraper.client
+
+    c.user.class
+    => Kickscraper::User
+
+    # List what data is available on an object
+    c.user.keys
+    => ["id", "name", "slug", "avatar", "urls", "location", 
+        "biography", "backed_projects_count", "created_projects_count", 
+        "unread_messages_count", "unanswered_surveys_count", 
+        "starred_projects_count", "social", "send_newsletters", 
+        "category_wheel", "notify_of_backings", "notify_of_updates", 
+        "notify_of_follower", "notify_of_friend_activity", 
+        "notify_of_comments", "notify_mobile_of_backings", 
+        "notify_mobile_of_updates", "notify_mobile_of_follower", 
+        "notify_mobile_of_friend_activity", "notify_mobile_of_comments", 
+        "updated_at", "created_at"]
+
+
+    # In addition, User types have "backed_projects" and "starred_projects" available.
+    c.user.backed_projects
+    => [<Project: 'The Veronica Mars Movie Project'>, <Project: 'RiffTrax Wants to Riff TWILIGHT Live in Theaters Nationwide!'>, <Project: 'To Be Or Not To Be: That Is The Adventure'>, <Project: 'Planetary Annihilation - A Next Generation RTS'>, <Project: 'OUYA: A New Kind of Video Game Console'>, <Project: 'Gotham Knight Terrors: Comedic Batman Short'>, <Project: 'Internet Meme Playing Cards'>, <Project: 'GOOD JOB, BRAIN! - A Trivia & Quiz Show Podcast'>, <Project: 'Elevation Dock: The Best Dock For iPhone'>, <Project: 'Trebuchette - the snap-together, desktop trebuchet'>]
+
+    # Some values are already converted to their appropriate types. Some aren't yet.
+    c.user.backed_projects.first.creator.class
+    => Kickscraper::User
+
+    # You can chain methods together and dig into the objects.
+    # So get the thumbnail of the person that created the latest project we backed
+    c.user.backed_projects.first.creator.avatar.thumb
+
+    # You can search for projects
+    c.search_projects('veronica mars')
+    => [<Project: 'The Veronica Mars Movie Project'>, <Project: 'Prodigal Daughter - TV Show Pilot'>]
+
+    # and view their rewards or updates, in addition to any of the data found in the "keys"
+    vm = c.search_projects('veronica mars').first
+    vm.updates
+    => [...]
+    vm.successful?
+    => true
+    vm.video.high
+    => "https://d2pq0u4uni88oo.cloudfront.net/projects/56284/video-217182-h264_high.mp4?2013"
+
+    # When searching for projects (or using convenient methods like recently_launched_projects()),
+    # you can continue the search until Kickstarter returns no more results
+    c.recently_launched_projects
+    => [ _array of projects_ ]
+    c.load_more_projects if c.more_projects_available?
+    => [ _array of additional projects_ ]
+
+    # Print all the updates for all the current user's projects
+    c.user.backed_projects.each { |project|
+        puts project.name.upcase
+        project.updates.reverse.each { |update|
+            # strip the HTML out of the body, since this outputs to a terminal
+            puts "Update #{update.sequence}: #{update.body.gsub(/<\/?[^>]*>/, "")}\n\n"
+        }
+        puts "\n\n"
+    }
+    
+    # Get info about Kickstarter categories or find projects by category
+    client.categories
+    => [<Category: 'Art'>, <Category: 'Conceptual Art'>, <Category: 'Crafts'>, <Category: 'Animation'>, <Category: 'Tabletop Games'>, <Category: 'Classical Music'>, <Category: 'Art Book'>, <Category: 'Hardware'>, <Category: 'Comics'>, <Category: 'Digital Art'>, <Category: 'Graphic Design'>, <Category: 'Documentary'>, <Category: 'Video Games'>, <Category: 'Country & Folk'>, <Category: 'Children's Book'>, <Category: 'Open Software'>, <Category: 'Dance'>, <Category: 'Illustration'>, <Category: 'Product Design'>, <Category: 'Narrative Film'>, <Category: 'Electronic Music'>, <Category: 'Fiction'>, <Category: 'Design'>, <Category: 'Journalism'>, <Category: 'Painting'>, <Category: 'Short Film'>, <Category: 'Hip-Hop'>, <Category: 'Fashion'>, <Category: 'Performance Art'>, <Category: 'Webseries'>, <Category: 'Indie Rock'>, <Category: 'Nonfiction'>, <Category: 'Film & Video'>, <Category: 'Jazz'>, <Category: 'Periodical'>, <Category: 'Food'>, <Category: 'Pop'>, <Category: 'Poetry'>, <Category: 'Mixed Media'>, <Category: 'Games'>, <Category: 'Rock'>, <Category: 'Public Art'>, <Category: 'Radio & Podcast'>, <Category: 'Music'>, <Category: 'Sculpture'>, <Category: 'World Music'>, <Category: 'Photography'>, <Category: 'Publishing'>, <Category: 'Technology'>, <Category: 'Theater'>] 
+
+    category = client.category('product design')
+    => <Category: 'Product Design'> 
+
+    category.keys
+    => ["id", "name", "position", "parent_id", "parent", "projects_count", "urls"] 
+
+    category.projects_count
+    => 234 
+
+    ps = category.projects
+    => [<Project: 'The Skinny Mirror - The Mirror That Compliments You'>, <Project: 'Rustic Fort'>, <Project: 'pushXpro: Revolutionizing the push-up exercise'>, <Project: 'Introducing Nubrella, the world's first hands-free umbrella.'>, <Project: 'Stream: The Pebble Smartwatch Dock'>, <Project: 'Get iPatched - Protect Yourself From Big Brother & Hackers'>, <Project: 'TrueRec introduces the DFP Kayak - Dive Fish Paddle'>, <Project: 'MagCozy: A Leash for Your MagSafe 2 Adapter'>, <Project: 'ECO Friendly Bamboo Party Trays'>, <Project: 'PJ reefs Miniature Saltwater Aquarium'>] 
+
+
+## Contributing
+
+There are two good ways to contribute: 
+
+First, by using it and creating Issues as you find problems or rough spots. Second, by taking matters into your own hands:
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,18 @@
+require "bundler/gem_tasks"
+
+task :console do
+  require 'irb'
+  require 'irb/completion'
+  require 'kickscraper'
+  
+  require './spec/test_constants'
+  
+  Kickscraper.configure do |config|
+      config.email = KICKSCRAPER_TEST_API_EMAIL
+      config.password = KICKSCRAPER_TEST_API_PASSWORD
+      config.proxy = LOCAL_PROXY if defined? LOCAL_PROXY
+  end
+  
+  ARGV.clear
+  IRB.start
+end

data/examples/my_projects.rb

@@ -0,0 +1,18 @@
+require 'kickscraper'
+
+p "configuring.."
+Kickscraper.configure do |config|
+    config.email = 'your-kickstarter-email-address@domain.com'
+    config.password = 'This is not my real password. Seriously.'
+end
+
+p "logging in.."
+c = Kickscraper.client
+p "got access token #{Kickscraper.token.gsub(/\w{30}$/, "X" * 30)}"
+puts " Active | Met Goal |"
+puts "--------------------"
+c.user.backed_projects.each {|x| 
+    print (x.active? ? '    X   |' : '        |')
+    print (x.successful? ? '    X     | ' : '          | ')
+    puts x.name
+}

data/kickscraper.gemspec

@@ -0,0 +1,31 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'kickscraper/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "kickscraper"
+  spec.version       = Kickscraper::VERSION
+  spec.authors       = ["Mark Olson", "Ben Rugg", "James Allen", "Mark Anderson"]
+  spec.email         = ["theothermarkolson@gmail.com"]
+  spec.description   = %q{Interact with Kickstarter through their API}
+  spec.summary       = %q{API library for Kickstarter}
+  spec.homepage      = "https://github.com/markolson/kickscraper"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec-core"
+
+  spec.add_runtime_dependency('faraday', ['>= 0.7', '< 0.9'])
+  spec.add_runtime_dependency('faraday_middleware', '~> 0.8')
+  spec.add_runtime_dependency('multi_json', '>= 1.0.3', '~> 1.0')
+  spec.add_runtime_dependency('hashie',  '~> 2')
+  spec.add_runtime_dependency('uri-query_params')
+
+end

data/lib/kickscraper.rb

@@ -0,0 +1,19 @@
+require 'hashie'
+require 'json'
+
+require "kickscraper/version"
+require "kickscraper/configure"
+require "kickscraper/response"
+require "kickscraper/connection"
+require "kickscraper/client"
+require "kickscraper/api"
+
+
+module Kickscraper
+  extend Configure
+  attr_accessor :client
+
+  def self.client
+    @client ||= Kickscraper::Client.new
+  end
+end

data/lib/kickscraper/api.rb

@@ -0,0 +1,32 @@
+module Kickscraper
+    class Api
+        extend Connection
+        include Hashie::Extensions::Coercion
+        attr_accessor :raw
+
+        def initialize(blob)
+            @raw = blob
+        end
+
+        def method_missing(name)
+            @raw.send(name) if @raw.respond_to? name
+        end
+
+        def self.coerce(raw)
+            a = self.new(raw)
+            self::do_coercion(a)
+            a
+        end
+        
+        def self.do_coercion(instance)
+            self.key_coercions.each{ |k,v| instance.raw[k] = v.coerce(instance.raw[k]) }
+        end
+        
+        def uid
+            self.id == Kickscraper.client.user.id ? 'self' : self.id
+        end
+
+    end
+end
+
+Dir[File.expand_path('../client/*.rb', __FILE__)].each{|f| load f}

data/lib/kickscraper/client.rb

@@ -0,0 +1,225 @@
+require 'uri'
+
+module Kickscraper
+    class Client
+        include Connection
+        
+        attr_accessor :user
+        
+        def initialize
+            @more_projects_available = false
+
+            if Kickscraper.email.nil?
+                @user = nil
+            else
+                if Kickscraper.token.nil?
+                    token_response = connection.post('xauth/access_token?client_id=2II5GGBZLOOZAA5XBU1U0Y44BU57Q58L8KOGM7H0E0YFHP3KTG', {'email' => Kickscraper.email, 'password' => Kickscraper.password }.to_json)
+                    if token_response.body.error_messages
+                        raise token_response.body.error_messages.join("\n")
+                        return
+                    end
+                    Kickscraper.token = token_response.body.access_token
+                    @user = User.coerce(token_response.body.user)
+                end
+            end
+        end
+
+        def find_user(id)
+            self::process_api_call "user", id.to_s
+        end
+
+        def find_project(id_or_slug)
+            self::process_api_call "project", id_or_slug.to_s
+        end
+
+        def search_projects(search_terms, page = nil)
+            self::process_api_call "projects", "advanced", search_terms, page
+        end
+
+        def ending_soon_projects(page = nil)
+            self::process_api_call "projects", "ending-soon", "", page
+        end
+
+        def popular_projects(page = nil)
+            self::process_api_call "projects", "popular", "", page
+        end
+
+        def recently_launched_projects(page = nil)
+            self::process_api_call "projects", "recently-launched", "", page
+        end
+
+        alias_method :newest_projects, :recently_launched_projects
+        
+        def more_projects_available?
+            @more_projects_available
+        end
+        
+        alias_method :can_load_more_projects, :more_projects_available?
+
+        def load_more_projects
+            if self::more_projects_available?
+                self::process_api_call @last_api_call_params[:request_for], @last_api_call_params[:additional_path], @last_api_call_params[:search_terms], (@last_api_call_params[:page] + 1)
+            else
+                []
+            end
+        end
+
+        def categories
+            self::process_api_call "categories", ""
+        end
+
+        def category(id_or_name = nil)
+            return categories.find{|i| i.name.downcase.start_with? id_or_name.downcase} if id_or_name.is_a? String
+                self::process_api_call "category", id_or_name.to_s
+        end
+
+
+        def process_api_call(request_for, additional_path, search_terms = "", page = nil)
+            
+            # save the parameters for this call, so we can repeat it to get the next page of results
+            @last_api_call_params = {
+                request_for: request_for, 
+                additional_path: additional_path, 
+                search_terms: search_terms,
+                page: page.nil? ? 1 : page
+            }
+            
+            # make the api call (to the API resource we want)
+            response = self::make_api_call(request_for, additional_path, search_terms, page)
+            
+            # handle the response, returning an object with the results
+            self::coerce_api_response(request_for, response)
+        end
+        
+        
+        def process_api_url(request_for, api_url, coerce_response = true)
+            
+            # make the api call to whatever url we specified
+            response = connection.get(api_url)
+            
+            
+            # if we want to coerce the response, do it now
+            if coerce_response
+                
+                self::coerce_api_response(request_for, response)
+                
+            # else, just return the raw body
+            else
+                
+                response.body
+            end
+        end
+        
+        
+        def coerce_api_response(expected_type, response)
+            
+            # define what we should return as an empty response, based on the expected type
+            types_that_should_return_an_array = ["projects", "comments", "updates", "categories"]
+            empty_response = (types_that_should_return_an_array.include? expected_type) ? [] : nil
+            
+            
+            # get the body from the response
+            body = response.body
+
+            
+            # if we got an error response back, stop here and return an empty response
+            return empty_response if response.headers['status'].to_i >= 400 || !response.headers['content-type'].start_with?('application/json')
+            return empty_response if (body.respond_to?("error_messages") && !body.error_messages.empty?) || (body.respond_to?("http_code") && body.http_code == 404)
+            
+            
+            # otherwise, take the response from the api and coerce it to the type we want
+            case expected_type.downcase
+            when "user"
+                
+                User.coerce body
+                
+            when "project"
+                
+                Project.coerce body
+                
+            when "projects"
+                
+                # if the body is just an array of projects, with no root keys, then coerce
+                # the array
+                if body.is_a?(Array)
+                    
+                    @more_projects_available = false
+                    body.map { |project| Project.coerce project }
+                    
+                    
+                # else, if the body doesn't have any projects, return an empty array
+                elsif body.projects.nil?
+                    
+                    @more_projects_available = false
+                    return empty_response
+                    
+                    
+                # else, determine if we can load more projects and then return an array of projects
+                else
+                    
+                    if @last_api_call_params && !body.total_hits.nil?
+                        @more_projects_available = @last_api_call_params[:page] * 20 < body.total_hits # (there is a huge assumption here that Kickstarter will always return 20 projects per page!)
+                    end
+                    
+                    return body.projects.map { |project| Project.coerce project }
+                end
+                
+            when "comments"
+                
+                return empty_response if body.comments.nil?
+                body.comments.map { |comment| Comment.coerce comment }
+                
+            when "updates"
+                
+                return empty_response if body.updates.nil?
+                body.updates.map { |update| Update.coerce update }
+                
+            when "categories"
+                
+                return empty_response if body.categories.nil?
+                body.categories.map { |category| Category.coerce category }
+            
+            when "category"
+                
+                Category.coerce body
+
+            else    
+                raise ArgumentError, "invalid api request type"
+            end
+        end
+        
+        
+        def make_api_call(request_for, additional_path = "", search_terms = "", page = nil)
+            
+            # set the url/path differently for each type of request
+            case request_for.downcase
+            when "user"
+                api_or_search = "api"
+                path = "/v1/users"
+            when "project"
+                api_or_search = "api"
+                path = "/v1/projects"
+            when "projects"
+                api_or_search = "search"
+                path = "/discover"
+            when "category", "categories"
+                api_or_search = "api"
+                path = "/v1/categories"
+            end
+            
+            
+            # add the additional path if we have it
+            path += "/" + CGI.escape(additional_path) unless additional_path.empty?
+            
+            
+            # create the params hash and add the params we want
+            params = {}
+            params[:term] = search_terms unless search_terms.empty?
+            params[:page] = page unless page.nil?
+            
+            
+            # make the connection and return the response
+            connection(api_or_search).get(path, params)
+        end
+    end
+end