wuclan 0.2.0

data/lib/wuclan/twitter/parse/public_timeline_parser.rb

@@ -0,0 +1,54 @@
+module Wuclan
+  module JsonModel
+
+    # ===========================================================================
+    #
+    # Public timeline is an array of tweets => users
+    #
+    #
+    class PublicTimelineParser < GenericJsonParser
+      attr_accessor :scraped_at
+      def initialize raw, context, scraped_at, *ignore
+        super raw
+        self.scraped_at = scraped_at
+      end
+
+      # Public timeline is an array of users with one tweet each
+      def healthy?() raw && raw.is_a?(Array) end
+      def each &block
+        raw.each do |hsh|
+          parsed = JsonTweet.new(hsh, nil)
+          next unless parsed && parsed.healthy?
+          [
+            parsed.generate_user_partial,
+            parsed.generate_tweet
+          ].each do |obj|
+            yield obj
+          end
+        end
+      end
+    end
+
+    class SingleStatusParser < GenericJsonParser
+      attr_accessor :scraped_at
+      def initialize raw, context, scraped_at, *ignore
+        super raw
+        self.scraped_at = scraped_at
+      end
+
+      # Public timeline is an array of users with one tweet each
+      def healthy?() raw && raw.is_a?(Hash) end
+      def each &block
+        hsh = raw
+        parsed = JsonTweet.new(hsh, nil)
+        next unless parsed && parsed.healthy?
+        [
+          parsed.generate_user_partial,
+          parsed.generate_tweet
+        ].each do |obj|
+          yield obj
+        end
+      end
+    end
+  end
+end

data/lib/wuclan/twitter/parse/twitter_search_parse.rb

@@ -0,0 +1,60 @@
+require 'wukong/encoding'
+module Wuclan
+  module Twitter
+    module Scrape
+
+      #
+      # ScrapeRequest for the twitter Search API.
+      #
+      # Examines the parsed contents to describe result
+      #
+      class TwitterSearchRequest < Monkeyshines::ScrapeRequest
+        include Wuclan::Twitter::Model
+
+        #
+        # Parse
+        #
+        def parse *args, &block
+          items.each do |item|
+            self.encode_and_sanitize!(item)
+            tweet = tweet_from_parse(item)
+            from_user_sid = TwitterUserSearchId.new(item['from_user'], item['from_user_id'])
+            to_user_sid   = TwitterUserSearchId.new(item['to_user'],   item['to_user_id']  ) if (! item['to_user_id'].blank?)
+            yield tweet
+            yield from_user_sid
+            if to_user_sid
+              yield to_user_sid
+              yield ARepliesBName.new(
+                from_user_sid.screen_name, to_user_sid.screen_name,
+                tweet.id, nil,
+                from_user_sid.sid, to_user_sid.sid)
+            end
+          end
+        end
+
+        def tweet_from_parse item
+          SearchTweet.new(item['id'], item['created_at'],
+            nil, nil, nil, # twitter_user_id, favorited, truncated
+            item[''], nil, item['text'],
+            item['source'],
+            item['to_user'],   item['to_user_id'],
+            item['from_user'], item['from_user_id'], item['iso_language_code'])
+        end
+
+        def encode_and_sanitize! item
+          item['from_user'].wukong_encode!(:url)
+          item['to_user'].wukong_encode!(:url) unless item['to_user'].blank?
+          item['text'].wukong_encode!
+          item['created_at'] = Time.parse(item['created_at']).utc.to_flat
+        end
+
+        def self.encode_screen_name screen_name
+          screen_name.wukong_encode!(:url) if (screen_name =~ /\W/)
+          screen_name
+        end
+
+      end # TwitterSearchRequest (parse)
+
+    end
+  end
+end

data/lib/wuclan/twitter/parse/user_parser.rb

@@ -0,0 +1,30 @@
+module Wuclan
+  module JsonModel
+
+    class UserParser < GenericJsonParser
+      attr_accessor :scraped_at
+      def initialize raw, context, scraped_at, *ignore
+        super raw
+        self.scraped_at = scraped_at
+      end
+      def healthy?() raw && raw.is_a?(Hash) end
+
+      def user
+        @user ||= JsonTwitterUser.new(raw, scraped_at)
+      end
+
+      def generate_twitter_user
+        user.generate_user_classes(TwitterUser).first
+      end
+
+      def each &block
+        user.generate_user_profile_and_style.each do |obj|
+          yield obj
+        end
+        tweet = user.generate_tweet
+        yield tweet if tweet
+      end
+    end
+
+  end
+end

data/lib/wuclan/twitter/scrape/base.rb

@@ -0,0 +1,97 @@
+require 'monkeyshines/scrape_request/raw_json_contents'
+module Wuclan
+  module Twitter
+    module Scrape
+      # Effectively unlimited request maximum
+      # NO_LIMIT = 2**31
+      NO_LIMIT = (50_000_000 / 100) # controlled insanity
+
+      #
+      # Base class for twitter API requests
+      #
+      class Base < TypedStruct.new(
+          [:twitter_user_id,  Integer],
+          [:page,             Integer],
+          [:moreinfo,         String],
+          [:url,              String],
+          [:scraped_at,       Bignum],
+          [:response_code,    Integer],
+          [:response_message, String],
+          [:contents,         String]
+          )
+        class_inheritable_accessor :resource_path, :page_limit, :max_items
+        include Monkeyshines::ScrapeRequestCore
+        # Let us be peers with AFollowsB and TwitterUser and etc.
+        include Wuclan::Twitter::Model
+        # Contents are JSON
+        include Monkeyshines::RawJsonContents
+        # Requests are paginated
+        include Monkeyshines::ScrapeRequest::Paginated
+
+        #
+        def healthy?
+          (! url.blank) && (            # has a URL and either:
+            scraped_at.blank?        || # hasn't been scraped,
+            (! response_code.blank?) || # or has, with response code
+            (! contents.blank?) )       # or has, with response
+        end
+
+        # Generate request URL from other attributes
+        def make_url
+          # This works for most of the twitter calls
+          "http://twitter.com/#{resource_path}/#{twitter_user_id}.json?page=#{page||1}"
+        end
+
+        def key
+          [twitter_user_id, page||1].join('-')
+        end
+
+        # Characters to scrub from contents.
+        # !! FIXME !! -- destructive.
+        BAD_CHARS = { "\r" => "&#13;", "\n" => "&#10;", "\t" => "&#9;" }
+        #
+        # Set the contents from the fetch payload
+        #
+        def response= response
+          self.contents = response.body.gsub(/[\r\n\t]/){|c| BAD_CHARS[c]}
+        end
+
+        #
+        # Pagination
+        #
+
+        # creates the paginated request
+        def request_for_page page, pageinfo=nil
+          (page.to_i > 1) ? self.class.new(twitter_user_id, page) : self
+        end
+
+        # Number of items
+        def num_items
+          parsed_contents.length rescue 0
+        end
+
+        # if from_result has something to say about the max_total_items, fix the
+        # value appropriately. (For example, a twitter_user's :statuses_count
+        # sets the max_total_items for a TwitterUserTimelineRequest)
+        def set_total_items from_result
+        end
+      end
+    end
+  end
+end
+
+# language: http://en.wikipedia.org/wiki/ISO_639-1
+#
+# * Find tweets containing a word:         http://search.twitter.com/search.atom?q=twitter
+# * Find tweets from a user:               http://search.twitter.com/search.atom?q=from%3Aalexiskold
+# * Find tweets to a user:                 http://search.twitter.com/search.atom?q=to%3Atechcrunch
+# * Find tweets referencing a user:        http://search.twitter.com/search.atom?q=%40mashable
+# * Find tweets containing a hashtag:      http://search.twitter.com/search.atom?q=%23haiku
+# * Combine any of the operators together: http://search.twitter.com/search.atom?q=movie+%3A%29
+#
+# * lang:      restricts tweets to the given language, given by an ISO 639-1 code. Ex: http://search.twitter.com/search.atom?lang=en&q=devo
+# * rpp:       the number of tweets to return per page, up to a max of 100. Ex: http://search.twitter.com/search.atom?lang=en&q=devo&rpp=15
+# * page:      the page number (starting at 1) to return, up to a max of roughly 1500 results (based on rpp * page)
+# * since_id:  returns tweets with status ids greater than the given id.
+# * geocode:   returns tweets by users located within a given radius of the given latitude/longitude, where the user's location is taken from their Twitter profile. The parameter value is specified by "latitide,longitude,radius", where radius units must be specified as either "mi" (miles) or "km" (kilometers). Ex: http://search.twitter.com/search.atom?geocode=40.757929%2C-73.985506%2C25km. Note that you cannot use the near operator via the API to geocode arbitrary locations; however you can use this geocode parameter to search near geocodes directly.
+# * show_user: when "true", adds "<user>:" to the beginning of the tweet. This is useful for readers that do not display Atom's author field. The default is "false".

data/lib/wuclan/twitter/scrape/old_skool_request_classes.rb

@@ -0,0 +1,40 @@
+module Wuclan::Twitter::Scrape
+  #
+  # Older versions of wuclan had a slightly different request naming scheme, and
+  # had additional specialized fields. This module adapts old to new; you're only
+  # likely to need this if you're me, @mrflip.
+  #
+  module OldSkoolRequest
+    def initialize(priority, twitter_user_id, page, screen_name, url, *args)
+      self.twitter_user_id = twitter_user_id
+      super(twitter_user_id, page, screen_name, url, *args)
+      self.url = make_url
+    end
+
+    def parse *args, &block
+      handle_special_cases!(*args, &block) or return
+      # super *args
+      yield self
+    end
+
+    def handle_special_cases! *args, &block
+      if scraped_at.to_s !~ /\d{14}/
+        yield BadRecord.new({:bad_date => scraped_at}.to_json, self.to_flat)
+        return nil
+      end
+      true
+    end
+  end
+
+  class Followers < TwitterFollowersRequest       ; include OldSkoolRequest ; end
+  class Friends   < TwitterFriendsRequest         ; include OldSkoolRequest ; end
+  class Favorites < TwitterFavoritesRequest       ; include OldSkoolRequest ; end
+  class UserTimeline < TwitterUserTimelineRequest ; include OldSkoolRequest ; end
+  class Bogus < BadRecord ;
+    def parse suffix=nil, *args
+      errors = suffix.split('-')
+      klass_name = errors.pop
+      yield BadRecord.new("%-23s"%errors.to_json, klass_name, record)
+    end
+  end
+end

data/lib/wuclan/twitter/scrape/twitter_fake_fetcher.rb

@@ -0,0 +1,31 @@
+
+#
+# returns json for a hash / array as appropriate
+#
+class TwitterFakeFetcher < Monkeyshines::Fetcher::FakeFetcher
+
+  def fake_contents req
+    case req
+    when TwitterFollowersIdsRequest, TwitterFriendsIdsRequest
+      (1 .. rand(50)).map{ rand(1e6) }.to_json
+    when TwitterUserRequest
+      { :id => (req.twitter_user_id.to_i == 0 ? rand(1e6) : req.twitter_user_id),
+        :created_at => Time.parse('08-08-2008 12:45'), :name => req.url, :protected => false,
+        :followers_count => rand(2001), :friends_count => rand(401), :statuses_count => rand(120), :favourites_count => rand(50) }.to_json
+    when TwitterFollowersRequest, TwitterFriendsRequest, TwitterFavoritesRequest
+      (0..req.max_items).map{|i| { :fetched => req.url, :id => i } }.to_json
+    else
+      if (req[:page].to_i > 1) && (rand(8) == 0)
+        [].to_json
+      else
+        (0..req.max_items).map{|i| { :fetched => req.url, :id => i } }.to_json
+      end
+    end
+  end
+
+  def get req
+    super req
+    req.contents = fake_contents(req)
+    req
+  end
+end

data/lib/wuclan/twitter/scrape/twitter_ff_ids_request.rb

@@ -0,0 +1,75 @@
+module Wuclan
+  module Twitter
+    module Scrape
+
+      #
+      # API request for the full list of a user's followers (IDs only).
+      # One call gives the whole list.
+      #
+      # Produces a (possibly very large) number of AFollowsB.
+      #
+      class TwitterFollowersIdsRequest < Wuclan::Twitter::Scrape::Base
+        self.resource_path       = 'followers/ids'
+        self.hard_request_limit  = 1
+        self.max_items      = NO_LIMIT
+        def items_count(thing) thing.followers_count == 0 ? 0 : 1 end
+        def make_url() "http://twitter.com/#{resource_path}/#{twitter_user_id}.json" end
+        def key
+          twitter_user_id
+        end
+
+        # followers_ids should be an array of user_ids
+        def healthy?()
+          parsed_contents && parsed_contents.is_a?(Array)
+        end
+
+        #
+        # unpacks the raw API response, yielding all the relationships.
+        #
+        def parse *args, &block
+          parsed_contents.each do |user_b_id|
+            user_b_id = "%010d"%user_b_id.to_i
+            # B is a follower: B follows user.
+            yield AFollowsB.new(user_b_id, user_a_id)
+          end
+        end
+      end
+
+      #
+      # API request for the full list of a user's friends (IDs only).
+      # One call gives the whole list.
+      #
+      # Produces a (possibly very large) number of AFollowsB.
+      #
+      class TwitterFriendsIdsRequest   < Wuclan::Twitter::Scrape::Base
+        self.resource_path       = 'friends/ids'
+        self.hard_request_limit  = 1
+        self.max_items      = NO_LIMIT
+        def items_count(thing) thing.friends_count == 0 ? 0 : 1 end
+        def make_url() "http://twitter.com/#{resource_path}/#{twitter_user_id}.json"  end
+        def key
+          twitter_user_id
+        end
+
+        #
+        # friends_ids should be an array of user_id's
+        #
+        def healthy?()
+          parsed_contents && parsed_contents.is_a?(Array)
+        end
+
+        #
+        # unpacks the raw API response, yielding all the relationships.
+        #
+        def parse *args, &block
+          parsed_contents.each do |user_b_id|
+            user_b_id = "%010d"%user_b_id.to_i
+            # B is a friend: user follows B
+            yield AFollowsB.new(user_a_id, user_b_id)
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/wuclan/twitter/scrape/twitter_followers_request.rb

@@ -0,0 +1,135 @@
+module Wuclan
+  module Twitter
+    module Scrape
+
+      #
+      # API request for the timeline from a user's followers.
+      #
+      # Produces max 100 TwitterUser,Profile,Style, their most recent Tweet,
+      # and an AFollowsB link
+      #
+      # Before early 2009, produced TwitterUserPartials, not full records
+      #
+      # http://apiwiki.twitter.com/Twitter-REST-API-Method%3A-statuses%C2%A0followers
+      #
+      class TwitterFollowersRequest    < Wuclan::Twitter::Scrape::Base
+        self.resource_path      = 'statuses/followers'
+        self.hard_request_limit = NO_LIMIT
+        self.max_items          = 100
+        def items_count(thing) thing.followers_count end
+
+        # set max_total_items from the favourites_count.
+        def set_total_items twitter_user_info
+          self.max_total_items = twitter_user_info['followers_count'].to_i rescue nil
+        end
+
+        # Extracted JSON should be an array
+        def healthy?()
+          parsed_contents && parsed_contents.is_a?(Array)
+        end
+
+        #
+        # unpacks the raw API response, yielding all the interesting objects
+        # and relationships within.
+        #
+        def parse *args, &block
+          return unless healthy?
+          parsed_contents.each do |hsh|
+            json_obj = JsonUserWithTweet.new(hsh, 'scraped_at' => scraped_at)
+            next unless json_obj && json_obj.healthy?
+            #
+            # Extract user, tweet and relationship
+            yield AFollowsB.new(json_obj.user.id, self.twitter_user_id) if json_obj.user
+            json_obj.each(&block)
+          end
+        end
+      end
+
+      #
+      # API request for the timeline from a user's friends.
+      #
+      # Produces max 100 TwitterUser,Profile,Style and their most recent Tweet
+      #
+      # Before early 2009, produced TwitterUserPartials, not full records
+      #
+      # http://apiwiki.twitter.com/Twitter-REST-API-Method%3A-statuses%C2%A0friends
+      #
+      class TwitterFriendsRequest      < Wuclan::Twitter::Scrape::Base
+        self.resource_path      = 'statuses/friends'
+        self.hard_request_limit = NO_LIMIT
+        self.max_items     = 100
+        def items_count(thing) thing.friends_count end
+
+        # set max_total_items from the friends_count.
+        def set_total_items twitter_user_info
+          self.max_total_items = twitter_user_info['friends_count'].to_i rescue nil
+        end
+
+        # Extracted JSON should be an array
+        def healthy?()
+          parsed_contents && parsed_contents.is_a?(Array)
+        end
+
+        #
+        # unpacks the raw API response, yielding all the interesting objects
+        # and relationships within.
+        #
+        def parse *args, &block
+          return unless healthy?
+          parsed_contents.each do |hsh|
+            json_obj = JsonUserWithTweet.new(hsh, 'scraped_at' => scraped_at)
+            next unless json_obj && json_obj.healthy?
+            #
+            # Extract user, tweet and relationship
+            yield AFollowsB.new(self.twitter_user_id, json_obj.user.id) if json_obj.user
+            json_obj.each(&block)
+          end
+        end
+      end
+
+      #
+      # API request for the tweets favorited by the given user.  At 20 requests
+      # per page, this is the worst bargain on the Twitter API call market.
+      #
+      # Produces max 20 TwitterUser,Profile,Style and the favorited Tweet.
+      #
+      # Before early 2009, produced TwitterUserPartials, not full records
+      #
+      # http://apiwiki.twitter.com/Twitter-REST-API-Method%3A-favorites
+      #
+      class TwitterFavoritesRequest    < Wuclan::Twitter::Scrape::Base
+        self.resource_path      = 'favorites'
+        self.hard_request_limit = NO_LIMIT
+        self.max_items     = 20
+        def items_count(thing) thing.favourites_count end
+
+        # Extracted JSON should be an array
+        def healthy?()
+          parsed_contents && parsed_contents.is_a?(Array)
+        end
+
+        # set max_total_items from the favourites_count.
+        def set_total_items twitter_user_info
+          self.max_total_items = twitter_user_info['favourites_count'].to_i rescue nil
+        end
+
+        #
+        # unpacks the raw API response, yielding all the interesting objects
+        # and relationships within.
+        #
+        def parse *args, &block
+          return unless healthy?
+          parsed_contents.each do |hsh|
+            json_obj = JsonTweetWithUser.new(hsh, 'scraped_at' => scraped_at)
+            next unless json_obj && json_obj.healthy?
+            #
+            # Extract user, tweet and relationship
+            yield AFavoritesB.new(self.twitter_user_id, json_obj.user.id, json_obj.tweet.id) if json_obj.user && json_obj.tweet
+            json_obj.each(&block)
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/wuclan/twitter/scrape/twitter_json_response.rb

@@ -0,0 +1,124 @@
+require 'wukong/encoding'
+module Wuclan::Twitter::Scrape
+  class JsonUserTweetPair
+    include Wuclan::Twitter::Model
+    attr_accessor :raw, :moreinfo
+    def initialize raw, moreinfo
+      self.raw = raw
+      self.moreinfo = moreinfo
+      fix_raw_user!
+      fix_raw_tweet!
+      # p ['new', self.class, raw, moreinfo]
+    end
+
+    # Extracted JSON should be an array
+    def healthy?()
+      raw && raw.is_a?(Hash) && (raw_tweet.nil? || raw_tweet.is_a?(Hash))
+    end
+
+    #
+    # generate all the contained TwitterXXX objects
+    #
+    def each
+      if is_partial?
+        yield user
+      else
+        yield user
+        yield user_profile
+        yield user_style
+      end
+      yield tweet if tweet
+    end
+
+    #
+    # Before mid-2009, most calls returned only the fields in
+    # TwitterUserPartial. After a mid-2009 API update, most calls return a full
+    # user record: TwitterUser, TwitterUserStyle and TwitterUserProfile
+    #
+    # This method tries to guess, based on the fields in the raw_user, which it has.
+    #
+    def is_partial?
+      not raw_user.include?('friends_count')
+    end
+
+
+    def tweet
+      Tweet.from_hash raw_tweet if raw_tweet
+    end
+
+    # create TwitterUser object from raw info
+    def user
+      if is_partial?
+        TwitterUserPartial.from_hash raw_user
+      else
+        TwitterUser.from_hash raw_user
+      end
+    end
+    def user_profile
+      TwitterUserProfile.from_hash raw_user
+    end
+    def user_style
+      TwitterUserStyle.from_hash raw_user
+    end
+
+    #
+    # Standardize the raw user hash's fields for further processing with Wukong
+    #
+    def fix_raw_user!
+      return unless raw_user
+      raw_user['scraped_at'] = self.moreinfo['scraped_at']
+      raw_user['created_at'] = ModelCommon.flatten_date(raw_user['created_at'])
+      raw_user['id']         = ModelCommon.zeropad_id(  raw_user['id'])
+      raw_user['protected']  = ModelCommon.unbooleanize(raw_user['protected'])
+      raw_user['profile_background_tile']  = ModelCommon.unbooleanize(raw['profile_background_tile']) unless raw_user['profile_background_tile'].nil?
+      Wukong.encode_components raw_user, 'name', 'location', 'description', 'url'
+      # There are several users with bogus screen names
+      # These we need to **URL encode** -- not XML-encode.
+      if raw_user['screen_name'] !~ /\A\w+\z/
+        raw_user['screen_name'] = Wukong.encode_str(raw_user['screen_name'], :url)
+      end
+    end
+
+    #
+    # Standardize the raw tweet hash's fields for further processing with Wukong
+    #
+    def fix_raw_tweet!
+      return unless raw_tweet
+      raw_tweet['id']                     = ModelCommon.zeropad_id(  raw_tweet['id'])
+      raw_tweet['created_at']             = ModelCommon.flatten_date(raw_tweet['created_at'])
+      raw_tweet['favorited']              = ModelCommon.unbooleanize(raw_tweet['favorited'])
+      raw_tweet['truncated']              = ModelCommon.unbooleanize(raw_tweet['truncated'])
+      raw_tweet['twitter_user_id']        = ModelCommon.zeropad_id(  raw_tweet['twitter_user_id'] )
+      raw_tweet['in_reply_to_user_id']    = ModelCommon.zeropad_id(  raw_tweet['in_reply_to_user_id'])   unless raw_tweet['in_reply_to_user_id'].blank?   || (raw_tweet['in_reply_to_user_id'].to_i   == 0)
+      raw_tweet['in_reply_to_status_id']  = ModelCommon.zeropad_id(  raw_tweet['in_reply_to_status_id']) unless raw_tweet['in_reply_to_status_id'].blank? || (raw_tweet['in_reply_to_status_id'].to_i == 0)
+      Wukong.encode_components raw_tweet, 'text', 'in_reply_to_screen_name'
+    end
+  end
+end
+
+
+class JsonUserWithTweet < JsonUserTweetPair
+
+  def raw_tweet
+    return @raw_tweet if @raw_tweet
+    @raw_tweet = raw['status']
+    @raw_tweet['twitter_user_id'] = raw_user['id'] if @raw_tweet
+    @raw_tweet
+  end
+  def raw_user
+    @raw_user ||= raw
+  end
+end
+
+
+class JsonTweetWithUser < JsonUserTweetPair
+
+  def raw_tweet
+    @raw_tweet ||= raw
+  end
+  def raw_user
+    return @raw_user if @raw_user
+    @raw_user = raw['user']
+    @raw_user
+  end
+end