wuclan 0.2.0

data/lib/wuclan/twitter/model/tweet/tweet_token.rb

@@ -0,0 +1,79 @@
+require 'wuclan/models/tweet/tweet_regexes'
+module Wuclan::Models
+
+  class TweetToken < TypedStruct.new(
+      [:word,           String],
+      [:user_id,        Integer],
+      [:tweet_id,       Integer],
+      [:freq,            Integer]
+      )
+    include ModelCommon
+    include TweetRegexes
+    class_inheritable_accessor :extract_re
+
+    def initialize *args
+      super *args
+      freq = 1 if freq.blank? && (! word.blank?)
+    end
+
+    def num_key_fields()     5  end
+    def numeric_id_fields()  [] ; end
+
+    # crawl through the string
+    # remove each token, leave a space behind
+    def self.extract_tokens! str
+      toks = []
+      str.gsub!(extract_re){|tok| toks << $1.strip ; ' ' }
+      toks
+    end
+  end
+
+  class SmilieToken < TweetToken
+    self.extract_re = RE_SMILIES
+  end
+  class UrlToken < TweetToken
+    self.extract_re = RE_URL
+  end
+  class RtToken < TweetToken
+    self.extract_re = RE_RETWEET
+    def self.extract_tokens! str
+      super.map{|str| str = 'RT_@'+str }
+    end
+  end
+  class AtsignToken < TweetToken
+    self.extract_re = RE_ATSIGNS
+    def self.extract_tokens! str
+      super.map{|str| str = '@'+str }
+    end
+  end
+  class HashtagToken < TweetToken
+    self.extract_re = RE_HASHTAGS
+    def self.extract_tokens! str
+      super.map{|str| str = '#'+str }
+    end
+  end
+  class WordToken < TweetToken
+    self.extract_re = nil
+    #
+    # This is pretty simpleminded.
+    #
+    # returns all words of three or more letters.
+    # * terminal 't and 's (as in "don't" and "it's") are tokenised together
+    # *
+    #
+    # * FIXME -- this doesn't leave str as blank, as it should to behave like
+    #   the other ! methods
+    def self.extract_tokens! str
+      return [] unless str
+      str = str.downcase;
+      # kill off all punctuation except 's
+      # this includes hyphens (words are split)
+      str = str.gsub(/[^\w\'@]+/, ' ').gsub(/\'([st])\b/, '!\1').gsub(/\'/, ' ').gsub(/!/, "'")
+      # Busticate at whitespace
+      words = str.strip.split(/\s+/)
+      #
+      words.reject{|w| w.blank? || (w.length < 3) }
+    end
+  end
+
+end

data/lib/wuclan/twitter/model/tweet.rb

@@ -0,0 +1,74 @@
+module Wuclan::Twitter::Model
+
+  #
+  # Tweet
+  #
+  # Text and metadata for a twitter status update
+  #
+  class Tweet < TypedStruct.new(
+      [:id,                      Integer     ],
+      [:created_at,              Bignum      ],
+      [:twitter_user_id,         Integer     ],
+      [:favorited,               Integer     ],
+      [:truncated,               Integer     ],
+      [:in_reply_to_user_id,     Integer     ],
+      [:in_reply_to_status_id,   Integer     ],
+      [:text,                    String      ],
+      [:source,                  String      ],
+      [:in_reply_to_screen_name, String      ]
+      )
+    include ModelCommon
+
+    #
+    # Memoized; if you change text you have to flush
+    #
+    def decoded_text
+      @decoded_text ||= text.wukong_decode
+    end
+
+    # Key on id
+    def num_key_fields()  1 end
+    def numeric_id_fields()     [:id, :twitter_user_id, :in_reply_to_status_id, :in_reply_to_user_id] ; end
+  end
+
+
+  #
+  # SearchTweet
+  #
+  # Text and metadata for a twitter status update pulled from the search API
+  #
+  class SearchTweet < TypedStruct.new(
+      [:id,                       Integer  ],
+      [:created_at,               Bignum   ],
+      [:twitter_user_id,          Integer  ],
+      [:favorited,                Integer  ],
+      [:truncated,                Integer  ],
+      [:in_reply_to_user_id,      Integer  ],
+      [:in_reply_to_status_id,    Integer  ],
+      [:text,                     String   ],
+      [:source,                   String   ],
+      [:in_reply_to_screen_name,  String   ],
+      [:in_reply_to_sid,          Integer  ],
+      [:twitter_user_screen_name, String   ],
+      [:twitter_user_sid,         Integer  ],
+      [:iso_language_code,        String   ]
+      )
+    include ModelCommon
+
+    #
+    # Memoized; if you change text you have to flush
+    #
+    def decoded_text
+      @decoded_text ||= text.wukong_decode
+    end
+
+    def from_sid()  twitter_user_sid         end
+    def from_user() twitter_user_screen_name end
+    def to_sid()    in_reply_to_sid          end
+    def to_sid()    in_reply_to_screen_name  end
+
+    # Key on id
+    def num_key_fields()  1 end
+    def numeric_id_fields()     [:id, :twitter_user_id, :in_reply_to_status_id, :in_reply_to_user_id] ; end
+  end
+end

data/lib/wuclan/twitter/model/twitter_user/style/color_to_hsv.rb

@@ -0,0 +1,57 @@
+#!/usr/bin/env ruby
+
+# Color::RGB.from_html("fed")
+# Color::RGB.from_html("#fed")
+# Color::RGB.from_html("#cabbed")
+# Color::RGB.from_html("cabbed")
+
+# Following stolen from the 'color-tools' gem
+
+
+# File lib/color/rgb.rb, line 45
+def from_html(html_colour)
+  html_colour = html_colour.gsub(%r{[#;]}, '')
+  case html_colour.size
+  when 3
+    colours = html_colour.scan(%r{[0-9A-Fa-f]}).map { |el| (el * 2).to_i(16) }
+  when 6
+    colours = html_colour.scan(%r<[0-9A-Fa-f]{2}>).map { |el| el.to_i(16) }
+  else
+    raise ArgumentError
+  end
+  colours
+end
+
+
+# File lib/color/rgb.rb, line 167
+def to_hsl r, g, b
+  min   = [ r, g, b ].min
+  max   = [ r, g, b ].max
+  delta = (max - min).to_f
+
+  lum   = (max + min) / 2.0
+
+  if delta <= 1e-5  # close to 0.0, so it's a grey
+    hue = 0
+    sat = 0
+  else
+    if (lum - 0.5) <= 1e-5
+      sat = delta / (max + min).to_f
+    else
+      sat = delta / (2 - max - min).to_f
+    end
+
+    if r == max
+      hue = (g - b) / delta.to_f
+    elsif g == max
+      hue = (2.0 + b - r) / delta.to_f
+    elsif (b - max) <= 1e-5
+      hue = (4.0 + r - g) / delta.to_f
+    end
+    hue /= 6.0
+
+    hue += 1 if hue < 0
+    hue -= 1 if hue > 1
+  end
+  [ hue, sat, lum ]
+end

data/lib/wuclan/twitter/model/twitter_user.rb

@@ -0,0 +1,145 @@
+
+module Wuclan::Twitter::Model
+
+  #
+  # Mixin: common methods for each of the user representations / partitions
+  #
+  module TwitterUserCommon
+    #
+    # Key on id
+    #
+    def num_key_fields()        1       end
+
+    #
+    # Fields that can change value
+    #
+    def mutable_fields
+      # everything but 0 (id) and 1 (scraped_at)
+      to_a[2..-1]
+    end
+
+    def decoded_name
+      @decoded_name        ||= (name        ? name.wukong_decode        : '')
+    end
+    def decoded_location
+      @decoded_location    ||= (location    ? location.wukong_decode    : '')
+    end
+    def decoded_description
+      @decoded_description ||= (description ? description.wukong_decode : '')
+    end
+
+  end
+
+  #
+  # Fundamental information on a user.
+  #
+  class TwitterUser        < TypedStruct.new(
+      [:id,                     Integer],
+      [:scraped_at,             Bignum],
+      [:screen_name,            String],
+      [:protected,              Integer],
+      [:followers_count,        Integer],
+      [:friends_count,          Integer],
+      [:statuses_count,         Integer],
+      [:favourites_count,       Integer],
+      [:created_at,             Bignum]
+      )
+    include ModelCommon
+    include TwitterUserCommon
+    alias_method :tweets_count,    :statuses_count
+    alias_method :favorites_count, :favourites_count
+    #
+    # Rate info
+    #
+    def friends_per_day()      friends_count.to_i   / days_since_created  end
+    def followers_per_day()    followers_count.to_i / days_since_created  end
+    def favorites_per_day()    favorites_count.to_i / days_since_created  end
+    def tweets_per_day()       tweets_count.to_i    / days_since_created  end
+  end
+
+  #
+  # Outside of a users/show page, when a user is mentioned
+  # only this subset of fields appear.
+  #
+  class TwitterUserPartial < TypedStruct.new(
+      [:id,                     Integer],       # appear in TwitterUser
+      [:scraped_at,             Bignum],
+      [:screen_name,            String],
+      [:protected,              Integer],
+      [:followers_count,        Integer],
+      [:name,                   String],        # appear in TwitterUserProfile
+      [:url,                    String],
+      [:location,               String],
+      [:description,            String],
+      [:profile_image_url,      String]         # appear in TwitterUserStyle
+      )
+    include ModelCommon
+    include TwitterUserCommon
+  end
+
+  #
+  # User-set information about a user
+  #
+  class TwitterUserProfile < TypedStruct.new(
+      [:id,                     Integer],
+      [:scraped_at,             Bignum],
+      [:name,                   String],
+      [:url,                    String],
+      [:location,               String],
+      [:description,            String],
+      [:time_zone,              String],
+      [:utc_offset,             String]
+      )
+    include ModelCommon
+    include TwitterUserCommon
+  end
+
+  #
+  # How the user has styled their page
+  #
+  class TwitterUserStyle   < TypedStruct.new(
+      [:id,                             Integer],
+      [:scraped_at,                     Bignum],
+      [:profile_background_color,       String],
+      [:profile_text_color,             String],
+      [:profile_link_color,             String],
+      [:profile_sidebar_border_color,   String],
+      [:profile_sidebar_fill_color,     String],
+      [:profile_background_tile,        String],
+      [:profile_background_image_url,   String],
+      [:profile_image_url,              String]
+      )
+    include ModelCommon
+    include TwitterUserCommon
+  end
+
+  #
+  # For passing around just screen_name => id mapping
+  #
+  class TwitterUserId      < TypedStruct.new(
+      [:id,                     Integer],
+      [:screen_name,            String],
+      [:full,                   Integer],
+      [:followers_count,        Integer],
+      [:created_at,             Bignum],
+      [:protected,              Integer],
+      [:status,                 String]
+      )
+    include ModelCommon
+    include TwitterUserCommon
+    def num_key_fields()  1  end
+  end
+
+  #
+  # For passing around just screen_name => id mapping
+  #
+  class TwitterUserSearchId < TypedStruct.new(
+      [:screen_name,            String],
+      [:sid,                    Integer],
+      [:id,                     Integer]
+      )
+    include ModelCommon
+    include TwitterUserCommon
+    def num_key_fields()  1  end
+  end
+end

data/lib/wuclan/twitter/model.rb

@@ -0,0 +1,21 @@
+module Wuclan
+  module Twitter
+    module Model
+      autoload :ModelCommon,         'wuclan/twitter/model/base'
+      autoload :TwitterUser,         'wuclan/twitter/model/twitter_user'
+      autoload :TwitterUserPartial,  'wuclan/twitter/model/twitter_user'
+      autoload :TwitterUserProfile,  'wuclan/twitter/model/twitter_user'
+      autoload :TwitterUserStyle,    'wuclan/twitter/model/twitter_user'
+      autoload :TwitterUserSearchId, 'wuclan/twitter/model/twitter_user'
+      autoload :TwitterUserId,       'wuclan/twitter/model/twitter_user'
+      autoload :Tweet,               'wuclan/twitter/model/tweet'
+      autoload :SearchTweet,         'wuclan/twitter/model/tweet'
+      autoload :AFollowsB,           'wuclan/twitter/model/relationship'
+      autoload :AFavoritesB,         'wuclan/twitter/model/relationship'
+      autoload :ARepliesB,           'wuclan/twitter/model/relationship'
+      autoload :AAtsignsB,           'wuclan/twitter/model/relationship'
+      autoload :AAtsignsBId,         'wuclan/twitter/model/relationship'
+      autoload :ARepliesBName,       'wuclan/twitter/model/relationship'
+    end
+  end
+end

data/lib/wuclan/twitter/parse/ff_ids_parser.rb

@@ -0,0 +1,27 @@
+module Wuclan
+  module JsonModel
+
+    # ===========================================================================
+    #
+    # Public timeline is an array of tweets => users
+    #
+    #
+    class FfIdsParser < GenericJsonParser
+
+      # friends_ids or followers_ids is an array of user_id's
+      def healthy?()
+        contents && contents.is_a?(Array)
+      end
+
+      def each &block
+        contents.each do |user_b_id|
+          user_b_id = "%010d"%user_b_id.to_i
+          case context.to_s
+          when 'followers_ids' then yield AFollowsB.new(user_b_id, user_a_id)
+          when 'friends_ids'   then yield AFollowsB.new(user_a_id, user_b_id)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/wuclan/twitter/parse/friends_followers_parser.rb

@@ -0,0 +1,52 @@
+module Wuclan
+  module JsonModel
+    # ===========================================================================
+    #
+    # Friends or Followers is a flat list of users => tweets
+    #
+    #
+    class FriendsFollowersParser < GenericJsonParser
+      attr_accessor :scraped_at, :context, :owning_user_id
+      def initialize raw, context, scraped_at, owning_user_id, *ignore
+        super raw
+        self.context         = context.to_sym
+        self.scraped_at      = scraped_at
+        self.owning_user_id  = owning_user_id
+      end
+
+      # Extracted JSON should be an array
+      def healthy?() raw && raw.is_a?(Array) end
+
+      def generate_relationship user, tweet
+        case context
+        when :followers then AFollowsB.new(  user.id,        owning_user_id)
+        when :friends   then AFollowsB.new(  owning_user_id, user.id)
+        when :favorites then AFavoritesB.new(owning_user_id, user.id, (tweet ? tweet.id : nil))
+        else raise "Can't make a relationship out of #{context}. Perhaps better communication is the key."
+        end
+      end
+
+      #
+      # Enumerate over users (each having one tweet)
+      #
+      def each &block
+        raw.each do |hsh|
+          case context
+          when :favorites then parsed = JsonTweet.new(      hsh, nil)
+          else                 parsed = JsonTwitterUser.new(hsh, scraped_at)
+          end
+          next unless parsed && parsed.healthy?
+          user_b = parsed.generate_user_partial
+          tweet  = parsed.generate_tweet
+          [ user_b,
+            tweet,
+            generate_relationship(user_b, tweet)
+          ].compact.each do |obj|
+            yield obj
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/wuclan/twitter/parse/generic_json_parser.rb

@@ -0,0 +1,26 @@
+module Wuclan
+  module JsonModel
+    class GenericJsonParser
+      attr_accessor :raw
+      def initialize raw
+        self.raw = raw
+      end
+      def healthy?() raw && raw.is_a?(Hash) end
+
+      #
+      # Safely parse the json object and instantiate with the raw hash
+      #
+      def self.new_from_json json_str, *args
+        return unless json_str
+        begin
+          raw = JSON.load(json_str) or return
+        rescue Exception => e;
+          warn e
+          return
+        end
+        self.new raw, *args
+      end
+    end
+
+  end
+end

data/lib/wuclan/twitter/parse/json_tweet.rb

@@ -0,0 +1,63 @@
+require 'wukong/encoding'
+module Wuclan
+  module JsonModel
+    #
+    # The JSON tweets records come off the wire a bit more heavyweight than we'd like.
+    #
+    # A sample JSON file, reformatted for clarity:
+    #
+    #
+    # {
+    #   "id"                           : 1012519767,
+    #   "created_at"                   : "Wed Nov 19 07:16:58 +0000 2008",
+    #   // twitter_user_id
+    #   "favorited"                    : false,
+    #   "truncated"                    : false,
+    #   "in_reply_to_user_id"          : null,
+    #   "in_reply_to_status_id"        : null,
+    #   "text"                         : "[Our lander (RIP) had the best name. The next rover to Mars, @MarsScienceLab, needs a name. A contest for kids: http:\/\/is.gd\/85rQ  ]"
+    #   "source"                       : "web",
+    # }
+    #
+    class JsonTweet < GenericJsonParser
+      attr_accessor :raw
+      def initialize raw, twitter_user_id = nil
+        self.raw = raw; return unless healthy?
+        if twitter_user_id
+          raw['twitter_user_id'] = twitter_user_id
+        elsif raw['user'] && raw['user']['id']
+          raw['twitter_user_id'] = raw['user']['id']
+        end
+        self.fix_raw!
+      end
+      def healthy?() raw && raw.is_a?(Hash) end
+
+      #
+      #
+      # Make the data easier for batch flat-record processing
+      #
+      def fix_raw!
+        raw['id']                     = ModelCommon.zeropad_id(  raw['id'])
+        raw['created_at']             = ModelCommon.flatten_date(raw['created_at'])
+        raw['favorited']              = ModelCommon.unbooleanize(raw['favorited'])
+        raw['truncated']              = ModelCommon.unbooleanize(raw['truncated'])
+        raw['twitter_user_id']        = ModelCommon.zeropad_id(raw['twitter_user_id'] )
+        raw['in_reply_to_user_id']    = ModelCommon.zeropad_id(raw['in_reply_to_user_id'])   unless raw['in_reply_to_user_id'].blank?   || (raw['in_reply_to_user_id'].to_i   == 0)
+        raw['in_reply_to_status_id']  = ModelCommon.zeropad_id(raw['in_reply_to_status_id']) unless raw['in_reply_to_status_id'].blank? || (raw['in_reply_to_status_id'].to_i == 0)
+        Wukong.encode_components raw, 'text'
+      end
+
+      def generate_tweet
+        return unless healthy?
+        Tweet.from_hash(raw)
+      end
+      #
+      # produce the included last tweet
+      #
+      def generate_user_partial
+        raw_user = raw['user'] or return
+        JsonTwitterUser.new(raw_user, raw['created_at']).generate_user_partial
+      end
+    end
+  end
+end

data/lib/wuclan/twitter/parse/json_twitter_user.rb

@@ -0,0 +1,122 @@
+require 'wukong/encoding'
+module Wuclan
+  module JsonModel
+
+    #
+    # The JSON user records come off the wire a bit more heavyweight than we'd like.
+    #
+    # We vertically partition the single user record into three, as described above:
+    # one with the fundamental info, one with user's personal info (name, location,
+    # etc) and one with the styling they've applied to their homepage.
+    #
+    # A sample JSON file, reformatted for clarity:
+    #
+    #    {
+    #      "id"                           : 14693823,
+    #      // scraped_at added in processing
+    #      "screen_name"                  : "MarsPhoenix"
+    #      "protected"                    : false,
+    #      "followers_count"              : 39452,
+    #      "friends_count"                : 3,
+    #      "statuses_count"               : 609,
+    #      "favourites_count"             : 5,
+    #      "created_at"                   : "Thu May 08 00:17:54 +0000 2008",
+    #
+    #      // "id"                        : 14693823,
+    #      // scraped_at added in processing
+    #      "name"                         : "MarsPhoenix",
+    #      "url"                          : "http:\/\/tinyurl.com\/5wwaru",
+    #      "location"                     : "Mars, Solar System",
+    #      "description"                  : "I dig Mars! ",
+    #      "time_zone"                    : "Pacific Time (US & Canada)",
+    #      "utc_offset"                   : -28800,
+    #
+    #      // "id"                        : 14693823,
+    #      // scraped_at added in processing
+    #      "profile_background_color"     : "9ae4e8",
+    #      "profile_text_color"           : "000000",
+    #      "profile_link_color"           : "0000ff",
+    #      "profile_sidebar_border_color" : "87bc44",
+    #      "profile_sidebar_fill_color"   : "e0ff92",
+    #      "profile_background_tile"      : true,
+    #      "profile_image_url"            : "http:\/\/s3.amazonaws.com\/twitter_production\/profile_images\/55133915\/PIA09942_normal.jpg",
+    #      "profile_background_image_url" : "http:\/\/s3.amazonaws.com\/twitter_production\/profile_background_images\/3069906\/PSP_008591_2485_RGB_Lander_Detail_516-387.jpg",
+    #
+    #      // Sometimes:
+    #      "status"                       :  { ... a tweet record: see tweet.tsv ... }
+    #
+    #    }
+    #
+    class JsonTwitterUser
+      attr_accessor :raw
+      def initialize raw, scraped_at
+        self.raw = raw; return unless healthy?
+        self.raw['scraped_at'] = scraped_at
+        self.fix_raw!
+      end
+      def healthy?() raw && raw.is_a?(Hash) end
+
+      # user id from the raw hash
+      def twitter_user_id
+        raw['id']
+      end
+
+      #
+      # Make the data easier for batch flat-record processing
+      #
+      def fix_raw!
+        raw['created_at'] = ModelCommon.flatten_date(raw['created_at'])
+        raw['id']         = ModelCommon.zeropad_id(raw['id'])
+        raw['protected']  = ModelCommon.unbooleanize(raw['protected'])
+        Wukong.encode_components raw, 'name', 'location', 'description', 'url'
+        # There are several users with bogus screen names
+        # These we need to **URL encode** -- not XML-encode.
+        if raw['screen_name'] !~ /\A\w+\z/
+          raw['screen_name'] = Wukong.encode_str(raw['screen_name'], :url)
+        end
+      end
+
+      #
+      #
+      # Expand a user .json record into model instances
+      #
+      # Ex.
+      #   # Parse a complete twitter users/show/foo.json record
+      #   twitter_user, twitter_user_profile, twitter_user_style =
+      #     JsonUser.generate_user_classes TwitterUser, TwitterUserProfile, TwitterUserStyle
+      #
+      #   # just get the id and screen_name
+      #   JsonUser.generate_user_classes TwitterUserId
+      #
+      def generate_user_classes *klasses
+        return [] unless healthy?
+        klasses.map do |klass|
+          klass.from_hash(raw)
+        end
+      end
+      #
+      # Create TwitterUser, TwitterUserProfile, and TwitterUserStyle
+      # instances from this hash
+      #
+      def generate_user_profile_and_style
+        generate_user_classes TwitterUser, TwitterUserProfile, TwitterUserStyle
+      end
+      #
+      # Create TwitterUserPartial from this hash -- use this when you only have a
+      # partial listing, for instance in the public timeline or another user's
+      # followers list
+      #
+      def generate_user_partial
+        generate_user_classes(TwitterUserPartial).first
+      end
+      #
+      # produce the included last tweet
+      #
+      def generate_tweet
+        raw_tweet = raw['status']
+        JsonTweet.new(raw_tweet, twitter_user_id).generate_tweet if raw_tweet
+      end
+    end
+
+  end
+end