wuclan 0.2.0

data/lib/wuclan/twitter/model/base.rb

@@ -0,0 +1,72 @@
+require 'date'
+module Wuclan::Twitter::Model
+  module ModelCommon
+
+    #
+    # By default, take the front num_key_fields of the flattened struct
+    #
+    def key
+      to_a[0..(num_key_fields-1)].join("-")
+    end
+
+    # ===========================================================================
+    #
+    # Metrics
+    #
+    def scrape_age
+      (DateTime.now - DateTime.parse_safely(scraped_at)).to_f
+    end
+
+    def days_since_created
+      (DateTime.now - DateTime.parse_safely(created_at)).to_f
+    end
+
+    # ===========================================================================
+    #
+    # Field conversions
+    #
+    # Make the data easier for batch flat-record processing
+    #
+
+    #
+    # Convert date into flat, uniform format
+    # This method is idempotent: repeated calls give same result.
+    #
+    def self.flatten_date dt
+      return dt if dt =~ /\d{14}/
+      DateTime.parse(dt).to_flat if dt
+    end
+
+    #
+    # Zero-pad IDs to a full 10 digits (the max digits for an unsigned 32-bit
+    # integer).
+    #
+    # nil id will be encoded as 0.  Shit happens and we'd rather be idempotent
+    # than picky.
+    #
+    # Note that sometime in 2010 (or sooner, depending on its growth rate: in 2008
+    # Dec it was 1.8M/day) the status_id will exceed 32 bits.  Something will
+    # happen then.
+    # This method is idempotent: repeated calls give same result.
+    #
+    def self.zeropad_id id
+      id ||= 0
+      '%010d' % [id.to_i]
+    end
+
+    #
+    # Express boolean as 1 (true) or 0 (false).  In contravention of typical ruby
+    # semantics (but in a way that is more robust for wukong-like batch
+    # processing), the number 0, the string '0', nil and false are all considered
+    # false. (This also makes the method idempotent: repeated calls give same result.)
+    #
+    def self.unbooleanize bool
+      case bool
+      when 0, '0', false, nil
+        then 0
+        else 1
+      end
+    end
+  end
+end
+

data/lib/wuclan/twitter/model/multi_edge.rb

@@ -0,0 +1,31 @@
+module Wuclan
+  module Models
+    class Edge < TypedStruct.new(
+        [:src,              Integer],
+        [:dest,             Integer]
+        )
+    end
+
+    class MultiEdge < TypedStruct.new(
+        [:src,              Integer],
+        [:dest,             Integer],
+        [:a_follows_b,      Integer],
+        [:b_follows_a,      Integer],
+        [:a_replies_b,      Integer],
+        [:b_replies_a,      Integer],
+        [:a_atsigns_b,      Integer],
+        [:b_atsigns_a,      Integer],
+        [:a_retweets_b,     Integer],
+        [:b_retweets_a,     Integer],
+        [:a_favorites_b,    Integer],
+        [:b_favorites_a,    Integer]
+        )
+      def strong?
+        # Symmetric relationship, easier
+        ( (a_follows_b && b_follows_a) &&
+          () )
+      end
+    end
+
+  end
+end

data/lib/wuclan/twitter/model/relationship.rb

@@ -0,0 +1,176 @@
+module Wuclan::Twitter::Model
+  # features common to all user-user relationships.
+  module RelationshipBase
+    module ClassMethods
+      def rel_name
+        @rel_name ||= resource_name.to_s.gsub(/a_(.*)_b/, '\1')
+      end
+    end
+
+    def self.included base
+      base.class_eval{ extend ClassMethods }
+    end
+  end
+
+  # Follower/Friend relationship
+  class AFollowsB           < TypedStruct.new(
+      [:user_a_id,              Integer],
+      [:user_b_id,              Integer]
+      )
+    include ModelCommon
+    include RelationshipBase
+    # Key on the user-user pair
+    def num_key_fields()  2  end
+    def numeric_id_fields()     [:user_a_id, :user_b_id] ; end
+  end
+
+  # User ==favorites_tweet=> tweet ==by_user=>b
+  class AFavoritesB        < TypedStruct.new(
+      [:user_a_id,              Integer],
+      [:user_b_id,              Integer],
+      [:status_id,              Integer]
+      )
+    include ModelCommon
+    include RelationshipBase
+    # Key on user_a-user_b-status_id (really just user_a-status_id is enough)
+    def num_key_fields()  3 end
+    def numeric_id_fields()     [:user_a_id, :user_b_id, :status_id] ; end
+  end
+
+  # Direct (threaded) replies: occur at the start of a tweet.
+  class ARepliesB           < TypedStruct.new(
+      [:user_a_id,              Integer],
+      [:user_b_id,              Integer],
+      [:status_id,              Integer],
+      [:in_reply_to_status_id,  Integer]
+      )
+    include ModelCommon
+    include RelationshipBase
+    # Key on user_a-user_b-status_id
+    def num_key_fields()  3  end
+    def numeric_id_fields()     [:user_a_id, :user_b_id, :status_id, :in_reply_to_status_id] ; end
+  end
+
+  # Direct (threaded) replies: occur at the start of a tweet.
+  class ARepliesBName       < TypedStruct.new(
+      [:user_a_name,            Integer],
+      [:user_b_name,            Integer],
+      [:status_id,              Integer],
+      [:in_reply_to_status_id,  Integer],
+      [:user_a_sid,             Integer],
+      [:user_b_sid,             Integer]
+      )
+    include ModelCommon
+    include RelationshipBase
+    # Key on user_a-user_b-status_id
+    def num_key_fields()  3  end
+    def numeric_id_fields()     [:user_a_id, :user_b_id, :status_id, :in_reply_to_status_id] ; end
+  end
+
+  # Atsign mentions anywhere in the tweet
+  # note we have no user_b_id for @foo
+  class AAtsignsB           < TypedStruct.new(
+      [:user_a_id,              Integer],
+      [:user_b_name,            String],
+      [:status_id,              Integer]
+      )
+    include ModelCommon
+    include RelationshipBase
+    # Key on user_a-user_b-status_id
+    def num_key_fields()  3 end
+    def numeric_id_fields()     [:user_a_id, :status_id] ; end
+  end
+
+  # Atsign mentions anywhere in the tweet
+  # note we have no user_b_id for @foo
+  class AAtsignsBId         < TypedStruct.new(
+      [:user_a_id,              Integer],
+      [:user_b_id,              Integer],
+      [:status_id,              Integer]
+      )
+    include ModelCommon
+    include RelationshipBase
+    # Key on user_a-user_b-status_id
+    def num_key_fields()  3 end
+    def numeric_id_fields()     [:user_a_id, :user_b_id, :status_id] ; end
+  end
+
+
+  #
+  # A re-tweet is /sent/ by user_a, repeating an earlier message by user_b
+  # Any tweet containing text roughly similar to
+  #   RT @user <stuff>
+  # with equivalently for RT: retweet, via, retweeting
+  #
+  #   !!! OR !!!
+  #
+  # A retweet whore request, something like
+  #   pls RT Hey lookit me
+  #
+  # We just pass along both in the same data structure; the heuristic is poor
+  # enough that we leave it to later steps to be clever.  (Note retweets and
+  # non-retweet-whore-requests have user_b_name set and unset respectively.)
+  #
+  # +user_a_id:+   the user who sent the re-tweet
+  # +status_id:+   the id of the tweet *containing* the re-tweet (for the ID of the original tweet you're on your own.)
+  # +user_b_name:+ the user citied as originating: RT @user_b_name
+  # +please_flag:+ a 1 if the text contains 'please' or 'plz' as a stand-alone word
+  # +text:+        the *full* text of the tweet
+  #
+  class ARetweetsB <  TypedStruct.new(
+      [:user_a_id,              Integer],
+      [:user_b_name,            String],
+      [:status_id,              Integer],
+      [:please_flag,            Integer],
+      [:text,                   String]
+      )
+    include ModelCommon
+    include RelationshipBase
+
+    def initialize *args
+      super *args
+      self.please_flag = ModelCommon.unbooleanize(self.please_flag)
+    end
+    # Key on retweeting_user-user-tweet_id
+    def num_key_fields()  3  end
+    def numeric_id_fields()     [:user_a_id, :status_id] ; end
+    #
+    # If there's no user we'll assume this
+    # is a retweet and not an rtwhore.
+    #
+    def is_retweet?
+      ! user_b_name.blank?
+    end
+  end
+
+  class ARetweetsBId <  TypedStruct.new(
+      [:user_a_id,              Integer],
+      [:user_b_id,              Integer],
+      [:status_id,              Integer],
+      [:please_flag,            Integer],
+      [:text,                   String]
+      )
+    include ModelCommon
+    include RelationshipBase
+
+    def initialize *args
+      super *args
+      self.please_flag = ModelCommon.unbooleanize(self.please_flag)
+    end
+
+    # Key on retweeting_user-user-tweet_id
+    def num_key_fields()  3  end
+    def numeric_id_fields()     [:user_a_id, :user_b_id, :status_id] ; end
+
+    #
+    # If there's no user we'll assume this
+    # is a retweet and not an rtwhore.
+    #
+    def is_retweet?
+      ! user_b_name.blank?
+    end
+  end
+
+
+
+end

data/lib/wuclan/twitter/model/text_element/extract_info_tests.rb

@@ -0,0 +1,83 @@
+
+# ===========================================================================
+#
+# tests = [
+#   'http://foo.us/',
+#   'http://foo.us/a',
+#   'http://foo.us/a?q=a',
+#   'http://foo.us/a#a',
+#   'http://foo.us/a?q=a&b=3#a',
+#   'http://foo.us/a;2/~m-_.\':%+@,;?q=a&b=3#a',
+#   'http://foo.us/a?q=a?',
+#   'http://foo.us/=a#a',
+#   'http://foo.us/a&?q=a&b=3#a',
+#   'http://foo.us/a;2/~m-_.\':%+@,;?q=a&b=3#a&',
+# ]
+# tests.each do |test_str|
+#   p test_str.scan(RE_URL)
+# end
+#
+# atsign_tests = [
+#   '@foo hello',
+#   ' @foo @hello  ',
+#   ' @foo, @hello  ',
+#   '-@foo,@hello',
+#   '@foo@bar ',
+#   'a  basdf@foo b',
+#   'http://@foo',
+#   'foo@bar @bar@foo @zz+',
+# ].each do |test_str|
+#   p test_str.scan(RE_ATSIGNS)
+# end
+#
+# hash_tag_tests = [
+#   '#downtown',
+#   '#downtown?',
+#   '#downtown.',
+#   '#downtown]',
+#   '#downtown}',
+#   '#downtown)',
+#   '#downtown,',
+#   '#downtown;',
+#   '#downtown\'',
+#   '#downtown\'s',
+#   '#downtown_',
+#   '#down+town',
+#   '#down_town',
+#   '#down-town',
+#   '#www.downtown.com',
+#   '#www.downtown.com.',
+#   '##',
+#   '#.',
+#   '#taxonomy:binomial=Alcedo_atthis',
+#   '#geo:lat=52.478342',
+#   '#geo:lon=53.609130',
+#   'a#www.downtown.com.',
+#   ' #www.downtown.com.',
+#   ' =#www.downtown.com.!',
+# ].each do |test_str|
+#   p test_str.scan(RE_HASHTAGS)
+# end
+#
+#
+# # #downtown
+# # #downtown
+# # #downtown
+# # #downtown
+# # #downtown
+# # #downtown
+# # #downtown
+# # #downtown
+# # #downtown
+# # #downtown
+# # #downtown_
+# # #down+town
+# # #down_town
+# # #down-town
+# # #www.downtown.com
+# # #www.downtown.com
+# #
+# #
+# # #taxonomy:binomial=Alcedo_atthis
+# # #geo:lat=52.478342
+# # #geo:lon=53.609130

data/lib/wuclan/twitter/model/text_element/grok_tweets.rb

@@ -0,0 +1,96 @@
+require 'wuclan/grok/tweet_regexes'
+require 'wuclan/models'
+require 'wukong/encoding'
+include Wuclan::Grok::TweetRegexes
+include Wuclan::Models
+
+Tweet.class_eval do
+
+  #
+  #
+  #
+  def tweet_len
+    decoded_text.length
+  end
+
+  #
+  # Any mention of another user, whether at the beginning of a line (and thus
+  # *also* an ARepliesB), a retweet, or just somewhere in the body of the text
+  #
+  def replies
+    unless (in_reply_to_user_id.blank?) || (in_reply_to_user_id.to_i == 0)
+      ARepliesB.new(twitter_user_id, in_reply_to_user_id, self.id, in_reply_to_status_id)
+    end
+  end
+
+  #
+  # Any mention of another user, whether at the beginning of a line (and thus
+  # *also* an ARepliesB), a retweet, or just somewhere in the body of the text
+  #
+  def atsigns
+    matches = decoded_text.scan(RE_ATSIGNS)
+    matches.map do |user_b_name|
+      user_b_name = user_b_name.first.wukong_encode
+      AAtsignsB.new(twitter_user_id, user_b_name, self.id)
+    end
+  end
+
+  #
+  # Remember that a retweet could be an actual retweet, a retweet whore request,
+  # or a retweet of a retweet whore request.
+  #
+  # Or, it could have just fooled us.
+  #
+  # Anyway you can take it from here.
+  #
+  def retweets
+    please_flag   = RE_RTWHORE.match(decoded_text)
+    retweet_match = RE_RETWEET.match(decoded_text)
+    return unless please_flag || retweet_match
+    user_b_name   = retweet_match.captures.first.wukong_encode if retweet_match
+    ARetweetsB.new(twitter_user_id, user_b_name, self.id, please_flag, self.text)
+  end
+
+  #
+  # Hashtags indicate a topic: #hashtag
+  #
+  def hashtags
+    matches = decoded_text.scan(RE_HASHTAGS)
+    matches.map do |hashtag_text|
+      hashtag_text = hashtag_text.first.wukong_encode
+      Hashtag.new(hashtag_text, self.id, twitter_user_id)
+    end
+  end
+
+  #
+  # URLs within a tweet.
+  # can be multiple per tweet.
+  #
+  # Uses a regexp more selective than all canonically allowed - see
+  # tweet_regexes
+  #
+  def tweet_urls
+    matches = decoded_text.scan(RE_URL)
+    matches.map do |tweet_url_text|
+      tweet_url_text = tweet_url_text.first.wukong_encode
+      TweetUrl.new(tweet_url_text, self.id, twitter_user_id)
+    end
+  end
+
+  def text_elements
+    # replies # done in tweet??
+    # atsigns
+    # tweet_url
+    # hashtags
+    # tweet length
+    # words
+    [
+      replies,
+      atsigns,
+      retweets,
+      hashtags,
+      tweet_urls,
+    ].compact.flatten
+  end
+
+end