activity_mapper 0.1.0

data/lib/activity_mapper/linguistics.rb

@@ -0,0 +1,147 @@
+# tagger.rb - a Ruby Part of Speech Tagger patterned on the work of Eric Brill.  Version 0.1
+#
+# Copyright 2005 Mark Watson.  All rights reserved.
+# This software is released under the LGPL
+#
+# Contributor: Pat Eyler
+#
+
+module ActivityMapper
+
+  module Linguistics
+
+    class Tagger
+  
+      UNINTERESTING_PARTS_OF_SPEECH = ['DT', 'PRP', 'IN', 'CC', 'MD']
+
+      def initialize()
+        @lexicon = {}
+        file = File.new(File.join(File.dirname(__FILE__), 'lexicon.txt'), 'r')
+        file.each_line {|line|
+                          toks=line.split
+                          @lexicon[toks.shift]=toks
+                       }
+        file.close
+      end
+
+      def tokenize(words)
+        words.split(/ |,|\.|\:|\;|\'/) #'
+      end
+  
+      def self.keywords_for_caption(caption)
+        @@tagger ||= self.new
+        keywords = []
+        all_keywords = @@tagger.tokenize(caption) 
+        pos_tags = @@tagger.part_of_speech_tag(all_keywords)
+        all_keywords.each_with_index do |keyword,i|
+          next if UNINTERESTING_PARTS_OF_SPEECH.include?(pos_tags[i])
+          next unless keyword.size > 4
+          keywords << keyword.downcase
+        end
+        keywords
+      rescue => e
+        all_keywords
+      end
+
+      def part_of_speech_tag(text)
+
+        ## start by tokenizing strings passed in
+        if text.class == String then
+          text = tokenize(text) 
+        end
+
+        ## we only work on arrays.  If text isn't an array, 
+        ## quit now.
+        if text.class != Array then
+          raise RuntimeError, "can't tokenize #{text.class}"
+        end
+
+        # this looks like an artifact of testing
+        #	puts "text:",text,"\n"
+        ret = []
+    
+        text.each do
+          |w| ret << (@lexicon[w] && @lexicon[w][0]) || 
+          (@lexicon[w.downcase] && words[w.downcase][0]) || 
+            'NN' 
+        end
+    
+        ## Now, apply transformational rules:
+        text.length.times do |i|
+      
+          ## rule 1: DT, {VBD | VBP} --> DT, NN
+          if i > 0 then
+            if ret[i - 1] == "DT" then
+              if ret[i] == "VBD" or ret[i] == "VBP" or ret[i] == "VB" then
+                ret[i] = "NN"
+              end
+            end
+          end
+
+          ## rule 2: convert a noun to a number (CD) if "." appears in the word
+          if ret[i] =~ /^N/ then
+            if text[i] =~ /\./ then
+              ret[i] = "CD"
+            end
+          end
+
+          ## rule 3: convert a noun to a past participle if words[i] ends 
+          ## with "ed"
+          if ret[i] =~ /^N/ && text[i] =~ /ed$/ then
+            ret[i] = "VBN"
+          end
+
+          ## rule 4: convert any type to adverb if it ends in "ly"
+          if text[i] =~ /ly$/ then
+            ret[i] = "RB"
+          end
+
+          ## rule 5: convert a common noun (NN or NNS) to a adjective if 
+          ## it ends with "al"
+          if ret[i] =~ /^NN/ && text[i] =~ /al$/ then
+            ret[i] = "JJ"
+          end
+
+          ## rule 6: convert a noun to a verb if the preceeding work is "would"
+          if i > 0 then
+            if ret[i] =~ /^NN/ then
+              if text[i-1].downcase == "would" then
+                ret[i] = "VB"
+              end
+            end
+          end
+
+          ## rule 7: if a word has been categorized as a common noun and 
+          ## it ends with "s", then set its type to plural common noun (NNS)
+          if ret[i] == "NN" && text[i] =~ /s$/ then
+            ret[i] = "NNS"
+          end
+
+          ## rule 8: convert a common noun to a present participle 
+          ## verb (i.e., a gerand)
+          if ret[i] =~ /^NN/ && text[i] =~ /ing$/ then
+            ret[i] = "VBG"
+          end
+
+          ## rule 9: <noun> <noun 2> --> <noun> <verb> if <noun 2> 
+          ## can also be a verb
+          if i > 0  then
+            if ( ( ret[i-1] =~ /^NN/ ) && ( ret[i] =~ /^NN/ ) )  then
+              if @lexicon[text[i]].include?("VBN")  then
+                ret[i] = "VBN"
+              end
+              if @lexicon[text[i]].include?("VBZ") then
+                ret[i] = "VBZ"
+              end
+            end
+          end
+
+       end
+       return ret
+      end # def getTag
+
+    end # class Tagger
+
+  end
+  
+end

data/lib/activity_mapper/service_module.rb

@@ -0,0 +1,194 @@
+
+module ActivityMapper
+
+  class ServiceModule
+  
+    COMMON_DIRECTIVES = ['user', 'users', 'person', 'people', 'traveller', 'in', 'profile', 'profiles']
+  
+    def initialize(profile)
+      @profile = profile
+    end
+  
+    def self.all_accepted_hosts
+      host_expressions = []
+      self.subclasses.each do |service_module_klass|
+        host_expressions << eval("#{service_module_klass.to_s}::ACCEPTED_HOSTS")
+      end
+      host_expressions.flatten!
+    end
+  
+    def self.klass_for(url)
+      self.subclasses.each do |service_module_klass|
+        if service_module_klass.accepts?(url)
+          return service_module_klass
+        end
+      end
+      nil
+    end
+    
+    # http://groups.google.com/group/comp.lang.ruby/browse_thread/thread/e812c7cef446a96?pli=1
+    def self.subclasses
+      class_hash = {}
+      ObjectSpace.each_object do |obj|
+        if Class == obj.class
+          if obj.ancestors.include? self
+            class_hash[obj] = true
+          end
+        end
+      end
+      class_hash.keys.reject { |ch| ch == self }
+    end
+  
+    # Based on a profile url, decide if I need to be responsible for this
+    def self.accepts?(url)  
+      u = uri(url)
+      return false unless u
+      
+      self::ACCEPTED_HOSTS.each do |host_re|
+        if (u.host =~ host_re)
+          return true
+        end
+      end
+      false
+    end
+  
+    def self.detect_username(profiles)
+      usernames = {}
+      profiles.each do |me_url|
+        username = username_from_url(me_url)
+        if username
+          usernames[username] ||= 0
+          usernames[username] += 1
+        end
+      end
+    
+      usernames = usernames.to_a
+      return [] if usernames.blank?
+    
+      usernames.sort! { |b,a| a.last <=> b.last }
+    
+      # Factor in the shortest username if top 2 matches have the same score
+      top_score = usernames[0].last
+      if (usernames[1] && usernames[1].last == top_score)
+        usernames = [usernames[0], usernames[1]]
+        usernames.sort! { |a,b| a.first.size <=> b.first.size }
+      end
+    
+      usernames.collect(&:first)
+    end
+  
+    # -- Implementables
+
+    # Update the long term data (eg top used software, biography)
+    def create_or_update_summary!(options = {}); raise "Method not implemented"; end
+  
+    # This is called as often as possible, this is to passively aggregate the latest activity
+    def aggregate_activity!(options = {}); raise "Method not implemented"; end
+  
+    # -- Optional Implementables
+  
+    # Analyze an individual activity object (eg photo, bookmark, book, software, slide...)
+  
+    def shallow_analysis_on(activity_object)
+      return unless activity_object.shallowly_analyzed_at.blank?
+    
+      # Extract social connections from body
+      #unless activity_object.body.blank?
+      #  SocialConnection.generate_from_nanoformats(@profile, activity_object.body)
+      #end
+    
+      # Extract tags from title
+      unless activity_object.title.blank?
+        tags = Linguistics::Tagger.keywords_for_caption(activity_object.title)
+        activity_object.tag_list = (activity_object.tag_list || []) + tags
+        activity_object.shallowly_analyzed_at = Time.now
+        activity_object.save
+      end
+    end
+  
+    def deep_analysis_on(activity_object)
+      return unless activity_object.deeply_analyzed_at.blank?
+      return unless activity_object.body.size > 100
+      unless activity_object.body.blank?
+        @extractor = ZemantaExtractor.new
+        tags = @extractor.extract_tags(activity_object.body)
+        activity_object.tag_list = activity_object.tag_list + tags
+        activity_object.save
+      end
+    end
+  
+    protected
+  
+    def self.username_from_url(url)
+      # Check for http://:host/COMMON_DIRECTIVES/:username
+      username_after_directive = url.match(/^http\:\/\/([^\/]+)\/([^\/]+)\/([^\/]+)/)
+      if username_after_directive && 
+         username_after_directive[3] && 
+         COMMON_DIRECTIVES.include?(username_after_directive[2])
+        return username_after_directive[3]
+      end
+    
+      # Check for http://:host/COMMON_DIRECTIVES?:var=:username
+      username_after_directive = url.match(/^http\:\/\/([^\/]+)\/([^\/]+)\?\w+\=([^\/]+)/)
+      if username_after_directive && 
+         username_after_directive[3] && 
+         COMMON_DIRECTIVES.include?(username_after_directive[2])
+        return username_after_directive[3]
+      end
+    
+      # Check for http://:host/username
+      username_ending = url.match(/^http\:\/\/([^\/]+)\/([^\/]+)\/*$/)
+      if username_ending
+        return username_ending[2]
+      end
+    
+      nil
+    end
+  
+    def self.uri(url)
+      URI.parse(url)
+    rescue => e
+      nil
+    end
+  
+    def create_activity(mapped_entry, activity_object_type, activity_verb, additional_activity_parameters = {}, &block)
+      @object_names ||= mapped_entry.keys.collect { |oa| oa.split('.').first }
+    
+      return unless mapped_entry['activity_object.url'] # Need proper warning here
+    
+      # Create entity pool
+      activity = Activity.create(additional_activity_parameters.merge(
+        :user_id         => @profile.user_id,
+        :verb            => activity_verb
+      ))
+      content_identifier = ActivityObject.content_identifier(mapped_entry['activity_object.url'])
+      activity_object = ActivityObject.fetch(content_identifier, activity_object_type.id)
+      activity_object ||= ActivityObject.create(:activity_object_type_id => activity_object_type.id)
+      media = @object_names.include?('media') ? Media.create : nil
+      rating_summary = @object_names.include?('rating_summary') ? RatingSummary.create : nil
+    
+      # Auto-populate attributes
+      mapped_entry.each do |destination, value|
+        eval("#{destination} = value")
+      end
+    
+      if block
+        block.call(activity, activity_object)
+      end
+    
+      shallow_analysis_on(activity_object)
+    
+      # Save all
+      activity_object.media = media unless media.blank?
+      activity_object.rating_summary = rating_summary unless rating_summary.blank?
+      activity_object.save
+      activity.object = activity_object
+      activity.save
+      
+      @profile.activities << activity
+      @profile.save
+    end
+  
+  end
+
+end

data/lib/activity_mapper/service_modules.rb

@@ -0,0 +1,4 @@
+
+Dir.open(File.join(File.dirname(__FILE__), 'service_modules')).each do |file|
+  require File.join(File.dirname(__FILE__), 'service_modules', file) if File.extname(file) == '.rb'
+end

data/lib/activity_mapper/service_modules/delicious.rb

@@ -0,0 +1,41 @@
+
+module ActivityMapper
+
+  class DeliciousServiceModule < ServiceModule
+    ACTIVITY_MAP = {
+      nil => {
+        'activity.occurred_at'          => 'dt',
+        'activity.caption'              => 'd',
+        'activity_object.title'         => 'd',
+        'activity_object.body'          => 'd',
+        'activity_object.tag_list'      => 't',
+        'activity_object.url'           => 'u',
+        'activity.url'                  => 'u'
+      }
+    }
+    ACCEPTED_HOSTS = [/delicious\.com/, /del\.icio\.us/]
+  
+    def create_or_update_summary!(options = {})
+      @profile.update_attributes(:username => self.class.username_from_url(@profile.url))
+    end
+  
+    def aggregate_activity!(options = {})
+      mapper = ActivityDataMapper.new(ACTIVITY_MAP)
+    
+      mapper.fetch!(
+        "http://feeds.delicious.com/v2/json/#{@profile.username}?count=20",
+        :format => :json
+      )
+      mapper.map!
+      mapper.entries.sort! { |e2,e1|
+        e1['activity.occurred_at'] <=> e2['activity.occurred_at']
+      }
+      mapper.entries.each do |entry|
+        break if Activity.exists?(@profile.user_id, entry)
+        create_activity(entry, ActivityObjectType::BOOKMARK, ActivityVerb::POST)
+      end
+    end
+  
+  end
+  
+end

data/lib/activity_mapper/service_modules/flickr.rb

@@ -0,0 +1,59 @@
+
+module ActivityMapper
+  
+  class FlickrServiceModule < ServiceModule
+    ACTIVITY_MAP = {
+      'items' => {
+        'activity.occurred_at'          => 'published',
+        'activity.caption'              => 'title',
+        'activity_object.title'         => 'title',
+        'activity_object.body'          => 'description',
+        'activity_object.spaced_tags'   => 'tags',
+        'activity_object.url'           => 'link',
+        'activity.url'                  => 'link',
+        'media.thumbnail_url'           => 'media/m',
+        'media.embed_url'               => 'media/m'
+      }
+    }
+    ACCEPTED_HOSTS = [/flickr\.com/]
+  
+    def create_or_update_summary!(options = {})
+      @profile.update_attributes(:username => self.class.username_from_url(@profile.url))
+      if @profile.native_id.blank?
+        response_body = Connector.fetch("http://api.flickr.com/services/rest/?method=flickr.urls.lookupUser&api_key=#{FLICKR_API_KEY}&url=#{CGI.escape(@profile.url)}&format=json&nojsoncallback=1")
+        profile = Connector.deserialize(response_body, :json)
+        if profile['user']
+          @profile.native_id = profile['user']['id']
+          @profile.username = profile['user']['username']['_content'] if profile['user']['username']
+          @profile.save
+        end
+      end
+    end
+  
+    def aggregate_activity!(options = {})
+      mapper = ActivityDataMapper.new(ACTIVITY_MAP)
+    
+      mapper.fetch!(
+        "http://api.flickr.com/services/feeds/photos_public.gne?id=#{@profile.native_id}&lang=en-us&format=json&nojsoncallback=1",
+        :format => :json
+      )
+      mapper.map!
+      mapper.entries.each do |entry|
+        next if Activity.exists?(@profile.user_id, entry)
+        create_activity(entry, ActivityObjectType::PHOTO, ActivityVerb::POST)
+      end
+    
+      mapper.fetch!(
+        "http://api.flickr.com/services/feeds/photos_faves.gne?nsid=#{@profile.native_id}&lang=en-us&format=json&nojsoncallback=1",
+        :format => :json
+      )
+      mapper.map!
+      mapper.entries.each do |entry|
+        next if Activity.exists?(@profile.user_id, entry)
+        create_activity(entry, ActivityObjectType::PHOTO, ActivityVerb::FAVORITE)
+      end
+    end
+  
+  end
+  
+end