curate_tumblr 1.0.3

data/lib/curate_tumblr/tumblr/extract_links.rb

@@ -0,0 +1,190 @@
+module CurateTumblr
+  module Tumblr
+	  module ExtractLinks
+		  attr_reader :all_external_links, :links_tofollow
+
+      REGEX_TUMBLR_URL = ".*.tumblr.com"
+      REGEX_TUMBLR_POST_SLUG_URL = "post/.*/"
+      REGEX_TUMBLR_POST_URL = "post/.*"
+      REGEX_TUMBLR_REBLOG_URL = "reblog/.*/"
+      REGEX_TUMBLR_URL_REBLOG_URL_SPEC = "http%3A%2F%2F.*.tumblr.com"
+      REGEX_TUMBLR_URL_REBLOG_URL = "http://.*.tumblr.com"
+      REGEX_TUMBLR_GLOBALKEY_REBLOG_URL = "reblog/.*"
+      REGEX_TUMBLR_KEY_REDIRECT_REBLOG_URL = "/.*\\?"
+      REGEX_TUMBLR_KEY_REBLOG_URL = "/.*"
+      # REGEX_EXTRACT_TUMBLR = "href=.*.tumblr.com"
+      REGEXS_EXTRACTS_TUMBLR = [ "\".*.tumblr.com\"", "\".*.tumblr.com/\"", "\".*.tumblr.com/post", ".*.tumblr.com/post" ]
+      #REGEX_EXTERNALS_FLICKR = [ "\".*flickr.com/photos/.*\"" ]
+      REGEX_EXTERNALS_LINKS = [ "href=\".*.com.*\""]
+
+      class << self
+        def get_tumblr_url( url )
+          return false if !tumblr_url?( url )      
+          tumblr_url = url.scan(/#{REGEX_TUMBLR_URL}/).first
+          CurateTumblr.format_tumblr_url!( tumblr_url )
+          tumblr_url
+        end
+
+        def get_tumblr_from_reblog_url( reblog_url )
+          return false if !tumblr_reblog_url?( reblog_url )  
+          if /#{REGEX_TUMBLR_URL_REBLOG_URL_SPEC}/ =~ reblog_url
+            tumblr_url = reblog_url.scan( /#{REGEX_TUMBLR_URL_REBLOG_URL_SPEC}/ ).first 
+          elsif /#{REGEX_TUMBLR_URL_REBLOG_URL}/ =~ reblog_url
+            tumblr_url = reblog_url.scan( /#{REGEX_TUMBLR_URL_REBLOG_URL}/ ).first
+          else
+            tumblr_url = ""
+          end 
+          CurateTumblr.format_tumblr_url!( tumblr_url )
+          tumblr_url
+        end
+
+        def get_post_id_from_post_url( url )
+          return false if !tumblr_post_url?( url )
+          return CurateTumblr.format_post_id( url.scan( /#{REGEX_TUMBLR_POST_SLUG_URL}/ ).first.gsub('post/', '') ) if /#{REGEX_TUMBLR_POST_SLUG_URL}/ =~ url
+          return CurateTumblr.format_post_id( url.scan( /#{REGEX_TUMBLR_POST_URL}/ ).first.gsub('post/', '') ) if /#{REGEX_TUMBLR_POST_URL}/ =~ url
+          false
+        end
+      
+        def get_post_id_from_reblog_url( url )
+          return false if !tumblr_reblog_url?( url )
+          CurateTumblr.format_post_id( url.scan(/#{REGEX_TUMBLR_REBLOG_URL}/).first.gsub('reblog/', '') )
+        end    
+      
+        def get_reblog_key_from_reblog_url( url )
+          return false if !tumblr_reblog_url?( url )
+          global_key = url.scan(/#{REGEX_TUMBLR_GLOBALKEY_REBLOG_URL}/).first.gsub('reblog/', '')
+          return false if global_key.empty?
+          return CurateTumblr.format_post_reblog_key( global_key.scan( /#{REGEX_TUMBLR_KEY_REDIRECT_REBLOG_URL}/ ).first ) if /#{REGEX_TUMBLR_KEY_REDIRECT_REBLOG_URL}/ =~ global_key
+          return CurateTumblr.format_post_reblog_key( global_key.scan( /#{REGEX_TUMBLR_KEY_REBLOG_URL}/ ).first ) if /#{REGEX_TUMBLR_KEY_REBLOG_URL}/ =~ global_key
+          false
+        end      
+
+        def simple_tumblr_url?( url )
+          return true if /#{REGEX_TUMBLR_URL}/ =~ url
+          false 
+        end
+
+        def tumblr_url?( url )
+          return true if simple_tumblr_url?( url ) &&
+            !url.index("<a ") && !url.index("</a>") && !url.index("</a>")
+          false 
+        end
+
+        def tumblr_post_url?( url )
+          return false if !tumblr_url?( url )
+          return true if /#{REGEX_TUMBLR_POST_SLUG_URL}/ =~ url 
+          return true if /#{REGEX_TUMBLR_POST_URL}/ =~ url 
+          false
+        end
+
+      def tumblr_reblog_url?( url )
+        return false if !tumblr_url?( url )
+        return true if /#{REGEX_TUMBLR_REBLOG_URL}/ =~ url 
+        false
+      end
+      
+      def valid_tumblr_url?( tumblr_url )
+        REGEXS_EXTRACTS_TUMBLR.each do |regex|
+          if /#{regex}/ =~ tumblr_url 
+          else
+            return false
+          end
+          true
+        end
+      end
+
+      def get_tumblr_links_from_regexs_caption( ar_regexs, caption )
+        ar_links = []
+        ar_regexs.each { |regex| ar_links += get_tumblr_link_from_regex_caption( regex, caption ) }      
+        ar_links        
+      end
+
+      def get_tumblr_link_from_regex_caption( regex, caption )
+        if /#{regex}/ =~ caption
+            return CurateTumblr.get_format_ar_tumblrs_urls( caption.scan( /#{regex}/ ) )
+        end
+        []
+      end
+
+      def get_links_from_regexs_caption( ar_regexs, caption )
+        ar_links = []
+        ar_regexs.each { |regex| ar_links += get_link_from_regex_caption( regex, caption ) }      
+        ar_links        
+      end
+
+      def get_link_from_regex_caption( regex, caption )
+        if /#{regex}/ =~ caption
+            return CurateTumblr.get_format_ar_urls( caption.scan( /#{regex}/ ) )
+        end
+        []
+      end    
+
+      def get_urls_only_tumblr( ar_urls )
+        ar_new_urls = []
+        ar_urls.each do |url|
+          ar_new_urls << url if tumblr_url?( url )
+        end
+        ar_new_urls
+      end
+
+     def get_urls_not_tumblr( ar_urls )
+        ar_new_urls = []
+        ar_urls.each do |url|
+          ar_new_urls << url if !simple_tumblr_url?( url ) 
+        end
+        ar_new_urls
+      end    
+
+      def get_tumblr_urls_from_text( text )
+        raise "text #{text.class} is not a String" if !text.is_a? String
+        ar_urls = get_tumblr_links_from_regexs_caption( REGEXS_EXTRACTS_TUMBLR, text )
+        ar_urls = get_urls_only_tumblr( ar_urls )
+        ar_urls = CurateTumblr.get_format_ar_tumblr_urls( ar_urls )
+        Set.new( ar_urls ).to_a
+      end
+
+      def get_external_urls_from_text( text )
+        raise "text #{text.class} is not a String" if !text.is_a? String
+        text.force_encoding 'utf-8'
+        ar_urls = get_links_from_regexs_caption( REGEX_EXTERNALS_LINKS, text )
+        ar_urls = get_urls_not_tumblr( ar_urls )
+        ar_urls = CurateTumblr.get_format_ar_urls( ar_urls )
+        Set.new( ar_urls ).to_a
+      end   
+    end
+
+    # --- end of class methods ---
+
+    def init_extract_links!( hash_config={} )
+      @all_external_links = Set.new
+    end
+
+    def add_external_links_from_caption( caption )
+      links = CurateTumblr::Tumblr::ExtractLinks.get_external_urls_from_text( caption )
+      links = Set.new( links ) if links.is_a? Array
+      @all_external_links += links
+      @all_external_links.count
+    end
+
+    def add_external_link( link )
+      CurateTumblr.format_url!( link )
+      @all_external_links << link
+      @all_external_links.count
+    end
+
+    def extract_links_caption_from_post( hash_post )
+      return false if !hash_post.has_key?( "caption" )
+      add_tofollow_tumblr_links_from_caption( hash_post["caption"], CurateTumblr.get_source_from_hash_post( hash_post ) )
+      add_external_links_from_caption( hash_post["caption"] )
+      true        
+    end
+
+    # --- utils ---
+  
+    def get_reblog_key_from_post( tumblr_url, post_id )
+      tumblr_url = @tumblr_name if tumblr_url.empty?
+      CurateTumblr.get_reblog_key_from_hash_post( get_hash_post( tumblr_url, post_id ) )    
+    end  
+  end
+ end
+end

data/lib/curate_tumblr/tumblr/infos.rb

@@ -0,0 +1,102 @@
+module CurateTumblr
+  module Tumblr
+    module Infos
+
+      HASH_CONFIG_INFOS = "infos"
+      HASH_CONFIG_INFOS_TAGS = "tags"
+      HASH_CONFIG_INFOS_TITLE = "title"
+
+      attr_accessor :infos_caption, :infos_tags, :infos_title
+      attr_reader :state
+
+      class << self
+        def get_infos_config_hash
+          {
+            HASH_CONFIG_INFOS => { }
+          }
+        end
+
+        def get_string_yaml_from_infos_config( hash_config )
+          hash_infos_config = hash_config[HASH_CONFIG_INFOS]
+          %Q{
+            #{Infos::HASH_CONFIG_INFOS}:
+          }
+        end        
+
+        def check_infos_config_hash( hash_config )
+          if hash_config.has_key?( HASH_CONFIG_INFOS )
+            hash_infos_config = hash_config[HASH_CONFIG_INFOS]
+            # nothing to check now
+          end
+        end
+      end
+
+      # --- config ---
+
+      def init_infos!( hash_config={} )
+        @infos_caption = ''
+        @infos_tags = ''
+        @infos_title = ''
+        @state = STATE_QUEUE
+        config_infos( hash_config ) if !hash_config.empty?
+      end
+
+      def config_infos( hash_config )
+        hash_infos = hash_config[HASH_CONFIG_INFOS]
+        return if hash_infos.nil?
+        @infos_tags = hash_infos[HASH_CONFIG_INFOS_TAGS] if hash_infos.has_key?( HASH_CONFIG_INFOS_TAGS )
+        @infos_title = hash_infos[HASH_CONFIG_INFOS_TITLE] if hash_infos.has_key?( HASH_CONFIG_INFOS_TITLE )
+        set_title_caption if !@infos_title.empty?
+      end
+
+      def check_infos_config
+      end    
+
+      # --- options ---
+
+      def get_tumblr_domain
+        @tumblr_name + ".tumblr.com"
+      end
+   
+      def post_in_published
+        @state = STATE_PUBLISHED
+      end
+       
+      def post_in_draft
+        @state = STATE_DRAFT
+      end
+       
+      def post_in_queue
+        @state = STATE_QUEUE
+      end
+       
+      def post_in_private
+        @state = STATE_PRIVATE
+      end    
+
+      def in_published?
+        @state == STATE_PUBLISHED
+      end   
+
+      def set_title_caption( title="", link="", style_begin="<i>", style_end="</i>", before="<p>&nbsp;</p>", after="")
+        title = @infos_title if title.empty?
+        raise "no title" if title.empty?
+        link = "http://www.#{@tumblr_name}.tumblr.com" if link.empty?
+        @infos_caption = "#{before}<p><a href='#{link}' target='_blank'>"
+        @infos_caption = @infos_caption + "#{style_begin}" if !style_begin.empty?
+        @infos_caption = @infos_caption + "#{title}"
+        @infos_caption = @infos_caption + "#{style_end}" if !style_end.empty?
+        @infos_caption = @infos_caption + "</a></p>#{after}"
+      end    
+            
+      def add_infos_ar_tags( ar_tags ) 
+        add_infos_tags( ar_tags.join(", ") )
+      end
+
+      def add_infos_tags( tags ) 
+        @infos_tags += ", " if !@infos_tags.empty?
+        @infos_tags += tags
+      end    
+    end
+  end
+end

data/lib/curate_tumblr/utilities/monkey.rb

@@ -0,0 +1,5 @@
+class Hash
+	def check_key( key )
+		raise "'#{key}' is empty in #{to_s}" if self[key].empty?
+	end
+end

data/lib/curate_tumblr/utilities/utilities.rb

@@ -0,0 +1,4 @@
+require File.expand_path File.join( File.dirname(__FILE__), 'utilities_client' )
+require File.expand_path File.join( File.dirname(__FILE__), 'utilities_file' )
+require File.expand_path File.join( File.dirname(__FILE__), 'utilities_format' )
+require File.expand_path File.join( File.dirname(__FILE__), 'utilities_validate' )

data/lib/curate_tumblr/utilities/utilities_client.rb

@@ -0,0 +1,103 @@
+module CurateTumblr
+  
+	def self.get_reblog_key_from_hash_post( hash_post )
+	  return false if !hash_post_valid?( hash_post )
+	  return false if !hash_post.has_key?( 'reblog_key' )
+	  hash_post['reblog_key']
+	end
+
+	def self.get_id_from_direct_post( hash_id )
+	  return false if !hash_id_valid?( hash_id )
+	  hash_id['id']
+	end	
+
+	def self.get_hash_url_from_post_url( post_url )
+		return false if !Tumblr::ExtractLinks.tumblr_post_url?( post_url )
+		{ tumblr_url: Tumblr::ExtractLinks.get_tumblr_url( post_url ), post_id: Tumblr::ExtractLinks.get_post_id_from_post_url( post_url ) } 
+	end
+	 
+	def self.get_hash_url_from_reblog_url( reblog_url )
+		return false if !Tumblr::ExtractLinks.tumblr_reblog_url?( reblog_url )
+		{ tumblr_url: Tumblr::ExtractLinks.get_tumblr_from_reblog_url( reblog_url ), post_id: Tumblr::ExtractLinks.get_post_id_from_reblog_url( reblog_url ) } 
+	end 
+
+	def self.get_source_from_hash_post( hash_post )
+	  get_url_from_hash_post( "source_url", hash_post )
+	end
+
+	def self.get_link_url_from_hash_post( hash_post )
+	  get_url_from_hash_post( "link_url", hash_post )
+	end
+
+	def self.get_url_from_hash_post( url_key, hash_post )
+	  return false if !hash_post_valid?( hash_post )
+	  return false if !hash_post.has_key?( url_key )
+	  #return false if !tumblr_url?( hash_post[url_key] )
+	  hash_post[url_key]
+	end
+
+	def self.hash_status_ok?( hash_status )
+	  hash_status?( hash_status, CLIENT_STATUS_OK )
+	end
+
+	def self.hash_status_not_found?( hash_status )
+	  hash_status?( hash_status, CLIENT_STATUS_NOT_FOUND )
+	end
+
+	def self.hash_status_rate_limit?( hash_status )
+	  hash_status?( hash_status, CLIENT_STATUS_RATE_LIMIT )
+	end
+
+	def self.hash_status_bad_request?( hash_status )
+	  hash_status?( hash_status, CLIENT_STATUS_BAD_REQUEST )
+	end
+
+	def self.hash_status?( hash_status, status )
+	  return true if !hash_status.has_key?( "status" )
+	  return true if hash_status["status"] == status
+	  false
+	end
+
+	def self.get_random_sleep( min, max )
+	  raise "min is empty" if !min
+	  raise "max is empty" if !max
+	  raise "min (#{min} is >=  max (#{max})" if min >= max
+	  seconds = 1 + rand( max )
+	  seconds = min if seconds < min
+	  seconds
+	end
+
+	def self.display_hash_post( hash_post, is_short=false )
+  	raise "hash_post #{hash_post} is not valid" if !hash_post_valid?( hash_post )
+  	title = "#{hash_post['type']} in #{hash_post['blog_name']} (#{hash_post['note_count']})"
+  	title = title + " (source #{hash_post['source_url']})" if hash_post.has_key?( 'source_url' )
+  	puts title
+  	return if is_short
+  	hash_post.each do |key, value|
+  	  puts "> #{key} : #{value}"
+  	end
+	end
+	  
+	def self.display_hash_multiple_posts( hash_posts, is_short=false )
+	  raise "hash_posts #{hash_posts} is not valid" if !hash_multiple_posts_valid?( hash_posts )
+	  hash_posts['posts'].each { |hash_post| display_hash_post( hash_post, is_short ) }
+	end
+
+	def self.get_summary_hash_post( hash_post )
+	  return hash_post if !hash_post_valid?( hash_post )
+	  summary = {}
+	  copy_hash_key( hash_post, summary, "blog_name")
+	  copy_hash_key( hash_post, summary, "id")
+	  copy_hash_key( hash_post, summary, "type")
+	  copy_hash_key( hash_post, summary, "post_url")
+	  copy_hash_key( hash_post, summary, "slug")
+	  copy_hash_key( hash_post, summary, "state")
+	  copy_hash_key( hash_post, summary, "reblog_key")
+	  summary
+	end
+
+	def self.copy_hash_key( hash_source, hash_target, key )
+	  return false if !hash_source.has_key?( key )
+	  hash_target[key] = hash_source[key]
+	end
+end	

data/lib/curate_tumblr/utilities/utilities_file.rb

@@ -0,0 +1,50 @@
+module CurateTumblr
+  def self.get_filepath_links_by_tumblr( tumblrname, path=PATH_LINKS )
+    path + tumblrname + "/" + tumblrname + "_links"
+  end
+
+  def self.get_filepath_errors_by_tumblr( tumblrname, path=PATH_LINKS )
+    path + tumblrname + "/" + tumblrname + "_errors"
+  end
+
+  def self.get_filepath_tofollow_by_tumblr( tumblrname, path=PATH_LINKS )
+    path + tumblrname + "/" + tumblrname + "_tofollow"
+  end
+
+  def self.get_filepath_externallinks_by_tumblr( tumblrname, path=PATH_LINKS )
+    path + tumblrname + "/" + tumblrname + "_externallinks"
+  end
+        
+  def self.get_ar_from_file( filename )
+    raise "filename is empty" if filename.nil? || filename.empty?
+    ::File.open( filename, "r" ).readlines
+  end       
+
+  def self.checkFile( filename )
+    raise "Filename is empty" if filename.empty?
+    raise "File #{filename} doesn't exist" if !::File.exists?( filename )
+    raise "#{filename} is not a true file" if !::File.file?( filename )
+    raise "File #{filename} is not readable" if !::File.readable?( filename )
+    true        
+  end  
+    
+  def self.backup_file( filename, backupname="" )
+    return false if !::File.exists?( filename )
+    backupname = filename + ".save" if backupname.empty?
+    ::File.open(backupname , "w+" ) { |file| file.puts get_ar_from_file( filename ) }
+    true
+  end
+
+  def self.add_set_tofile_without_repeat( filename, set_tofile )
+    set_toadd = set_tofile.dup
+    ::File.open( filename, "w" ) if !::File.exists?( filename )
+    ar_file = get_ar_from_file( filename )
+    ar_file.each { |line| format_tumblr_url!( line ) }
+    set_toadd.merge( ar_file )
+    ::File.open( filename, "w" ) { |file| file.puts( set_toadd.to_a )  }
+    true
+  end
+
+  def create_config_file( filename, hash_config )
+  end
+end