curl 0.0.4 → 0.0.9

data/lib/curl.rb

@@ -4,6 +4,7 @@ require "open3"
 require 'fileutils' 
 require 'ap' 
 require 'digest/md5'
+require 'string_cleaner'
   
 
 include Open3
@@ -31,15 +32,20 @@ class CURL
     attr_accessor :user_agent
     
   def initialize(keys={})
+    @socks_hostname = keys[:socks_hostname] ||= false
     @cache = ( keys[:cache] ? keys[:cache] : false  )
+    @cache_time = ( keys[:cache_time] ? keys[:cache_time] : 3600*24*1  ) # 1 day cache life
+    @connect_timeout = keys[:connect_timeout] || 6
+    @max_time = keys[:max_time] || 8
+    @retry = keys[:retry] || 1
     @cookies_enable = ( keys[:cookies_disable] ? false : true  )
           @user_agent     = AGENT_ALIASES["Google"]#AGENT_ALIASES[AGENT_ALIASES.keys[rand(6)]]
           FileUtils.makedirs("/tmp/curl/")
     @cookies_file = keys[:cookies] || "/tmp/curl/curl_#{rand}_#{rand}.jar"
-		# @cookies_file	= "/home/ruslan/curl.jar"		
-		#--header "Accept-Encoding: deflate"
-		@setup_params	= ' --header "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" --header "Accept-Language: en-us,en;q=0.5" --header "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7" '
-		@setup_params	= ' --connect-timeout 6 --max-time 8 --retry 1  --location --compressed --silent -k  '
+    # @cookies_file	= "/home/ruslan/curl.jar"		
+    #--header "Accept-Encoding: deflate"
+#    @setup_params	= ' --header "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" --header "Accept-Language: en-us,en;q=0.5" --header "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7" '
+    @setup_params	= " --connect-timeout #{@connect_timeout}  --max-time #{@max_time} --retry #{@retry}  --location --compressed --silent -k "
 #		@setup_params	= ' --location --silent  '
 		yield self if block_given?		
 	end
@@ -60,10 +66,11 @@ class CURL
     end
     
     def socks(socks_uri)
-    	socks = ( socks_uri.is_a?(URI) ? socks_uri : URI.parse("http://#{socks_uri}") )
-    	@setup_params = "#{@setup_params} --socks5-hostname \"#{socks.host}:#{socks.port}\" "
-    	@setup_params = "#{@setup_params} --proxy-user \"#{socks.user}:#{socks.password}\" " if socks.user
-    	@setup_params
+      socks = ( socks_uri.is_a?(URI) ? socks_uri : URI.parse("http://#{socks_uri}") )
+      s = @socks_hostname ? "--socks5-hostname" : "--socks5"
+      @setup_params = "#{@setup_params} #{s} \"#{socks.host}:#{socks.port}\" "
+      @setup_params = "#{@setup_params} --proxy-user \"#{socks.user}:#{socks.password}\" " if socks.user
+      @setup_params
     end
     
     def self.check(proxy)
@@ -84,27 +91,44 @@ class CURL
     	@debug
     end
     
-    def get(url, count=3, ref=nil, keys={})
+    def cache_path(url)
+      "#{@cache}/#{Digest::MD5.hexdigest(url)[0..1]}/#{Digest::MD5.hexdigest(url)[2..3]}/#{Digest::MD5.hexdigest(url)[4..5]}/#{Digest::MD5.hexdigest(url)[6..7]}"
+    end
+    def cache_file(url)
+      cache_path(url)+"/#{Digest::MD5.hexdigest(url)}.html"
+    end
+    
+    def get(url, keys={})
+      ref = keys[:ref] ||= nil
+      count = keys[:count] ||= 3
+      encoding = keys[:encoding] ||= "utf-8"
+      raw = ( keys[:raw]==nil ? false : keys[:raw] )
+      
       if @cache
-        filename = "#{@cache}/#{Digest::MD5.hexdigest(url)[0..3]}/#{Digest::MD5.hexdigest(url)}.html"
-        unless File.exists?(filename)
-          FileUtils.mkdir_p("#{@cache}/#{Digest::MD5.hexdigest(url)[0..3]}/")
-          result = get_raw(url,count,ref)
-          puts "cache to file '#{filename}'"
+        filename = cache_file(url)
+        unless File.exists?(filename) && (File.exists?(filename) && File.ctime(filename) > Time.now-@cache_time)
+          FileUtils.mkdir_p(cache_path(url))
+          result = get_raw(url, {:count=>count, :ref=>ref, :encoding=>encoding} ) #+" --output \"#{filename}\" ")
+          puts "cache to file '#{filename}'" if @debug
           File.open(filename,"w"){|f| f.puts result}
           return result
         else
-          puts "read from cache file '#{filename}'"
+          puts "read from cache file '#{filename}'" if @debug
           return open(filename).read
         end
       else
-        return get_raw(url,count,ref)
+        return get_raw(url, {:count=>count , :ref=>ref, :encoding=>encoding, :raw=>raw})
       end
       
     end
     
-    def get_raw(url,count=3,ref=nil)
-    	cmd = "curl #{cookies_store} #{browser_type} #{@setup_params} #{ref}  \"#{url}\"  "
+    def get_raw(url, keys={})
+      ref = keys[:ref] ||= nil
+      count = keys[:count] ||= 3
+      encoding = keys[:encoding] ||= "utf-8"
+      raw = ( keys[:raw]==nil ? false : keys[:raw] )
+      
+      cmd = "curl #{cookies_store} #{browser_type} #{@setup_params} #{ref}  \"#{url}\"  "
     	if @debug
     		puts cmd.red  
     	end
@@ -114,8 +138,13 @@ class CURL
     			count -= 1
     			result = self.get(url,count) if count > 0
                 end
-      result = result.gsub(/\\x../,'')
-
+      #      result.force_encoding(encoding)
+      if raw
+        return result
+      else
+        return ( encoding=="utf-8" ? result.clean : Iconv.new("UTF-8", "WINDOWS-1251").iconv(result) )
+      end
+        
     end
     
 # 	формат данных для поста
@@ -155,26 +184,29 @@ class CURL
 #			}    
     def send(url,post_data, ref = nil,count=5 )
     	
-			post_q = '' # "  -F \"method\"=\"post\"  "
-			post_data.each do |key,val|
-				pre = ""
-				if key
-						pre = "@" if key.scan("file").size>0 or key.scan("photo").size>0 
-					val = val.gsub('"','\"')
-					post_q += " -F \"#{key}\"=#{pre}\"#{val}\" " 
-				end
-			end
-			
-		cmd = "curl   #{cookies_store} #{browser_type} #{post_q}  #{@setup_params} #{ref}  \"#{url}\" "		
-		puts cmd.red if @debug
-		
-		result = open_pipe(cmd)
-    		#if result.to_s.strip.size == 0 
-    		#	puts "empty result, left #{count} try".yellow  if @debug
-    		#	count -= 1
-    		#	result = self.send(url,post_data,nil,count) if count > 0
-			#end
-    	result
+      post_q = '' # "  -F \"method\"=\"post\"  "
+      post_data.each do |key,val|
+        pre = ""
+        if key
+          key = key.to_s
+          pre = "@" if key.scan("file").size>0 or key.scan("photo").size>0 or key.scan("@").size>0
+          key = key.to_s.gsub("@",'')
+          val = val.to_s
+          val = val.gsub('"','\"')
+          post_q += " -F \"#{key}\"=#{pre}\"#{val}\" " 
+        end
+      end
+      
+      cmd = "curl   #{cookies_store} #{browser_type} #{post_q}  #{@setup_params} #{ref}  \"#{url}\" "		
+      puts cmd.red if @debug
+      
+      result = open_pipe(cmd)
+      #if result.to_s.strip.size == 0 
+      #	puts "empty result, left #{count} try".yellow  if @debug
+      #	count -= 1
+      #	result = self.send(url,post_data,nil,count) if count > 0
+      #end
+      result
     end
     
     

data/lib/string_cleaner.rb

@@ -0,0 +1,181 @@
+# encoding: UTF-8
+require "unidecoder"
+
+module String::Cleaner
+
+  def clean
+    fix_encoding.fix_endlines.fix_invisible_chars
+  end
+
+  def fix_encoding
+    utf8 = dup
+    if utf8.respond_to?(:force_encoding)
+      utf8.force_encoding("UTF-8") # for Ruby 1.9+
+      unless utf8.valid_encoding? # if invalid UTF-8
+        utf8 = utf8.force_encoding("ISO8859-1")
+        utf8.encode!("UTF-8", :invalid => :replace, :undef => :replace, :replace => "")
+      end
+      utf8.gsub!(/\u0080|¤/, "€") # special case for euro sign from Windows-1252
+      utf8
+    else
+      require "iconv"
+      utf8 << " "
+      begin
+        Iconv.new("UTF-8", "UTF-8").iconv(utf8)
+      rescue
+        utf8.gsub!(/\x80/n, "\xA4")
+        Iconv.new("UTF-8//IGNORE", "ISO8859-1").iconv(utf8).gsub("¤", "€")
+      end
+    end
+  end
+
+  def fix_endlines
+    gsub(/(?:\r\n|\r)/u, "\n")
+  end
+  
+  SPECIAL_SPACES = [
+    0x00A0,                # NO-BREAK SPACE
+    0x1680,                # OGHAM SPACE MARK
+    0x180E,                # MONGOLIAN VOWEL SEPARATOR
+    (0x2000..0x200A).to_a, # EN QUAD..HAIR SPACE
+    0x2028,                # LINE SEPARATOR
+    0x2029,                # PARAGRAPH SEPARATOR
+    0x202F,                # NARROW NO-BREAK SPACE
+    0x205F,                # MEDIUM MATHEMATICAL SPACE
+    0x3000,                # IDEOGRAPHIC SPACE
+  ].flatten.collect{|e| [e].pack 'U*'}
+
+  ZERO_WIDTH = [
+    0x200B,                # ZERO WIDTH SPACE
+    0x200C,                # ZERO WIDTH NON-JOINER
+    0x200D,                # ZERO WIDTH JOINER
+    0x2060,                # WORD JOINER
+    0xFEFF,                # ZERO WIDTH NO-BREAK SPACE
+  ].flatten.collect{|e| [e].pack 'U*'}
+
+  def fix_invisible_chars
+    utf8 = self.dup
+    utf8.gsub!(Regexp.new(ZERO_WIDTH.join("|")), "")
+    utf8 = if utf8.respond_to?(:force_encoding)
+      utf8 = (utf8 << " ").split(/\n/u).each{|line|
+        line.gsub!(/[\s\p{C}]/u, " ")
+      }.join("\n").chop!
+    else
+      require "oniguruma"
+      utf8.split(/\n/n).collect{|line|
+        Oniguruma::ORegexp.new("[\\p{C}]", {:encoding => Oniguruma::ENCODING_UTF8}).gsub(line, " ")
+      }.join("\n").chop!
+    end
+    utf8.gsub!(Regexp.new(SPECIAL_SPACES.join("|") + "|\s"), " ")
+    utf8
+  end
+
+  def trim(chars = "")
+    chars.size>0 ? gsub(/\A[#{chars}]+|[#{chars}]+\z/, "") : strip
+  end
+
+  def to_permalink(separator="-")
+    clean.to_ascii(chartable).downcase.gsub(/[^a-z0-9]+/, separator).trim(separator)
+  end
+
+  def nl2br
+    gsub("\n", "<br/>\n")
+  end
+  
+  def to_nicer_sym
+    to_permalink("_").to_sym
+  end
+
+  def chartable(options = {})
+    options = {
+      :clean_binary => true,
+      :translit_symbols => true,
+    }.merge(options)
+    char = "%c"
+    table = {
+      "`" => "'",  # dec = 96
+      "¦" => "|",  # dec = 166, broken vertical bar
+      "¨" => "",   # dec = 168, spacing diaeresis - umlaut
+      "ª" => "",   # dec = 170, feminine ordinal indicator
+      "«" => "\"", # dec = 171, left double angle quotes
+      "¬" => "!",  # dec = 172, not sign
+      "­" => "-",  # dec = 173, soft hyphen
+      "¯" => "-",  # dec = 175, spacing macron - overline
+      "²" => "2",  # dec = 178, superscript two - squared
+      "³" => "3",  # dec = 179, superscript three - cubed
+      "´" => "'",  # dec = 180, acute accent - spacing acute
+      "·" => "",   # dec = 183, middle dot - Georgian comma
+      "¸" => "",   # dec = 184, spacing cedilla
+      "¹" => "1",  # dec = 185, superscript one
+      "º" => "0",  # dec = 186, masculine ordinal indicator
+      "»" => "\"", # dec = 187, right double angle quotes
+      "¿" => "",   # dec = 191, inverted question mark
+      "Ý" => "Y",  # dec = 221
+      "–" => "-",  # hex = 2013, en dash
+      "—" => "-",  # hex = 2014, em dash
+      "‚" => "'",  # hex = 201A, single low-9 quotation mark
+      "„" => "\"", # hex = 201E, double low-9 quotation mark
+    }
+    if options[:clean_binary]
+      table[char %   0] = ""  # null
+      table[char %   1] = ""  # start of heading
+      table[char %   2] = ""  # start of text
+      table[char %   3] = ""  # end of text
+      table[char %   4] = ""  # end of transmission
+      table[char %   5] = ""  # enquiry
+      table[char %   6] = ""  # acknowledge
+      table[char %   7] = ""  # bell
+      table[char %   8] = ""  # backspace
+      table[char %   9] = " " # tab
+      table[char %  11] = ""  # vertical tab
+      table[char %  12] = ""  # form feed
+      table[char %  14] = ""  # shift out
+      table[char %  15] = ""  # shift in
+      table[char %  16] = ""  # data link escape
+      table[char %  17] = ""  # device control 1
+      table[char %  18] = ""  # device control 2
+      table[char %  19] = ""  # device control 3
+      table[char %  20] = ""  # device control 4
+      table[char %  21] = ""  # negative acknowledgement
+      table[char %  22] = ""  # synchronous idle
+      table[char %  23] = ""  # end of transmission block
+      table[char %  24] = ""  # cancel
+      table[char %  25] = ""  # end of medium
+      table[char %  26] = ""  # substitute
+      table[char %  27] = ""  # escape
+      table[char %  28] = ""  # file separator
+      table[char %  29] = ""  # group separator
+      table[char %  30] = ""  # record separator
+      table[char %  31] = ""  # unit separator
+      table[char % 127] = ""  # delete
+    end
+    if options[:translit_symbols]
+      table["$"]        = " dollars "              # dec = 36, dollar sign
+      table["%"]        = " percent "              # dec = 37, percent sign
+      table["&"]        = " and "                  # dec = 38, ampersand
+      table["@"]        = " at "                   # dec = 64, at symbol
+      table[char % 128] = " euros "                # windows euro
+      table["¢"]        = " cents "                # dec = 162, cent sign
+      table["£"]        = " pounds "               # dec = 163, pound sign
+      table["¤"]        = " euros "                # dec = 164, currency sign
+      table["¥"]        = " yens "                 # dec = 165, yen sign
+      table["§"]        = " section "              # dec = 167, section sign
+      table["©"]        = " copyright "            # dec = 169, copyright sign
+      table["®"]        = " registered trademark " # dec = 174, registered trade mark sign
+      table["°"]        = " degrees "              # dec = 176, degree sign
+      table["±"]        = " approx "               # dec = 177, plus-or-minus sign
+      table["µ"]        = " micro "                # dec = 181, micro sign
+      table["¶"]        = " paragraph "            # dec = 182, pilcrow sign - paragraph sign
+      table["¼"]        = " 1/4 "                  # dec = 188, fraction one quarter
+      table["½"]        = " 1/2 "                  # dec = 189, fraction one half
+      table["¾"]        = " 3/4 "                  # dec = 190, fraction three quarters
+      table["€"]        = " euros "                # hex = 20AC, unicode euro
+      table["™"]        = " trademark "            # hex = 2122, trade mark
+    end
+    table
+  end
+
+end
+class String
+  include String::Cleaner
+end

metadata

@@ -1,12 +1,8 @@
 --- !ruby/object:Gem::Specification 
 name: curl
 version: !ruby/object:Gem::Version 
-  prerelease: false
-  segments: 
-  - 0
-  - 0
-  - 4
-  version: 0.0.4
+  prerelease: 
+  version: 0.0.9
 platform: ruby
 authors: 
 - tg0
@@ -14,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-10-13 00:00:00 +03:00
+date: 2011-03-10 00:00:00 +02:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -25,13 +21,20 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        segments: 
-        - 0
-        - 2
-        - 1
         version: 0.2.1
   type: :runtime
   version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: unidecoder
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.1.1
+  type: :runtime
+  version_requirements: *id002
 description: Some simple methods to use shell curl
 email: email@tg0.ru
 executables: []
@@ -43,6 +46,7 @@ extra_rdoc_files: []
 files: 
 - README
 - lib/curl.rb
+- lib/string_cleaner.rb
 has_rdoc: true
 homepage: http://github.com/tg0/curl
 licenses: []
@@ -58,21 +62,17 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      segments: 
-      - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      segments: 
-      - 0
       version: "0"
 requirements: []
 
 rubyforge_project: curl
-rubygems_version: 1.3.7
+rubygems_version: 1.5.2
 signing_key: 
 specification_version: 3
 summary: shell CURL ruby wrapper.