curl 0.0.6 → 0.0.7

data/lib/curl.rb

@@ -33,14 +33,18 @@ class CURL
     
   def initialize(keys={})
     @cache = ( keys[:cache] ? keys[:cache] : false  )
+    @cache_time = ( keys[:cache_time] ? keys[:cache_time] : 3600*24*1  ) # 1 day cache life
+    @connect_timeout = keys[:connect_timeout] || 6
+    @max_time = keys[:max_time] || 8
+    @retry = keys[:retry] || 1
     @cookies_enable = ( keys[:cookies_disable] ? false : true  )
           @user_agent     = AGENT_ALIASES["Google"]#AGENT_ALIASES[AGENT_ALIASES.keys[rand(6)]]
           FileUtils.makedirs("/tmp/curl/")
     @cookies_file = keys[:cookies] || "/tmp/curl/curl_#{rand}_#{rand}.jar"
-		# @cookies_file	= "/home/ruslan/curl.jar"		
-		#--header "Accept-Encoding: deflate"
-		@setup_params	= ' --header "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" --header "Accept-Language: en-us,en;q=0.5" --header "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7" '
-		@setup_params	= ' --connect-timeout 6 --max-time 8 --retry 1  --location --compressed --silent -k  '
+    # @cookies_file	= "/home/ruslan/curl.jar"		
+    #--header "Accept-Encoding: deflate"
+#    @setup_params	= ' --header "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" --header "Accept-Language: en-us,en;q=0.5" --header "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7" '
+    @setup_params	= ' --connect-timeout #{@connect_timeout}  --max-time #{@max_time} --retry #{@retry}  --location --compressed --silent -k  '
 #		@setup_params	= ' --location --silent  '
 		yield self if block_given?		
 	end
@@ -92,12 +96,16 @@ class CURL
       cache_path(url)+"/#{Digest::MD5.hexdigest(url)}.html"
     end
     
-    def get(url, count=3, ref=nil, keys={})
+    def get(url, keys={})
+      ref = keys[:ref] ||= nil
+      count = keys[:count] ||= 3
+      encoding = keys[:encoding] ||= "utf-8"
+      
       if @cache
         filename = cache_file(url)
-        unless File.exists?(filename)
+        unless File.exists?(filename) && (File.exists?(filename) && File.ctime(filename) > Time.now-@cache_time)
           FileUtils.mkdir_p(cache_path(url))
-          result = get_raw(url,count,ref) #+" --output \"#{filename}\" ")
+          result = get_raw(url, {:count=>count, :ref=>ref, :encoding=>encoding} ) #+" --output \"#{filename}\" ")
           puts "cache to file '#{filename}'" if @debug
           File.open(filename,"w"){|f| f.puts result}
           return result
@@ -106,13 +114,17 @@ class CURL
           return open(filename).read
         end
       else
-        return get_raw(url,count,ref)
+        return get_raw(url, {:count=>count , :ref=>ref, :encoding=>encoding})
       end
       
     end
     
-    def get_raw(url,count=3,ref=nil)
-    	cmd = "curl #{cookies_store} #{browser_type} #{@setup_params} #{ref}  \"#{url}\"  "
+    def get_raw(url, keys={})
+      ref = keys[:ref] ||= nil
+      count = keys[:count] ||= 3
+      encoding = keys[:encoding] ||= "utf-8"
+      
+      cmd = "curl #{cookies_store} #{browser_type} #{@setup_params} #{ref}  \"#{url}\"  "
     	if @debug
     		puts cmd.red  
     	end
@@ -122,8 +134,8 @@ class CURL
     			count -= 1
     			result = self.get(url,count) if count > 0
                 end
-      result.clean
-
+      result.force_encoding(encoding)
+      ( encoding=="utf-8" ? result.clean : Iconv.new("UTF-8", "WINDOWS-1251").iconv(result) )
     end
     
 # 	формат данных для поста

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 0
-  - 6
-  version: 0.0.6
+  - 7
+  version: 0.0.7
 platform: ruby
 authors: 
 - tg0
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-10-30 00:00:00 +03:00
+date: 2010-12-07 00:00:00 +02:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency