zeitungen 0.2.3 → 0.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3a846b2505bc2a7bf100f9b1f4a3c8641759ae3c
-  data.tar.gz: db99b50955e6b51b613381e4d01125ce24c5e6a3
+  metadata.gz: 7f0b320e2eb2f2e765162d7e7ea1785ed34dd360
+  data.tar.gz: a596e27b09d564b4ec625057cc36734fd086f74d
 SHA512:
-  metadata.gz: 63e38445c086d9db9852c7c1303ac76074586385a64b2b9d6c5db497751b1312712f6521e01b4dbaae8335cfa7d46c4a2da6f8a6b9cc5d8c0aea13876fd65fc6
-  data.tar.gz: f33fe04a75b4f3465a2cb1005857a799100c91ce6def1dcdae5f9757a1ab468be6897ad1fa6908ecdec1d002c9076f793bc7bd870f45a2f0ccfe7e352d4c7642
+  metadata.gz: fae374fd14e3b047743719995a2058babeb5b7e7dbe4b2c4fdf373274040c56c8e5a1ff701762cbf6477413859f343059a3b971b87fc764770cae7bd8fc016b9
+  data.tar.gz: 15d3c52e0f960b11812d84869077804dac613bc40622ad5d2121f465310ee8c297c7d8e84ecb061b5e86a95c8b0aff5f1658703309da3986cee85d8b9f2c88f5

data/.gitignore

@@ -10,4 +10,6 @@
 /spec/reports/
 /tmp/
 
-execute.rb
+execute.rb
+test_execute.rb
+test_http.rb

data/lib/zeitungen.rb

@@ -5,6 +5,7 @@ require 'fileutils'
 require 'mechanize'
 require 'tempfile'
 require 'dropbox_sdk'
+require 'http'
 
 
 %w(

data/lib/zeitungen/downloader.rb

@@ -1,10 +1,11 @@
 module Zeitungen
   class Downloader
-    def initialize(url, zeitungen, client, passwords)
+    def initialize(url, zeitungen, client, passwords, verbose: false)
       @url       = url
-      @zeitungen  = zeitungen
+      @zeitungen = zeitungen
       @client    = client
       @passwords = passwords
+      @verbose   = verbose
     end
     
     def run(date=Date.today, options={})
@@ -14,6 +15,7 @@ module Zeitungen
 
       page = IndexPage.new(@url, @passwords)
       zeitungen_links = page.links(date)
+      puts "zeitungen_links.size: #{zeitungen_links.size}" if @verbose
       # puts zeitungen_links.inspect
       queue = enqueue(zeitungen_links)
 
@@ -37,9 +39,12 @@ module Zeitungen
       @zeitungen.each do |z|
         if link = zeitungen_links.find{|l| z.regexp.match l.text } # se c'è un link per il zeitungen corrente
           uri = link.uri
+          puts "uri: #{uri}" if @verbose
           if uri.host=="t.umblr.com"
             h = Hash[uri.query.split("&").map{|e| e.split("=")}]
-            z.uri = URI(URI.unescape(h["z"])+"\?directDownload\=true")
+            u = URI(URI.unescape(h["z"])+"\?directDownload\=true")
+            puts "u: #{u}" if @verbose
+            z.uri = u
           else
             z.uri = uri+"\?directDownload\=true"   # zeitungen.uri+"\?directDownload\=true"
           end
@@ -56,14 +61,32 @@ module Zeitungen
         Thread.new do    
           while !queue.empty? && z = queue.pop
             begin
-              uri = z.uri
+              url = z.uri
               file = Tempfile.new('zeitungen')
-              Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme=='https') do |http|
-                request = Net::HTTP::Get.new(uri)
-                puts "downloading #{z.final_name} (#{z.uri})..."
-                response = http.request request
-                file.write(response.body)
+
+
+              res = HTTP.get(url)
+              i = 0
+              puts "Response status: #{res.status.to_s}" if @verbose
+              while res.status.to_s=="302 Found" and i<5 # max 5 redirect
+                url = res.headers.get("Location")
+                url = url.first if url.is_a? Array
+                puts "Redirect URL: #{url}" if @verbose
+                res = HTTP.get(url)
+                puts "Response status: #{res.status.to_s}" if @verbose
+                i += 1
               end
+              puts "downloading #{z.final_name} (#{url})..."
+              file.write(res.to_s)
+
+
+              # res = HTTP.get()
+              # Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme=='https') do |http|
+              #   request = Net::HTTP::Get.new(uri)
+              #   puts "downloading #{z.final_name} (#{z.uri})..."
+              #   response = http.request request
+              #   file.write(response.body)
+              # end
 
               filename = filename_w_date(z.final_name)
               z.upload ? @client.mv_file_in_public_dest(filename, file) : @client.mv_file_in_private_dest(filename, file)

data/lib/zeitungen/version.rb

@@ -1,3 +1,3 @@
 module Zeitungen
-  VERSION = "0.2.3"
+  VERSION = "0.2.4"
 end

data/zeitungen.gemspec

@@ -42,4 +42,5 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "mechanize"
   spec.add_dependency "dropbox-sdk"
+  spec.add_dependency "http"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zeitungen
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.2.4
 platform: ruby
 authors:
 - Iwan Buetti
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-01-20 00:00:00.000000000 Z
+date: 2016-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -178,6 +178,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: http
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email:
 - iwan.buetti@gmail.com