image_puller 0.2.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d133a19581ee2fb6f89e4e3a5bc2817ac506ebb0c6f44d318c72a6dc47e51201
-  data.tar.gz: 290ea3e4333c5661c12ac71a72572ca8aff7d9028d74e9c982cabebb8d818f5b
+  metadata.gz: f4d6f8b3aa69adcf0996129eaf61997b38c572a8790a28b073b9b57ed443a4cf
+  data.tar.gz: 89bc1c0fcacaf36278147e4a99500b89b318efc2b5103fc107ab05254f80e387
 SHA512:
-  metadata.gz: c48c85d593d022257c28e755792a8c8f95df4453ad576e74ff6c8c55e94c16484eff75a7ced5aa2f68e7dfa62adf3fca98a28847f6a29afcc087e06c99c33450
-  data.tar.gz: 43195582ea12aeea215168901cf986a4bfe81e3bdf21be8b2b7bfa97ac87f845d77fab39027d7273f1533161104c3209edf68239c8251ead4771e06caa0e565d
+  metadata.gz: afabc83d0bd69cd3ca1ee343a792e94d3f968c4373efec8c754a1075a7521518b11ee4ed387ce690a0f72bf92a9bd0bc37ab171a3a2bb9053303f0cd5ead9236
+  data.tar.gz: 7258cc5690096c4bab21ee6a76e087cf1ece5cc0874dbdf69d4d6a985e948a44621bcfac22143d1618a5ee0b408ef122f5fc7963fd8254d030cd50efab54b04e

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    image_puller (0.2.1)
+    image_puller (0.2.2)
       activesupport (~> 5.2)
       flickraw
       ruby-progressbar (~> 1.10)

data/README.md

@@ -39,7 +39,6 @@ Example settings file
   "OUTPUT_PATH":  "path to export to",
   "FLICKR_USER_ID": "https://www.flickr.com/services/api/explore/?method=flickr.people.getInfo under 'Your User ID'",
   "PHOTO": "false",
-  "VIDEO": "true",
   "PULLER_TYPE": "flickr",
   "FLICKR_STARTING_PAGE": 0
 }

data/failed items

@@ -0,0 +1,18 @@
+cannot download https://www.flickr.com/photos/105860661@N02/24198129478/play/orig/c8e5749bc2/ to /Users/nicklongmore/Pictures/flickr_backup_2//Wedding & Florida & Mexico/MK_TSHOLIDAY_20171013_8109792074.jpg error: 404 Not Found         |
+cannot download https://www.flickr.com/photos/105860661@N02/24198129478/play/orig/c8e5749bc2/ to /Users/nicklongmore/Pictures/flickr_backup_2//Wedding & Florida & Mexico/MK_TSHOLIDAY_20171013_8109792074.jpg error: 404 Not Found         |
+cannot download https://www.flickr.com/photos/105860661@N02/38051132251/play/orig/d70bbe8237/ to /Users/nicklongmore/Pictures/flickr_backup_2//Wedding & Florida & Mexico/EPCOT_ECENTR1_20171019_8112946569.jpg error: 404 Not Found        |
+Page 8/14: |================================================================================================================================================================================================================================|
+cannot download https://www.flickr.com/photos/105860661@N02/26244735229/play/orig/cc9b2d2a58/ to /Users/nicklongmore/Pictures/flickr_backup_2//Wedding & Florida & Mexico/VID-20171026-WA0001.jpg error: 404 Not Found                      |
+cannot download https://www.flickr.com/photos/105860661@N02/37989896432/play/orig/6c446f28d3/ to /Users/nicklongmore/Pictures/flickr_backup_2//Wedding & Florida & Mexico/VID-20171026-WA0002.jpg error: 404 Not Found                      |
+cannot download https://www.flickr.com/photos/105860661@N02/37311335954/play/orig/661f938092/ to /Users/nicklongmore/Pictures/flickr_backup_2//Wedding & Florida & Mexico/VID-20171027-WA0012.jpg error: 404 Not Found                      |
+cannot download https://www.flickr.com/photos/105860661@N02/24168590038/play/orig/c5e4c14c31/ to /Users/nicklongmore/Pictures/flickr_backup_2//Wedding & Florida & Mexico/VID-20171029-WA0000.jpg error: 404 Not Found                      |
+cannot download https://www.flickr.com/photos/105860661@N02/37966724626/play/orig/4f13cc1315/ to /Users/nicklongmore/Pictures/flickr_backup_2//Wedding & Florida & Mexico/VID-20171029-WA0001.jpg error: 404 Not Found                      |
+cannot download https://www.flickr.com/photos/105860661@N02/37310930364/play/orig/dee2bc3f6a/ to /Users/nicklongmore/Pictures/flickr_backup_2//Wedding & Florida & Mexico/VID-20171029-WA0002.jpg error: 404 Not Found                      |
+cannot download https://www.flickr.com/photos/105860661@N02/37924109381/play/orig/5d218dfd58/ to /Users/nicklongmore/Pictures/flickr_backup_2//Wedding & Florida & Mexico/VID-20171016-WA0000.jpg error: 404 Not Found                      |
+cannot download https://www.flickr.com/photos/105860661@N02/26244735229/play/orig/cc9b2d2a58/ to /Users/nicklongmore/Pictures/flickr_backup_2//Wedding & Florida & Mexico/VID-20171026-WA0001.jpg error: 404 Not Found                      |
+cannot download https://www.flickr.com/photos/105860661@N02/37989896432/play/orig/6c446f28d3/ to /Users/nicklongmore/Pictures/flickr_backup_2//Wedding & Florida & Mexico/VID-20171026-WA0002.jpg error: 404 Not Found                      |
+cannot download https://www.flickr.com/photos/105860661@N02/37311335954/play/orig/661f938092/ to /Users/nicklongmore/Pictures/flickr_backup_2//Wedding & Florida & Mexico/VID-20171027-WA0012.jpg error: 404 Not Found                      |
+cannot download https://www.flickr.com/photos/105860661@N02/24168590038/play/orig/c5e4c14c31/ to /Users/nicklongmore/Pictures/flickr_backup_2//Wedding & Florida & Mexico/VID-20171029-WA0000.jpg error: 404 Not Found                      |
+cannot download https://www.flickr.com/photos/105860661@N02/37966724626/play/orig/4f13cc1315/ to /Users/nicklongmore/Pictures/flickr_backup_2//Wedding & Florida & Mexico/VID-20171029-WA0001.jpg error: 404 Not Found                      |
+cannot download https://www.flickr.com/photos/105860661@N02/37310930364/play/orig/dee2bc3f6a/ to /Users/nicklongmore/Pictures/flickr_backup_2//Wedding & Florida & Mexico/VID-20171029-WA0002.jpg error: 404 Not Found                      |
+cannot download https://www.flickr.com/photos/105860661@N02/37924109381/play/orig/5d218dfd58/ to /Users/nicklongmore/Pictures/flickr_backup_2//Wedding & Florida & Mexico/VID-20171016-WA0000.jpg error: 404 Not Found   

data/image_puller.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = 'image_puller'
-  spec.version       =  '0.2.1'
+  spec.version       =  '0.2.2'
   spec.authors       = ['Nick Longmore']
   spec.email         = ['neeko99@gmail.com']
   spec.summary       = 'Pulls all images from Flickr under photostreams '

data/lib/flickr/downloader.rb

@@ -6,79 +6,74 @@ module Flickr
     end
 
     MAX_RETRIES = 10
+    PER_PAGE = 500
+
 
     def download
       page = ENV['FLICKR_STARTING_PAGE'].to_i
-      per_page = 100
 
       loop do
         page += 1
 
         begin
           retry_count ||= 0
-          images = @flickr.people.getPhotos(user_id: ENV['FLICKR_USER_ID'], page: page, per_page: per_page)
+          images = @flickr.people.getPhotos(user_id: ENV['FLICKR_USER_ID'], page: page, per_page: PER_PAGE)
 
-          pb = ProgressBar.create(title: "Images Page #{page}")
-          images.each do |photoset|
-            get_photo(photoset['id'])
-            pb.increment
-          end
+          pb = ProgressBar.create(title: "Page #{page}/#{images['pages']}", total: PER_PAGE)
+          images.each { |photoset| get_photo(photoset['id']); pb.increment }
         rescue StandardError => e
-          puts "Failed on #{page} exception: #{e.message}" if retry_count.zero?
-
           retry_count += 1
-          sleep(1.minute)
-          puts "Retry retry #{retry_count}/#{MAX_RETRIES}"
+          sleep(30.seconds)
           retry unless retry_count == MAX_RETRIES
+
+          puts "Failure on page #{page}"
         end
 
-        break if images.nil? || images.size < per_page
+        break if images.nil? || images.size < PER_PAGE
       end
     end
 
     private
 
     def get_photo(id)
-      # transform to json and then back to a hash so we can use hash methods
-      photoset_details = JSON.parse(@flickr.photos.getAllContexts(photo_id: id).to_json)
-      album_title = photoset_details.dig('set', 0, 'title') || 'misc'
-
       info = @flickr.photos.getInfo(photo_id: id)
 
-      filename = "#{info['title']}.#{info['originalformat']}"
-
       case info['media']
       when 'photo'
         return unless ENV['PHOTO'] == 'true'
 
         url = FlickRaw.url_o(info)
-      when 'video'
-        return unless ENV['VIDEO'] == 'true'
-        return unless original_video = @flickr.photos.getSizes(photo_id: id).detect{ |size| size['label'] == 'Video Original' }
-
-        url = original_video['source']
+        filename = url.split('/')[-1]
+        #TODO: add support for video
       end
 
       return unless url
 
-      url_download(url, album_title, filename)
-
+      url_download(url, path(album_title(id), filename))
     end
 
-    def url_download(url, album_title, filename)
+    def url_download(url, path)
       retry_count ||= 0
-      open(url) do |image|
-        path = "#{ENV['OUTPUT_PATH']}/#{album_title}/#{filename}"
 
+      open(url) do |image|
         FileUtils.mkdir_p(File.dirname(path))
         open(path, 'wb') { |file| file.write(image.read) }
       end
     rescue StandardError => e
-      puts "cannot download #{url} error: #{e.message}" if retry_count.zero?
-
       retry_count += 1
-      puts "Retry  #{retry_count}/#{MAX_RETRIES}"
       retry unless retry_count == MAX_RETRIES
+
+      puts "cannot download #{url} to #{path} error: #{e.message}"
+    end
+
+    def album_title(id)
+      # transform to json and then back to a hash so we can use hash methods
+      photoset_details = JSON.parse(@flickr.photos.getAllContexts(photo_id: id).to_json)
+      photoset_details.dig('set', 0, 'title') || 'misc'
+    end
+
+    def path(album_title, filename)
+      "#{ENV['OUTPUT_PATH']}/#{album_title}/#{filename}"
     end
   end
 end

data/lib/image_puller.rb

@@ -29,7 +29,7 @@ class ImagePuller
       end
 
       settings = JSON.parse(File.read(settings_file_path))
-      settings.each { |key, value| ENV[key] = value }
+      settings.each { |key, value| ENV[key] = value.to_s }
 
       true
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: image_puller
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors:
 - Nick Longmore
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-15 00:00:00.000000000 Z
+date: 2020-03-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pry-byebug
@@ -99,6 +99,7 @@ files:
 - bin/image_puller
 - bin/setup
 - example.json
+- failed items
 - image_puller.gemspec
 - lib/flickr/authenticator.rb
 - lib/flickr/downloader.rb