image_puller 0.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 0cf21c6b917480a5d73e7ede9a7f909700feb550bf9cd2f1d6e4bc81730e1801
+  data.tar.gz: 29df877be494b58f32b43d2755279431a8e743504379b161d361326032d75653
+SHA512:
+  metadata.gz: 71289dae5bf021efce0c2521b4364ef6963c441b513e719d327310e7b6fd18ba3d0e3e980e8ec229d59f0cb2fa7ab59bdf5f01324b9349b08f0c6b3b75ee1341
+  data.tar.gz: 8e92c4abbb3ac0eaaa1c8b3521ca6999f1ee6c5200897860f16ff5f0ef9a059a6fa198b9aa13b75a729c9090e4292d161f0f488dc161d7a36290bbcd351ab8ee

data/.env

@@ -0,0 +1,6 @@
+FLICKRAW_API_KEY='b545208674db23ddefeb8c498e1a8743'
+FLICKRAW_SHARED_SECRET='6f8e82010108372a'
+OUTPUT_PATH='/Users/nicklongmore/Pictures/flickr_backup/'
+FLICKR_USER_ID='105860661@N02'
+PHOTO='false'
+VIDEO='true'

data/.gitignore

@@ -0,0 +1,12 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+authkey.json
+vendor/*
+my_params.json
+.idea/*

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in ImagePuller.gemspec
+gemspec
+
+gem "rake", "~> 12.0"

data/Gemfile.lock

@@ -0,0 +1,61 @@
+PATH
+  remote: .
+  specs:
+    image_puller (0.2.0)
+      activesupport (~> 5.2)
+      flickraw
+      ruby-progressbar (~> 1.10)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (5.2.3)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+    byebug (11.0.1)
+    coderay (1.1.2)
+    concurrent-ruby (1.1.5)
+    diff-lcs (1.3)
+    flickraw (0.9.10)
+    i18n (1.6.0)
+      concurrent-ruby (~> 1.0)
+    method_source (0.9.2)
+    minitest (5.13.0)
+    pry (0.12.2)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
+    pry-byebug (3.7.0)
+      byebug (~> 11.0)
+      pry (~> 0.10)
+    rake (12.3.3)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.1)
+      rspec-support (~> 3.9.1)
+    rspec-expectations (3.9.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.2)
+    ruby-progressbar (1.10.1)
+    thread_safe (0.3.6)
+    tzinfo (1.2.5)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  image_puller!
+  pry-byebug
+  rake (~> 12.0)
+  rspec (~> 3.2)
+
+BUNDLED WITH
+   2.1.4

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2020 Nick Longmore
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,54 @@
+# ImagePuller
+
+## *** Currently only supports Flickr ***
+Gem designed to pull all images/videos from hosting providers.  
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'image_puller'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install image_puller
+
+## Usage
+
+# Flickr 
+- Require API key and Secret via https://www.flickr.com/services/apps/create
+- Add settings into env variables (see Settings)
+
+Run `image_puller $filepath` in a command prompt and follow onscreen instructions
+
+## Settings
+
+# Flickr Specific Settings
+
+Example settings file
+```json
+{
+  "FLICKR_API_KEY": "Flickr API",
+  "FLICKR_SHARED_SECRET": "Flickr secret",
+  "OUTPUT_PATH":  "path to export to",
+  "FLICKR_USER_ID": "https://www.flickr.com/services/api/explore/?method=flickr.people.getInfo under 'Your User ID'",
+  "PHOTO": "false",
+  "VIDEO": "true",
+  "PULLER_TYPE": "flickr"
+}
+```
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/neeko99/image_puller.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "image_puller"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/image_puller

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'image_puller'
+
+ImagePuller.pull(ARGV[0])

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/example.json

@@ -0,0 +1,9 @@
+{
+  "FLICKR_API_KEY": "Flickr API",
+  "FLICKR_SHARED_SECRET": "Flickr secret",
+  "OUTPUT_PATH":  "path to export to",
+  "FLICKR_USER_ID": "https://www.flickr.com/services/api/explore/?method=flickr.people.getInfo under 'Your User ID'",
+  "PHOTO": "false",
+  "VIDEO": "true",
+  "PULLER_TYPE": "flickr"
+}

data/image_puller.gemspec

@@ -0,0 +1,24 @@
+Gem::Specification.new do |spec|
+  spec.name          = 'image_puller'
+  spec.version       =  '0.2.0'
+  spec.authors       = ['Nick Longmore']
+  spec.email         = ['neeko99@gmail.com']
+  spec.summary       = 'Pulls all images from Flickr under photostreams '
+  spec.description   = 'Image Puller'
+  spec.license       = 'MIT'
+  spec.required_ruby_version = Gem::Requirement.new('>= 2.3.0')
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+
+  spec.require_paths = ['lib']
+  spec.executables << 'image_puller'
+  spec.add_development_dependency 'pry-byebug'
+  spec.add_development_dependency 'rspec', '~> 3.2'
+  spec.add_dependency 'activesupport', '~> 5.2'
+  spec.add_dependency 'flickraw'
+  spec.add_dependency 'ruby-progressbar', '~> 1.10'
+end

data/lib/flickr/authenticator.rb

@@ -0,0 +1,47 @@
+module Flickr
+  class Authenticator
+
+    def flickr
+      @flickr ||= get_tokens
+    end
+
+    def get_tokens
+      flickr = FlickRaw::Flickr.new(api_key: ENV['FLICKR_API_KEY'], shared_secret: ENV['FLICKR_SHARED_SECRET'])
+
+      if File.exist? 'authkey.json'
+        auth_key = JSON.parse(File.read('authkey.json'))
+
+        flickr.access_token = auth_key['access_token']
+        flickr.access_secret = auth_key['secret']
+      end
+
+      begin
+        flickr.test.login
+        flickr
+      rescue  FlickRaw::FailedResponse => e
+        authenticate
+        # retry
+        get_tokens
+      end
+    end
+
+    def authenticate
+      token = flickr.get_request_token
+      auth_url = flickr.get_authorize_url(token['oauth_token'], :perms => 'delete')
+
+      puts "Open this url in your browser to complete the authentication process : #{auth_url}"
+      puts "Copy here the number given when you complete the process."
+      verify = gets.strip
+
+      begin
+        flickr.get_access_token(token['oauth_token'], token['oauth_token_secret'], verify)
+        login = flickr.test.login
+        puts "You are now authenticated as #{login.username} with token #{flickr.access_token} and secret #{flickr.access_secret}"
+        File.write('authkey.json', { access_token: flickr.access_token, secret: flickr.access_secret }.to_json)
+        flickr
+      rescue FlickRaw::FailedResponse => e
+        puts "Authentication failed : #{e.msg}"
+      end
+    end
+  end
+end

data/lib/flickr/downloader.rb

@@ -0,0 +1,85 @@
+module Flickr
+  class Downloader
+
+    def initialize(flickr)
+      @flickr = flickr
+    end
+
+    MAX_RETRIES = 10
+
+    def download
+      page = 0
+      per_page = 100
+
+      loop do
+        page += 1
+
+        begin
+          retry_count ||= 0
+          images = @flickr.people.getPhotos(user_id: ENV['FLICKR_USER_ID'], page: page, per_page: per_page)
+
+          pb = ProgressBar.create(title: "Images Page #{page}")
+          images.each do |photoset|
+            get_photo(photoset['id'])
+            pb.increment
+          end
+        rescue StandardError => e
+          puts "Failed on #{page} exception: #{e.message}" if retry_count.zero?
+
+          retry_count += 1
+          sleep(1.minute)
+          puts "Retry retry #{retry_count}/#{MAX_RETRIES}"
+          retry unless retry_count == MAX_RETRIES
+        end
+
+        break if images.size < per_page
+      end
+    end
+
+    private
+
+    def get_photo(id)
+      # transform to json and then back to a hash so we can use hash methods
+      photoset_details = JSON.parse(@flickr.photos.getAllContexts(photo_id: id).to_json)
+      album_title = photoset_details.dig('set', 0, 'title') || 'misc'
+
+      info = @flickr.photos.getInfo(photo_id: id)
+
+      filename = "#{info['title']}.#{info['originalformat']}"
+
+      case info['media']
+      when 'photo'
+        return unless ENV['PHOTO'] == 'true'
+
+        url = FlickRaw.url_o(info)
+      when 'video'
+        return unless ENV['VIDEO'] == 'true'
+        return unless original_video = @flickr.photos.getSizes(photo_id: id).detect{ |size| size['label'] == 'Video Original' }
+
+        url = original_video['source']
+      end
+
+      return unless url
+
+      url_download(url, album_title, filename)
+
+    end
+
+    def url_download(url, album_title, filename)
+      retry_count ||= 0
+      open(url) do |image|
+        path = "#{ENV['OUTPUT_PATH']}/#{album_title}/#{filename}"
+
+        FileUtils.mkdir_p(File.dirname(path))
+        open(path, 'wb') { |file| file.write(image.read) }
+      end
+    rescue StandardError => e
+      puts "cannot download #{url} error: #{e.message}" if retry_count.zero?
+
+      retry_count += 1
+      puts "Retry retry #{retry_count}/#{MAX_RETRIES}"
+      retry unless retry_count == MAX_RETRIES
+    end
+  end
+end
+

data/lib/image_puller.rb

@@ -0,0 +1,46 @@
+class StupidUserError < StandardError
+end
+
+class ImagePuller
+  require 'pry-byebug'
+  require 'open-uri'
+  require 'fileutils'
+  require 'ruby-progressbar'
+  require 'active_support/all'
+  require 'puller_factory'
+
+  class << self
+
+    SUPPORTED_PULLERS = ['flickr'].freeze
+
+    def pull(file_path = nil)
+      return unless valid_settings_file?(file_path) && valid_type
+
+      factory = PullerFactory.create(ENV['PULLER_TYPE'])
+      factory.exec
+    end
+
+    private
+
+    def valid_settings_file?(settings_file_path)
+      unless settings_file_path && File.exist?(settings_file_path)
+        puts 'Invalid settings file'
+        return false
+      end
+
+      settings = JSON.parse(File.read(settings_file_path))
+      settings.each { |key, value| ENV[key] = value }
+
+      true
+    end
+
+    def valid_type
+      unless SUPPORTED_PULLERS.include? ENV['PULLER_TYPE']
+        puts 'Invalid Type'
+        return false
+      end
+
+      true
+    end
+  end
+end

data/lib/puller_factory.rb

@@ -0,0 +1,13 @@
+class PullerFactory
+
+  require 'strategies/flickr'
+
+  def self.create(type)
+    case type.downcase
+    when 'flickr'
+      Strategy::Flickr
+    end
+    # TODO:  add other strategies
+  end
+end
+

data/lib/strategies/flickr.rb

@@ -0,0 +1,24 @@
+module Strategy
+  class Flickr
+    require 'flickraw'
+    require 'flickr/authenticator'
+    require 'flickr/downloader'
+
+    class << self
+      def exec
+        return puts 'Missing Requirements for Flickr Download please see documentation' unless has_requirements?
+
+        auth = ::Flickr::Authenticator.new
+        downloader = ::Flickr::Downloader.new(auth.flickr)
+
+        downloader.download
+      end
+
+      private
+
+      def has_requirements?
+        ENV['FLICKR_SHARED_SECRET'] && ENV['FLICKR_API_KEY'] && ENV['FLICKR_USER_ID'] && ENV['OUTPUT_PATH']
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,132 @@
+--- !ruby/object:Gem::Specification
+name: image_puller
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+platform: ruby
+authors:
+- Nick Longmore
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-03-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.2'
+- !ruby/object:Gem::Dependency
+  name: flickraw
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby-progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+description: Image Puller
+email:
+- neeko99@gmail.com
+executables:
+- image_puller
+extensions: []
+extra_rdoc_files: []
+files:
+- ".env"
+- ".gitignore"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/image_puller
+- bin/setup
+- example.json
+- image_puller.gemspec
+- lib/flickr/authenticator.rb
+- lib/flickr/downloader.rb
+- lib/image_puller.rb
+- lib/puller_factory.rb
+- lib/strategies/flickr.rb
+homepage: 
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.10
+signing_key: 
+specification_version: 4
+summary: Pulls all images from Flickr under photostreams
+test_files: []