image_puller 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 0cf21c6b917480a5d73e7ede9a7f909700feb550bf9cd2f1d6e4bc81730e1801
4
+ data.tar.gz: 29df877be494b58f32b43d2755279431a8e743504379b161d361326032d75653
5
+ SHA512:
6
+ metadata.gz: 71289dae5bf021efce0c2521b4364ef6963c441b513e719d327310e7b6fd18ba3d0e3e980e8ec229d59f0cb2fa7ab59bdf5f01324b9349b08f0c6b3b75ee1341
7
+ data.tar.gz: 8e92c4abbb3ac0eaaa1c8b3521ca6999f1ee6c5200897860f16ff5f0ef9a059a6fa198b9aa13b75a729c9090e4292d161f0f488dc161d7a36290bbcd351ab8ee
data/.env ADDED
@@ -0,0 +1,6 @@
1
+ FLICKRAW_API_KEY='b545208674db23ddefeb8c498e1a8743'
2
+ FLICKRAW_SHARED_SECRET='6f8e82010108372a'
3
+ OUTPUT_PATH='/Users/nicklongmore/Pictures/flickr_backup/'
4
+ FLICKR_USER_ID='105860661@N02'
5
+ PHOTO='false'
6
+ VIDEO='true'
data/.gitignore ADDED
@@ -0,0 +1,12 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /_yardoc/
4
+ /coverage/
5
+ /doc/
6
+ /pkg/
7
+ /spec/reports/
8
+ /tmp/
9
+ authkey.json
10
+ vendor/*
11
+ my_params.json
12
+ .idea/*
data/Gemfile ADDED
@@ -0,0 +1,6 @@
1
+ source "https://rubygems.org"
2
+
3
+ # Specify your gem's dependencies in ImagePuller.gemspec
4
+ gemspec
5
+
6
+ gem "rake", "~> 12.0"
data/Gemfile.lock ADDED
@@ -0,0 +1,61 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ image_puller (0.2.0)
5
+ activesupport (~> 5.2)
6
+ flickraw
7
+ ruby-progressbar (~> 1.10)
8
+
9
+ GEM
10
+ remote: https://rubygems.org/
11
+ specs:
12
+ activesupport (5.2.3)
13
+ concurrent-ruby (~> 1.0, >= 1.0.2)
14
+ i18n (>= 0.7, < 2)
15
+ minitest (~> 5.1)
16
+ tzinfo (~> 1.1)
17
+ byebug (11.0.1)
18
+ coderay (1.1.2)
19
+ concurrent-ruby (1.1.5)
20
+ diff-lcs (1.3)
21
+ flickraw (0.9.10)
22
+ i18n (1.6.0)
23
+ concurrent-ruby (~> 1.0)
24
+ method_source (0.9.2)
25
+ minitest (5.13.0)
26
+ pry (0.12.2)
27
+ coderay (~> 1.1.0)
28
+ method_source (~> 0.9.0)
29
+ pry-byebug (3.7.0)
30
+ byebug (~> 11.0)
31
+ pry (~> 0.10)
32
+ rake (12.3.3)
33
+ rspec (3.9.0)
34
+ rspec-core (~> 3.9.0)
35
+ rspec-expectations (~> 3.9.0)
36
+ rspec-mocks (~> 3.9.0)
37
+ rspec-core (3.9.1)
38
+ rspec-support (~> 3.9.1)
39
+ rspec-expectations (3.9.0)
40
+ diff-lcs (>= 1.2.0, < 2.0)
41
+ rspec-support (~> 3.9.0)
42
+ rspec-mocks (3.9.1)
43
+ diff-lcs (>= 1.2.0, < 2.0)
44
+ rspec-support (~> 3.9.0)
45
+ rspec-support (3.9.2)
46
+ ruby-progressbar (1.10.1)
47
+ thread_safe (0.3.6)
48
+ tzinfo (1.2.5)
49
+ thread_safe (~> 0.1)
50
+
51
+ PLATFORMS
52
+ ruby
53
+
54
+ DEPENDENCIES
55
+ image_puller!
56
+ pry-byebug
57
+ rake (~> 12.0)
58
+ rspec (~> 3.2)
59
+
60
+ BUNDLED WITH
61
+ 2.1.4
data/LICENSE.txt ADDED
@@ -0,0 +1,21 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2020 Nick Longmore
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,54 @@
1
+ # ImagePuller
2
+
3
+ ## *** Currently only supports Flickr ***
4
+ Gem designed to pull all images/videos from hosting providers.
5
+
6
+ ## Installation
7
+
8
+ Add this line to your application's Gemfile:
9
+
10
+ ```ruby
11
+ gem 'image_puller'
12
+ ```
13
+
14
+ And then execute:
15
+
16
+ $ bundle install
17
+
18
+ Or install it yourself as:
19
+
20
+ $ gem install image_puller
21
+
22
+ ## Usage
23
+
24
+ # Flickr
25
+ - Require API key and Secret via https://www.flickr.com/services/apps/create
26
+ - Add settings into env variables (see Settings)
27
+
28
+ Run `image_puller $filepath` in a command prompt and follow onscreen instructions
29
+
30
+ ## Settings
31
+
32
+ # Flickr Specific Settings
33
+
34
+ Example settings file
35
+ ```json
36
+ {
37
+ "FLICKR_API_KEY": "Flickr API",
38
+ "FLICKR_SHARED_SECRET": "Flickr secret",
39
+ "OUTPUT_PATH": "path to export to",
40
+ "FLICKR_USER_ID": "https://www.flickr.com/services/api/explore/?method=flickr.people.getInfo under 'Your User ID'",
41
+ "PHOTO": "false",
42
+ "VIDEO": "true",
43
+ "PULLER_TYPE": "flickr"
44
+ }
45
+ ```
46
+
47
+ ## Contributing
48
+
49
+ Bug reports and pull requests are welcome on GitHub at https://github.com/neeko99/image_puller.
50
+
51
+
52
+ ## License
53
+
54
+ The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
data/Rakefile ADDED
@@ -0,0 +1,2 @@
1
+ require "bundler/gem_tasks"
2
+ task :default => :spec
data/bin/console ADDED
@@ -0,0 +1,14 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "bundler/setup"
4
+ require "image_puller"
5
+
6
+ # You can add fixtures and/or initialization code here to make experimenting
7
+ # with your gem easier. You can also use a different console, if you like.
8
+
9
+ # (If you use this, don't forget to add pry to your Gemfile!)
10
+ # require "pry"
11
+ # Pry.start
12
+
13
+ require "irb"
14
+ IRB.start(__FILE__)
data/bin/image_puller ADDED
@@ -0,0 +1,5 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'image_puller'
4
+
5
+ ImagePuller.pull(ARGV[0])
data/bin/setup ADDED
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+ set -vx
5
+
6
+ bundle install
7
+
8
+ # Do any other automated setup that you need to do here
data/example.json ADDED
@@ -0,0 +1,9 @@
1
+ {
2
+ "FLICKR_API_KEY": "Flickr API",
3
+ "FLICKR_SHARED_SECRET": "Flickr secret",
4
+ "OUTPUT_PATH": "path to export to",
5
+ "FLICKR_USER_ID": "https://www.flickr.com/services/api/explore/?method=flickr.people.getInfo under 'Your User ID'",
6
+ "PHOTO": "false",
7
+ "VIDEO": "true",
8
+ "PULLER_TYPE": "flickr"
9
+ }
@@ -0,0 +1,24 @@
1
+ Gem::Specification.new do |spec|
2
+ spec.name = 'image_puller'
3
+ spec.version = '0.2.0'
4
+ spec.authors = ['Nick Longmore']
5
+ spec.email = ['neeko99@gmail.com']
6
+ spec.summary = 'Pulls all images from Flickr under photostreams '
7
+ spec.description = 'Image Puller'
8
+ spec.license = 'MIT'
9
+ spec.required_ruby_version = Gem::Requirement.new('>= 2.3.0')
10
+
11
+ # Specify which files should be added to the gem when it is released.
12
+ # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
13
+ spec.files = Dir.chdir(File.expand_path(__dir__)) do
14
+ `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
15
+ end
16
+
17
+ spec.require_paths = ['lib']
18
+ spec.executables << 'image_puller'
19
+ spec.add_development_dependency 'pry-byebug'
20
+ spec.add_development_dependency 'rspec', '~> 3.2'
21
+ spec.add_dependency 'activesupport', '~> 5.2'
22
+ spec.add_dependency 'flickraw'
23
+ spec.add_dependency 'ruby-progressbar', '~> 1.10'
24
+ end
@@ -0,0 +1,47 @@
1
+ module Flickr
2
+ class Authenticator
3
+
4
+ def flickr
5
+ @flickr ||= get_tokens
6
+ end
7
+
8
+ def get_tokens
9
+ flickr = FlickRaw::Flickr.new(api_key: ENV['FLICKR_API_KEY'], shared_secret: ENV['FLICKR_SHARED_SECRET'])
10
+
11
+ if File.exist? 'authkey.json'
12
+ auth_key = JSON.parse(File.read('authkey.json'))
13
+
14
+ flickr.access_token = auth_key['access_token']
15
+ flickr.access_secret = auth_key['secret']
16
+ end
17
+
18
+ begin
19
+ flickr.test.login
20
+ flickr
21
+ rescue FlickRaw::FailedResponse => e
22
+ authenticate
23
+ # retry
24
+ get_tokens
25
+ end
26
+ end
27
+
28
+ def authenticate
29
+ token = flickr.get_request_token
30
+ auth_url = flickr.get_authorize_url(token['oauth_token'], :perms => 'delete')
31
+
32
+ puts "Open this url in your browser to complete the authentication process : #{auth_url}"
33
+ puts "Copy here the number given when you complete the process."
34
+ verify = gets.strip
35
+
36
+ begin
37
+ flickr.get_access_token(token['oauth_token'], token['oauth_token_secret'], verify)
38
+ login = flickr.test.login
39
+ puts "You are now authenticated as #{login.username} with token #{flickr.access_token} and secret #{flickr.access_secret}"
40
+ File.write('authkey.json', { access_token: flickr.access_token, secret: flickr.access_secret }.to_json)
41
+ flickr
42
+ rescue FlickRaw::FailedResponse => e
43
+ puts "Authentication failed : #{e.msg}"
44
+ end
45
+ end
46
+ end
47
+ end
@@ -0,0 +1,85 @@
1
+ module Flickr
2
+ class Downloader
3
+
4
+ def initialize(flickr)
5
+ @flickr = flickr
6
+ end
7
+
8
+ MAX_RETRIES = 10
9
+
10
+ def download
11
+ page = 0
12
+ per_page = 100
13
+
14
+ loop do
15
+ page += 1
16
+
17
+ begin
18
+ retry_count ||= 0
19
+ images = @flickr.people.getPhotos(user_id: ENV['FLICKR_USER_ID'], page: page, per_page: per_page)
20
+
21
+ pb = ProgressBar.create(title: "Images Page #{page}")
22
+ images.each do |photoset|
23
+ get_photo(photoset['id'])
24
+ pb.increment
25
+ end
26
+ rescue StandardError => e
27
+ puts "Failed on #{page} exception: #{e.message}" if retry_count.zero?
28
+
29
+ retry_count += 1
30
+ sleep(1.minute)
31
+ puts "Retry retry #{retry_count}/#{MAX_RETRIES}"
32
+ retry unless retry_count == MAX_RETRIES
33
+ end
34
+
35
+ break if images.size < per_page
36
+ end
37
+ end
38
+
39
+ private
40
+
41
+ def get_photo(id)
42
+ # transform to json and then back to a hash so we can use hash methods
43
+ photoset_details = JSON.parse(@flickr.photos.getAllContexts(photo_id: id).to_json)
44
+ album_title = photoset_details.dig('set', 0, 'title') || 'misc'
45
+
46
+ info = @flickr.photos.getInfo(photo_id: id)
47
+
48
+ filename = "#{info['title']}.#{info['originalformat']}"
49
+
50
+ case info['media']
51
+ when 'photo'
52
+ return unless ENV['PHOTO'] == 'true'
53
+
54
+ url = FlickRaw.url_o(info)
55
+ when 'video'
56
+ return unless ENV['VIDEO'] == 'true'
57
+ return unless original_video = @flickr.photos.getSizes(photo_id: id).detect{ |size| size['label'] == 'Video Original' }
58
+
59
+ url = original_video['source']
60
+ end
61
+
62
+ return unless url
63
+
64
+ url_download(url, album_title, filename)
65
+
66
+ end
67
+
68
+ def url_download(url, album_title, filename)
69
+ retry_count ||= 0
70
+ open(url) do |image|
71
+ path = "#{ENV['OUTPUT_PATH']}/#{album_title}/#{filename}"
72
+
73
+ FileUtils.mkdir_p(File.dirname(path))
74
+ open(path, 'wb') { |file| file.write(image.read) }
75
+ end
76
+ rescue StandardError => e
77
+ puts "cannot download #{url} error: #{e.message}" if retry_count.zero?
78
+
79
+ retry_count += 1
80
+ puts "Retry retry #{retry_count}/#{MAX_RETRIES}"
81
+ retry unless retry_count == MAX_RETRIES
82
+ end
83
+ end
84
+ end
85
+
@@ -0,0 +1,46 @@
1
+ class StupidUserError < StandardError
2
+ end
3
+
4
+ class ImagePuller
5
+ require 'pry-byebug'
6
+ require 'open-uri'
7
+ require 'fileutils'
8
+ require 'ruby-progressbar'
9
+ require 'active_support/all'
10
+ require 'puller_factory'
11
+
12
+ class << self
13
+
14
+ SUPPORTED_PULLERS = ['flickr'].freeze
15
+
16
+ def pull(file_path = nil)
17
+ return unless valid_settings_file?(file_path) && valid_type
18
+
19
+ factory = PullerFactory.create(ENV['PULLER_TYPE'])
20
+ factory.exec
21
+ end
22
+
23
+ private
24
+
25
+ def valid_settings_file?(settings_file_path)
26
+ unless settings_file_path && File.exist?(settings_file_path)
27
+ puts 'Invalid settings file'
28
+ return false
29
+ end
30
+
31
+ settings = JSON.parse(File.read(settings_file_path))
32
+ settings.each { |key, value| ENV[key] = value }
33
+
34
+ true
35
+ end
36
+
37
+ def valid_type
38
+ unless SUPPORTED_PULLERS.include? ENV['PULLER_TYPE']
39
+ puts 'Invalid Type'
40
+ return false
41
+ end
42
+
43
+ true
44
+ end
45
+ end
46
+ end
@@ -0,0 +1,13 @@
1
+ class PullerFactory
2
+
3
+ require 'strategies/flickr'
4
+
5
+ def self.create(type)
6
+ case type.downcase
7
+ when 'flickr'
8
+ Strategy::Flickr
9
+ end
10
+ # TODO: add other strategies
11
+ end
12
+ end
13
+
@@ -0,0 +1,24 @@
1
+ module Strategy
2
+ class Flickr
3
+ require 'flickraw'
4
+ require 'flickr/authenticator'
5
+ require 'flickr/downloader'
6
+
7
+ class << self
8
+ def exec
9
+ return puts 'Missing Requirements for Flickr Download please see documentation' unless has_requirements?
10
+
11
+ auth = ::Flickr::Authenticator.new
12
+ downloader = ::Flickr::Downloader.new(auth.flickr)
13
+
14
+ downloader.download
15
+ end
16
+
17
+ private
18
+
19
+ def has_requirements?
20
+ ENV['FLICKR_SHARED_SECRET'] && ENV['FLICKR_API_KEY'] && ENV['FLICKR_USER_ID'] && ENV['OUTPUT_PATH']
21
+ end
22
+ end
23
+ end
24
+ end
metadata ADDED
@@ -0,0 +1,132 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: image_puller
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.2.0
5
+ platform: ruby
6
+ authors:
7
+ - Nick Longmore
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2020-03-14 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: pry-byebug
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: '0'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ">="
25
+ - !ruby/object:Gem::Version
26
+ version: '0'
27
+ - !ruby/object:Gem::Dependency
28
+ name: rspec
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '3.2'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '3.2'
41
+ - !ruby/object:Gem::Dependency
42
+ name: activesupport
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '5.2'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '5.2'
55
+ - !ruby/object:Gem::Dependency
56
+ name: flickraw
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ">="
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: '0'
69
+ - !ruby/object:Gem::Dependency
70
+ name: ruby-progressbar
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - "~>"
74
+ - !ruby/object:Gem::Version
75
+ version: '1.10'
76
+ type: :runtime
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - "~>"
81
+ - !ruby/object:Gem::Version
82
+ version: '1.10'
83
+ description: Image Puller
84
+ email:
85
+ - neeko99@gmail.com
86
+ executables:
87
+ - image_puller
88
+ extensions: []
89
+ extra_rdoc_files: []
90
+ files:
91
+ - ".env"
92
+ - ".gitignore"
93
+ - Gemfile
94
+ - Gemfile.lock
95
+ - LICENSE.txt
96
+ - README.md
97
+ - Rakefile
98
+ - bin/console
99
+ - bin/image_puller
100
+ - bin/setup
101
+ - example.json
102
+ - image_puller.gemspec
103
+ - lib/flickr/authenticator.rb
104
+ - lib/flickr/downloader.rb
105
+ - lib/image_puller.rb
106
+ - lib/puller_factory.rb
107
+ - lib/strategies/flickr.rb
108
+ homepage:
109
+ licenses:
110
+ - MIT
111
+ metadata: {}
112
+ post_install_message:
113
+ rdoc_options: []
114
+ require_paths:
115
+ - lib
116
+ required_ruby_version: !ruby/object:Gem::Requirement
117
+ requirements:
118
+ - - ">="
119
+ - !ruby/object:Gem::Version
120
+ version: 2.3.0
121
+ required_rubygems_version: !ruby/object:Gem::Requirement
122
+ requirements:
123
+ - - ">="
124
+ - !ruby/object:Gem::Version
125
+ version: '0'
126
+ requirements: []
127
+ rubyforge_project:
128
+ rubygems_version: 2.7.10
129
+ signing_key:
130
+ specification_version: 4
131
+ summary: Pulls all images from Flickr under photostreams
132
+ test_files: []