fyt 1.2.0 → 1.3.0.pre

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: b068b9aa73605f8d48c04d802b8e1cf022b0086cd021fa2f3009291af483b526
4
- data.tar.gz: a4a3b584e7f1357aeca75fcff15abd4fd914c8f717a409871421aab5121d3219
3
+ metadata.gz: c2c92a3bbe1b82badeaf428859b9c7e4236882459511079a62f06c4961238d33
4
+ data.tar.gz: d585d6bd01c90721299aab2c7e74c48780c0aea7890992595a301096ac936c33
5
5
  SHA512:
6
- metadata.gz: 2b97b1dcf367d77dde813d078b40b52a649482ad55b9cdc6d079267c498bce10d9294309b343b9b287aaf331b410a832b709931e50b8a7ef1e1166a22f709eee
7
- data.tar.gz: be2eba64e737c8d634dc0883ec400278c23410ae5e3fc8f1b2e85c5599e90e5d4ef6935b574936d0b3801744def3fd39c7d309d79a74cda94ae186259fc7b75b
6
+ metadata.gz: 03daa1ca65ea44500f5102c986e2ef48280339579d4e100f401a7965f2bcc1d4f0eaa81965c35040c05ccda2e817f269865056e7b56ce3486545ef444b8b1503
7
+ data.tar.gz: 0d12c9f946cbe4a63f2a59a665a0a5ccb49c01149dd0f4405fefe5c4b89425c4b383ee83ab23867c5527b99ec1d3d43972d2d32a89ea1c13b5a69fb85e1622a5
data/lib/fyt/base.rb CHANGED
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+
2
3
  require 'logger'
3
4
 
4
5
  module FYT
data/lib/fyt/builder.rb CHANGED
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+
2
3
  module FYT
3
4
  # processes the Youtube feed
4
5
  class Builder < FYT::Base
@@ -55,7 +56,7 @@ module FYT
55
56
  rescue
56
57
  @proxy_manager.remove(proxy)
57
58
 
58
- add_image(youtube_url, title) if @proxy_manager.proxies.size > 0
59
+ add_image(youtube_url, title) unless @proxy_manager.proxies.empty?
59
60
  end
60
61
 
61
62
  def add_item(link, title, filename)
data/lib/fyt/config.rb CHANGED
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+
2
3
  require 'yaml/store'
3
4
 
4
5
  module FYT
@@ -0,0 +1,119 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'aws-sdk-s3'
4
+
5
+ module FYT
6
+ # Manages file downloads and storage
7
+ class S3Storage < FYT::Base
8
+ include FYT::StorageHelper
9
+
10
+ def initialize(tmp_path, format_options, output_format, proxy_manager)
11
+ @tmp_path = tmp_path || ''
12
+ @format_options = format_options
13
+ @output_format = output_format
14
+ @proxy_manager = proxy_manager
15
+ @known_files = []
16
+ end
17
+
18
+ def add(item)
19
+ url = item.link.href
20
+
21
+ filename_for(item).tap do |filename|
22
+ unless files_on_s3.include? filename
23
+ download_file!(url, filename)
24
+ upload_to_s3(filename)
25
+ delete_file_from_disk(filename)
26
+ end
27
+
28
+ @known_files << filename
29
+ end
30
+ end
31
+
32
+ def add_feed(feedname, feed)
33
+ logger.debug feed.to_s
34
+ logger.debug @known_files
35
+
36
+ feed_filename = "#{feedname}.feed.rss20.xml"
37
+
38
+ tmp_path_for(feed_filename).tap do |path|
39
+ File.write(path, feed)
40
+ upload_to_s3(feed_filename)
41
+ delete_file_from_disk(feed_filename)
42
+ @known_files << feed_filename
43
+ end
44
+ end
45
+
46
+ def cleanup!
47
+ logger.debug 'Files to delete:'
48
+ logger.debug files_to_delete
49
+
50
+ files_to_delete.each do |filename|
51
+ delete_from_s3 filename
52
+ end
53
+ end
54
+
55
+ def mtime(filename)
56
+ bucket.object(filename).last_modified
57
+ end
58
+
59
+ def size(filename)
60
+ bucket.object(filename).content_length
61
+ end
62
+
63
+ private
64
+
65
+ def download_file!(url, filename)
66
+ proxy = @proxy_manager.get!
67
+
68
+ command = "#{timeout_cmd} #{youtube_dl_cmd(proxy, url, filename)}"
69
+ logger.debug "Executing: #{command}"
70
+
71
+ begin
72
+ execute command
73
+ rescue
74
+ @proxy_manager.remove(proxy)
75
+
76
+ download_file!(url, filename) unless @proxy_manager.proxies.empty?
77
+ end
78
+ end
79
+
80
+ def delete_file_from_disk(filename)
81
+ tmp_path_for(filename).tap do |path|
82
+ logger.debug "Deleting file: #{path}"
83
+ File.delete(path)
84
+ end
85
+ end
86
+
87
+ def filename_for(item)
88
+ "#{item.id.content}.mp4"
89
+ end
90
+
91
+ def files_to_delete
92
+ files_on_s3 - @known_files
93
+ end
94
+
95
+ def files_on_s3
96
+ bucket.objects.limit(100).map(&:key)
97
+ end
98
+
99
+ def upload_to_s3(filename)
100
+ bucket.object(filename).upload_file(
101
+ tmp_path_for(filename), acl: 'public-read'
102
+ )
103
+ end
104
+
105
+ def delete_from_s3(name)
106
+ bucket.object(name).delete
107
+ end
108
+
109
+ def tmp_path_for(filename)
110
+ File.join(@tmp_path, filename)
111
+ end
112
+
113
+ def bucket
114
+ Aws::S3::Resource
115
+ .new(region: 'eu-central-1')
116
+ .bucket('fyt-storage')
117
+ end
118
+ end
119
+ end
data/lib/fyt/storage.rb CHANGED
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+
2
3
  module FYT
3
4
  # Manages file downloads and storage
4
5
  class Storage < FYT::Base
@@ -66,7 +67,7 @@ module FYT
66
67
  rescue
67
68
  @proxy_manager.remove(proxy)
68
69
 
69
- download_file!(url, output_path) if @proxy_manager.proxies.size > 0
70
+ download_file!(url, output_path) unless @proxy_manager.proxies.empty?
70
71
  end
71
72
 
72
73
  def execute(command_string)
@@ -0,0 +1,38 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FYT
4
+ module StorageHelper
5
+ private
6
+
7
+ def execute(command_string)
8
+ IO.pipe do |read_io, write_io|
9
+ break if system(command_string, out: write_io, err: write_io)
10
+
11
+ write_io.close
12
+ logger.debug read_io.read
13
+ raise
14
+ end
15
+ end
16
+
17
+ def timeout_cmd
18
+ if system('which timeout', out: '/dev/null')
19
+ return 'timeout --preserve-status 300'
20
+ end
21
+
22
+ raise unless system('which gtimeout', out: '/dev/null')
23
+
24
+ 'gtimeout --preserve-status 300'
25
+ end
26
+
27
+ def youtube_dl_cmd(proxy, url, filename)
28
+ [
29
+ 'youtube-dl',
30
+ "-f '#{@format_options}'",
31
+ "--merge-output-format '#{@output_format}'",
32
+ "-o '#{tmp_path_for(filename)}'",
33
+ "--proxy 'http:#{proxy.url}'",
34
+ "'#{url}'"
35
+ ].join(' ')
36
+ end
37
+ end
38
+ end
data/lib/fyt.rb CHANGED
@@ -4,10 +4,13 @@ require_relative 'fyt/builder'
4
4
  require_relative 'fyt/config'
5
5
  require_relative 'fyt/parser'
6
6
  require_relative 'fyt/storage'
7
+ require_relative 'fyt/storage_helper'
8
+ require_relative 'fyt/s3_storage'
7
9
 
8
10
  require 'fileutils'
9
11
 
10
12
  require 'proxy_fetcher'
13
+ require 'aws-sdk-s3'
11
14
 
12
15
  module ProxyFetcher
13
16
  class Manager
@@ -36,12 +39,31 @@ module FYT
36
39
  config = FYT::Config.new
37
40
  manager =
38
41
  ProxyFetcher::Manager.new(filters: { country: 'DE', maxtime: '500' })
39
- storage = FYT::Storage.new(
40
- config[:storage_path],
41
- config[:format_options],
42
- config[:output_format],
43
- manager
44
- )
42
+
43
+ case config[:storage_type]
44
+ when :local
45
+ storage = FYT::S3Storage.new(
46
+ config[:storage_path],
47
+ config[:format_options],
48
+ config[:output_format],
49
+ manager
50
+ )
51
+ when :aws
52
+ storage = FYT::S3Storage.new(
53
+ config[:tmp_path],
54
+ config[:format_options],
55
+ config[:output_format],
56
+ manager
57
+ )
58
+
59
+ Aws.config.update(
60
+ credentials: Aws::Credentials.new(
61
+ config[:aws][:access_key], config[:aws][:secret_key]
62
+ )
63
+ )
64
+ else
65
+ raise 'no storage_type configured'
66
+ end
45
67
 
46
68
  config[:feeds].each do |feed_config|
47
69
  source_feed = FYT::Parser.new(feed_config[:url], manager).read
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fyt
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.2.0
4
+ version: 1.3.0.pre
5
5
  platform: ruby
6
6
  authors:
7
7
  - Steffen Schröder
@@ -10,20 +10,34 @@ bindir: bin
10
10
  cert_chain: []
11
11
  date: 2017-10-10 00:00:00.000000000 Z
12
12
  dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: aws-sdk-s3
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - '='
18
+ - !ruby/object:Gem::Version
19
+ version: 1.13.0
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - '='
25
+ - !ruby/object:Gem::Version
26
+ version: 1.13.0
13
27
  - !ruby/object:Gem::Dependency
14
28
  name: proxy_fetcher
15
29
  requirement: !ruby/object:Gem::Requirement
16
30
  requirements:
17
- - - "~>"
31
+ - - '='
18
32
  - !ruby/object:Gem::Version
19
- version: '0.5'
33
+ version: 0.6.5
20
34
  type: :runtime
21
35
  prerelease: false
22
36
  version_requirements: !ruby/object:Gem::Requirement
23
37
  requirements:
24
- - - "~>"
38
+ - - '='
25
39
  - !ruby/object:Gem::Version
26
- version: '0.5'
40
+ version: 0.6.5
27
41
  description: Downloads a youtube channel and its videos and stores them locally as
28
42
  a podcatcher-friendly feed
29
43
  email: steffen@schröder.xyz
@@ -38,7 +52,9 @@ files:
38
52
  - lib/fyt/builder.rb
39
53
  - lib/fyt/config.rb
40
54
  - lib/fyt/parser.rb
55
+ - lib/fyt/s3_storage.rb
41
56
  - lib/fyt/storage.rb
57
+ - lib/fyt/storage_helper.rb
42
58
  homepage: https://github.com/ChaosSteffen/fyt
43
59
  licenses:
44
60
  - BSD-2-Clause
@@ -54,9 +70,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
54
70
  version: '0'
55
71
  required_rubygems_version: !ruby/object:Gem::Requirement
56
72
  requirements:
57
- - - ">="
73
+ - - ">"
58
74
  - !ruby/object:Gem::Version
59
- version: '0'
75
+ version: 1.3.1
60
76
  requirements: []
61
77
  rubyforge_project:
62
78
  rubygems_version: 2.7.6