fyt 1.2.0 → 1.3.0.pre
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/fyt/base.rb +1 -0
- data/lib/fyt/builder.rb +2 -1
- data/lib/fyt/config.rb +1 -0
- data/lib/fyt/s3_storage.rb +119 -0
- data/lib/fyt/storage.rb +2 -1
- data/lib/fyt/storage_helper.rb +38 -0
- data/lib/fyt.rb +28 -6
- metadata +23 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c2c92a3bbe1b82badeaf428859b9c7e4236882459511079a62f06c4961238d33
|
4
|
+
data.tar.gz: d585d6bd01c90721299aab2c7e74c48780c0aea7890992595a301096ac936c33
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 03daa1ca65ea44500f5102c986e2ef48280339579d4e100f401a7965f2bcc1d4f0eaa81965c35040c05ccda2e817f269865056e7b56ce3486545ef444b8b1503
|
7
|
+
data.tar.gz: 0d12c9f946cbe4a63f2a59a665a0a5ccb49c01149dd0f4405fefe5c4b89425c4b383ee83ab23867c5527b99ec1d3d43972d2d32a89ea1c13b5a69fb85e1622a5
|
data/lib/fyt/base.rb
CHANGED
data/lib/fyt/builder.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
module FYT
|
3
4
|
# processes the Youtube feed
|
4
5
|
class Builder < FYT::Base
|
@@ -55,7 +56,7 @@ module FYT
|
|
55
56
|
rescue
|
56
57
|
@proxy_manager.remove(proxy)
|
57
58
|
|
58
|
-
add_image(youtube_url, title)
|
59
|
+
add_image(youtube_url, title) unless @proxy_manager.proxies.empty?
|
59
60
|
end
|
60
61
|
|
61
62
|
def add_item(link, title, filename)
|
data/lib/fyt/config.rb
CHANGED
@@ -0,0 +1,119 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'aws-sdk-s3'
|
4
|
+
|
5
|
+
module FYT
|
6
|
+
# Manages file downloads and storage
|
7
|
+
class S3Storage < FYT::Base
|
8
|
+
include FYT::StorageHelper
|
9
|
+
|
10
|
+
def initialize(tmp_path, format_options, output_format, proxy_manager)
|
11
|
+
@tmp_path = tmp_path || ''
|
12
|
+
@format_options = format_options
|
13
|
+
@output_format = output_format
|
14
|
+
@proxy_manager = proxy_manager
|
15
|
+
@known_files = []
|
16
|
+
end
|
17
|
+
|
18
|
+
def add(item)
|
19
|
+
url = item.link.href
|
20
|
+
|
21
|
+
filename_for(item).tap do |filename|
|
22
|
+
unless files_on_s3.include? filename
|
23
|
+
download_file!(url, filename)
|
24
|
+
upload_to_s3(filename)
|
25
|
+
delete_file_from_disk(filename)
|
26
|
+
end
|
27
|
+
|
28
|
+
@known_files << filename
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def add_feed(feedname, feed)
|
33
|
+
logger.debug feed.to_s
|
34
|
+
logger.debug @known_files
|
35
|
+
|
36
|
+
feed_filename = "#{feedname}.feed.rss20.xml"
|
37
|
+
|
38
|
+
tmp_path_for(feed_filename).tap do |path|
|
39
|
+
File.write(path, feed)
|
40
|
+
upload_to_s3(feed_filename)
|
41
|
+
delete_file_from_disk(feed_filename)
|
42
|
+
@known_files << feed_filename
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def cleanup!
|
47
|
+
logger.debug 'Files to delete:'
|
48
|
+
logger.debug files_to_delete
|
49
|
+
|
50
|
+
files_to_delete.each do |filename|
|
51
|
+
delete_from_s3 filename
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
def mtime(filename)
|
56
|
+
bucket.object(filename).last_modified
|
57
|
+
end
|
58
|
+
|
59
|
+
def size(filename)
|
60
|
+
bucket.object(filename).content_length
|
61
|
+
end
|
62
|
+
|
63
|
+
private
|
64
|
+
|
65
|
+
def download_file!(url, filename)
|
66
|
+
proxy = @proxy_manager.get!
|
67
|
+
|
68
|
+
command = "#{timeout_cmd} #{youtube_dl_cmd(proxy, url, filename)}"
|
69
|
+
logger.debug "Executing: #{command}"
|
70
|
+
|
71
|
+
begin
|
72
|
+
execute command
|
73
|
+
rescue
|
74
|
+
@proxy_manager.remove(proxy)
|
75
|
+
|
76
|
+
download_file!(url, filename) unless @proxy_manager.proxies.empty?
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
def delete_file_from_disk(filename)
|
81
|
+
tmp_path_for(filename).tap do |path|
|
82
|
+
logger.debug "Deleting file: #{path}"
|
83
|
+
File.delete(path)
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
def filename_for(item)
|
88
|
+
"#{item.id.content}.mp4"
|
89
|
+
end
|
90
|
+
|
91
|
+
def files_to_delete
|
92
|
+
files_on_s3 - @known_files
|
93
|
+
end
|
94
|
+
|
95
|
+
def files_on_s3
|
96
|
+
bucket.objects.limit(100).map(&:key)
|
97
|
+
end
|
98
|
+
|
99
|
+
def upload_to_s3(filename)
|
100
|
+
bucket.object(filename).upload_file(
|
101
|
+
tmp_path_for(filename), acl: 'public-read'
|
102
|
+
)
|
103
|
+
end
|
104
|
+
|
105
|
+
def delete_from_s3(name)
|
106
|
+
bucket.object(name).delete
|
107
|
+
end
|
108
|
+
|
109
|
+
def tmp_path_for(filename)
|
110
|
+
File.join(@tmp_path, filename)
|
111
|
+
end
|
112
|
+
|
113
|
+
def bucket
|
114
|
+
Aws::S3::Resource
|
115
|
+
.new(region: 'eu-central-1')
|
116
|
+
.bucket('fyt-storage')
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
data/lib/fyt/storage.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
module FYT
|
3
4
|
# Manages file downloads and storage
|
4
5
|
class Storage < FYT::Base
|
@@ -66,7 +67,7 @@ module FYT
|
|
66
67
|
rescue
|
67
68
|
@proxy_manager.remove(proxy)
|
68
69
|
|
69
|
-
download_file!(url, output_path)
|
70
|
+
download_file!(url, output_path) unless @proxy_manager.proxies.empty?
|
70
71
|
end
|
71
72
|
|
72
73
|
def execute(command_string)
|
@@ -0,0 +1,38 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module FYT
|
4
|
+
module StorageHelper
|
5
|
+
private
|
6
|
+
|
7
|
+
def execute(command_string)
|
8
|
+
IO.pipe do |read_io, write_io|
|
9
|
+
break if system(command_string, out: write_io, err: write_io)
|
10
|
+
|
11
|
+
write_io.close
|
12
|
+
logger.debug read_io.read
|
13
|
+
raise
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def timeout_cmd
|
18
|
+
if system('which timeout', out: '/dev/null')
|
19
|
+
return 'timeout --preserve-status 300'
|
20
|
+
end
|
21
|
+
|
22
|
+
raise unless system('which gtimeout', out: '/dev/null')
|
23
|
+
|
24
|
+
'gtimeout --preserve-status 300'
|
25
|
+
end
|
26
|
+
|
27
|
+
def youtube_dl_cmd(proxy, url, filename)
|
28
|
+
[
|
29
|
+
'youtube-dl',
|
30
|
+
"-f '#{@format_options}'",
|
31
|
+
"--merge-output-format '#{@output_format}'",
|
32
|
+
"-o '#{tmp_path_for(filename)}'",
|
33
|
+
"--proxy 'http:#{proxy.url}'",
|
34
|
+
"'#{url}'"
|
35
|
+
].join(' ')
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
data/lib/fyt.rb
CHANGED
@@ -4,10 +4,13 @@ require_relative 'fyt/builder'
|
|
4
4
|
require_relative 'fyt/config'
|
5
5
|
require_relative 'fyt/parser'
|
6
6
|
require_relative 'fyt/storage'
|
7
|
+
require_relative 'fyt/storage_helper'
|
8
|
+
require_relative 'fyt/s3_storage'
|
7
9
|
|
8
10
|
require 'fileutils'
|
9
11
|
|
10
12
|
require 'proxy_fetcher'
|
13
|
+
require 'aws-sdk-s3'
|
11
14
|
|
12
15
|
module ProxyFetcher
|
13
16
|
class Manager
|
@@ -36,12 +39,31 @@ module FYT
|
|
36
39
|
config = FYT::Config.new
|
37
40
|
manager =
|
38
41
|
ProxyFetcher::Manager.new(filters: { country: 'DE', maxtime: '500' })
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
42
|
+
|
43
|
+
case config[:storage_type]
|
44
|
+
when :local
|
45
|
+
storage = FYT::S3Storage.new(
|
46
|
+
config[:storage_path],
|
47
|
+
config[:format_options],
|
48
|
+
config[:output_format],
|
49
|
+
manager
|
50
|
+
)
|
51
|
+
when :aws
|
52
|
+
storage = FYT::S3Storage.new(
|
53
|
+
config[:tmp_path],
|
54
|
+
config[:format_options],
|
55
|
+
config[:output_format],
|
56
|
+
manager
|
57
|
+
)
|
58
|
+
|
59
|
+
Aws.config.update(
|
60
|
+
credentials: Aws::Credentials.new(
|
61
|
+
config[:aws][:access_key], config[:aws][:secret_key]
|
62
|
+
)
|
63
|
+
)
|
64
|
+
else
|
65
|
+
raise 'no storage_type configured'
|
66
|
+
end
|
45
67
|
|
46
68
|
config[:feeds].each do |feed_config|
|
47
69
|
source_feed = FYT::Parser.new(feed_config[:url], manager).read
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fyt
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.3.0.pre
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Steffen Schröder
|
@@ -10,20 +10,34 @@ bindir: bin
|
|
10
10
|
cert_chain: []
|
11
11
|
date: 2017-10-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: aws-sdk-s3
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - '='
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 1.13.0
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - '='
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 1.13.0
|
13
27
|
- !ruby/object:Gem::Dependency
|
14
28
|
name: proxy_fetcher
|
15
29
|
requirement: !ruby/object:Gem::Requirement
|
16
30
|
requirements:
|
17
|
-
- -
|
31
|
+
- - '='
|
18
32
|
- !ruby/object:Gem::Version
|
19
|
-
version:
|
33
|
+
version: 0.6.5
|
20
34
|
type: :runtime
|
21
35
|
prerelease: false
|
22
36
|
version_requirements: !ruby/object:Gem::Requirement
|
23
37
|
requirements:
|
24
|
-
- -
|
38
|
+
- - '='
|
25
39
|
- !ruby/object:Gem::Version
|
26
|
-
version:
|
40
|
+
version: 0.6.5
|
27
41
|
description: Downloads a youtube channel and its videos and stores them locally as
|
28
42
|
a podcatcher-friendly feed
|
29
43
|
email: steffen@schröder.xyz
|
@@ -38,7 +52,9 @@ files:
|
|
38
52
|
- lib/fyt/builder.rb
|
39
53
|
- lib/fyt/config.rb
|
40
54
|
- lib/fyt/parser.rb
|
55
|
+
- lib/fyt/s3_storage.rb
|
41
56
|
- lib/fyt/storage.rb
|
57
|
+
- lib/fyt/storage_helper.rb
|
42
58
|
homepage: https://github.com/ChaosSteffen/fyt
|
43
59
|
licenses:
|
44
60
|
- BSD-2-Clause
|
@@ -54,9 +70,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
54
70
|
version: '0'
|
55
71
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
56
72
|
requirements:
|
57
|
-
- - "
|
73
|
+
- - ">"
|
58
74
|
- !ruby/object:Gem::Version
|
59
|
-
version:
|
75
|
+
version: 1.3.1
|
60
76
|
requirements: []
|
61
77
|
rubyforge_project:
|
62
78
|
rubygems_version: 2.7.6
|