fyt 1.2.0 → 1.3.0.pre

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b068b9aa73605f8d48c04d802b8e1cf022b0086cd021fa2f3009291af483b526
-  data.tar.gz: a4a3b584e7f1357aeca75fcff15abd4fd914c8f717a409871421aab5121d3219
+  metadata.gz: c2c92a3bbe1b82badeaf428859b9c7e4236882459511079a62f06c4961238d33
+  data.tar.gz: d585d6bd01c90721299aab2c7e74c48780c0aea7890992595a301096ac936c33
 SHA512:
-  metadata.gz: 2b97b1dcf367d77dde813d078b40b52a649482ad55b9cdc6d079267c498bce10d9294309b343b9b287aaf331b410a832b709931e50b8a7ef1e1166a22f709eee
-  data.tar.gz: be2eba64e737c8d634dc0883ec400278c23410ae5e3fc8f1b2e85c5599e90e5d4ef6935b574936d0b3801744def3fd39c7d309d79a74cda94ae186259fc7b75b
+  metadata.gz: 03daa1ca65ea44500f5102c986e2ef48280339579d4e100f401a7965f2bcc1d4f0eaa81965c35040c05ccda2e817f269865056e7b56ce3486545ef444b8b1503
+  data.tar.gz: 0d12c9f946cbe4a63f2a59a665a0a5ccb49c01149dd0f4405fefe5c4b89425c4b383ee83ab23867c5527b99ec1d3d43972d2d32a89ea1c13b5a69fb85e1622a5

data/lib/fyt/base.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'logger'
 
 module FYT

data/lib/fyt/builder.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module FYT
   # processes the Youtube feed
   class Builder < FYT::Base
@@ -55,7 +56,7 @@ module FYT
     rescue
       @proxy_manager.remove(proxy)
 
-      add_image(youtube_url, title) if @proxy_manager.proxies.size > 0
+      add_image(youtube_url, title) unless @proxy_manager.proxies.empty?
     end
 
     def add_item(link, title, filename)

data/lib/fyt/config.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'yaml/store'
 
 module FYT

data/lib/fyt/s3_storage.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+require 'aws-sdk-s3'
+
+module FYT
+  # Manages file downloads and storage
+  class S3Storage < FYT::Base
+    include FYT::StorageHelper
+
+    def initialize(tmp_path, format_options, output_format, proxy_manager)
+      @tmp_path = tmp_path || ''
+      @format_options = format_options
+      @output_format = output_format
+      @proxy_manager = proxy_manager
+      @known_files = []
+    end
+
+    def add(item)
+      url = item.link.href
+
+      filename_for(item).tap do |filename|
+        unless files_on_s3.include? filename
+          download_file!(url, filename)
+          upload_to_s3(filename)
+          delete_file_from_disk(filename)
+        end
+
+        @known_files << filename
+      end
+    end
+
+    def add_feed(feedname, feed)
+      logger.debug feed.to_s
+      logger.debug @known_files
+
+      feed_filename = "#{feedname}.feed.rss20.xml"
+
+      tmp_path_for(feed_filename).tap do |path|
+        File.write(path, feed)
+        upload_to_s3(feed_filename)
+        delete_file_from_disk(feed_filename)
+        @known_files << feed_filename
+      end
+    end
+
+    def cleanup!
+      logger.debug 'Files to delete:'
+      logger.debug files_to_delete
+
+      files_to_delete.each do |filename|
+        delete_from_s3 filename
+      end
+    end
+
+    def mtime(filename)
+      bucket.object(filename).last_modified
+    end
+
+    def size(filename)
+      bucket.object(filename).content_length
+    end
+
+    private
+
+    def download_file!(url, filename)
+      proxy = @proxy_manager.get!
+
+      command = "#{timeout_cmd} #{youtube_dl_cmd(proxy, url, filename)}"
+      logger.debug "Executing: #{command}"
+
+      begin
+        execute command
+      rescue
+        @proxy_manager.remove(proxy)
+
+        download_file!(url, filename) unless @proxy_manager.proxies.empty?
+      end
+    end
+
+    def delete_file_from_disk(filename)
+      tmp_path_for(filename).tap do |path|
+        logger.debug "Deleting file: #{path}"
+        File.delete(path)
+      end
+    end
+
+    def filename_for(item)
+      "#{item.id.content}.mp4"
+    end
+
+    def files_to_delete
+      files_on_s3 - @known_files
+    end
+
+    def files_on_s3
+      bucket.objects.limit(100).map(&:key)
+    end
+
+    def upload_to_s3(filename)
+      bucket.object(filename).upload_file(
+        tmp_path_for(filename), acl: 'public-read'
+      )
+    end
+
+    def delete_from_s3(name)
+      bucket.object(name).delete
+    end
+
+    def tmp_path_for(filename)
+      File.join(@tmp_path, filename)
+    end
+
+    def bucket
+      Aws::S3::Resource
+        .new(region: 'eu-central-1')
+        .bucket('fyt-storage')
+    end
+  end
+end

data/lib/fyt/storage.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module FYT
   # Manages file downloads and storage
   class Storage < FYT::Base
@@ -66,7 +67,7 @@ module FYT
     rescue
       @proxy_manager.remove(proxy)
 
-      download_file!(url, output_path) if @proxy_manager.proxies.size > 0
+      download_file!(url, output_path) unless @proxy_manager.proxies.empty?
     end
 
     def execute(command_string)

data/lib/fyt/storage_helper.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module FYT
+  module StorageHelper
+    private
+
+    def execute(command_string)
+      IO.pipe do |read_io, write_io|
+        break if system(command_string, out: write_io, err: write_io)
+
+        write_io.close
+        logger.debug read_io.read
+        raise
+      end
+    end
+
+    def timeout_cmd
+      if system('which timeout', out: '/dev/null')
+        return 'timeout --preserve-status 300'
+      end
+
+      raise unless system('which gtimeout', out: '/dev/null')
+
+      'gtimeout --preserve-status 300'
+    end
+
+    def youtube_dl_cmd(proxy, url, filename)
+      [
+        'youtube-dl',
+        "-f '#{@format_options}'",
+        "--merge-output-format '#{@output_format}'",
+        "-o '#{tmp_path_for(filename)}'",
+        "--proxy 'http:#{proxy.url}'",
+        "'#{url}'"
+      ].join(' ')
+    end
+  end
+end

data/lib/fyt.rb

@@ -4,10 +4,13 @@ require_relative 'fyt/builder'
 require_relative 'fyt/config'
 require_relative 'fyt/parser'
 require_relative 'fyt/storage'
+require_relative 'fyt/storage_helper'
+require_relative 'fyt/s3_storage'
 
 require 'fileutils'
 
 require 'proxy_fetcher'
+require 'aws-sdk-s3'
 
 module ProxyFetcher
   class Manager
@@ -36,12 +39,31 @@ module FYT
     config = FYT::Config.new
     manager =
       ProxyFetcher::Manager.new(filters: { country: 'DE', maxtime: '500' })
-    storage = FYT::Storage.new(
-      config[:storage_path],
-      config[:format_options],
-      config[:output_format],
-      manager
-    )
+
+    case config[:storage_type]
+    when :local
+      storage = FYT::S3Storage.new(
+        config[:storage_path],
+        config[:format_options],
+        config[:output_format],
+        manager
+      )
+    when :aws
+      storage = FYT::S3Storage.new(
+        config[:tmp_path],
+        config[:format_options],
+        config[:output_format],
+        manager
+      )
+
+      Aws.config.update(
+        credentials: Aws::Credentials.new(
+          config[:aws][:access_key], config[:aws][:secret_key]
+        )
+      )
+    else
+      raise 'no storage_type configured'
+    end
 
     config[:feeds].each do |feed_config|
       source_feed = FYT::Parser.new(feed_config[:url], manager).read

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fyt
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.3.0.pre
 platform: ruby
 authors:
 - Steffen Schröder
@@ -10,20 +10,34 @@ bindir: bin
 cert_chain: []
 date: 2017-10-10 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.13.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.13.0
 - !ruby/object:Gem::Dependency
   name: proxy_fetcher
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: 0.6.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: 0.6.5
 description: Downloads a youtube channel and its videos and stores them locally as
   a podcatcher-friendly feed
 email: steffen@schröder.xyz
@@ -38,7 +52,9 @@ files:
 - lib/fyt/builder.rb
 - lib/fyt/config.rb
 - lib/fyt/parser.rb
+- lib/fyt/s3_storage.rb
 - lib/fyt/storage.rb
+- lib/fyt/storage_helper.rb
 homepage: https://github.com/ChaosSteffen/fyt
 licenses:
 - BSD-2-Clause
@@ -54,9 +70,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.7.6