pulse-downloader 0.1.0 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 82910e4a2a00c0519958083ae3380241bb4c0cd3166d513d311c338227e4184f
-  data.tar.gz: c7642a1d110693f56ce933a466312fe851647ede790cba74a18c8157137c2a7e
+  metadata.gz: 143241844753713ded3ebdb7fccf727a20ab2055ad9c40e9936b88f9ecd4a0a5
+  data.tar.gz: 471f4432385de2dc96223ed014e5d3c88adccfda7c98a591b1cd2c915dbe3256
 SHA512:
-  metadata.gz: 1ec8ec9d18dabd67e9c7e01abf5718206512e9169b2cc38c4ec2654b1ae289d7955945995ea55fda637c7a6363f8f053baa681a6ba463c1b6f3a2519bf1b714c
-  data.tar.gz: 37dc7aa8612c40f696c6902e3e791811ac801106a1c45243ba5597f73bbecd7198b077d8d37340a7e010c995252c092ea3f8a869498cac0844762d5d72412ab3
+  metadata.gz: f26e86e5e59e24be532fc542e2de26c33d69dd2e4bb264ef6fe1f28f46d68782d5d0838a80db769912fc202598dff4e40aad10a856e4659e01105c73997b7ed3
+  data.tar.gz: a29a979395ce6b6d7311e7150e22f1ed11a62cd317e19aa42a16241674e7abe33397e8d210882aeb4fa6f7835e3a32e321ccdadf13ecc20219c7a9065a38f9f5

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    pulse-downloader (0.1.0)
+    pulse-downloader (0.1.3)
       active_attr (~> 0.15)
       httparty (~> 0.18)
       nokogiri (~> 1.10.9)

data/README.md

@@ -1,8 +1,6 @@
 # Pulse::Downloader
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/pulse/downloader`. To experiment with that code, run `bin/console` for an interactive prompt.
-
-TODO: Delete this and the text above, and describe your gem
+This is a library to download a specific group of files linked to on an html page.
 
 ## Installation
 
@@ -22,7 +20,21 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
+```ruby
+require 'pulse/downloader'
+
+client = Pulse::Downloader::Client.new(
+  url: '',
+  file_type: 'zip',
+  save_data: true,
+  save_path: '',
+  read_from_save_path: false,
+  verify_ssl: true,
+  drop_exitsing_files_in_path: false,
+  save_and_dont_return: true
+  report_time: false
+)
+```
 
 ## Development
 
@@ -34,7 +46,6 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/pulse-downloader. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/[USERNAME]/pulse-downloader/blob/master/CODE_OF_CONDUCT.md).
 
-
 ## License
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/lib/pulse/downloader.rb

@@ -3,6 +3,7 @@ require 'nokogiri'
 
 require "pulse/downloader/version"
 require 'pulse/downloader/web_page_parser'
+require 'pulse/downloader/file_checker'
 require 'pulse/downloader/file_downloader'
 require 'pulse/downloader/client'
 

data/lib/pulse/downloader/client.rb

@@ -2,16 +2,45 @@ module Pulse
   module Downloader
     class Client
       include ::Pulse::Downloader::WebPageParser
+      include ::Pulse::Downloader::FileChecker
       include ::Pulse::Downloader::FileDownloader
 
-      attr_reader :path, :file_type, :save_data, :save_path, :read_from_save_path
+      attr_reader :url,
+        :file_type,
+        :save_data,
+        :save_path,
+        :read_from_save_path,
+        :verify_ssl,
+        :drop_exitsing_files_in_path,
+        :report_time,
+        :start_time,
+        :end_time
 
-      def initialize(path:, file_type:, save_data: false, save_path: '', read_from_save_path: false)
-        @path = path
+      # Does not continue downloads-
+      # Will only save once the file has been downloaded in memory
+
+      # TODO: Add in progress bar
+      # TODO: Validation
+      # TODO: Retry
+      # TODO: DNS
+      def initialize(url:,
+        file_type:,
+        save_data: false,
+        save_path: '',
+        read_from_save_path: false,
+        verify_ssl: true,
+        drop_exitsing_files_in_path: false,
+        save_and_dont_return: true,
+        report_time: false)
+
+        @url = url
         @file_type = file_type
         @save_data = save_data
         @save_path = save_path
         @read_from_save_path = read_from_save_path
+        @verify_ssl = verify_ssl
+        @drop_exitsing_files_in_path = drop_exitsing_files_in_path
+        @report_time = report_time
       end
 
       def call!
@@ -33,11 +62,11 @@ module Pulse
       private
 
       def get_micro_second_time
-        (Time.now.to_f * 1000000).to_i
+        (Time.now.to_f * 1000).to_i
       end
 
-      def compute_filename(file_path)
-        file_path.scan(/[\/]\S+/).last
+      def print_time
+        puts "Request time: #{end_time - start_time} ms."
       end
     end
   end

data/lib/pulse/downloader/file_checker.rb

@@ -0,0 +1,27 @@
+module Pulse
+  module Downloader
+    module FileChecker
+      def file_path_in_file_list?(file_path)
+        return false unless drop_exitsing_files_in_path && save_data
+
+        list_files_in(save_path).include?(compute_save_path(file_path))
+      end
+
+      private
+
+      def compute_save_path(url)
+        "#{save_path}/#{compute_filename(url)}".gsub('//', '/')
+      end
+
+      def compute_filename(file_path)
+        file_path.scan(/[\/]\S+/).last
+      end
+
+      def list_files_in(path)
+        `ls #{path}`.split("\n").map do |filename|
+          "#{path}/#{filename}".gsub('//', '/')
+        end
+      end
+    end
+  end
+end

data/lib/pulse/downloader/file_downloader.rb

@@ -1,29 +1,71 @@
 module Pulse
   module Downloader
     module FileDownloader
-      # save_path are defined in client.rb
+      # save_path and verify_ssl are defined in client.rb
       def download(file_path)
         raise "save_path is undefined" if save_data && save_path == ''
+        return if file_path_in_file_list?(file_path) # skip downloading the file
 
-        start_time = get_micro_second_time
+        @start_time = get_micro_second_time
 
-        file_data = HTTParty.get(file_path)
+        file_data = HTTParty.get(compute_file_link(file_path), verify: verify_ssl)
 
-        # TODO: Use the time
-        end_time = get_micro_second_time
+        @end_time = get_micro_second_time
+
+        if report_time
+          print_time
+        end
 
         if save_data
-          File.open("#{save_path}/#{compute_filename(file_path)}", 'wb') do |file|
+          File.open(compute_save_path(file_path), 'wb') do |file|
             file.write(file_data.body)
           end
+
+          return true if save_and_dont_return
         end
 
         file_data
       end
 
+      def fetch_save_paths
+        fetch_file_paths.map do |file_path|
+          "#{save_path}/#{compute_filename(file_path)}"
+        end
+      end
+
       def compute_hash_of(data)
         { data: data }.hash
       end
+
+      private
+
+      def compute_file_link(file_path)
+        if section?(file_path)
+          raise 'invalid download path'
+        elsif absolute?(file_path)
+          file_path
+        elsif relative?(file_path)
+          "#{url}/#{file_path}"
+        else
+          "#{url}/#{file_path}"
+        end
+      end
+
+      def absolute?(file_path)
+        file_path.include?('http://') ||
+          file_path.include?('https://') ||
+          file_path.include?('ftp://') ||
+          file_path.include?('sftp://')||
+          file_path.include?('file://')
+      end
+
+      def relative?(file_path)
+        file_path[0] == '/'
+      end
+
+      def section?(file_path)
+        file_path[0] == '#'
+      end
     end
   end
 end

data/lib/pulse/downloader/version.rb

@@ -1,5 +1,5 @@
 module Pulse
   module Downloader
-    VERSION = "0.1.0"
+    VERSION = "0.1.3"
   end
 end

data/lib/pulse/downloader/web_page_parser.rb

@@ -2,24 +2,30 @@ module Pulse
   module Downloader
     module WebPageParser
       def fetch_file_paths
-        start_time = get_micro_second_time
+        @start_time = get_micro_second_time
 
-        response = HTTParty.get(@path)
+        response = HTTParty.get(url, verify: verify_ssl)
 
-        # TODO: Use the time
-        end_time = get_micro_second_time
-        extract_file_urls(response, start_time, end_time)
+        @end_time = get_micro_second_time
+
+        if report_time
+          print_time
+        end
+
+        extract_file_urls(response)
       end
 
       private
 
-      def extract_file_urls(response, start_time, end_time)
-        parse_html(response)
+      def extract_file_urls(response)
+        return [] if response.body.nil? || response.body.empty?
+
+        parse_html(response.body)
           .css('a')
           .to_a
           .map { |link| link['href'] }
           .compact
-          .select { |link| link.include? @file_type }
+          .select { |link| link.include? file_type }
       end
 
       def parse_html(raw_html)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pulse-downloader
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.3
 platform: ruby
 authors:
 - trex22
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-07-15 00:00:00.000000000 Z
+date: 2020-07-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httparty
@@ -197,6 +197,7 @@ files:
 - bin/setup
 - lib/pulse/downloader.rb
 - lib/pulse/downloader/client.rb
+- lib/pulse/downloader/file_checker.rb
 - lib/pulse/downloader/file_downloader.rb
 - lib/pulse/downloader/version.rb
 - lib/pulse/downloader/web_page_parser.rb