pulse-downloader 0.1.0 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 82910e4a2a00c0519958083ae3380241bb4c0cd3166d513d311c338227e4184f
-  data.tar.gz: c7642a1d110693f56ce933a466312fe851647ede790cba74a18c8157137c2a7e
+  metadata.gz: 528b545a62356f1b1d9c1786f975b059f0387c3fecd58d6fe211b62405922d56
+  data.tar.gz: a173671066bb527aaf9abdd8e68be48ab0e849a22d70cce4b4abaf0271ba425a
 SHA512:
-  metadata.gz: 1ec8ec9d18dabd67e9c7e01abf5718206512e9169b2cc38c4ec2654b1ae289d7955945995ea55fda637c7a6363f8f053baa681a6ba463c1b6f3a2519bf1b714c
-  data.tar.gz: 37dc7aa8612c40f696c6902e3e791811ac801106a1c45243ba5597f73bbecd7198b077d8d37340a7e010c995252c092ea3f8a869498cac0844762d5d72412ab3
+  metadata.gz: 8028861136a23be938d906811375d24dd4da26ad25c33b97ed8872316610daf34ca8e83a285a16e5b76445bab5fe8164592848442a51d7c3dd00696e9092f688
+  data.tar.gz: 50b64203c44b39b458fd8ea5dc7610ba6d1e22418a74762d6ebae78c06a14a31ea5c3e7606eec21cf1a60bf6c82f1d1be6549b3eb8a7f3945777f35d93b27714

data/.github/workflows/semgrep.yml

@@ -0,0 +1,18 @@
+on:
+  pull_request: {}
+  push:
+    branches:
+    - main
+    - master
+name: Semgrep
+jobs:
+  semgrep:
+    name: Scan
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: returntocorp/semgrep-action@v1
+      with:
+        auditOn: push
+        publishToken: ${{ secrets.SEMGREP_APP_TOKEN }}
+        publishDeployment: 607

data/Gemfile.lock

@@ -1,79 +1,79 @@
 PATH
   remote: .
   specs:
-    pulse-downloader (0.1.0)
+    pulse-downloader (0.1.4)
       active_attr (~> 0.15)
       httparty (~> 0.18)
-      nokogiri (~> 1.10.9)
+      nokogiri (~> 1.11)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    actionpack (6.0.3.2)
-      actionview (= 6.0.3.2)
-      activesupport (= 6.0.3.2)
-      rack (~> 2.0, >= 2.0.8)
+    actionpack (6.1.3.1)
+      actionview (= 6.1.3.1)
+      activesupport (= 6.1.3.1)
+      rack (~> 2.0, >= 2.0.9)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actionview (6.0.3.2)
-      activesupport (= 6.0.3.2)
+    actionview (6.1.3.1)
+      activesupport (= 6.1.3.1)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.1, >= 1.2.0)
-    active_attr (0.15.0)
-      actionpack (>= 3.0.2, < 6.1)
-      activemodel (>= 3.0.2, < 6.1)
-      activesupport (>= 3.0.2, < 6.1)
-    activemodel (6.0.3.2)
-      activesupport (= 6.0.3.2)
-    activesupport (6.0.3.2)
+    active_attr (0.15.3)
+      actionpack (>= 3.0.2, < 7.0)
+      activemodel (>= 3.0.2, < 7.0)
+      activesupport (>= 3.0.2, < 7.0)
+    activemodel (6.1.3.1)
+      activesupport (= 6.1.3.1)
+    activesupport (6.1.3.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (>= 0.7, < 2)
-      minitest (~> 5.1)
-      tzinfo (~> 1.1)
-      zeitwerk (~> 2.2, >= 2.2.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
     ansi (1.5.0)
     builder (3.2.4)
     coderay (1.1.3)
-    concurrent-ruby (1.1.6)
-    crack (0.4.3)
-      safe_yaml (~> 1.0.0)
+    concurrent-ruby (1.1.8)
+    crack (0.4.5)
+      rexml
     crass (1.0.6)
-    erubi (1.9.0)
+    erubi (1.10.0)
     hashdiff (1.0.1)
     httparty (0.18.1)
       mime-types (~> 3.0)
       multi_xml (>= 0.5.2)
-    i18n (1.8.3)
+    i18n (1.8.10)
       concurrent-ruby (~> 1.0)
-    loofah (2.6.0)
+    loofah (2.9.1)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     method_source (1.0.0)
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2020.0512)
-    mini_portile2 (2.4.0)
-    minitest (5.14.1)
+    mime-types-data (3.2021.0225)
+    minitest (5.14.4)
     minitest-focus (1.1.2)
       minitest (>= 4, < 6)
-    minitest-reporters (1.4.2)
+    minitest-reporters (1.4.3)
       ansi
       builder
       minitest (>= 5.0)
       ruby-progressbar
     mocha (1.11.2)
     multi_xml (0.6.0)
-    nokogiri (1.10.10)
-      mini_portile2 (~> 2.4.0)
-    pry (0.13.1)
+    nokogiri (1.11.3-x86_64-linux)
+      racc (~> 1.4)
+    pry (0.14.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    public_suffix (4.0.5)
+    public_suffix (4.0.6)
+    racc (1.5.2)
     rack (2.2.3)
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
@@ -83,23 +83,22 @@ GEM
     rails-html-sanitizer (1.3.0)
       loofah (~> 2.3)
     rake (12.3.3)
-    ruby-progressbar (1.10.1)
-    safe_yaml (1.0.5)
-    thread_safe (0.3.6)
-    timecop (0.9.1)
-    tzinfo (1.2.7)
-      thread_safe (~> 0.1)
+    rexml (3.2.5)
+    ruby-progressbar (1.11.0)
+    timecop (0.9.4)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
     webmock (3.8.3)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
-    zeitwerk (2.4.0)
+    zeitwerk (2.4.2)
 
 PLATFORMS
-  ruby
+  x86_64-linux
 
 DEPENDENCIES
-  bundler (~> 2.1.4)
+  bundler (~> 2.2.16)
   minitest (~> 5.0)
   minitest-focus (~> 1.1.2)
   minitest-reporters (~> 1.4.2)
@@ -111,4 +110,4 @@ DEPENDENCIES
   webmock (~> 3.8.3)
 
 BUNDLED WITH
-   2.1.4
+   2.2.16

data/README.md

@@ -1,8 +1,6 @@
 # Pulse::Downloader
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/pulse/downloader`. To experiment with that code, run `bin/console` for an interactive prompt.
-
-TODO: Delete this and the text above, and describe your gem
+This is a library to download a specific group of files linked to on an html page.
 
 ## Installation
 
@@ -22,7 +20,24 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
+```ruby
+require 'pulse/downloader'
+
+client = Pulse::Downloader::Client.new(
+  url: '',
+  file_type: 'zip',
+  save_data: true,
+  save_path: '',
+  read_from_save_path: false,
+  verify_ssl: true,
+  drop_exitsing_files_in_path: false,
+  save_and_dont_return: true,
+  report_time: false,
+  progress_bar: true
+)
+
+client.call!
+```
 
 ## Development
 
@@ -34,7 +49,6 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/pulse-downloader. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/[USERNAME]/pulse-downloader/blob/master/CODE_OF_CONDUCT.md).
 
-
 ## License
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/lib/pulse/downloader.rb

@@ -3,6 +3,7 @@ require 'nokogiri'
 
 require "pulse/downloader/version"
 require 'pulse/downloader/web_page_parser'
+require 'pulse/downloader/file_checker'
 require 'pulse/downloader/file_downloader'
 require 'pulse/downloader/client'
 

data/lib/pulse/downloader/client.rb

@@ -2,16 +2,52 @@ module Pulse
   module Downloader
     class Client
       include ::Pulse::Downloader::WebPageParser
+      include ::Pulse::Downloader::FileChecker
       include ::Pulse::Downloader::FileDownloader
 
-      attr_reader :path, :file_type, :save_data, :save_path, :read_from_save_path
+      attr_reader :url,
+        :file_type,
+        :save_data,
+        :save_path,
+        :read_from_save_path,
+        :verify_ssl,
+        :drop_exitsing_files_in_path,
+        :save_and_dont_return,
+        :report_time,
+        :start_time,
+        :end_time,
+        :progress_bar
 
-      def initialize(path:, file_type:, save_data: false, save_path: '', read_from_save_path: false)
-        @path = path
+      # Does not continue downloads-
+      # Will only save once the file has been downloaded in memory
+
+      # TODO: Validation
+      # TODO: Retry
+      # TODO: DNS
+      def initialize(url:,
+        file_type:,
+        save_data: false,
+        save_path: '',
+        read_from_save_path: false,
+        verify_ssl: true,
+        drop_exitsing_files_in_path: false,
+        save_and_dont_return: true,
+        report_time: false,
+        progress_bar: false)
+
+        @url = url
         @file_type = file_type
         @save_data = save_data
         @save_path = save_path
         @read_from_save_path = read_from_save_path
+        @verify_ssl = verify_ssl
+        @drop_exitsing_files_in_path = drop_exitsing_files_in_path
+        @save_and_dont_return = save_and_dont_return
+        @report_time = report_time
+
+        if progress_bar
+          @progress_bar = ProgressBar.new
+        end
       end
 
       def call!
@@ -21,8 +57,13 @@ module Pulse
       def call
         return false unless valid?
 
+        if @progress_bar
+          @progress_bar = ProgressBar.new(fetch_file_paths.size)
+        end
+
         fetch_file_paths.map do |file_path|
-          download(file_path)
+          download(file_path, @progress_bar)
+          @progress_bar.increment!
         end
       end
 
@@ -33,11 +74,17 @@ module Pulse
       private
 
       def get_micro_second_time
-        (Time.now.to_f * 1000000).to_i
+        (Time.now.to_f * 1000).to_i
       end
 
-      def compute_filename(file_path)
-        file_path.scan(/[\/]\S+/).last
+      def print_time(progress_bar=nil)
+        output = "Request time: #{end_time - start_time} ms."
+
+        if progress_bar
+          progress_bar.puts output
+        else
+          puts output
+        end
       end
     end
   end

data/lib/pulse/downloader/file_checker.rb

@@ -0,0 +1,27 @@
+module Pulse
+  module Downloader
+    module FileChecker
+      def file_path_in_file_list?(file_path)
+        return false unless drop_exitsing_files_in_path && save_data
+
+        list_files_in(save_path).include?(compute_save_path(file_path))
+      end
+
+      private
+
+      def compute_save_path(url)
+        "#{save_path}/#{compute_filename(url)}".gsub('//', '/')
+      end
+
+      def compute_filename(file_path)
+        file_path.scan(/[\/]\S+/).last
+      end
+
+      def list_files_in(path)
+        `ls #{path}`.split("\n").map do |filename|
+          "#{path}/#{filename}".gsub('//', '/')
+        end
+      end
+    end
+  end
+end

data/lib/pulse/downloader/file_downloader.rb

@@ -1,29 +1,71 @@
 module Pulse
   module Downloader
     module FileDownloader
-      # save_path are defined in client.rb
-      def download(file_path)
+      # save_path and verify_ssl are defined in client.rb
+      def download(file_path, progress_bar=nil)
         raise "save_path is undefined" if save_data && save_path == ''
+        return if file_path_in_file_list?(file_path) # skip downloading the file
 
-        start_time = get_micro_second_time
+        @start_time = get_micro_second_time
 
-        file_data = HTTParty.get(file_path)
+        file_data = HTTParty.get(compute_file_link(file_path), verify: verify_ssl)
 
-        # TODO: Use the time
-        end_time = get_micro_second_time
+        @end_time = get_micro_second_time
+
+        if report_time
+          print_time(progress_bar)
+        end
 
         if save_data
-          File.open("#{save_path}/#{compute_filename(file_path)}", 'wb') do |file|
+          File.open(compute_save_path(file_path), 'wb') do |file|
             file.write(file_data.body)
           end
+
+          return true if save_and_dont_return
         end
 
         file_data
       end
 
+      def fetch_save_paths
+        fetch_file_paths.map do |file_path|
+          "#{save_path}/#{compute_filename(file_path)}"
+        end
+      end
+
       def compute_hash_of(data)
         { data: data }.hash
       end
+
+      private
+
+      def compute_file_link(file_path)
+        if section?(file_path)
+          raise 'invalid download path'
+        elsif absolute?(file_path)
+          file_path
+        elsif relative?(file_path)
+          "#{url}/#{file_path}"
+        else
+          "#{url}/#{file_path}"
+        end
+      end
+
+      def absolute?(file_path)
+        file_path.include?('http://') ||
+          file_path.include?('https://') ||
+          file_path.include?('ftp://') ||
+          file_path.include?('sftp://')||
+          file_path.include?('file://')
+      end
+
+      def relative?(file_path)
+        file_path[0] == '/'
+      end
+
+      def section?(file_path)
+        file_path[0] == '#'
+      end
     end
   end
 end

data/lib/pulse/downloader/version.rb

@@ -1,5 +1,5 @@
 module Pulse
   module Downloader
-    VERSION = "0.1.0"
+    VERSION = "0.1.5"
   end
 end

data/lib/pulse/downloader/web_page_parser.rb

@@ -2,24 +2,30 @@ module Pulse
   module Downloader
     module WebPageParser
       def fetch_file_paths
-        start_time = get_micro_second_time
+        @start_time = get_micro_second_time
 
-        response = HTTParty.get(@path)
+        response = HTTParty.get(url, verify: verify_ssl)
 
-        # TODO: Use the time
-        end_time = get_micro_second_time
-        extract_file_urls(response, start_time, end_time)
+        @end_time = get_micro_second_time
+
+        if report_time
+          print_time
+        end
+
+        extract_file_urls(response)
       end
 
       private
 
-      def extract_file_urls(response, start_time, end_time)
-        parse_html(response)
+      def extract_file_urls(response)
+        return [] if response.body.nil? || response.body.empty?
+
+        parse_html(response.body)
           .css('a')
           .to_a
           .map { |link| link['href'] }
           .compact
-          .select { |link| link.include? @file_type }
+          .select { |link| link.include? file_type }
       end
 
       def parse_html(raw_html)

data/pulse-downloader.gemspec

@@ -22,10 +22,11 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "httparty", "~> 0.18"
   spec.add_dependency "active_attr", "~> 0.15"
-  spec.add_dependency "nokogiri", "~> 1.10.9"
+  spec.add_dependency "nokogiri", "~> 1.11"
+  spec.add_dependency "progress_bar", "~> 1.3.3"
 
   # Development dependancies
-  spec.add_development_dependency "bundler", "~> 2.1.4"
+  spec.add_development_dependency "bundler", "~> 2.2.16"
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "minitest", "~> 5.0"
   spec.add_development_dependency "minitest-focus", "~> 1.1.2"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pulse-downloader
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.5
 platform: ruby
 authors:
 - trex22
-autorequire:
+autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-07-15 00:00:00.000000000 Z
+date: 2021-04-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httparty
@@ -44,28 +44,42 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.10.9
+        version: '1.11'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.10.9
+        version: '1.11'
+- !ruby/object:Gem::Dependency
+  name: progress_bar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.3
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.1.4
+        version: 2.2.16
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.1.4
+        version: 2.2.16
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -185,6 +199,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/semgrep.yml"
 - ".gitignore"
 - CODE_OF_CONDUCT.md
 - Gemfile
@@ -197,6 +212,7 @@ files:
 - bin/setup
 - lib/pulse/downloader.rb
 - lib/pulse/downloader/client.rb
+- lib/pulse/downloader/file_checker.rb
 - lib/pulse/downloader/file_downloader.rb
 - lib/pulse/downloader/version.rb
 - lib/pulse/downloader/web_page_parser.rb
@@ -205,7 +221,7 @@ homepage: https://github.com/TRex22/pulse-downloader
 licenses:
 - MIT
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -220,8 +236,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
-signing_key:
+rubygems_version: 3.2.3
+signing_key: 
 specification_version: 4
 summary: Client to download datasets from webpages.
 test_files: []