csr_data_inventory 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: f9ff97cb355f70a1598d06969ac5fea7aab3e691d67b770eb3eebeba8a32fb2f
+  data.tar.gz: fade09a5ae0526452fb1d5b7a60fe3d76795b184b56529ea95c05056bdbe57dc
+SHA512:
+  metadata.gz: 2568aae3ef8facb79484ca7b02f6271fd96db9915d5e36a80f68781124b9fc1cbad6effeecef03cfadb3cce1a6a8f3cc1e88a2df3213e738b23973f2bdfab873
+  data.tar.gz: 144289409d88c0d26eeed2fc7a013ce9189cef3436a1782ba93f550aeb342ca83cabfb7e3c7b127be4bbb2fc4172bd14e1e9588fdd68e6cb4fdbc8b321e8531e

data/lib/csr_data_inventory.rb

@@ -0,0 +1,41 @@
+require_relative './download_request.rb'
+class CSRDataInventory
+  def self.download
+    puts "Please enter your token:"
+    token = gets.strip
+    download_url = "https://datascience.uth.edu/databases/download_file_list?"
+    params_part = URI.encode_www_form([["token",token]])
+    download_url = download_url + params_part
+    download_folder = download_folder = ::File.join(Dir.pwd, "#{token}_file_list.txt")
+    download_request = DownloadRequest.new(download_url, download_folder)
+    download_request.get
+    if download_request.error.to_s == ""
+    	puts "  File list downloaded"
+    	File.readlines("#{Dir.pwd}/#{token}_file_list.txt").each do |line|
+    		path_tokens = line.strip.split
+
+    		path = path_tokens[0]
+    		file_size = path_tokens[1]
+    		 
+    		puts "Downloading file: #{path}"
+    		download_url = "https://datascience.uth.edu/databases/download_file?"
+		    params_part = URI.encode_www_form([["token",token], ["path", path]])
+		    download_url = download_url + params_part
+		    download_folder = ::File.join(Dir.pwd, path[14..-1])
+		    if File.exists?(download_folder) and (File.size(download_folder).to_s == file_size)
+		    	puts "File exists"
+		    	next
+		    end
+		    download_request = DownloadRequest.new(download_url, download_folder)
+		    download_request.get
+	      if download_request.error.to_s == ""
+	      	puts "Successful"
+	      else
+	      	puts "Failed: #{download_request.error.to_s}"
+	      end
+    	end
+    else
+    	puts "  #{download_request.error.to_s}"
+    end
+  end
+end

data/lib/download_request.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require "openssl"
+require "net/http"
+require "uri"
+require 'fileutils'
+# Downloads a file to a specified folder.
+class DownloadRequest
+  class << self
+    def get(*args)
+      new(*args).get
+    end
+  end
+
+  attr_reader :url, :error, :file_size
+
+  def initialize(url, download_folder)
+    @url = URI.parse(url)
+    puts @url.path
+    @http = Net::HTTP.new(@url.host, @url.port)
+    if @url.scheme == "https"
+      @http.use_ssl = true
+      @http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+    end
+    @download_folder = download_folder
+    @file_size = 0
+  end
+
+  # Writes file segments to disk immediately instead of storing in memory
+  def get
+    return unless @error.nil?
+
+    dirname = File.dirname(@download_folder)
+    unless File.directory?(dirname)
+      FileUtils.mkdir_p(dirname)
+    end
+
+    local_file = ::File.open(@download_folder, "wb")
+    partial = true
+    @http.request_get(@url) do |response|
+      case response.code
+      when "200"
+        response.read_body do |segment|
+          local_file.write(segment)
+        end
+        @file_size = ::File.size(@download_folder)
+        partial = false
+      when "302"
+        @error = "Token Not Authorized to Access Specified File"
+      else
+        @error = "#{response.code} #{response.class.name}"
+      end
+    end
+  rescue => e # Net::ReadTimeout, SocketError
+    @error = "(#{e.class}) #{e.message}"
+  ensure
+    local_file.close if local_file
+    ::File.delete(@download_folder) if partial && ::File.exist?(@download_folder)
+  end
+end
+ 

metadata

@@ -0,0 +1,44 @@
+--- !ruby/object:Gem::Specification
+name: csr_data_inventory
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Shiqiang Tao
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2021-06-25 00:00:00.000000000 Z
+dependencies: []
+description: A gem to download CSR Data in a scalable way.
+email: shiqiang.tao@uth.tmc.edu
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/csr_data_inventory.rb
+- lib/download_request.rb
+homepage: https://rubygems.org/gems/csr_data_inventory
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.8
+signing_key:
+specification_version: 4
+summary: Download CSR Data
+test_files: []