nsrr 0.1.0.beta1 → 0.1.0.beta2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 155cc2e206c5b662c49888859663f40c4e376c21
-  data.tar.gz: 21461c274e3324827ed970425e91647d9fca2ad9
+  metadata.gz: 3924a64a6b689e857473a865bbc08a9b0753c575
+  data.tar.gz: 54066220485da3828fee75c6864f95bf91e30ae6
 SHA512:
-  metadata.gz: 396491cfb57a83a188a9a4abd1e455ff95b3abff322f2a1301694d80334f85e016b49920ef2829e0b2650706c08d06cd2440f4a88d71793fb46a30cc612d43c9
-  data.tar.gz: c0d6949622fc0ec591013f2b8260b86f5f2b4df51410c80d6a2f6041487a57bd5a92a5951a3bf474229cb618ebc357ee2bcb1269aad5b9e1e3333c824e47bfbd
+  metadata.gz: 21f4d39b700cc218e210ca77fafec61ade8fed81da9ee48ba2ec59e650539c5c9642f6490b5394a7da58c335d391c8f1fae4d2adc641f818bb468bf1e38de8b8
+  data.tar.gz: 16686364fce98068357195fdbd790661cfa1615fe051d2aaba08445643def2dd0f298ba4ce1ecd19e5df4ec390d331fdb2cb453ab9984f3f502ad2b60fc0029d

data/CHANGELOG.md

@@ -1,5 +1,25 @@
 ## 0.1.0
 
 ### Enhancements
+- Added a `nsrr console` command that allows users to access and download datasets and files
+  - Datasets can be loaded in the console environment
+    - `d = Dataset.find 'shhs'`
+  - Dataset files can be downloaded as well
+    - `d.download`
+  - The download function can include a path, method, and depth
+    - **path**
+      - can be `nil` to download entire dataset or a string to specify a folder
+    - **method**
+      - 'md5' [default]
+        - Checks if a downloaded file exists with the exact md5 as the online version, if so, skips that file
+      - 'fresh'
+        - Downloads every file without checking if it was already downloaded
+      - 'fast'
+        - Only checks if a download file exists with the same file size as the online version, if so, skips that file
+    - **depth**
+      - 'recursive' [default]
+        - Downloads files in selected path folder and all subfolders
+      - 'shallow'
+        - Only downloads files in selected path folder
 - Added a `nsrr version` command the returns the current version of the nsrr gem
 - Added testing framework to more easily add new tests for new features

data/README.md

@@ -26,6 +26,104 @@ Or install it yourself as:
 
 ## Usage
 
+### Open the console and download entire datasets
+
+```console
+$ nsrr console
+```
+
+```
+> d = Dataset.find 'shhs'
+> d.download
+
+     File Integrity Check Method: md5
+                           Depth: recursive
+             Get your token here: https://sleepdata.org/token
+Please enter your download token:
+      create shhs/
+      create shhs/datasets
+   identical shhs-cvd-dataset-0.4.0.csv
+   identical shhs-data-dictionary-0.4.0-domains.csv
+   identical shhs-data-dictionary-0.4.0-forms.csv
+   identical shhs-data-dictionary-0.4.0-variables.csv
+   identical shhs1-dataset-0.4.0.csv
+   ...
+```
+
+**method**
+  - 'md5' [default]
+    - Checks if a downloaded file exists with the exact md5 as the online version, if so, skips that file
+  - 'fresh'
+    - Downloads every file without checking if it was already downloaded
+  - 'fast'
+    - Only checks if a download file exists with the same file size as the online version, if so, skips that file
+
+**depth**
+  - 'recursive' [default]
+    - Downloads files in selected path folder and all subfolders
+  - 'shallow'
+    - Only downloads files in selected path folder
+
+For example to download only the shhs1 edfs folder and skip MD5 file validation:
+
+```
+> d = Dataset.find 'shhs'
+> d.download('edfs/shhs1', method: 'fast', depth: 'shallow')
+
+     File Integrity Check Method: fast
+                           Depth: shallow
+             Get your token here: https://sleepdata.org/token
+Please enter your download token:
+
+      create shhs/edfs/shhs1
+    download 100001.EDF
+    download 100002.EDF
+    download 100003.EDF
+    download 100004.EDF
+    ...
+```
+
+You can type `Ctrl-C` to pause the download, and retype the command to restart:
+
+```
+> d = Dataset.find 'shhs'
+> d.download
+
+     File Integrity Check Method: md5
+                           Depth: recursive
+             Get your token here: https://sleepdata.org/token
+Please enter your download token:
+
+      create shhs/
+      create shhs/datasets
+    download shhs-cvd-dataset-0.4.0.csv
+    download shhs-data-dictionary-0.4.0-domains.csv
+^C
+    Interrupted
+
+Finished in 4.384734 seconds.
+
+1 folder created, 2 files downloaded, 60 MiBs downloaded, 0 files skipped, 0 files failed
+
+> d.download
+
+     File Integrity Check Method: md5
+                           Depth: recursive
+             Get your token here: https://sleepdata.org/token
+Please enter your download token:
+
+      create shhs/
+      create shhs/datasets
+   identical shhs-cvd-dataset-0.4.0.csv
+   identical shhs-data-dictionary-0.4.0-domains.csv
+^C
+    Interrupted
+
+Finished in 2.384734 seconds.
+
+1 folder created, 0 files downloaded, 0 MiBs downloaded, 2 files skipped, 0 files failed
+```
+
 ### Show the version of the NSRR gem
 
 ```

data/lib/nsrr.rb

@@ -1,6 +1,7 @@
 require "nsrr/version"
 
 Nsrr::COMMANDS = {
+  'c' => :console,
   'v' => :version
 }
 
@@ -9,12 +10,36 @@ module Nsrr
     self.send((Nsrr::COMMANDS[argv.first.to_s.scan(/\w/).first] || :help), argv)
   end
 
+  def self.console(argv)
+    require 'nsrr/commands/console'
+    Nsrr::Commands::Console.start(argv)
+    # console = Nsrr::Commands::Console.new(argv)
+    # console.start
+    # `#{File.expand_path('../', __FILE__)}/nsrr/commands/console2`
+
+
+    # require 'irb'
+    # require 'irb/completion'
+    # IRB.setup nil
+    # IRB.conf[:MAIN_CONTEXT] = IRB::Irb.new.context
+    # require 'irb/ext/multi-irb'
+    # IRB.irb nil, self
+
+    # require 'irb'
+    # ARGV.clear
+    # @a = "hello"
+    # IRB.start
+
+  end
+
   def self.help(argv)
     puts <<-EOT
 
 Usage: nsrr COMMAND [ARGS]
 
 The most common nsrr commands are:
+  [c]onsole         Load an interactive console to access
+                    and download datasets and files
   [v]ersion         Returns the version of nsrr gem
 
 Commands can be referenced by the first letter:

data/lib/nsrr/commands/console.rb

@@ -0,0 +1,28 @@
+require 'irb'
+require 'irb/completion'
+require 'nsrr/models/all'
+
+module Nsrr
+  module Commands
+    class Console
+      class << self
+        def start(*args)
+          new(*args).start
+        end
+      end
+
+      attr_reader :console
+
+      def initialize(argv)
+        ARGV.clear
+        @console = IRB
+      end
+
+      def start
+        puts "Loading environment (Nsrr #{Nsrr::VERSION::STRING})"
+        @console.start
+      end
+
+    end
+  end
+end

data/lib/nsrr/helpers/constants.rb

@@ -0,0 +1 @@
+Nsrr::WEBSITE = "https://sleepdata.org"

data/lib/nsrr/helpers/download_request.rb

@@ -0,0 +1,48 @@
+require 'net/http'
+require 'uri'
+
+module Nsrr
+  module Helpers
+    class DownloadRequest
+      class << self
+        def get(*args)
+          new(*args).get
+        end
+      end
+
+      attr_reader :url, :error, :file_size
+
+      def initialize(url, download_folder)
+        begin
+          escaped_url = URI.escape(url)
+          @url = URI.parse(escaped_url)
+          @http = Net::HTTP.new(@url.host, @url.port)
+          @http.use_ssl = true if (@url.scheme == 'https')
+          @download_folder = download_folder
+          @file_size = 0
+        rescue => e
+          @error = 'Invalid Token'
+        end
+      end
+
+      def get
+        req = Net::HTTP::Get.new(@url.path)
+        response = @http.start do |http|
+          http.request(req)
+        end
+        case response.code when '200'
+          ::File.open(@download_folder, 'wb') do |local_file|
+            local_file.write( response.body )
+          end
+          @file_size = ::File.size(@download_folder)
+        when '302'
+          @error = 'Token Not Authorized to Access Specified File'
+        else
+          @error = "#{response.code} #{response.class.name}"
+        end
+      end
+    end
+  end
+end
+
+

data/lib/nsrr/helpers/hash_helper.rb

@@ -0,0 +1,9 @@
+module Nsrr
+  module Helpers
+    class HashHelper
+      def self.symbolize_keys(hash)
+        hash.inject({}){|memo,(k,v)| memo[k.to_sym] = v; memo}
+      end
+    end
+  end
+end

data/lib/nsrr/helpers/json_request.rb

@@ -0,0 +1,32 @@
+require 'net/http'
+require 'json'
+
+module Nsrr
+  module Helpers
+    class JsonRequest
+      class << self
+        def get(*args)
+          new(*args).get
+        end
+      end
+
+      attr_reader :url
+
+      def initialize(url)
+        @url = URI.parse(url)
+        @http = Net::HTTP.new(@url.host, @url.port)
+        @http.use_ssl = true if (@url.scheme == 'https')
+      end
+
+      def get
+        req = Net::HTTP::Get.new(@url.path)
+        response = @http.start do |http|
+          http.request(req)
+        end
+        JSON.parse(response.body) rescue nil
+      end
+    end
+  end
+end
+
+

data/lib/nsrr/models/all.rb

@@ -0,0 +1 @@
+require 'nsrr/models/dataset'

data/lib/nsrr/models/dataset.rb

@@ -0,0 +1,126 @@
+require 'colorize'
+require 'fileutils'
+
+require 'nsrr/helpers/constants'
+require 'nsrr/helpers/hash_helper'
+require 'nsrr/helpers/json_request'
+
+require 'nsrr/models/file'
+
+module Nsrr
+  module Models
+    class Dataset
+      def self.find(slug)
+        json = Nsrr::Helpers::JsonRequest.get("#{Nsrr::WEBSITE}/datasets/#{slug}.json")
+        if json
+          new(json)
+        else
+          nil
+        end
+      end
+
+      attr_reader :slug, :name
+
+      def initialize(json = {})
+        @slug = json['slug']
+        @name = json['name']
+        @files = {}
+        @download_token = nil
+      end
+
+      def files(path = nil)
+        @files[path] ||= begin
+          json = Nsrr::Helpers::JsonRequest.get("#{Nsrr::WEBSITE}/datasets/#{@slug}/json_manifest/#{path}")
+          (json || []).collect{|file_json| Nsrr::Models::File.new(file_json)}
+        end
+      end
+
+      def folders(path = nil)
+        self.files(path).select{|f| !f.is_file}.collect{|f| f.name}
+      end
+
+      # Options include:
+      # method:
+      #   'md5'       => [default] Checks if a downloaded file exists with the exact md5 as the online version, if so, skips that file
+      #   'fresh'     => Downloads every file without checking if it was already downloaded
+      #   'fast'      => Only checks if a download file exists with the same file size as the online version, if so, skips that file
+      # depth:
+      #   'recursive' => [default] Downloads files in selected path folder and all subfolders
+      #   'shallow'   => Only downloads files in selected path folder
+      def download(path = nil, *args)
+        options = Nsrr::Helpers::HashHelper.symbolize_keys(args.first || {})
+        options[:method] ||= 'md5'
+        options[:depth] ||= 'recursive'
+        @folders_created = 0
+        @files_downloaded = 0
+        @downloaded_bytes = 0
+        @files_skipped = 0
+        @files_failed = 0
+
+        begin
+          if @download_token.to_s == ''
+            puts "     File Integrity Check Method: " + options[:method].to_s.colorize(:white)
+            puts "                           Depth: " + options[:depth].to_s.colorize(:white)
+            set_download_token()
+          end
+
+          @start_time = Time.now
+
+          download_helper(path, options)
+        rescue Interrupt, IRB::Abort
+          puts "\n   Interrupted".colorize(:red)
+        end
+
+        @downloaded_megabytes = @downloaded_bytes / (1024 * 1024)
+
+        puts "\nFinished in #{Time.now - @start_time} seconds."
+        puts "\n#{@folders_created} folder#{"s" if @folders_created != 1} created".colorize(:white) + ", " +
+             "#{@files_downloaded} file#{"s" if @files_downloaded != 1} downloaded".colorize(:green) + ", " +
+             "#{@downloaded_megabytes} MiB#{"s" if @downloaded_megabytes != 1} downloaded".colorize(:green) + ", " +
+             "#{@files_skipped} file#{"s" if @files_skipped != 1} skipped".colorize(:blue) + ", " +
+             "#{@files_failed} file#{"s" if @files_failed != 1} failed".colorize(:red) + "\n\n"
+        nil
+      end
+
+      def download_helper(path, options)
+        current_folder = ::File.join(self.slug.to_s, path.to_s)
+        create_folder(current_folder)
+
+        self.files(path).select{|f| f.is_file}.each do |file|
+          result = file.download(options[:method], current_folder, @download_token)
+          case result when 'fail'
+            @files_failed += 1
+          when 'skip'
+            @files_skipped += 1
+          else
+            @files_downloaded += 1
+            @downloaded_bytes += result
+          end
+        end
+
+        if options[:depth] == 'recursive'
+          self.files(path).select{|f| !f.is_file}.each do |file|
+            folder = [path, file.name].compact.join('/')
+            self.download_helper(folder, options)
+          end
+        end
+      end
+
+      def create_folder(folder)
+        puts "      create".colorize( :white ) + " #{folder}"
+        FileUtils.mkdir_p folder
+        @folders_created += 1
+      end
+
+      def set_download_token
+        puts  "             Get your token here: " + "https://sleepdata.org/token".colorize(:blue)
+        print "Please enter your download token: "
+        @download_token = gets.chomp
+      end
+
+    end
+  end
+end
+
+class Dataset < Nsrr::Models::Dataset
+end

data/lib/nsrr/models/file.rb

@@ -0,0 +1,98 @@
+# Models a downloadable file or folder
+require 'digest/md5'
+
+require 'nsrr/helpers/constants'
+require 'nsrr/helpers/download_request'
+
+module Nsrr
+  module Models
+    class File
+      attr_reader :name, :is_file, :web_file_size, :web_checksum
+
+      def initialize(json = {})
+        @name = json['file_name']
+        @web_checksum = json['checksum']
+        @is_file = json['is_file']
+        @web_file_size = json['file_size']
+        @dataset_slug = json['dataset']
+        @file_path = json['file_path']
+      end
+
+      # method:
+      #   'md5'       => [default] Checks if a downloaded file exists with the exact md5 as the online version, if so, skips that file
+      #   'fresh'     => Downloads every file without checking if it was already downloaded
+      #   'fast'      => Only checks if a download file exists with the same file size as the online version, if so, skips that file
+      def download(method, path, token)
+        do_md5_check = (method == 'md5')
+        redownload_all = (method == 'fresh')
+
+        if do_md5_check
+          if md5_matches?(path)
+            skip
+          else
+            force_download(path, token)
+          end
+        else
+          if redownload_all
+            force_download(path, token)
+          else
+            if file_size_matches?(path)
+              skip
+            else
+              force_download(path, token)
+            end
+          end
+        end
+      end
+
+      def force_download(path, token)
+        download_folder = ::File.join(Dir.pwd, path.to_s, @name.to_s)
+        download_url = "#{Nsrr::WEBSITE}/datasets/#{@dataset_slug}/files/a/#{token}/m/nsrr-gem-v#{Nsrr::VERSION::STRING.gsub('.', '-')}/#{@file_path.to_s}"
+        download_request = Nsrr::Helpers::DownloadRequest.new(download_url, download_folder)
+        download_request.get
+
+        if download_request.error.to_s == ''
+          puts "    download".colorize( :green ) + " #{@name}"
+          download_request.file_size
+        else
+          puts "      failed".colorize( :red ) + " #{@name}"
+          puts "             #{download_request.error}"
+          'fail'
+        end
+      end
+
+      def skip
+        puts "   identical".colorize( :blue ) + " #{self.name}"
+        'skip'
+      end
+
+      def md5_matches?(path)
+        @web_checksum == local_checksum(path)
+      end
+
+      def local_checksum(path)
+        download_folder = ::File.join(Dir.pwd, path.to_s, self.name.to_s)
+        if ::File.exist?(download_folder)
+          Digest::MD5.file(download_folder).hexdigest
+        else
+          ""
+        end
+      end
+
+      def file_size_matches?(path)
+        @web_file_size == local_filesize(path)
+      end
+
+      def local_filesize(path)
+        download_folder = ::File.join(Dir.pwd, path.to_s, self.name.to_s)
+        if ::File.exist?(download_folder)
+          ::File.size(download_folder)
+        else
+          -1
+        end
+      end
+
+    end
+  end
+end
+

data/lib/nsrr/version.rb

@@ -3,7 +3,7 @@ module Nsrr
     MAJOR = 0
     MINOR = 1
     TINY = 0
-    BUILD = "beta1" # nil, "pre", "rc", "rc2"
+    BUILD = "beta2" # nil, "pre", "rc", "rc2"
 
     STRING = [MAJOR, MINOR, TINY, BUILD].compact.join('.')
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: nsrr
 version: !ruby/object:Gem::Version
-  version: 0.1.0.beta1
+  version: 0.1.0.beta2
 platform: ruby
 authors:
 - Remo Mueller
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-25 00:00:00.000000000 Z
+date: 2014-07-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -95,6 +95,14 @@ files:
 - Rakefile
 - bin/nsrr
 - lib/nsrr.rb
+- lib/nsrr/commands/console.rb
+- lib/nsrr/helpers/constants.rb
+- lib/nsrr/helpers/download_request.rb
+- lib/nsrr/helpers/hash_helper.rb
+- lib/nsrr/helpers/json_request.rb
+- lib/nsrr/models/all.rb
+- lib/nsrr/models/dataset.rb
+- lib/nsrr/models/file.rb
 - lib/nsrr/version.rb
 - nsrr.gemspec
 homepage: https://github.com/nsrr/nsrr-gem