cloudapp-export 0.2.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 1579ad02b2a2789e95e6e5c743765006091e9c67
+  data.tar.gz: bfac4398babc3afc70aa4d1a44c57085128c0d52
+SHA512:
+  metadata.gz: d20379fcab2828e82ee1e70e6fd35c246c9f00804b9b85a78a7e2fe680ae278d1c6e32c9905ab0e4636cc3f6bcb08e242973e6a0930749c45b82803e49361b7d
+  data.tar.gz: 9ea24ea8e415140425fe60c9c16e522acea2b3e4c8c5b4514e68ef38f734f51ebda8da9c294f50d4ab021b8e332f34298be468cf15f5de4073063bf2dbe54cfc

data/.circleci/config.yml

@@ -0,0 +1,36 @@
+# Ruby CircleCI 2.0 configuration file
+#
+# Check https://circleci.com/docs/2.0/language-ruby/ for more details
+#
+version: 2
+jobs:
+  build:
+    docker:
+      - image: circleci/ruby:2.4.2
+    working_directory: ~/repo
+
+    steps:
+      - checkout
+
+      # Download and cache dependencies
+      - restore_cache:
+          keys:
+          - v1-dependencies-{{ checksum "Gemfile.lock" }}
+          # fallback to using the latest cache if no exact match is found
+          - v1-dependencies-
+
+      - run:
+          name: install dependencies
+          command: |
+            bundle install --jobs=4 --retry=3 --path vendor/bundle
+
+      - save_cache:
+          paths:
+            - ./vendor/bundle
+          key: v1-dependencies-{{ checksum "Gemfile.lock" }}
+
+      # run tests!
+      - run:
+          name: run tests
+          command: |
+            rake

data/.gitignore

@@ -0,0 +1,17 @@
+.env
+items
+!items/.gitkeep
+items.json
+
+# RubyGem
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+.ruby-version
+*.gem

data/.rubocop.yml

@@ -0,0 +1,58 @@
+AllCops:
+  DisplayCopNames: true
+  Exclude:
+    - 'Gemfile'
+    - 'Gemfile.lock'
+    - '*.gemspec'
+    - 'vendor/**/*'
+
+Style/FrozenStringLiteralComment:
+  Enabled: false
+
+Style/StringLiterals:
+  Enabled: false
+
+Style/MutableConstant:
+  Enabled: false
+
+Style/Documentation:
+  Enabled: false
+
+Style/DoubleNegation:
+  Enabled: false
+
+Style/MultilineTernaryOperator:
+  Enabled: false
+
+Style/RegexpLiteral:
+  Enabled: false
+
+# Screens are big, even when they're small
+Metrics/LineLength:
+  Enabled: false
+
+Style/TrailingCommaInArguments:
+  EnforcedStyleForMultiline: consistent_comma
+
+Style/TrailingCommaInArrayLiteral:
+  EnforcedStyleForMultiline: consistent_comma
+
+Style/TrailingCommaInHashLiteral:
+  EnforcedStyleForMultiline: consistent_comma
+
+Style/ClassAndModuleChildren:
+  Exclude:
+    - '**/*_test.rb'
+Metrics/MethodLength:
+  Max: 16
+  Exclude:
+    - '**/*_test.rb'
+
+# Namespace blocks contain multiple tasks
+Metrics/BlockLength:
+  Exclude:
+    - '**/*.rake'
+
+# TODO: Refactor to reduce this
+Metrics/AbcSize:
+  Max: 26

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in cloudapp-export.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 Marc Qualie
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,43 @@
+# CloudApp Export
+
+A quick way to grab all the data you have stored in [CloudApp](https://www.getcloudapp.com/).
+
+
+## Credentials
+
+The script requiests the following environment variables. These are easiest to
+set via the .env file in this directory which will be automatically detected.
+
+```
+export CLOUDAPP_API_HOST="my.cl.ly"
+export CLOUDAPP_USERNAME="YOUR_EMAIL"
+export CLOUDAPP_PASSWORD="YOUR_PASSWORD"
+```
+
+
+## API Output Format
+
+Each item will have the following attributes
+
+```
+slug: 3a1X1J383340
+name: marcqualie-nyc-2017.jpg
+created_at: '2017-07-06T17:48:01Z'
+updated_at: '2017-07-06T17:48:02Z'
+long_link: true
+item_type: image
+view_counter: 0
+content_url: http://share.marcqualie.com/3a1X1J383340/marcqualie-nyc-2017.jpg
+redirect_url:
+remote_url: http://f.cl.ly/items/292o1N3J3s0A281O3Y2x/marcqualie-nyc-2017.jpg
+source: CloudApp/4.2.4 (1054) (Mac OS X 10.12.5)
+thumbnail_url: https://d1dqt6infcio59.cloudfront.net/3a1X1J383340/010428f967be2109bac41acfae90cb55
+share_url: http://share.marcqualie.com/3a1X1J383340
+```
+
+You can download the direct file via `remote_url`
+
+
+## LICENSE
+
+[MIT](LICENSE)

data/Rakefile

@@ -0,0 +1,18 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+require "rubocop/rake_task"
+
+lib = File.expand_path("lib", __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+load "tasks/cloudapp_export.rake"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/*_test.rb"]
+end
+
+RuboCop::RakeTask.new
+
+task default: %w[rubocop test]

data/bin/cloudapp-export

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+lib = File.expand_path("../lib", __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+require "cloudapp_export"
+require "dotenv"
+
+Dotenv.load
+CloudappExport::CLI.start(ARGV)

data/cloudapp-export.gemspec

@@ -0,0 +1,30 @@
+require_relative "lib/cloudapp_export/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "cloudapp-export"
+  spec.version       = CloudappExport::VERSION
+  spec.authors       = ["Marc Qualie"]
+  spec.email         = ["marc@marcqualie.com"]
+
+  spec.summary       = "A quick way to export all the data you have stored in CloudApp"
+  spec.homepage      = "https://github.com/marcqualie/cloudapp-export"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "bin"
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.required_ruby_version = ">= 2.3.3"
+
+  spec.add_dependency "dotenv", "2.2.1"
+  spec.add_dependency "net-http-digest_auth", "1.4.1"
+  spec.add_dependency "thor", "0.20.0"
+
+  spec.add_development_dependency "bundler", "~> 1.16"
+  spec.add_development_dependency "rake", "~> 12.3"
+  spec.add_development_dependency "rubocop", "~> 0.54.0"
+  spec.add_development_dependency "minitest", "~> 5.11"
+end

data/lib/cloudapp_export.rb

@@ -0,0 +1,9 @@
+require "cloudapp_export/api"
+require "cloudapp_export/cli"
+require "cloudapp_export/exporter"
+require "cloudapp_export/item"
+require "cloudapp_export/item_list"
+require "cloudapp_export/version"
+
+module CloudappExport
+end

data/lib/cloudapp_export/api.rb

@@ -0,0 +1,64 @@
+require 'json'
+require 'net/http'
+require 'net/http/digest_auth'
+require 'open-uri'
+require 'uri'
+
+module CloudappExport
+  class Api
+    DEFAULT_HOST = "my.cl.ly"
+
+    def initialize(options = {})
+      @options = options
+    end
+
+    def request(path)
+      digest_auth = Net::HTTP::DigestAuth.new
+      uri = URI.parse("https://#{host}/v3/#{path}")
+      uri.user = CGI.escape(username)
+      uri.password = password
+
+      Net::HTTP.start(uri.host, uri.port, use_ssl: true) do |http|
+        # First request is a 401 response with a WWW-Authenticate header
+        request = Net::HTTP::Get.new(uri.request_uri)
+        response = http.request(request)
+
+        # Create a new request with the Authorization header
+        auth = digest_auth.auth_header(uri, response['www-authenticate'], 'GET')
+        request = Net::HTTP::Get.new(uri.request_uri)
+        request.add_field('Authorization', auth)
+
+        # re-issue request with Authorization
+        ApiResponse.new(http.request(request))
+      end
+    end
+
+    protected
+
+    def host
+      @options['host'] || DEFAULT_HOST
+    end
+
+    def username
+      @options['username'] || raise('Username is required for API use')
+    end
+
+    def password
+      @options['password'] || raise('Password is required for API use')
+    end
+  end
+
+  class ApiResponse
+    def initialize(http_response)
+      @http_response = http_response
+    end
+
+    def body
+      @http_response.body
+    end
+
+    def data
+      JSON.parse(@http_response.body)['data']
+    end
+  end
+end

data/lib/cloudapp_export/cli.rb

@@ -0,0 +1,83 @@
+require "base64"
+require "fileutils"
+require "thor"
+
+module CloudappExport
+  class CLI < Thor
+    class_option :username, type: :string, desc: "Account username"
+    class_option :password, type: :string, desc: "Account password (not recommended to pass via command line)"
+
+    desc :all, "Export all data from your Cloudapp account"
+    option :limit, default: 5, type: :numeric
+    option :dir, default: "#{ENV['HOME']}/Downloads/CloudappExport", type: :string, desc: "Directory to download all files to"
+    option :cache, type: :boolean, default: true
+    def all
+      items = CloudappExport::ItemList.new(
+        api,
+        'limit' => options['limit'],
+        'cache' => options['cache'],
+      )
+
+      exporter = ::CloudappExport::Exporter.new(
+        items,
+        'dir' => options['dir'],
+      )
+      exporter.on_log do |message|
+        print message
+      end
+      exporter.export_all
+    end
+
+    # rubocop:disable Metrics/AbcSize
+    # rubocop:disable Layout/TrailingWhitespace
+    desc :stats, "Show stats for CloudApp items"
+    option :dir, default: "#{ENV['HOME']}/Downloads/CloudappExport", type: :string, desc: "Directory where your files were downloaded to"
+    option :cache, type: :boolean, default: true
+    def stats
+      items = CloudappExport::ItemList.new(
+        api,
+        'cache' => options['cache'],
+      )
+      
+      downloaded_items = items.data.select { |item| File.exist?("#{options['dir']}/#{item.filename}") }
+      downloaded_items_size = downloaded_items.inject(0) { |sum, item| sum + File.size("#{options['dir']}/#{item.filename}") }
+      
+      say("Dir         #{options['dir']}")
+      say("Items       #{items.count}")
+      say("Downloaded  #{downloaded_items.count}")
+      say("            #{(downloaded_items_size.to_f / 1_000_000).round 2} mb")
+    end
+    # rubocop:enable Metrics/AbcSize
+    # rubocop:enable Layout/TrailingWhitespace
+
+    no_commands do
+      def api
+        @api ||= begin
+          CloudappExport::Api.new(
+            'username' => username,
+            'password' => password,
+          )
+        end
+      end
+
+      def username
+        @username ||= begin
+          username = options['username']
+          username ||= ENV['CLOUDAPP_USERNAME']
+          username ||= ask("Username:")
+          username || raise("Username is required")
+        end
+      end
+
+      def password
+        say("WARNING: Passing password via the command line is not recommended!", :yellow) if options['password']
+        @password ||= begin
+          password = options['password']
+          password ||= ENV['CLOUDAPP_PASSWORD']
+          password ||= (ask("Password:", echo: false) && say('*' * 20))
+          password || raise("Password is required")
+        end
+      end
+    end
+  end
+end

data/lib/cloudapp_export/exporter.rb

@@ -0,0 +1,80 @@
+module CloudappExport
+  class Exporter
+    attr_reader :items
+
+    def initialize(items, options = {})
+      @items = items
+      @options = options
+      @log_callbacks = []
+    end
+
+    def export_all
+      FileUtils.mkdir_p(download_dir) unless Dir.exist?(download_dir)
+
+      items_count = items.count
+      items.each_with_index do |item, index|
+        percent = (index + 1).to_f / items_count * 100
+        log "[#{(index + 1).to_s.rjust(4, '0')} / #{items_count.to_s.rjust(4, '0')}  #{percent.round.to_s.rjust(3, ' ')}%]"
+        export_item(item)
+      end
+    end
+
+    def export_item(item)
+      log "  #{item.name} -> #{item.filename}".ljust(80, ' ')
+
+      filepath = "#{download_dir}/#{item.filename}"
+      if File.exist?(filepath)
+        log "  SK  #{item_filesize_human(item)}"
+      else
+        begin
+          log "  DL"
+          copy_file(item['remote_url'], filepath)
+          log "  #{item_filesize_human(item)}"
+        rescue StandardError => error
+          log "  ER #{error.message}"
+        end
+      end
+      log "\n"
+    end
+
+    def on_log(&block)
+      @log_callbacks.push(block)
+    end
+
+    protected
+
+    def copy_file(remote_url, local_path)
+      remote_uri = URI(remote_url)
+      File.open(local_path, 'wb') do |file|
+        file << Net::HTTP.get(remote_uri)
+      end
+    end
+
+    def item_filesize(item)
+      filepath = "#{download_dir}/#{item.filename}"
+      File.exist?(filepath) ? File.size(filepath) : 0
+    end
+
+    def item_filesize_human(item)
+      filesize = item_filesize(item)
+      return '--' if filesize.zero?
+      size_human(filesize)
+    end
+
+    def size_human(filesize)
+      filesize >= (1024 * 1024) \
+        ? "#{(filesize.to_f / (1024 * 1024)).round 2} mb"
+        : "#{(filesize.to_f / 1024).round 2} kb"
+    end
+
+    def log(message)
+      @log_callbacks.each do |callback|
+        callback.call(message)
+      end
+    end
+
+    def download_dir
+      @options['dir']
+    end
+  end
+end

data/lib/cloudapp_export/item.rb

@@ -0,0 +1,22 @@
+module CloudappExport
+  class Item
+    attr_reader :attributes
+
+    def initialize(attributes)
+      @attributes = attributes
+    end
+
+    def [](key)
+      @attributes[key]
+    end
+
+    def name
+      @attributes['name']
+    end
+
+    def filename
+      ext = @attributes['name'].split('.').last
+      "#{@attributes['slug']}.#{ext}"
+    end
+  end
+end

data/lib/cloudapp_export/item_list.rb

@@ -0,0 +1,59 @@
+require "json"
+
+module CloudappExport
+  class ItemList
+    def initialize(api, options = {})
+      @api = api
+      @items = []
+      @use_cache = !!options['cache']
+      @limit = (options['limit'] || 999_999_999).to_i
+      @offset = (options['offset'] || 0).to_i
+    end
+
+    def data
+      load
+      @items
+    end
+
+    def count
+      load
+      [@items.count, @limit].min
+    end
+
+    def each(&block)
+      load
+      @items[@offset..(@limit - 1)].each(&block)
+    end
+
+    def each_with_index(&block)
+      load
+      @items[@offset..(@limit - 1)].each_with_index(&block)
+    end
+
+    # Restrict items to a subset from 0 to $number
+    # @param number [Integer] Number of items to return
+    # @return Integer
+    attr_writer :limit
+
+    protected
+
+    def load
+      @items = begin
+        if @use_cache && File.exist?(cache_file_path)
+          items = ::JSON.parse(::File.read(cache_file_path))
+        else
+          response = @api.request("items?per_page=100000")
+          items = response.data
+          ::File.write(cache_file_path, ::JSON.dump(items))
+        end
+        items.map do |attributes|
+          ::CloudappExport::Item.new(attributes)
+        end
+      end
+    end
+
+    def cache_file_path
+      "#{ENV['HOME']}/.cloudapp-export-items.json"
+    end
+  end
+end

data/lib/cloudapp_export/version.rb

@@ -0,0 +1,3 @@
+module CloudappExport
+  VERSION = "0.2.1"
+end

data/lib/tasks/cloudapp_export.rake

@@ -0,0 +1,10 @@
+require 'dotenv'
+require 'json'
+require 'cloudapp_export'
+
+namespace :cloudapp do
+  desc "Export data from CloudApp"
+  task :export do |_t, _args|
+    puts "Please use `cloudapp-export all` instead of this rake task"
+  end
+end

metadata

@@ -0,0 +1,162 @@
+--- !ruby/object:Gem::Specification
+name: cloudapp-export
+version: !ruby/object:Gem::Version
+  version: 0.2.1
+platform: ruby
+authors:
+- Marc Qualie
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-04-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: dotenv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.2.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.2.1
+- !ruby/object:Gem::Dependency
+  name: net-http-digest_auth
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.4.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.4.1
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.20.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.20.0
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.3'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.54.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.54.0
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.11'
+description: 
+email:
+- marc@marcqualie.com
+executables:
+- cloudapp-export
+extensions: []
+extra_rdoc_files: []
+files:
+- ".circleci/config.yml"
+- ".gitignore"
+- ".rubocop.yml"
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- bin/cloudapp-export
+- cloudapp-export.gemspec
+- items/.gitkeep
+- lib/cloudapp_export.rb
+- lib/cloudapp_export/api.rb
+- lib/cloudapp_export/cli.rb
+- lib/cloudapp_export/exporter.rb
+- lib/cloudapp_export/item.rb
+- lib/cloudapp_export/item_list.rb
+- lib/cloudapp_export/version.rb
+- lib/tasks/cloudapp_export.rake
+homepage: https://github.com/marcqualie/cloudapp-export
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.3
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2
+signing_key: 
+specification_version: 4
+summary: A quick way to export all the data you have stored in CloudApp
+test_files: []