wetransfer 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a6bb6ae2971c42681f0f887d3bcd76f3e19041c5
-  data.tar.gz: 02e73980a479f35655e0a775072a0a2728d3f023
+  metadata.gz: 3e0f327dda7a10e3f6b84555d9a3345b1542061c
+  data.tar.gz: 3800f908a5b774c44e297cdc1d5047fcdbb95ef7
 SHA512:
-  metadata.gz: a9badb333a33924c63de6ba39f0c40f05ce3444618164a166a08a21ff3d16e46b5f7c0bbb2d046f2674a8948146fdec26b6386716172fdd81cc71bdf9d9e916c
-  data.tar.gz: 8c6c6641be4a570ea300e75e24512896b0d55c4ea1efabb5f45c2277bd7378a19fc8196e6aab9d5079a42da8a7befbb5dfda8f7bc4dc56682c71746cc51b4db6
+  metadata.gz: 32550fa1fccece36ff52b75a95f7fc51e6d3d67bf90b6c63962b99671be5b6fa81b6576bbee71a786ee49dea7701450fc3a28007743edd30379bbea0917d3eb5
+  data.tar.gz: 4625e08a9cce1b5bf3f8df1ac209499cc488791b96bb2c18f9086f76ecb999261d3a8482ed2ac878af017f50813b4e6a14a20664f3521dfdaa1c37196634082e

data/.travis.yml

@@ -1,8 +1,7 @@
 sudo: false
 language: ruby
 rvm:
-- 2.3.7
-- 2.4.4
+- 2.1.0
 - 2.5.1
 - jruby-9.0
 before_install: gem install bundler -v 1.16.1
@@ -11,5 +10,5 @@ matrix:
   allow_failures:
     - rvm: jruby-9.0
 script:
-  - bundle exec rubocop -c .rubocop.yml --force-exclusion
   - bundle exec rspec
+  - bundle exec rubocop -c .rubocop.yml --force-exclusion

data/README.md

@@ -19,7 +19,7 @@ For your API key please visit our [developer portal](https://developers.wetransf
 Add this line to your application's Gemfile:
 
 ```ruby
-gem 'wetransfer'
+gem 'we_transfer_client'
 ```
 
 And then execute:
@@ -32,56 +32,32 @@ Or install it yourself as:
 
 ## Usage
 
-### Configuration
-
-The gem allows you to configure several settings using environment variables.
-
-- `WT_API_LOGGING_ON` can be set to (a string) "true" if you want to switch Faraday's default logging on.
-
-- `WT_API_URL` can be set to a staging or test URL (something we do not offer yet, but plan to in the future)
-
-- `WT_API_CONNECTION_PATH` can be set to prefix the paths passed to faraday - for example if you're testing against a test API or a different version.
-
 ### Super simple transfers
 
 You'll need to retrieve an API key from [our developer portal](https://developers.wetransfer.com).
 
-Be sure to not commit this key to github! If you do though, no worries, you can always revoke & create a new key from within the portal. You will most likely want to pass this to the client setter using an environment variable.
+Be sure to not commit this key to Github! If you do though, no worries, you can always revoke & create a new key from within the portal. You will most likely want to pass this to the client setter using an environment variable.
 
 Now that you've got a wonderful WeTransfer API key, you can create a Client object like so:
 
 ```ruby
-# In a .env or other secret handling file, not checked in to version control:
-WT_API_KEY=<your API key>
-
 # In your project file:
-@client = WeTransfer::Client.new(api_key: ENV['WT_API_KEY'])
+@client = WeTransferClient.new(api_key: ENV.fetch('WT_API_KEY'))
 ```
 
 Now that you've got the client set up you can use the `create_transfer` to, well, create a transfer!
 
-If you pass item paths to the method it will handle the upload process itself, otherwise you can omit them and
-use the `add_items` method once the transfer has been created.
-
 ```ruby
-transfer = @client.create_transfer(name: "My wonderful transfer", description: "I'm so excited to share this", items: ["/path/to/local/file_1.jpg", "/path/to/local/file_2.png", "/path/to/local/file_3.key"])
+transfer = @client.create_transfer(name: "My wonderful transfer", description: "I'm so excited to share this") do |upload|
+  upload.add_file_at(path: '/path/to/local/file.jpg')
+  upload.add_file_at(path: '/path/to/another/local/file.jpg')
+  upload.add_file(name: 'README.txt', io: StringIO.new("This is the contents of the file"))
+end
 
-transfer.shortened_url = "https://we.tl/SSBsb3ZlIHJ1Ynk="
+transfer.shortened_url => "https://we.tl/SSBsb3ZlIHJ1Ynk="
 ```
 
-## Item upload flow
-
-### `add_items`
-
-If you want slightly more granular control over your transfer, create it without an `items` array, and then use `add_items` with the resulting transfer object.
-
-```ruby
-transfer = @client.create_transfer(name: "My wonderful transfer", description: "I'm so excited to share this")
-
-@client.add_items(transfer: @transfer, items: ["/path/to/local/file_1.jpg", "/path/to/local/file_2.png", "/path/to/local/file_3.key"])
-
-transfer.shortened_url = "https://we.tl/d2V0cmFuc2Zlci5ob21lcnVuLmNv"
-```
+The upload will be performed at the end of the block.
 
 ## Development
 
@@ -99,4 +75,4 @@ The gem is available as open source under the terms of the [MIT License](https:/
 
 ## Code of Conduct
 
-Everyone interacting in the WetransferRubySdk project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/wetransfer/wetransfer/blob/master/CODE_OF_CONDUCT.md).
+Everyone interacting in the WetransferRubySdk project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/wetransfer/wetransfer/blob/master/CODE_OF_CONDUCT.md).

data/examples/create_transfer.rb

@@ -0,0 +1,13 @@
+require_relative 'we_transfer_client'
+require 'dotenv'
+Dotenv.load
+
+client = WeTransferClient.new(api_key: ENV.fetch('WT_API_KEY')) # , logger: Logger.new($stderr))
+transfer = client.create_transfer(title: 'My amazing board', message: 'Hi there!') do |builder|
+  builder.add_file(name: File.basename(__FILE__), io: File.open(__FILE__, 'rb'))
+  builder.add_file(name: 'amazing.txt', io: StringIO.new('This is unbelievable'))
+  builder.add_file(name: 'huge.bin', io: File.open('/path/to/local/file.jpg', 'rb'))
+  builder.add_file_at(path: __FILE__)
+end
+
+puts transfer.shortened_url

data/lib/we_transfer_client.rb

@@ -0,0 +1,194 @@
+require 'faraday'
+require 'logger'
+require 'ks'
+require 'securerandom'
+require 'json'
+
+class WeTransferClient
+  require_relative 'we_transfer_client/version'
+
+  class Error < StandardError
+  end
+
+  NULL_LOGGER = Logger.new(nil)
+  MAGIC_PART_SIZE = 6 * 1024 * 1024
+  EXPOSED_COLLECTION_ATTRIBUTES = [:id, :version_identifier, :state, :shortened_url, :name, :description, :size, :items]
+  EXPOSED_ITEM_ATTRIBUTES = [:id, :local_identifier, :content_identifier, :name, :size, :mime_type]
+
+  class FutureFileItem < Ks.strict(:name, :io, :local_identifier)
+    def initialize(**kwargs)
+      super(local_identifier: SecureRandom.uuid, **kwargs)
+    end
+
+    def to_item_request_params
+      # Ideally the content identifier should stay the same throughout multiple
+      # calls if the file contents doesn't change.
+      {
+        content_identifier: 'file',
+        local_identifier: local_identifier,
+        filename: name,
+        filesize: io.size,
+      }
+    end
+  end
+
+  class TransferBuilder
+    attr_reader :items
+
+    def initialize
+      @items = []
+    end
+
+    def add_file(name:, io:)
+      ensure_io_compliant!(io)
+      @items << FutureFileItem.new(name: name, io: io)
+      true
+    end
+
+    def add_file_at(path:)
+      add_file(name: File.basename(path), io: File.open(path, 'rb'))
+    end
+
+    def ensure_io_compliant!(io)
+      io.seek(0)
+      io.read(1) # Will cause things like Errno::EACCESS to happen early, before the upload begins
+      io.seek(0) # Also rewinds the IO for later uploading action
+      size = io.size # Will cause a NoMethodError
+      raise Error, 'The IO object given to add_file has a size of 0' if size <= 0
+    rescue NoMethodError
+      raise Error, "The IO object given to add_file must respond to seek(), read() and size(), but #{io.inspect} did not"
+    end
+  end
+
+  class FutureTransfer < Ks.strict(:name, :description, :items)
+    def to_create_transfer_params
+      {
+        name: name,
+        description: description,
+        items: items.map(&:to_item_request_params),
+      }
+    end
+  end
+
+  class RemoteTransfer < Ks.strict(*EXPOSED_COLLECTION_ATTRIBUTES)
+  end
+
+  class RemoteItem < Ks.strict(*EXPOSED_ITEM_ATTRIBUTES)
+  end
+
+  def initialize(api_key:, logger: NULL_LOGGER)
+    @api_url_base = 'https://dev.wetransfer.com'
+    @api_key = api_key.to_str
+    @bearer_token = nil
+    @logger = logger
+  end
+
+  def create_transfer(title:, message:)
+    builder = TransferBuilder.new
+    yield(builder)
+    raise 'The transfer you have tried to create contains no items' if builder.items.empty?
+    future_transfer = FutureTransfer.new(name: title, description: message, items: builder.items)
+    create_and_upload(future_transfer)
+  end
+
+  def create_and_upload(xfer)
+    authorize_if_no_bearer_token!
+    response = faraday.post(
+      '/v1/transfers',
+      JSON.pretty_generate(xfer.to_create_transfer_params),
+      auth_headers.merge('Content-Type' => 'application/json')
+    )
+    ensure_ok_status!(response)
+    create_transfer_response = JSON.parse(response.body, symbolize_names: true)
+
+    remote_transfer = hash_to_struct(create_transfer_response, RemoteTransfer)
+    remote_transfer.items = remote_transfer.items.map do |remote_item_hash|
+      hash_to_struct(remote_item_hash, RemoteItem)
+    end
+
+    item_id_map = Hash[xfer.items.map(&:local_identifier).zip(xfer.items)]
+
+    create_transfer_response.fetch(:items).each do |remote_item|
+      local_item = item_id_map.fetch(remote_item.fetch(:local_identifier))
+      remote_item_id = remote_item.fetch(:id)
+      put_io_in_parts(
+        remote_item_id,
+        remote_item.fetch(:meta).fetch(:multipart_parts),
+        remote_item.fetch(:meta).fetch(:multipart_upload_id),
+        local_item.io
+      )
+
+      complete_response = faraday.post(
+        "/v1/files/#{remote_item_id}/uploads/complete",
+        '{}',
+        auth_headers.merge('Content-Type' => 'application/json')
+      )
+      ensure_ok_status!(complete_response)
+    end
+
+    remote_transfer
+  end
+
+  def hash_to_struct(hash, struct_class)
+    members = struct_class.members
+    struct_attrs = Hash[members.zip(hash.values_at(*members))]
+    struct_class.new(**struct_attrs)
+  end
+
+  def put_io_in_parts(item_id, n_parts, multipart_id, io)
+    chunk_size = MAGIC_PART_SIZE
+    (1..n_parts).each do |part_n_one_based|
+      response = faraday.get("/v1/files/#{item_id}/uploads/#{part_n_one_based}/#{multipart_id}", {}, auth_headers)
+      ensure_ok_status!(response)
+      response = JSON.parse(response.body, symbolize_names: true)
+
+      upload_url = response.fetch(:upload_url)
+      part_io = StringIO.new(io.read(chunk_size)) # needs a lens
+      part_io.rewind
+      response = faraday.put(upload_url, part_io, 'Content-Type' => 'binary/octet-stream', 'Content-Length' => part_io.size.to_s)
+      ensure_ok_status!(response)
+    end
+  end
+
+  def faraday
+    Faraday.new(@api_url_base) do |c|
+      c.response :logger, @logger
+      c.adapter Faraday.default_adapter
+      c.headers = { 'User-Agent' => "WetransferRubySdk/#{WeTransferClient::VERSION} Ruby #{RUBY_VERSION}"}
+    end
+  end
+
+  def authorize_if_no_bearer_token!
+    return if @bearer_token
+    response = faraday.post('/v1/authorize', '{}', 'Content-Type' => 'application/json', 'X-API-Key' => @api_key)
+    ensure_ok_status!(response)
+    @bearer_token = JSON.parse(response.body, symbolize_names: true)[:token]
+    if @bearer_token.nil? || @bearer_token.empty?
+      raise Error, "The authorization call returned #{response.body} and no usable :token key could be found there"
+    end
+  end
+
+  def auth_headers
+    raise 'No bearer token retrieved yet' unless @bearer_token
+    {
+      'X-API-Key' => @api_key,
+      'Authorization' => ('Bearer %s' % @bearer_token),
+    }
+  end
+
+  def ensure_ok_status!(response)
+    case response.status
+    when 200..299
+      nil
+    when 400..499
+      @logger.error { response.body }
+      raise Error, "Response had a #{response.status} code, the server will not accept this request even if retried"
+    when 500..504
+      @logger.error { response.body }
+      raise Error, "Response had a #{response.status} code, we could retry"
+    else
+      @logger.error { response.body }
+      raise Error, "Response had a #{response.status} code, no idea what to do with that"
+    end
+  end
+end

data/lib/we_transfer_client/version.rb

@@ -0,0 +1,3 @@
+class WeTransferClient
+  VERSION = '0.2.0'
+end

data/spec/integration_spec.rb

@@ -0,0 +1,98 @@
+require 'tempfile'
+require 'bundler'
+Bundler.setup
+
+require 'dotenv'
+Dotenv.load
+
+require_relative '../lib/we_transfer_client.rb'
+
+describe WeTransferClient do
+  let :test_logger do
+    Logger.new($stderr).tap { |log| log.level = Logger::WARN }
+  end
+
+  let :very_large_file do
+    tf = Tempfile.new('test-upload')
+    20.times { tf << Random.new.bytes(1024 * 1024) }
+    tf << Random.new.bytes(rand(1..512))
+    tf.rewind
+    tf
+  end
+
+  it 'exposes VERSION' do
+    expect(WeTransferClient::VERSION).to be_kind_of(String)
+  end
+
+  it 'is able to create a transfer start to finish, both with small and large files' do
+    client = WeTransferClient.new(api_key: ENV.fetch('WT_API_KEY'), logger: test_logger)
+    transfer = client.create_transfer(title: 'My amazing board', message: 'Hi there!') do |builder|
+      # Upload ourselves
+      add_result = builder.add_file(name: File.basename(__FILE__), io: File.open(__FILE__, 'rb'))
+      expect(add_result).to eq(true)
+
+      # Upload ourselves again, but using add_file_at
+      add_result = builder.add_file_at(path: __FILE__) # Upload ourselves again, but this time via path
+      expect(add_result).to eq(true)
+
+      # Upload the large file
+      add_result = builder.add_file(name: 'large.bin', io: very_large_file)
+      expect(add_result).to eq(true)
+
+      expect(add_result).to eq(true)
+    end
+
+    expect(transfer).to be_kind_of(WeTransferClient::RemoteTransfer)
+    expect(transfer.id).to be_kind_of(String)
+
+    # expect(transfer.version_identifier).to be_kind_of(String)
+    expect(transfer.state).to be_kind_of(String)
+    expect(transfer.name).to eq('My amazing board')
+    expect(transfer.description).to eq('Hi there!')
+    expect(transfer.items).to be_kind_of(Array)
+    expect(transfer.items.length).to eq(3)
+
+    item = transfer.items.first
+    expect(item).to be_kind_of(WeTransferClient::RemoteItem)
+
+    expect(transfer.shortened_url).to be_kind_of(String)
+    response = Faraday.get(transfer.shortened_url)
+    expect(response.status).to eq(302)
+    expect(response['location']).to start_with('https://wetransfer')
+  end
+
+  it 'refuses to create a transfer with no items' do
+    client = WeTransferClient.new(api_key: ENV.fetch('WT_API_KEY'), logger: test_logger)
+    expect {
+      client.create_transfer(title: 'My amazing board', message: 'Hi there!') do |builder|
+        # ...do nothing
+      end
+    }.to raise_error(/no items/)
+  end
+
+  it 'refuses to create a transfer when reading an IO raises an error' do
+    broken = StringIO.new('hello')
+    def broken.read(*)
+      raise 'This failed somehow'
+    end
+
+    client = WeTransferClient.new(api_key: ENV.fetch('WT_API_KEY'), logger: test_logger)
+    expect(client).not_to receive(:faraday) # Since we will not be doing any requests - we fail earlier
+    expect {
+      client.create_transfer(title: 'My amazing board', message: 'Hi there!') do |builder|
+        builder.add_file(name: 'broken', io: broken)
+      end
+    }.to raise_error(/failed somehow/)
+  end
+
+  it 'refuses to create a transfer when given an IO of 0 size' do
+    broken = StringIO.new('')
+
+    client = WeTransferClient.new(api_key: ENV.fetch('WT_API_KEY'), logger: test_logger)
+    expect {
+      client.create_transfer(title: 'My amazing board', message: 'Hi there!') do |builder|
+        builder.add_file(name: 'broken', io: broken)
+      end
+    }.to raise_error(/has a size of 0/)
+  end
+end

data/wetransfer.gemspec

@@ -1,11 +1,11 @@
 
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'wetransfer/version'
+require 'we_transfer_client/version'
 
 Gem::Specification.new do |spec|
   spec.name          = 'wetransfer'
-  spec.version       = WeTransfer::VERSION
+  spec.version       = WeTransferClient::VERSION
   spec.authors       = ['Noah Berman', 'David Bosveld']
   spec.email         = ['noah@wetransfer.com', 'david@wetransfer.com', 'developers@wetransfer.com']
 
@@ -23,18 +23,15 @@ Gem::Specification.new do |spec|
       'public gem pushes.'
   end
 
-  spec.files = `git ls-files -z`.split("\x0").reject do |f|
-    # Make sure large fixture files are not packaged with the gem every time
-    f.match(%r{^spec/fixtures/})
-  end
+  spec.files = `git ls-files -z`.split("\x0")
   spec.bindir        = 'exe'
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_dependency 'faraday', '~> 0.13'
-  spec.add_dependency 'dotenv', '~> 2.2'
+  spec.add_dependency 'faraday', '~> 0.15'
   spec.add_dependency 'ks', '~> 0.0.1'
 
+  spec.add_development_dependency 'dotenv', '~> 2.2'
   spec.add_development_dependency 'bundler', '~> 1.16'
   spec.add_development_dependency 'pry', '~> 0.11'
   spec.add_development_dependency 'rake', '~> 10.0'