verm-client 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 3a616e96483beae73fd10062005d641d30fb5352
+  data.tar.gz: 6b9fffb843d7d12a75c7cdb3658b193a740e5f32
+SHA512:
+  metadata.gz: 0aa23231f3b808815b44a89bf9cd81adf3e919affded5249c58ed6486aa41c4891f6d02427672c87b90e760047aed62b5033ec2dd8f1c9efe922c27d15997793
+  data.tar.gz: 9a333cbd52d2fc22b4e9389f0bb49f44592ffc26388bff151cb0795c9b54da62c89fd760f4556dbacb83ab57b0a3fedfdf974f1082bf75ffffe9bbd4ecf2ea4a

data/.gitignore

@@ -0,0 +1,34 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalisation:
+/.bundle/
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2015 Will Bryant, Sekuda Ltd
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

data/README.md

@@ -0,0 +1,115 @@
+Verm client library for Ruby
+============================
+
+Verm (https://github.com/willbryant/verm) is a WORM (write-once, read-many) file store to make it easy to reliably store and replicate files.  Clients talk to Verm over simple HTTP POST and GET calls.
+
+This tiny client library adds one-line convenience methods for storing files in Verm and retrieving/streaming them back again.
+
+
+Saving content
+--------------
+
+```ruby
+mydata = "data to store"
+VERM_CLIENT = Verm::Client.new("my-verm-server")
+VERM_CLIENT.store("/important_gifs/january", mydata, "text/plain") # => String
+```
+
+The `store` call will return a path like `/important_gifs/january/ab/cdefgh.txt`, only longer (Verm makes the filename using an URL-safe base64 encoding of the SHA-256 of the content).
+
+
+Saving files
+------------
+
+You don't need to read the whole file into memory before calling store, just pass the IO object in:
+
+```ruby
+VERM_CLIENT = Verm::Client.new("my-verm-server")
+
+File.open("example.jpg", "rb") do |f|
+  VERM_CLIENT.store("/important_gifs/january", f, "image/jpeg") # => String
+end
+```
+
+
+Retrieving content
+------------------
+
+```ruby
+VERM_CLIENT = Verm::Client.new("my-verm-server")
+
+file, content_type = VERM_CLIENT.load("/important_gifs/january/ab/cdefgh.jpg")
+```
+
+This basically does the same thing as the built-in Net::HTTP get method (aside from timeouts, error handling, etc.):
+
+```ruby
+Net::HTTP.get('my-verm-server:3404', '/important_gifs/january/ab/cdefgh.jpg') # => String - the file content
+```
+
+So you really don't need a client library for this - but it's usually more convenient to configure the Verm server somewhere central, instead of sprinkling it through your application.
+
+Verm stores all content without touching its character encoding, but Ruby 1.9+ treats all strings as having a specific character encoding.  Most applications use UTF-8 for all text, so by default, the `load` method will set the character encoding of `text/*` content returned by Verm to `UTF-8`.  You can override this or disable it entirely:
+
+```ruby
+file, content_type = VERM_CLIENT.load("/important_gifs/january/ab/cdefgh.csv", force_encoding: 'ISO-8859-1')
+file, content_type = VERM_CLIENT.load("/important_gifs/january/ab/cdefgh.csv", force_encoding: nil)
+```
+
+
+Retrieving large content
+------------------------
+
+If the data is large, reading it all into memory as a string is inefficient.  Instead you can use `stream`, which will yield the file contents in chunks:
+
+```ruby
+VERM_CLIENT = Verm::Client.new("my-verm-server")
+
+File.open("out.csv", "w") do |f|
+  VERM_CLIENT.stream("/big_data/ab/cdefgh.csv") do |chunk|
+    f.write chunk
+  end
+end
+```
+
+Because the chunks may not be broken on a character encoding boundary, the character encoding can't be set as for `load`, so they will still have the default `ASCII-8BIT` binary encoding.
+
+
+File compression
+----------------
+
+Like HTTP and web browsers, Verm treats gzip compression as being a way of making transfers smaller and faster, not a separate file type.  So if you want to store gzipped files, you still get to say what content-type the actual file content has.
+
+Saving files that are already gzip-compressed just requires one extra argument:
+
+```ruby
+File.open("acme_20150101.csv.gz", "rb") do |f|
+  VERM_CLIENT.store("/third_party_files/acme/2015", f, "text/csv", encoding: "gzip")
+end
+```
+
+It's important to note that Ruby's Net::HTTP client will automatically uncompress gzip transfer-encoding on responses - so when you use `load` or `store`, you'll get the uncompressed content back, not the gzip file.
+
+You can suppress that behavior by passing `Accept-Encoding: gzip` as a header (Net::HTTP would accept gzip anyway, but setting this header has the side effect of disabling uncompression):
+
+```ruby
+file, content_type = VERM_CLIENT.load("/third_party_files/acme/2015/ab/cdefgh.csv", 'Accept-Encoding' => 'gzip')
+```
+
+
+Network stuff
+-------------
+
+Verm runs its HTTP server on port 3404 by default.  You can override this in the client configuration:
+
+```ruby
+VERM_CLIENT = Verm::Client.new("my-verm-server", port: 80)
+```
+
+Verm::Client defaults to a 15s timeout, which you can override:
+
+```ruby
+VERM_CLIENT = Verm::Client.new("my-verm-server", timeout: 60)
+```
+
+Like Net::HTTP, this timeout applies to individual network operations, not the whole method.

data/Rakefile

@@ -0,0 +1,8 @@
+require "bundler/gem_tasks"
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.pattern = "test/*_test.rb"
+end
+
+task :default => :test

data/lib/verm.rb

@@ -0,0 +1 @@
+require 'verm/client'

data/lib/verm/client.rb

@@ -0,0 +1,59 @@
+require 'net/http'
+
+module Verm
+  class Client
+    attr_reader :http_client
+
+    def initialize(hostname, port: 3404, timeout: 15)
+      @http_client = Net::HTTP.new(hostname, port)
+      @http_client.open_timeout = timeout
+      @http_client.read_timeout = timeout
+      @http_client.ssl_timeout = timeout
+    end
+
+    def store(directory, io_or_data, content_type, encoding: nil)
+      directory = "/#{directory}" unless directory[0] == '/'
+      request = Net::HTTP::Post.new(directory, 'Content-Type' => content_type)
+      request['Content-Encoding'] = encoding.to_s if encoding
+
+      if io_or_data.respond_to?(:read)
+        request.body_stream = io_or_data
+        if io_or_data.respond_to?(:size)
+          request['Content-Length'] = io_or_data.size
+        else
+          request['Transfer-Encoding'] = 'chunked'
+        end
+
+        io_or_data.rewind if io_or_data.respond_to?(:rewind)
+        response = http_client.request(request)
+      else
+        response = http_client.request(request, io_or_data)
+      end
+
+      response.error! unless response.is_a?(Net::HTTPSuccess)
+      raise "Got a HTTP #{response.code} when trying to store content - should always be 201" unless response.code.to_i == 201
+      raise "No location was returned when trying to store content - should always be given" unless response['location']
+      response['location']
+    end
+
+    def load(path, initheader = {}, force_text_encoding: "UTF-8")
+      response = http_client.request_get(path, initheader)
+      response.error! unless response.is_a?(Net::HTTPSuccess)
+
+      if force_text_encoding && response.content_type =~ /text\//
+        [response.body.force_encoding(force_text_encoding), response.content_type]
+      else
+        [response.body, response.content_type]
+      end
+    end
+
+    def stream(path, initheader = {})
+      http_client.request_get(path, initheader) do |response|
+        response.error! unless response.is_a?(Net::HTTPSuccess)
+        response.read_body do |chunk|
+          yield chunk
+        end
+      end
+    end
+  end
+end

data/lib/verm/version.rb

@@ -0,0 +1,3 @@
+module Verm
+  VERSION = '1.0.0'
+end

data/test/verm_test.rb

@@ -0,0 +1,109 @@
+require 'rubygems'
+require 'minitest/autorun'
+require 'verm/client'
+require 'byebug' rescue nil
+
+class TestVerm < Minitest::Test
+  def setup
+    @client = Verm::Client.new(ENV["VERM_TEST_HOSTNAME"] || "localhost")
+  end
+
+  def fixture_file_path(filename)
+    File.join(File.dirname(__FILE__), 'fixtures', filename)
+  end
+
+  def test_stores_content_and_returns_location
+    assert_equal "/test/files/F0/ZdYVIlyqOiCKtR_oQF_9y8G8_9qAWhR9Fw5hzK8UM.txt",
+      @client.store("/test/files", "this is a test", "text/plain")
+
+    assert_equal "/test/files/F0/ZdYVIlyqOiCKtR_oQF_9y8G8_9qAWhR9Fw5hzK8UM.txt",
+      @client.store("/test/files/", "this is a test", "text/plain")
+
+    assert_equal "/test/files/F0/ZdYVIlyqOiCKtR_oQF_9y8G8_9qAWhR9Fw5hzK8UM",
+      @client.store("/test/files", "this is a test", "application/octet-stream")
+
+    assert_equal "/test/files/SL/VHO4KGrhBRAAjinYUEdqilkOrS4JD6akWSeJb5Tra.txt",
+      @client.store("/test/files", "this is a test\n", "text/plain")
+  end
+
+  def test_stores_files_and_returns_location
+    File.open fixture_file_path("binary_file") do |f|
+      assert_equal "/bins/IF/P8unS2JIuR6_UZI5pZ0lxWHhfvR2ocOcRAma_lEiA",
+        @client.store("/bins/", f, "application/octet-stream")
+
+      assert_equal "/bins/IF/P8unS2JIuR6_UZI5pZ0lxWHhfvR2ocOcRAma_lEiA.png",
+        @client.store("/bins/", f, "image/png")
+    end
+  end
+
+  def test_stores_compressed_files_and_returns_location_from_uncompressed_content
+    File.open fixture_file_path("binary_file.gz") do |f|
+      # note the hashes are the same here as the uncompressed file above
+      assert_equal "/bins/IF/P8unS2JIuR6_UZI5pZ0lxWHhfvR2ocOcRAma_lEiA",
+        @client.store("/bins/", f, "application/octet-stream", encoding: 'gzip')
+
+      assert_equal "/bins/IF/P8unS2JIuR6_UZI5pZ0lxWHhfvR2ocOcRAma_lEiA.png",
+        @client.store("/bins/", f, "image/png", encoding: 'gzip')
+    end
+
+    File.open fixture_file_path("large_compressed_csv_file.gz") do |f|
+      assert_equal "/csvs/VP/YY9KXv6cc6XksLTHOQJywCHnzyQDCd7Xh7HHUO2hd.csv",
+        @client.store("/csvs/", f, "text/csv", encoding: 'gzip')
+    end
+  end
+
+  def test_loads_content_and_type
+    content, type = "this is a test", "text/plain"
+    assert_equal [content, type],
+      @client.load(@client.store("/test/files_to_load", content, type))
+
+    content, type = File.binread(fixture_file_path("binary_file")), "application/octet-stream"
+    assert_equal [content, type],
+      @client.load(@client.store("/test/files_to_load", content, type))
+  end
+
+  def test_loads_and_uncompresses_compressed_content_and_type
+    compressed_content, type = File.binread(fixture_file_path("binary_file.gz")), "application/octet-stream"
+    uncompressed_content = File.binread(fixture_file_path("binary_file"))
+
+    assert_equal [uncompressed_content, type],
+      @client.load(@client.store("/test/compressed", compressed_content, type, encoding: 'gzip'))
+  end
+
+  def test_optionally_doesnt_uncompress_compressed_content
+    compressed_content, type = File.binread(fixture_file_path("binary_file.gz")), "application/octet-stream"
+
+    assert_equal [compressed_content, type],
+      @client.load(@client.store("/test/compressed", compressed_content, type, encoding: 'gzip'), 'Accept-Encoding' => 'gzip')
+  end
+
+  def test_marks_loaded_text_as_utf_8_by_default
+    compressed_content, type = File.binread(fixture_file_path("large_compressed_csv_file.gz")), "text/csv"
+    uncompressed_content = Zlib::GzipReader.new(StringIO.new(compressed_content)).read
+
+    assert_equal [uncompressed_content, type],
+      @client.load(@client.store("/test/compressed", compressed_content, type, encoding: 'gzip'))
+  end
+
+  def test_optionally_doesnt_mark_loaded_text_as_anything
+    compressed_content, type = File.binread(fixture_file_path("large_compressed_csv_file.gz")), "text/csv"
+    uncompressed_content = Zlib::GzipReader.new(StringIO.new(compressed_content)).read.force_encoding("ASCII-8BIT")
+
+    assert_equal [uncompressed_content, type],
+      @client.load(@client.store("/test/compressed", compressed_content, type, encoding: 'gzip'), force_text_encoding: nil)
+  end
+
+  def test_streams_content
+    compressed_content, type = File.binread(fixture_file_path("large_compressed_csv_file.gz")), "application/octet-stream"
+    uncompressed_content = Zlib::GzipReader.new(StringIO.new(compressed_content)).read.force_encoding("ASCII-8BIT")
+
+    result = ""
+    chunks = 0
+    @client.stream(@client.store("/test/compressed", uncompressed_content, type)) do |chunk|
+      result << chunk
+      chunks += 1
+    end
+    assert_equal uncompressed_content, result
+    assert chunks > 1, "should have been given the content in multiple chunks"
+  end
+end

data/verm.gemspec

@@ -0,0 +1,21 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/verm/version', __FILE__)
+
+spec = Gem::Specification.new do |gem|
+  gem.name         = 'verm-client'
+  gem.version      = Verm::VERSION
+  gem.license      = "MIT"
+  gem.summary      = "Tiny Ruby HTTP client for the Verm immutable, WORM filestore."
+  gem.description  = <<-EOF
+Adds one-line methods for storing files in Verm and retrieving them again.
+EOF
+  gem.has_rdoc     = false
+  gem.author       = "Will Bryant"
+  gem.email        = "will.bryant@gmail.com"
+  gem.homepage     = "http://github.com/willbryant/verm-client-ruby"
+
+  gem.executables  = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.files        = `git ls-files`.split("\n")
+  gem.test_files   = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.require_path = "lib"
+end

metadata

@@ -0,0 +1,60 @@
+--- !ruby/object:Gem::Specification
+name: verm-client
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Will Bryant
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-01-11 00:00:00.000000000 Z
+dependencies: []
+description: |
+  Adds one-line methods for storing files in Verm and retrieving them again.
+email: will.bryant@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- MIT-LICENSE
+- README.md
+- Rakefile
+- lib/verm.rb
+- lib/verm/client.rb
+- lib/verm/version.rb
+- test/fixtures/binary_file
+- test/fixtures/binary_file.gz
+- test/fixtures/large_compressed_csv_file.gz
+- test/verm_test.rb
+- verm.gemspec
+homepage: http://github.com/willbryant/verm-client-ruby
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Tiny Ruby HTTP client for the Verm immutable, WORM filestore.
+test_files:
+- test/fixtures/binary_file
+- test/fixtures/binary_file.gz
+- test/fixtures/large_compressed_csv_file.gz
+- test/verm_test.rb