zipline 0.0.7

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 3dced1a62ccd13637dce8896c6809922661aa25d
+  data.tar.gz: 32b3a8cb4809dedd83248501f8ac2eb973482a39
+SHA512:
+  metadata.gz: 89bf143bc7f27eddf9cba7064d2b43a248b594a35363643796ece7a51bdf96dcc1a61fa2f5fcd946f3b505b5afb2b1be8c5e189d107ddbfa44251f7b355bd414
+  data.tar.gz: 032abbeebc338a1959f95d35a3fd21e43d1a06190445bf2861387b53e5b2916f4b56d0157b299b9670f432f47e5983d6c28ed4655d0bdc19ba0e7f4cc3dd269e

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in zipline.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Ram Dobson
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,53 @@
+# Zipline
+
+A gem to stream dynamically generated zip files from a rails application. Unlike other solutions that generate zips for user download, zipline does not wait for the entire zip file to be created (or even for the entire input file in the cloud to be downloaded) before it begins sending the zip file to the user. It does this by never seeking backwards during zip creation, and streaming the zip file over http as it is constructed. The advantages of this are:
+
+- Removes need for large disk space or memory allocation to generate zips, even huge zips. So it works on Heroku.
+- The user begins downloading immediately, which decreaceses latency, download time, and timeouts on Heroku.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'zipline'
+
+And then execute:
+
+    $ bundle
+
+## Usage
+
+set up some models with [carrierwave](https://github.com/jnicklas/carrierwave) or [paperclip](https://github.com/thoughtbot/paperclip).
+ Right now only plain file storage and S3 are supported in the case of [carrierwave](https://github.com/jnicklas/carrierwave) and only plain file storage, not S3 in the case of [paperclip](https://github.com/thoughtbot/paperclip). Alternatively, you can pass in plain old File objects. Not to be judgy, but plain old file objects are probably not the best option most of the time.
+
+You'll need to be using [unicorn](http://unicorn.bogomips.org/) or rainbows or some other server that supports streaming output.
+
+    class MyController < ApplicationController
+      # enable streaming responses
+      include ActionController::Streaming
+      # enable zipline
+      include Zipline
+      
+      def index
+        users= User.all
+        files =  users.map{ |user| [user.avatar, "#{user.username}.png"] }
+        zipline( files, 'avatars.zip')
+      end
+    end
+
+For directories, just give the files names like "directory/file".
+    
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+## TODO (possible contributions?)
+
+* tests!
+* support rails 4.0 streaming
+* extract library for plain ruby streaming zips, which this will depend on.
+* get my changes to support streaming zips checked in to the rubyzip library.

data/Rakefile

@@ -0,0 +1,2 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"

data/lib/zipline.rb

@@ -0,0 +1,28 @@
+require "zipline/version"
+
+require 'zip'
+require 'curb'
+
+require "zipline/fake_stream"
+require "zipline/output_stream"
+require "zipline/zip_generator"
+
+# class MyController < ApplicationController
+#   include Zipline
+#   def index
+#     users= User.all
+#     files =  users.map{ |user| [user.avatar, "#{user.username}.png"] }
+#     zipline( files, 'avatars.zip')
+#   end
+# end
+module Zipline
+  def zipline(files, zipname = 'zipline.zip')
+    zip_generator = ZipGenerator.new(files)
+    headers['Content-Disposition'] = "attachment; filename=#{zipname}"
+    headers['Content-Type'] = Mime::Type.lookup_by_extension('zip').to_s
+    response.sending_file = true
+    response.cache_control[:public] ||= false
+    self.response_body = zip_generator
+    self.response.headers['Last-Modified'] = Time.now.to_s
+  end
+end

data/lib/zipline/fake_stream.rb

@@ -0,0 +1,43 @@
+#this is a class that acts like an IO::Stream, but really puts to the browser
+module Zipline
+  class FakeStream
+
+    # &block is the block that each gets from rails... we pass it strings to send data
+    def initialize(&block)
+      @block = block
+      @pos = 0
+    end
+
+    def tell
+      @pos
+    end
+
+    def pos
+      @pos
+    end
+
+    def seek
+      throw :fit
+    end
+
+    def pos=
+      throw :fit
+    end
+
+    def to_s
+      throw :fit
+    end
+
+    def <<(x)
+      return if x.nil?
+      throw "bad class #{x.class}" unless x.class == String
+      @pos += x.bytesize
+      @block.call(x.to_s)
+    end
+
+    def close
+      nil
+    end
+
+  end
+end

data/lib/zipline/output_stream.rb

@@ -0,0 +1,51 @@
+# a ZipOutputStream that never rewinds output
+# in order for that to be possible we store only uncompressed files
+module Zipline
+  class OutputStream < Zip::OutputStream
+
+    #we need to be able to hand out own custom output in order to stream to browser
+    def initialize(io)
+      # Create an io stream thing
+      super StringIO.new, true
+      # Overwrite it with my own
+      @output_stream = io
+    end
+
+    def stream
+      @output_stream
+    end
+
+    def put_next_entry(entry_name, size)
+      new_entry = Zip::Entry.new(@file_name, entry_name)
+      new_entry.size = size
+
+      #THIS IS THE MAGIC, tells zip to look after data for size, crc
+      new_entry.gp_flags = new_entry.gp_flags | 0x0008
+
+      super(new_entry)
+    end
+
+    # just reset state, no rewinding required
+    def finalize_current_entry
+      if current_entry
+        entry = current_entry
+        super
+        write_local_footer(entry)
+      end
+    end
+
+    def write_local_footer(entry)
+      @output_stream << [ 0x08074b50, entry.crc, entry.compressed_size, entry.size].pack('VVVV')
+    end
+
+    #never need to do this because we set correct sizes up front
+    def update_local_headers
+      nil
+    end
+
+    # helper to deal with difference between rubyzip 1.0 and 1.1
+    def current_entry
+      @currentEntry || @current_entry
+    end
+  end
+end

data/lib/zipline/version.rb

@@ -0,0 +1,3 @@
+module Zipline
+  VERSION = "0.0.7"
+end

data/lib/zipline/zip_generator.rb

@@ -0,0 +1,113 @@
+# this class acts as a streaming body for rails
+# initialize it with an array of the files you want to zip
+# right now only carrierwave is supported with file storage or S3
+module Zipline
+  class ZipGenerator
+    # takes an array of pairs [[uploader, filename], ... ]
+    def initialize(files)
+      @files = files
+    end
+
+    #this is supposed to be streamed!
+    def to_s
+      throw "stop!"
+    end
+
+    def each(&block)
+      output = new_output(&block)
+      OutputStream.open(output) do |zip|
+        @files.each do |file, name|
+	  file = file.file if file.respond_to? :file
+
+          #normalize file
+          if file.class.to_s == 'CarrierWave::Storage::Fog::File'
+            file = file.send(:file)
+          end
+          if file.class.to_s == 'CarrierWave::SanitizedFile'
+            path = file.send(:file)
+            file = File.open(path)
+          end
+	  if file.class.to_s == 'Paperclip::Attachment'
+            path = file.send(:path)
+            file = File.open(path)
+	  end
+          throw "bad_file" unless %w{Fog::Storage::AWS::File File}.include? file.class.to_s
+
+          name = uniquify_name(name)
+          write_file(zip, file, name)
+        end
+      end
+    end
+
+    def new_output(&block)
+      FakeStream.new(&block)
+    end
+
+    def write_file(zip, file, name)
+      size = get_size(file)
+
+      zip.put_next_entry name, size
+
+      if file.is_a? File
+        while buffer = file.read(2048)
+          zip << buffer
+        end
+      else
+        the_remote_url = file.url(Time.now + 1.minutes)
+        c = Curl::Easy.new(the_remote_url) do |curl|
+          curl.on_body do |data|
+            zip << data
+            data.bytesize
+          end
+        end
+        c.perform
+      end
+    end
+
+    def get_size(file)
+      case file.class.to_s
+      when 'File'
+        file.size
+      when 'Fog::Storage::AWS::FILE'
+        file.content_length
+      end
+    end
+
+    def uniquify_name(name)
+      @used_names ||= Set.new
+
+
+      if @used_names.include?(name)
+
+        #remove suffix e.g. ".foo"
+        parts = name.split '.'
+        name, extension =
+          if parts.length == 1
+            #no suffix, e.g. README
+            parts << ''
+          else
+            extension = parts.pop
+            [parts.join('.'), ".#{extension}"]
+          end
+
+        #trailing _#{number}
+        pattern = /_(\d+)$/
+
+        unless name.match pattern
+          name = "#{name}_1"
+        end
+
+        while @used_names.include? name + extension
+          #increment trailing number
+          name = name.sub( pattern ) { |x| "_#{$1.to_i + 1}" }
+        end
+
+        #reattach suffix
+        name += extension
+      end
+
+      @used_names << name
+      name
+    end
+  end
+end

data/zipline.gemspec

@@ -0,0 +1,21 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/zipline/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Ram Dobson"]
+  gem.email         = ["ram.dobson@solsystemscompany.com"]
+  gem.description   = %q{this is a giant pile of hax that may let you stream dynamically generated zip files}
+  gem.summary       = %q{stream zip files from rails}
+  gem.homepage      = "http://github.com/fringd/zipline"
+
+  gem.files         = `git ls-files`.split($\) - %w{.gitignore}
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "zipline"
+  gem.require_paths = ["lib"]
+  gem.version       = Zipline::VERSION
+
+  gem.add_dependency 'rubyzip', ['>= 1.0', '<= 1.1.2']
+  gem.add_dependency 'rails', '>= 3.2.1'
+  gem.add_dependency 'curb'
+end

metadata

@@ -0,0 +1,102 @@
+--- !ruby/object:Gem::Specification
+name: zipline
+version: !ruby/object:Gem::Version
+  version: 0.0.7
+platform: ruby
+authors:
+- Ram Dobson
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-03-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - <=
+      - !ruby/object:Gem::Version
+        version: 1.1.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - <=
+      - !ruby/object:Gem::Version
+        version: 1.1.2
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 3.2.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 3.2.1
+- !ruby/object:Gem::Dependency
+  name: curb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: this is a giant pile of hax that may let you stream dynamically generated
+  zip files
+email:
+- ram.dobson@solsystemscompany.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- lib/zipline.rb
+- lib/zipline/fake_stream.rb
+- lib/zipline/output_stream.rb
+- lib/zipline/version.rb
+- lib/zipline/zip_generator.rb
+- zipline.gemspec
+homepage: http://github.com/fringd/zipline
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.0.rc.2
+signing_key: 
+specification_version: 4
+summary: stream zip files from rails
+test_files: []