em-zipper 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 64af8eac1c0ad6532476956a5b9852c8cc6d8acf
-  data.tar.gz: 16ef90ab41a5ee6416f5ac4b0979aa4b6ef71f21
+  metadata.gz: 9cb4c9228359aa7f2d20d70f6ba45d8065fe3a7d
+  data.tar.gz: b2b4bdbbe95c3b5f97864e3cdf63999c087c1543
 SHA512:
-  metadata.gz: ece25c89c5c737918b9bc3a43c0aada19034166523091441d3c52e1c5ebde846ba35d9e21ffeefe1f3ad0b18c07cb85f80a1d95301cf9ff7946f415851ce36e2
-  data.tar.gz: 08fa21a4d870487480b779d616fdcd0cf265e27d27aaa526d0fc827466144110a169d7871558c6d8b0141deaf0b35f13e88b9060d7dd0ac4504eb55392daafa2
+  metadata.gz: 2f853b18fc71e141fabacba3f2134c8af4e2a1ddb20c0801291f065008930aaeb9f8eb73845b4242923803e5475c016e1cb1eb826cb553cbfa9bc6d7dd523ec6
+  data.tar.gz: d2b930387a4d820a79f8f7fcc67a23150722fef8b280680354a99a0605fa17792d1a4ff989387212eed6ac65aea2aa55f4b399ea079efa4f0c8609a430c73a88

data/benchmarkz/em-zipper-bm.rb

@@ -0,0 +1,105 @@
+$: << File.expand_path('../lib', __FILE__)
+require 'rubygems'
+require "benchmark"
+require 'em-zipper'
+require 'net/http'
+
+
+sites = %w(http://s3.amazonaws.com/ping-em-assets/sample1.txt http://s3.amazonaws.com/ping-em-assets/sample2.txt)
+files = %w(sample1.txt sample2.txt).map do |file_name|
+  File.join(File.expand_path("./examples/files"), file_name)
+end
+
+n     = 10
+@count = 0
+
+def em_zipper(collection, n)
+  zip  = EM::Zipper.new(collection, StringIO.new).zip!
+  zip.callback do |io|
+    io.close
+    @count = @count + 1
+    EM.stop if @count == n
+  end
+end
+
+def rubyzip(collection, n)
+  ::Zip::ZipOutputStream.write_buffer do |zip|
+    collection.each do |file_path|
+      zip.put_next_entry(File.basename(file_path))
+
+      begin
+        if file_path.match(/\Ahttps?:\/\//)
+          uri = URI file_path
+          Net::HTTP.start(uri.host, uri.port) do |http|
+            request = Net::HTTP::Get.new uri
+
+            http.request request do |response|
+              response.read_body { |chunk| zip << chunk }
+            end
+          end
+        else
+          file = File.open file_path
+          while chunk = file.read_nonblock(16384)
+            zip << chunk
+          end
+        end
+      rescue EOFError
+        # /
+      ensure
+        file && file.close
+      end
+    end
+  end
+end
+
+
+# //
+
+
+marks = Benchmark.benchmark do |b|
+  b.report("[file] em-zipper") do
+    @count = 0
+
+    EM.run do
+      n.times do
+        EM.next_tick { em_zipper(files, n) }
+      end
+    end
+  end
+
+  b.report('[file] rubyzip') do
+    n.times do
+      rubyzip(files, n)
+    end
+  end
+
+  b.report("[net] em-zipper") do
+    @count = 0
+
+    EM.run do
+      n.times do
+        EM.next_tick { em_zipper(sites, n) }
+      end
+    end
+  end
+
+  b.report('[net] rubyzip') do
+    n.times do
+      rubyzip(sites, n)
+    end
+  end
+end
+
+puts marks
+
+# //
+#
+# [file] em-zipper  2.780000   0.040000   2.820000 (  2.829317)
+# [file] rubyzip    2.690000   0.030000   2.720000 (  2.740578)
+# [net]  em-zipper  3.060000   0.150000   3.210000 (  5.076801)
+# [net]  rubyzip    3.020000   0.090000   3.110000 (  6.962918)
+#   2.780000   0.040000   2.820000 (  2.829317)
+#   2.690000   0.030000   2.720000 (  2.740578)
+#   3.060000   0.150000   3.210000 (  5.076801)
+#   3.020000   0.090000   3.110000 (  6.962918)
+

data/benchmarkz/em-zipper.rb

@@ -13,7 +13,13 @@ end
 
 Tach.meter(10) do 
   tach('[file] em-zipper') do |n|
-    em_zipper files, n
+    @count = 0
+
+    EM.run do
+      n.times do |i|
+        EM.next_tick { em_zipper files, n }
+      end
+    end
   end
 
   tach('[file] rubyzip') do |n|
@@ -21,7 +27,13 @@ Tach.meter(10) do
   end
 
   tach('[net] em-zipper') do |n|
-    em_zipper sites, n
+    @count = 0
+
+    EM.run do
+      n.times do |i|
+        EM.next_tick { em_zipper files, n }
+      end
+    end
   end
 
   tach('[net] rubyzip') do |n|
@@ -30,21 +42,11 @@ Tach.meter(10) do
 
 
   def em_zipper(collection, n)
-    EventMachine.run do
-      count = 0
-
-      em_zipper = proc do
-        zip = EM::Zipper.new(collection, StringIO.new).zip!
-        zip.callback do |io|
-          io.close
-          count = count + 1
-          EM.stop if count == n
-        end
-      end
-
-      n.times do |i|
-        EM.next_tick(&em_zipper)
-      end
+    zip = EM::Zipper.new(collection, StringIO.new).zip!
+    zip.callback do |io|
+      io.close
+      @count = @count + 1
+      EM.stop if @count == n
     end
   end
 
@@ -85,27 +87,27 @@ end
 # File sizes 1.5mb, 503kb
 #
 # x10
-# +------------------+----------+
-# | tach             | total    |
-# +------------------+----------+
-# | [file] em-zipper | 2.849514 |
-# +------------------+----------+
-# | [net] em-zipper  | 3.646211 |
-# +------------------+----------+
-# | [net] rubyzip    | 5.765788 |
-# +------------------+----------+
-# | [file] rubyzip   | 9.334782 |
-# +------------------+----------+
+# +------------------+-----------+
+# | tach             | total     |
+# +------------------+-----------+
+# | [net] em-zipper  | 2.727529  |
+# +------------------+-----------+
+# | [file] em-zipper | 2.753560  |
+# +------------------+-----------+
+# | [file] rubyzip   | 8.035955  |
+# +------------------+-----------+
+# | [net] rubyzip    | 10.780907 |
+# +------------------+-----------+
 #
 # x100
 # +------------------+-----------+
 # | tach             | total     |
 # +------------------+-----------+
-# | [file] em-zipper | 27.344414 |
+# | [file] em-zipper | 27.435623 |
 # +------------------+-----------+
-# | [net] em-zipper  | 48.246017 |
+# | [net] em-zipper  | 27.858313 |
 # +------------------+-----------+
-# | [net] rubyzip    | 60.066137 |
+# | [net] rubyzip    | 58.306401 |
 # +------------------+-----------+
-# | [file] rubyzip   | 66.197528 |
+# | [file] rubyzip   | 60.937767 |
 # +------------------+-----------+

data/lib/em-zipper.rb

@@ -1,6 +1,8 @@
 require 'eventmachine'
 
 require 'em-zipper/base'
+require 'em-zipper/entry'
+require 'em-zipper/chunks'
 require 'em-zipper/zip_output_stream'
 require 'em-zipper/stream_io'
 

data/lib/em-zipper/base.rb

@@ -11,11 +11,14 @@ module EventMachine
         @files  = files
         @zos    = zos
         @fqueue = EM::Queue.new
+        @count  = 0
       end
 
       def zip!
-        files.each { |file_path| fqueue.push file_path }
-        fqueue.pop(&next_entry_non_block)
+        files.each do |file_path|
+          EM.next_tick { chunk_data(file_path) }
+        end
+        fqueue.pop(&write_entry) # begin queue, first file to chunk will be issued this call
 
         self # ...new(..., ...).zip!
       end
@@ -23,65 +26,34 @@ module EventMachine
 
       private
 
-      def file_name(file_path)
-        File.basename file_path
-      end
-
       def close_zos
         zos.close if zos.io.is_a?(File) # close the io if it's a file
-        succeed zos.close_buffer # return the zos
+        succeed zos.close_buffer        # return the zos
       end
 
-      def next_entry_non_block
-        proc do |file_path|
-          cqueue = EM::Queue.new
-          chunky = proc do |chunk|
-            zos << chunk
+      def write_entry
+        proc do |args|
+          file_path = args[0]
+          q         = args[1]
+
+          entry = Entry.new(file_path, q, zos).write!
+          entry.callback do
+            @count = @count + 1
 
-            if cqueue.empty?
-              close_zos and return if fqueue.empty?
-              EM.next_tick { fqueue.pop(&next_entry_non_block) }
+            if files.size == @count
+              close_zos
             else
-              EM.next_tick { cqueue.pop(&chunky) }
+              EM.next_tick { fqueue.pop(&write_entry) }
             end
           end
-
-          defer = EM::DefaultDeferrable.new
-          defer.callback do
-            # begin the entry when we have all the data
-            new_entry = ::Zip::ZipEntry.new '-', file_name(file_path)
-            zos.put_next_entry(new_entry)
-
-            EM.next_tick { cqueue.pop(&chunky) } if cqueue.size > 0
-          end
-          defer.errback  { puts 'error' }
-
-          _chunk_method = file_path.match(/\Ahttps?:\/\//) ? :chunk_http : :chunk_file
-          __send__ _chunk_method, file_path, cqueue, defer
         end
       end
 
-      def chunk_file(file_path, q, d)
-        file = File.open file_path
-
-        fr = proc do
-          begin
-            q.push file.read_nonblock(16384)
-            EM.next_tick(&fr)
-          rescue EOFError
-            file.close
-            EM.next_tick { d.succeed }
-          end
+      def chunk_data(file_path)
+        chunks = Chunks.new(file_path).chop!
+        chunks.callback do |chunk_q|
+          EM.next_tick { fqueue.push [file_path, chunk_q] }
         end
-
-        EM.next_tick(&fr)
-      end
-
-      def chunk_http(uri, q, d)
-        http = EventMachine::HttpRequest.new(uri).get
-        http.errback  { EM.next_tick { d.fail } }
-        http.callback { EM.next_tick { d.succeed } }
-        http.stream   { |chunk| q.push chunk }
       end
     end
   end

data/lib/em-zipper/chunks.rb

@@ -0,0 +1,47 @@
+module EventMachine
+  module Zipper
+    class Chunks
+      include EventMachine::Deferrable
+
+      attr_reader :file_path, :q
+
+      def initialize(file_path)
+        @file_path = file_path
+        @q         = EM::Queue.new
+      end
+
+      def chop!
+        _chunk_method = file_path.match(/\Ahttps?:\/\//) ? :chunk_http : :chunk_file
+        __send__ _chunk_method
+
+        self
+      end
+
+
+      private
+
+      def chunk_file
+        file = File.open file_path
+
+        fr = proc do
+          begin
+            q.push file.read_nonblock(16384)
+            EM.next_tick(&fr)
+          rescue EOFError
+            file.close
+            succeed(q)
+          end
+        end
+
+        EM.next_tick(&fr)
+      end
+
+      def chunk_http
+        http = EventMachine::HttpRequest.new(file_path).get
+        http.errback  { fail }
+        http.callback { succeed(q) }
+        http.stream   { |chunk| q.push chunk }
+      end
+    end
+  end
+end

data/lib/em-zipper/entry.rb

@@ -0,0 +1,46 @@
+module EventMachine
+  module Zipper
+    class Entry
+      include EventMachine::Deferrable
+
+      attr_reader :file_path, :chunks, :out
+
+      def initialize(file_path, chunks, out)
+        @file_path = file_path
+        @chunks    = chunks
+        @out       = out
+      end
+
+      def write!
+        create_entry
+        EM.next_tick { chunks.pop(&write_chunks) }
+
+        self
+      end
+
+
+      private
+
+      def write_chunks
+        proc do |chunk|
+          out << chunk
+
+          if chunks.empty?
+            EM.next_tick { succeed }
+          else
+            EM.next_tick { chunks.pop(&write_chunks) }
+          end
+        end
+      end
+
+      def file_name
+        @file_namem ||= File.basename file_path
+      end
+
+      def create_entry
+        new_entry = ::Zip::ZipEntry.new '-', file_name
+        out.put_next_entry(new_entry)
+      end
+    end
+  end
+end

data/lib/em-zipper/version.rb

@@ -1,5 +1,5 @@
 module EventMachine
   module Zipper
-    VERSION = "0.0.3"
+    VERSION = "0.0.4"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: em-zipper
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Yung Hwa Kwon
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-16 00:00:00.000000000 Z
+date: 2013-10-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rubyzip
@@ -63,6 +63,7 @@ files:
 - .ruby-version
 - Gemfile
 - Gemfile.lock
+- benchmarkz/em-zipper-bm.rb
 - benchmarkz/em-zipper.rb
 - em-zipper.gemspec
 - examples/async_sinatra.md
@@ -72,6 +73,8 @@ files:
 - examples/io.rb
 - lib/em-zipper.rb
 - lib/em-zipper/base.rb
+- lib/em-zipper/chunks.rb
+- lib/em-zipper/entry.rb
 - lib/em-zipper/stream_io.rb
 - lib/em-zipper/version.rb
 - lib/em-zipper/zip_output_stream.rb