active_model_archive 0.0.3 → 0.0.4

data/lib/active_model_archive/version.rb

@@ -1,3 +1,3 @@
 module ActiveModelArchive
-  VERSION = "0.0.3"
+  VERSION = "0.0.4"
 end

data/lib/active_model_archive.rb

@@ -3,12 +3,71 @@ require 'active_support/concern'
 
 module ActiveModel
   module Archive
-    extend ActiveSupport::Concern
-
     MAGIC = 42098
     CURRENT_VERSION = 1
     SUPPORTED_VERSIONS = [1]
 
+    class FileManager
+      UNLIMITED_PER_FILE = 999999999
+      attr_accessor :item_count, :file_number, :base_filename, :items_per_file
+      def initialize(base_filename, items_per_file)
+        @item_count = 0
+        @file_number = 0
+        @base_filename = base_filename
+        @items_per_file = (items_per_file.presence || UNLIMITED_PER_FILE).to_i
+      end
+
+      def add(object)
+        Rails.logger.info "Adding #{object.id}"
+
+        if item_count % items_per_file == 0
+          close_current_file
+          @current_file = create_file
+        end
+
+        self.item_count += 1
+
+        encoder.encode(object.as_archive, nil) do |json|
+          # write size, record, and crc
+          write_int(@current_file, json.bytesize)
+          @current_file.write(json)
+          write_int(@current_file, Zlib::crc32(json))
+        end
+      end
+
+      def create_file
+        self.file_number += 1
+        
+        filename = items_per_file == UNLIMITED_PER_FILE ? base_filename : "#{base_filename}.#{file_number}"
+        file = File.open(filename, "w")
+        file.set_encoding("ASCII-8BIT")
+
+        write_int(file, MAGIC)
+        write_int(file, CURRENT_VERSION)
+
+        file
+      end
+
+      def close_current_file
+        if @current_file
+          write_int(@current_file, 0)
+          write_int(@current_file, item_count - ((file_number - 1) * items_per_file))
+          write_int(@current_file, MAGIC)
+
+          @current_file.close
+        end
+      end
+      
+      def write_int(file, int)
+        file.write([int].pack("N"))
+      end
+
+      def encoder
+        @encoder ||= Yajl::Encoder.new
+      end
+    end
+
+    extend ActiveSupport::Concern
     # dump file format:
     # version 1:
     #
@@ -38,57 +97,42 @@ module ActiveModel
       # uses paginated_each with query params
       # executes the block with each dumped record if given
       # the block can be used to log progress
-      def dump!(stream, query_params={}, &block)
-        # set the encoding so we can write packed ints
-        stream.set_encoding("ASCII-8BIT")
-
-        archive_write_int(stream, MAGIC)
-        archive_write_int(stream, CURRENT_VERSION)
+      def dump!(filename, options = {})
+        query = options[:query]
+        per_file = options[:per_file]
 
-        count = 0
-        encoder = Yajl::Encoder.new
+        file_manager = FileManager.new(filename, per_file)
 
-        writer = lambda do |json|
-          # write size, record, and crc
-          archive_write_int(stream, json.bytesize)
-          stream.write(json)
-          archive_write_int(stream, Zlib::crc32(json))
-          count += 1
-        end
-
-        if query_params.blank?
+        if query.blank?
           find_each do |object|
-            encoder.encode(object.as_archive, nil, &writer)
-            block.call(object) if block_given?
+            file_manager.add(object)
           end
         else
-          paginated_each(query_params) do |object|
-            encoder.encode(object.as_archive, nil, &writer)
-            block.call(object) if block_given?
+          paginated_each(query) do |object|
+            file_manager.add(object)
           end
         end
 
-        archive_write_int(stream, 0)
-        archive_write_int(stream, count)
-        archive_write_int(stream, MAGIC)
+        file_manager.close_current_file
+        file_manager.item_count
+      end
 
-        stream.flush
-        return count
+      def restore!(stream, options={})
+        each_instance do |object|
+          object.save(:validate => !!options[:validate])
+          yield object if block_given?
+        end
       end
 
-      # Restore records from a stream.
-      # require attributes= and save accepting :validate option
-      # set :check true to only check the archive and not restore. if no exceptions are thrown, archive is valid.
-      # set :validate true to validate records as they are saved
-      # executes the block with each restored record if given
-      # the block can be used to log progress and warn about invalid objects
-      def restore!(stream, options={}, &block)
+      def each_instance(stream)
+        each_archive(stream) do |hash|
+          yield from_archive(hash)
+        end
+      end
 
-        # for each hash read, create an object
+      def each_record(stream)
         each_archive(stream) do |hash|
-          o = from_archive(hash)
-          o.save(:validate => !!options[:validate])
-          yield o if block_given?
+          yield find(hash['id'])
         end
       end
 
@@ -121,7 +165,7 @@ module ActiveModel
         raise "Unexpected end of archive file" unless magic == MAGIC
         raise "Record counts didn't match: #{record_count} in stream, #{count} read" unless record_count == count
 
-        return count
+        count
       end
 
       # returns true if the archive is valid
@@ -147,10 +191,6 @@ module ActiveModel
       def archive_read_int(stream)
         stream.read(4).unpack("N").first
       end
-
-      def archive_write_int(stream, int)
-        stream.write([int].pack("N"))
-      end
     end
   
     # instance method to return archived hash

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: active_model_archive
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,12 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-03-25 00:00:00.000000000 -07:00
-default_executable: 
+date: 2011-07-27 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &21767140 !ruby/object:Gem::Requirement
+  requirement: &2160497260 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -22,10 +21,10 @@ dependencies:
         version: '3'
   type: :runtime
   prerelease: false
-  version_requirements: *21767140
+  version_requirements: *2160497260
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
-  requirement: &21766380 !ruby/object:Gem::Requirement
+  requirement: &2160496760 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -33,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *21766380
+  version_requirements: *2160496760
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &21765700 !ruby/object:Gem::Requirement
+  requirement: &2160496300 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -44,7 +43,7 @@ dependencies:
         version: '2'
   type: :development
   prerelease: false
-  version_requirements: *21765700
+  version_requirements: *2160496300
 description: Adds dump and restore functionality to activemodel classes
 email:
 - grantr@gmail.com
@@ -61,7 +60,6 @@ files:
 - lib/active_model_archive/version.rb
 - spec/archive_spec.rb
 - spec/spec_helper.rb
-has_rdoc: true
 homepage: ''
 licenses: []
 post_install_message: 
@@ -82,7 +80,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: active_model_archive
-rubygems_version: 1.6.2
+rubygems_version: 1.8.5
 signing_key: 
 specification_version: 3
 summary: Dump and restore activemodel objects