active_model_archive 0.0.7 → 0.9.0

data/Gemfile

@@ -1,4 +1,3 @@
 source "http://rubygems.org"
 
-# Specify your gem's dependencies in active_model_archive.gemspec
 gemspec

data/active_model_archive.gemspec

@@ -1,10 +1,8 @@
 # -*- encoding: utf-8 -*-
-$:.push File.expand_path("../lib", __FILE__)
-require "active_model_archive/version"
 
 Gem::Specification.new do |s|
   s.name        = "active_model_archive"
-  s.version     = ActiveModelArchive::VERSION
+  s.version     = '0.9.0'
   s.platform    = Gem::Platform::RUBY
   s.authors     = ["Grant Rodgers"]
   s.email       = ["grantr@gmail.com"]
@@ -20,5 +18,4 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
   s.add_dependency('activesupport', '~> 3')
   s.add_dependency('yajl-ruby', '>= 0')
-  s.add_development_dependency('rspec', '~> 2')
 end

data/lib/active_model_archive.rb

@@ -1,209 +1,22 @@
-require 'zlib'
+require 'active_support/dependencies/autoload'
 require 'active_support/concern'
+require 'yajl'
 
-module ActiveModel
-  module Archive
-    MAGIC = 42098
-    CURRENT_VERSION = 1
-    SUPPORTED_VERSIONS = [1]
-
-    class FileManager
-      UNLIMITED_PER_FILE = 999999999
-      attr_accessor :item_count, :file_number, :base_filename, :items_per_file
-      def initialize(base_filename, items_per_file)
-        @item_count = 0
-        @file_number = 0
-        @base_filename = base_filename
-        @items_per_file = (items_per_file.presence || UNLIMITED_PER_FILE).to_i
-      end
-
-      def add(object)
-        Rails.logger.info "Adding #{object.id}"
-
-        if item_count % items_per_file == 0
-          close_current_file
-          @current_file = create_file
-        end
-
-        self.item_count += 1
-
-        encoder.encode(object.as_archive, nil) do |json|
-          # write size, record, and crc
-          write_int(@current_file, json.bytesize)
-          @current_file.write(json)
-          write_int(@current_file, Zlib::crc32(json))
-        end
-      end
-
-      def create_file
-        self.file_number += 1
-        
-        filename = items_per_file == UNLIMITED_PER_FILE ? base_filename : "#{base_filename}.#{file_number}"
-        file = File.open(filename, "w")
-        file.set_encoding("ASCII-8BIT")
-
-        write_int(file, MAGIC)
-        write_int(file, CURRENT_VERSION)
+module ActiveModelArchive
+  extend ActiveSupport::Autoload
 
-        file
-      end
-
-      def close_current_file
-        if @current_file
-          write_int(@current_file, 0)
-          write_int(@current_file, item_count - ((file_number - 1) * items_per_file))
-          write_int(@current_file, MAGIC)
-
-          @current_file.close
-        end
-      end
-      
-      def write_int(file, int)
-        file.write([int].pack("N"))
-      end
-
-      def encoder
-        @encoder ||= Yajl::Encoder.new
-      end
-    end
+  autoload :FileManager
+  autoload :Dump
+  autoload :Restore
+end
 
+module ActiveModel
+  module Archive
     extend ActiveSupport::Concern
-    # dump file format:
-    # version 1:
-    #
-    # magic 4 bytes
-    # version 4 bytes
-    # 
-    # record length 4 bytes
-    # record + \n
-    # crc 4 bytes
-    # record length 4 bytes
-    # record + \n
-    # crc 4 bytes
-    # ...
-    #
-    # null 4 bytes
-    # record count 4 bytes
-    # magic 4 bytes
-    #
-
-    # version 2:
-    # TODO add sync markers every N records
-    # OR keep the human-readable format simple, and use avro for splittable files
 
-    module ClassMethods
-      # Dump records to a stream.
-      # uses find_each without query params
-      # uses paginated_each with query params
-      # executes the block with each dumped record if given
-      # the block can be used to log progress
-      def dump!(filename, options = {})
-        query = options[:query]
-        per_file = options[:per_file]
-
-        file_manager = FileManager.new(filename, per_file)
-
-        if query.blank?
-          find_each do |object|
-            yield(object) if block_given?
-            file_manager.add(object)
-          end
-        else
-          paginated_each(query) do |object|
-            yield(object) if block_given?
-            file_manager.add(object)
-          end
-        end
-
-        file_manager.close_current_file
-        file_manager.item_count
-      end
-
-      def restore!(stream, options={})
-        each_instance(stream) do |object|
-          object.save(:validate => !!options[:validate])
-          yield object if block_given?
-        end
-      end
-
-      def each_instance(stream)
-        each_archive(stream) do |hash|
-          yield from_archive(hash)
-        end
-      end
-
-      def each_record(stream)
-        each_archive(stream) do |hash|
-          yield find(hash.keys.first)
-        end
-      end
-
-      def each_archive(stream, &block)
-        # check magic byte
-        magic = archive_read_int(stream)
-        raise "Not an archive stream" unless magic == MAGIC
-
-        # check version byte
-        version = archive_read_int(stream)
-        raise "Unsupported version #{version}" unless SUPPORTED_VERSIONS.include?(version)
-
-        count = 0
-        parser = Yajl::Parser.new
-        parser.on_parse_complete = block if block_given?
-
-        # loop through records until a null is read
-        record_size = archive_read_int(stream)
-        while record_size != 0
-          json = stream.read(record_size)
-          crc = archive_read_int(stream)
-          raise "Invalid CRC at #{stream.pos-1}" unless crc == Zlib::crc32(json)
-          count += 1
-          parser << json
-          record_size = archive_read_int(stream)
-        end
-
-        record_count = archive_read_int(stream)
-        magic = archive_read_int(stream)
-        raise "Unexpected end of archive file" unless magic == MAGIC
-        raise "Record counts didn't match: #{record_count} in stream, #{count} read" unless record_count == count
-
-        count
-      end
-
-      # returns true if the archive is valid
-      def valid_archive?(stream)
-        begin
-          each_archive(stream)
-        rescue
-          return false
-        end
-        true
-      end
-
-      # generates an object from the archived hash
-      def from_archive(hash)
-        new.tap do |o|
-          o.id = hash.keys.first
-          o.attributes = hash.values.first[model_name.element]
-        end
-      end
-
-      private
-
-      def archive_read_int(stream)
-        stream.read(4).unpack("N").first
-      end
-    end
-  
-    # instance method to return archived hash
-    def as_archive
-      # force include_root_in_json
-      # allows archives to include multiple object types
-      if include_root_in_json
-        {id => as_json}
-      else
-        {id => {self.class.model_name.element => as_json}}
-      end
+    included do
+      include ActiveModelArchive::Dump
+      include ActiveModelArchive::Restore
     end
   end
 end

data/lib/active_model_archive/dump.rb

@@ -0,0 +1,33 @@
+module ActiveModelArchive
+  module Dump
+    extend ActiveSupport::Concern
+
+    module ClassMethods
+      def dump!(filename, options = {})
+        query = options[:query]
+        per_file = options[:per_file]
+
+        file_manager = FileManager.new(filename, per_file)
+
+        if query.blank?
+          find_each do |object|
+            yield(object) if block_given?
+            file_manager.add(object)
+          end
+        else
+          paginated_each(query) do |object|
+            yield(object) if block_given?
+            file_manager.add(object)
+          end
+        end
+
+        file_manager.flush
+        file_manager.item_count
+      end
+    end
+
+    def as_archive
+      attributes.dup.update('id' => id)
+    end
+  end
+end

data/lib/active_model_archive/file_manager.rb

@@ -0,0 +1,44 @@
+module ActiveModelArchive
+  class FileManager
+    UNLIMITED_PER_FILE = 999999999
+    attr_accessor :item_count, :file_number, :base_filename, :items_per_file
+    def initialize(base_filename, items_per_file)
+      @item_count = 0
+      @file_number = 0
+      @base_filename = base_filename
+      @items_per_file = (items_per_file.presence || UNLIMITED_PER_FILE).to_i
+    end
+
+    def add(object)
+      if item_count % items_per_file == 0
+        flush
+        @current_file = create_file
+      end
+
+      self.item_count += 1
+
+      encoder.encode(object.as_archive, nil) do |json|
+        @current_file.write(json)
+      end
+    end
+
+    def create_file
+      self.file_number += 1
+      
+      filename = items_per_file == UNLIMITED_PER_FILE ? base_filename : "#{base_filename}.#{file_number}"
+      file = File.open(filename, "w")
+
+      file
+    end
+
+    def flush
+      if @current_file
+        @current_file.close
+      end
+    end
+    
+    def encoder
+      @encoder ||= Yajl::Encoder.new
+    end
+  end
+end

data/lib/active_model_archive/restore.rb

@@ -0,0 +1,38 @@
+module ActiveModelArchive
+  module Restore
+    extend ActiveSupport::Concern
+
+    module ClassMethods
+      def restore!(io, options={})
+        each_instance(io) do |object|
+          object.save(validate: !!options[:validate])
+          yield object if block_given?
+        end
+      end
+
+      def each_instance(io)
+        each_archive(io) do |hash|
+          yield new(hash)
+        end
+      end
+
+      def each_record(io)
+        each_archive(io) do |hash|
+          yield find(hash['id'])
+        end
+      end
+
+      def each_archive(io)
+        count = 0
+        parser = Yajl::Parser.new
+
+        parser.parse(io) do |hash|
+          count += 1
+          yield hash
+        end
+
+        count
+      end
+    end
+  end
+end

data/test/dump_test.rb

@@ -0,0 +1,11 @@
+require 'test_helper'
+
+class ActiveModelArchive::DumpTest < MiniTest::Spec
+  def test_as_archive
+    model = TestModel.new(id: 5, color: 'blue')
+    assert_equal(
+      {'id' => 5, 'color' => 'blue'},
+      model.as_archive
+    )
+  end
+end

data/test/restore_test.rb

@@ -0,0 +1,55 @@
+require 'test_helper'
+
+class ActiveModelArchive::RestoreTest < MiniTest::Spec
+  def test_each_archive
+    json_io = StringIO.new %{
+       {
+         "id":     42,
+         "color": "green"
+       }
+       {
+         "id":     84,
+         "color": "yellow"
+       }
+     }
+
+    results = []
+    TestModel.each_archive(json_io) do |hash|
+      results << hash
+    end
+    
+    expected = [
+      {
+        "id"    =>     42,
+        "color" => "green"
+      },
+      {
+        "id"    => 84,
+        "color" => "yellow"
+      }
+    ]
+    assert_equal expected, results
+  end
+
+  def test_each_instance
+    json_io = StringIO.new %{
+       {
+         "id":     42,
+         "color": "green"
+       }
+     }
+
+     results = []
+     TestModel.each_instance(json_io) do |object|
+       results << object
+     end
+
+     assert_equal 1, results.size
+     assert_equal 42, results.first.id
+     assert_equal 'green', results.first.color
+  end
+
+  def test_restore!
+    
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,36 @@
+require 'rubygems'
+require 'bundler/setup'
+require 'minitest/autorun'
+Bundler.require :default
+
+class TestModel
+  include ActiveModel::Archive
+
+  class << self
+    def find(id)
+      new(id: id, color: 'blue')
+    end
+  end
+
+  def initialize(attrs = {})
+    @attributes = {}
+    self.attributes = attrs
+  end
+
+  def id
+    @attributes['id']
+  end
+
+  def color
+    @attributes['color']
+  end
+
+  def attributes=(attrs)
+    @attributes['color'] = attrs[:color] || attrs['color']
+    @attributes['id'] = attrs[:id] || attrs['id']
+  end
+
+  def attributes
+    @attributes
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: active_model_archive
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.9.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-07-28 00:00:00.000000000Z
+date: 2011-11-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &2164669460 !ruby/object:Gem::Requirement
+  requirement: &70251111770820 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: '3'
   type: :runtime
   prerelease: false
-  version_requirements: *2164669460
+  version_requirements: *70251111770820
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
-  requirement: &2164668960 !ruby/object:Gem::Requirement
+  requirement: &70251111770120 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,18 +32,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2164668960
-- !ruby/object:Gem::Dependency
-  name: rspec
-  requirement: &2164668500 !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '2'
-  type: :development
-  prerelease: false
-  version_requirements: *2164668500
+  version_requirements: *70251111770120
 description: Adds dump and restore functionality to activemodel classes
 email:
 - grantr@gmail.com
@@ -57,9 +46,12 @@ files:
 - Rakefile
 - active_model_archive.gemspec
 - lib/active_model_archive.rb
-- lib/active_model_archive/version.rb
-- spec/archive_spec.rb
-- spec/spec_helper.rb
+- lib/active_model_archive/dump.rb
+- lib/active_model_archive/file_manager.rb
+- lib/active_model_archive/restore.rb
+- test/dump_test.rb
+- test/restore_test.rb
+- test/test_helper.rb
 homepage: ''
 licenses: []
 post_install_message: 
@@ -80,10 +72,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: active_model_archive
-rubygems_version: 1.8.5
+rubygems_version: 1.8.10
 signing_key: 
 specification_version: 3
 summary: Dump and restore activemodel objects
 test_files:
-- spec/archive_spec.rb
-- spec/spec_helper.rb
+- test/dump_test.rb
+- test/restore_test.rb
+- test/test_helper.rb

data/lib/active_model_archive/version.rb

@@ -1,3 +0,0 @@
-module ActiveModelArchive
-  VERSION = "0.0.7"
-end

data/spec/archive_spec.rb

@@ -1,3 +0,0 @@
-require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
-
-

data/spec/spec_helper.rb

@@ -1,12 +0,0 @@
-$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
-$LOAD_PATH.unshift(File.dirname(__FILE__))
-require 'rspec'
-require 'active_model_archive'
-
-# Requires supporting files with custom matchers and macros, etc,
-# in ./support/ and its subdirectories.
-Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
-
-RSpec.configure do |config|
-  config.mock_with :rspec
-end