active_model_archive 0.0.7 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/Gemfile CHANGED
@@ -1,4 +1,3 @@
1
1
  source "http://rubygems.org"
2
2
 
3
- # Specify your gem's dependencies in active_model_archive.gemspec
4
3
  gemspec
@@ -1,10 +1,8 @@
1
1
  # -*- encoding: utf-8 -*-
2
- $:.push File.expand_path("../lib", __FILE__)
3
- require "active_model_archive/version"
4
2
 
5
3
  Gem::Specification.new do |s|
6
4
  s.name = "active_model_archive"
7
- s.version = ActiveModelArchive::VERSION
5
+ s.version = '0.9.0'
8
6
  s.platform = Gem::Platform::RUBY
9
7
  s.authors = ["Grant Rodgers"]
10
8
  s.email = ["grantr@gmail.com"]
@@ -20,5 +18,4 @@ Gem::Specification.new do |s|
20
18
  s.require_paths = ["lib"]
21
19
  s.add_dependency('activesupport', '~> 3')
22
20
  s.add_dependency('yajl-ruby', '>= 0')
23
- s.add_development_dependency('rspec', '~> 2')
24
21
  end
@@ -1,209 +1,22 @@
1
- require 'zlib'
1
+ require 'active_support/dependencies/autoload'
2
2
  require 'active_support/concern'
3
+ require 'yajl'
3
4
 
4
- module ActiveModel
5
- module Archive
6
- MAGIC = 42098
7
- CURRENT_VERSION = 1
8
- SUPPORTED_VERSIONS = [1]
9
-
10
- class FileManager
11
- UNLIMITED_PER_FILE = 999999999
12
- attr_accessor :item_count, :file_number, :base_filename, :items_per_file
13
- def initialize(base_filename, items_per_file)
14
- @item_count = 0
15
- @file_number = 0
16
- @base_filename = base_filename
17
- @items_per_file = (items_per_file.presence || UNLIMITED_PER_FILE).to_i
18
- end
19
-
20
- def add(object)
21
- Rails.logger.info "Adding #{object.id}"
22
-
23
- if item_count % items_per_file == 0
24
- close_current_file
25
- @current_file = create_file
26
- end
27
-
28
- self.item_count += 1
29
-
30
- encoder.encode(object.as_archive, nil) do |json|
31
- # write size, record, and crc
32
- write_int(@current_file, json.bytesize)
33
- @current_file.write(json)
34
- write_int(@current_file, Zlib::crc32(json))
35
- end
36
- end
37
-
38
- def create_file
39
- self.file_number += 1
40
-
41
- filename = items_per_file == UNLIMITED_PER_FILE ? base_filename : "#{base_filename}.#{file_number}"
42
- file = File.open(filename, "w")
43
- file.set_encoding("ASCII-8BIT")
44
-
45
- write_int(file, MAGIC)
46
- write_int(file, CURRENT_VERSION)
5
+ module ActiveModelArchive
6
+ extend ActiveSupport::Autoload
47
7
 
48
- file
49
- end
50
-
51
- def close_current_file
52
- if @current_file
53
- write_int(@current_file, 0)
54
- write_int(@current_file, item_count - ((file_number - 1) * items_per_file))
55
- write_int(@current_file, MAGIC)
56
-
57
- @current_file.close
58
- end
59
- end
60
-
61
- def write_int(file, int)
62
- file.write([int].pack("N"))
63
- end
64
-
65
- def encoder
66
- @encoder ||= Yajl::Encoder.new
67
- end
68
- end
8
+ autoload :FileManager
9
+ autoload :Dump
10
+ autoload :Restore
11
+ end
69
12
 
13
+ module ActiveModel
14
+ module Archive
70
15
  extend ActiveSupport::Concern
71
- # dump file format:
72
- # version 1:
73
- #
74
- # magic 4 bytes
75
- # version 4 bytes
76
- #
77
- # record length 4 bytes
78
- # record + \n
79
- # crc 4 bytes
80
- # record length 4 bytes
81
- # record + \n
82
- # crc 4 bytes
83
- # ...
84
- #
85
- # null 4 bytes
86
- # record count 4 bytes
87
- # magic 4 bytes
88
- #
89
-
90
- # version 2:
91
- # TODO add sync markers every N records
92
- # OR keep the human-readable format simple, and use avro for splittable files
93
16
 
94
- module ClassMethods
95
- # Dump records to a stream.
96
- # uses find_each without query params
97
- # uses paginated_each with query params
98
- # executes the block with each dumped record if given
99
- # the block can be used to log progress
100
- def dump!(filename, options = {})
101
- query = options[:query]
102
- per_file = options[:per_file]
103
-
104
- file_manager = FileManager.new(filename, per_file)
105
-
106
- if query.blank?
107
- find_each do |object|
108
- yield(object) if block_given?
109
- file_manager.add(object)
110
- end
111
- else
112
- paginated_each(query) do |object|
113
- yield(object) if block_given?
114
- file_manager.add(object)
115
- end
116
- end
117
-
118
- file_manager.close_current_file
119
- file_manager.item_count
120
- end
121
-
122
- def restore!(stream, options={})
123
- each_instance(stream) do |object|
124
- object.save(:validate => !!options[:validate])
125
- yield object if block_given?
126
- end
127
- end
128
-
129
- def each_instance(stream)
130
- each_archive(stream) do |hash|
131
- yield from_archive(hash)
132
- end
133
- end
134
-
135
- def each_record(stream)
136
- each_archive(stream) do |hash|
137
- yield find(hash.keys.first)
138
- end
139
- end
140
-
141
- def each_archive(stream, &block)
142
- # check magic byte
143
- magic = archive_read_int(stream)
144
- raise "Not an archive stream" unless magic == MAGIC
145
-
146
- # check version byte
147
- version = archive_read_int(stream)
148
- raise "Unsupported version #{version}" unless SUPPORTED_VERSIONS.include?(version)
149
-
150
- count = 0
151
- parser = Yajl::Parser.new
152
- parser.on_parse_complete = block if block_given?
153
-
154
- # loop through records until a null is read
155
- record_size = archive_read_int(stream)
156
- while record_size != 0
157
- json = stream.read(record_size)
158
- crc = archive_read_int(stream)
159
- raise "Invalid CRC at #{stream.pos-1}" unless crc == Zlib::crc32(json)
160
- count += 1
161
- parser << json
162
- record_size = archive_read_int(stream)
163
- end
164
-
165
- record_count = archive_read_int(stream)
166
- magic = archive_read_int(stream)
167
- raise "Unexpected end of archive file" unless magic == MAGIC
168
- raise "Record counts didn't match: #{record_count} in stream, #{count} read" unless record_count == count
169
-
170
- count
171
- end
172
-
173
- # returns true if the archive is valid
174
- def valid_archive?(stream)
175
- begin
176
- each_archive(stream)
177
- rescue
178
- return false
179
- end
180
- true
181
- end
182
-
183
- # generates an object from the archived hash
184
- def from_archive(hash)
185
- new.tap do |o|
186
- o.id = hash.keys.first
187
- o.attributes = hash.values.first[model_name.element]
188
- end
189
- end
190
-
191
- private
192
-
193
- def archive_read_int(stream)
194
- stream.read(4).unpack("N").first
195
- end
196
- end
197
-
198
- # instance method to return archived hash
199
- def as_archive
200
- # force include_root_in_json
201
- # allows archives to include multiple object types
202
- if include_root_in_json
203
- {id => as_json}
204
- else
205
- {id => {self.class.model_name.element => as_json}}
206
- end
17
+ included do
18
+ include ActiveModelArchive::Dump
19
+ include ActiveModelArchive::Restore
207
20
  end
208
21
  end
209
22
  end
@@ -0,0 +1,33 @@
1
+ module ActiveModelArchive
2
+ module Dump
3
+ extend ActiveSupport::Concern
4
+
5
+ module ClassMethods
6
+ def dump!(filename, options = {})
7
+ query = options[:query]
8
+ per_file = options[:per_file]
9
+
10
+ file_manager = FileManager.new(filename, per_file)
11
+
12
+ if query.blank?
13
+ find_each do |object|
14
+ yield(object) if block_given?
15
+ file_manager.add(object)
16
+ end
17
+ else
18
+ paginated_each(query) do |object|
19
+ yield(object) if block_given?
20
+ file_manager.add(object)
21
+ end
22
+ end
23
+
24
+ file_manager.flush
25
+ file_manager.item_count
26
+ end
27
+ end
28
+
29
+ def as_archive
30
+ attributes.dup.update('id' => id)
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,44 @@
1
+ module ActiveModelArchive
2
+ class FileManager
3
+ UNLIMITED_PER_FILE = 999999999
4
+ attr_accessor :item_count, :file_number, :base_filename, :items_per_file
5
+ def initialize(base_filename, items_per_file)
6
+ @item_count = 0
7
+ @file_number = 0
8
+ @base_filename = base_filename
9
+ @items_per_file = (items_per_file.presence || UNLIMITED_PER_FILE).to_i
10
+ end
11
+
12
+ def add(object)
13
+ if item_count % items_per_file == 0
14
+ flush
15
+ @current_file = create_file
16
+ end
17
+
18
+ self.item_count += 1
19
+
20
+ encoder.encode(object.as_archive, nil) do |json|
21
+ @current_file.write(json)
22
+ end
23
+ end
24
+
25
+ def create_file
26
+ self.file_number += 1
27
+
28
+ filename = items_per_file == UNLIMITED_PER_FILE ? base_filename : "#{base_filename}.#{file_number}"
29
+ file = File.open(filename, "w")
30
+
31
+ file
32
+ end
33
+
34
+ def flush
35
+ if @current_file
36
+ @current_file.close
37
+ end
38
+ end
39
+
40
+ def encoder
41
+ @encoder ||= Yajl::Encoder.new
42
+ end
43
+ end
44
+ end
@@ -0,0 +1,38 @@
1
+ module ActiveModelArchive
2
+ module Restore
3
+ extend ActiveSupport::Concern
4
+
5
+ module ClassMethods
6
+ def restore!(io, options={})
7
+ each_instance(io) do |object|
8
+ object.save(validate: !!options[:validate])
9
+ yield object if block_given?
10
+ end
11
+ end
12
+
13
+ def each_instance(io)
14
+ each_archive(io) do |hash|
15
+ yield new(hash)
16
+ end
17
+ end
18
+
19
+ def each_record(io)
20
+ each_archive(io) do |hash|
21
+ yield find(hash['id'])
22
+ end
23
+ end
24
+
25
+ def each_archive(io)
26
+ count = 0
27
+ parser = Yajl::Parser.new
28
+
29
+ parser.parse(io) do |hash|
30
+ count += 1
31
+ yield hash
32
+ end
33
+
34
+ count
35
+ end
36
+ end
37
+ end
38
+ end
data/test/dump_test.rb ADDED
@@ -0,0 +1,11 @@
1
+ require 'test_helper'
2
+
3
+ class ActiveModelArchive::DumpTest < MiniTest::Spec
4
+ def test_as_archive
5
+ model = TestModel.new(id: 5, color: 'blue')
6
+ assert_equal(
7
+ {'id' => 5, 'color' => 'blue'},
8
+ model.as_archive
9
+ )
10
+ end
11
+ end
@@ -0,0 +1,55 @@
1
+ require 'test_helper'
2
+
3
+ class ActiveModelArchive::RestoreTest < MiniTest::Spec
4
+ def test_each_archive
5
+ json_io = StringIO.new %{
6
+ {
7
+ "id": 42,
8
+ "color": "green"
9
+ }
10
+ {
11
+ "id": 84,
12
+ "color": "yellow"
13
+ }
14
+ }
15
+
16
+ results = []
17
+ TestModel.each_archive(json_io) do |hash|
18
+ results << hash
19
+ end
20
+
21
+ expected = [
22
+ {
23
+ "id" => 42,
24
+ "color" => "green"
25
+ },
26
+ {
27
+ "id" => 84,
28
+ "color" => "yellow"
29
+ }
30
+ ]
31
+ assert_equal expected, results
32
+ end
33
+
34
+ def test_each_instance
35
+ json_io = StringIO.new %{
36
+ {
37
+ "id": 42,
38
+ "color": "green"
39
+ }
40
+ }
41
+
42
+ results = []
43
+ TestModel.each_instance(json_io) do |object|
44
+ results << object
45
+ end
46
+
47
+ assert_equal 1, results.size
48
+ assert_equal 42, results.first.id
49
+ assert_equal 'green', results.first.color
50
+ end
51
+
52
+ def test_restore!
53
+
54
+ end
55
+ end
@@ -0,0 +1,36 @@
1
+ require 'rubygems'
2
+ require 'bundler/setup'
3
+ require 'minitest/autorun'
4
+ Bundler.require :default
5
+
6
+ class TestModel
7
+ include ActiveModel::Archive
8
+
9
+ class << self
10
+ def find(id)
11
+ new(id: id, color: 'blue')
12
+ end
13
+ end
14
+
15
+ def initialize(attrs = {})
16
+ @attributes = {}
17
+ self.attributes = attrs
18
+ end
19
+
20
+ def id
21
+ @attributes['id']
22
+ end
23
+
24
+ def color
25
+ @attributes['color']
26
+ end
27
+
28
+ def attributes=(attrs)
29
+ @attributes['color'] = attrs[:color] || attrs['color']
30
+ @attributes['id'] = attrs[:id] || attrs['id']
31
+ end
32
+
33
+ def attributes
34
+ @attributes
35
+ end
36
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: active_model_archive
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.7
4
+ version: 0.9.0
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,11 +9,11 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2011-07-28 00:00:00.000000000Z
12
+ date: 2011-11-03 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: activesupport
16
- requirement: &2164669460 !ruby/object:Gem::Requirement
16
+ requirement: &70251111770820 !ruby/object:Gem::Requirement
17
17
  none: false
18
18
  requirements:
19
19
  - - ~>
@@ -21,10 +21,10 @@ dependencies:
21
21
  version: '3'
22
22
  type: :runtime
23
23
  prerelease: false
24
- version_requirements: *2164669460
24
+ version_requirements: *70251111770820
25
25
  - !ruby/object:Gem::Dependency
26
26
  name: yajl-ruby
27
- requirement: &2164668960 !ruby/object:Gem::Requirement
27
+ requirement: &70251111770120 !ruby/object:Gem::Requirement
28
28
  none: false
29
29
  requirements:
30
30
  - - ! '>='
@@ -32,18 +32,7 @@ dependencies:
32
32
  version: '0'
33
33
  type: :runtime
34
34
  prerelease: false
35
- version_requirements: *2164668960
36
- - !ruby/object:Gem::Dependency
37
- name: rspec
38
- requirement: &2164668500 !ruby/object:Gem::Requirement
39
- none: false
40
- requirements:
41
- - - ~>
42
- - !ruby/object:Gem::Version
43
- version: '2'
44
- type: :development
45
- prerelease: false
46
- version_requirements: *2164668500
35
+ version_requirements: *70251111770120
47
36
  description: Adds dump and restore functionality to activemodel classes
48
37
  email:
49
38
  - grantr@gmail.com
@@ -57,9 +46,12 @@ files:
57
46
  - Rakefile
58
47
  - active_model_archive.gemspec
59
48
  - lib/active_model_archive.rb
60
- - lib/active_model_archive/version.rb
61
- - spec/archive_spec.rb
62
- - spec/spec_helper.rb
49
+ - lib/active_model_archive/dump.rb
50
+ - lib/active_model_archive/file_manager.rb
51
+ - lib/active_model_archive/restore.rb
52
+ - test/dump_test.rb
53
+ - test/restore_test.rb
54
+ - test/test_helper.rb
63
55
  homepage: ''
64
56
  licenses: []
65
57
  post_install_message:
@@ -80,10 +72,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
80
72
  version: '0'
81
73
  requirements: []
82
74
  rubyforge_project: active_model_archive
83
- rubygems_version: 1.8.5
75
+ rubygems_version: 1.8.10
84
76
  signing_key:
85
77
  specification_version: 3
86
78
  summary: Dump and restore activemodel objects
87
79
  test_files:
88
- - spec/archive_spec.rb
89
- - spec/spec_helper.rb
80
+ - test/dump_test.rb
81
+ - test/restore_test.rb
82
+ - test/test_helper.rb
@@ -1,3 +0,0 @@
1
- module ActiveModelArchive
2
- VERSION = "0.0.7"
3
- end
data/spec/archive_spec.rb DELETED
@@ -1,3 +0,0 @@
1
- require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
2
-
3
-
data/spec/spec_helper.rb DELETED
@@ -1,12 +0,0 @@
1
- $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
2
- $LOAD_PATH.unshift(File.dirname(__FILE__))
3
- require 'rspec'
4
- require 'active_model_archive'
5
-
6
- # Requires supporting files with custom matchers and macros, etc,
7
- # in ./support/ and its subdirectories.
8
- Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
9
-
10
- RSpec.configure do |config|
11
- config.mock_with :rspec
12
- end