active_model_archive 0.0.7 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +0 -1
- data/active_model_archive.gemspec +1 -4
- data/lib/active_model_archive.rb +13 -200
- data/lib/active_model_archive/dump.rb +33 -0
- data/lib/active_model_archive/file_manager.rb +44 -0
- data/lib/active_model_archive/restore.rb +38 -0
- data/test/dump_test.rb +11 -0
- data/test/restore_test.rb +55 -0
- data/test/test_helper.rb +36 -0
- metadata +16 -23
- data/lib/active_model_archive/version.rb +0 -3
- data/spec/archive_spec.rb +0 -3
- data/spec/spec_helper.rb +0 -12
data/Gemfile
CHANGED
@@ -1,10 +1,8 @@
|
|
1
1
|
# -*- encoding: utf-8 -*-
|
2
|
-
$:.push File.expand_path("../lib", __FILE__)
|
3
|
-
require "active_model_archive/version"
|
4
2
|
|
5
3
|
Gem::Specification.new do |s|
|
6
4
|
s.name = "active_model_archive"
|
7
|
-
s.version =
|
5
|
+
s.version = '0.9.0'
|
8
6
|
s.platform = Gem::Platform::RUBY
|
9
7
|
s.authors = ["Grant Rodgers"]
|
10
8
|
s.email = ["grantr@gmail.com"]
|
@@ -20,5 +18,4 @@ Gem::Specification.new do |s|
|
|
20
18
|
s.require_paths = ["lib"]
|
21
19
|
s.add_dependency('activesupport', '~> 3')
|
22
20
|
s.add_dependency('yajl-ruby', '>= 0')
|
23
|
-
s.add_development_dependency('rspec', '~> 2')
|
24
21
|
end
|
data/lib/active_model_archive.rb
CHANGED
@@ -1,209 +1,22 @@
|
|
1
|
-
require '
|
1
|
+
require 'active_support/dependencies/autoload'
|
2
2
|
require 'active_support/concern'
|
3
|
+
require 'yajl'
|
3
4
|
|
4
|
-
module
|
5
|
-
|
6
|
-
MAGIC = 42098
|
7
|
-
CURRENT_VERSION = 1
|
8
|
-
SUPPORTED_VERSIONS = [1]
|
9
|
-
|
10
|
-
class FileManager
|
11
|
-
UNLIMITED_PER_FILE = 999999999
|
12
|
-
attr_accessor :item_count, :file_number, :base_filename, :items_per_file
|
13
|
-
def initialize(base_filename, items_per_file)
|
14
|
-
@item_count = 0
|
15
|
-
@file_number = 0
|
16
|
-
@base_filename = base_filename
|
17
|
-
@items_per_file = (items_per_file.presence || UNLIMITED_PER_FILE).to_i
|
18
|
-
end
|
19
|
-
|
20
|
-
def add(object)
|
21
|
-
Rails.logger.info "Adding #{object.id}"
|
22
|
-
|
23
|
-
if item_count % items_per_file == 0
|
24
|
-
close_current_file
|
25
|
-
@current_file = create_file
|
26
|
-
end
|
27
|
-
|
28
|
-
self.item_count += 1
|
29
|
-
|
30
|
-
encoder.encode(object.as_archive, nil) do |json|
|
31
|
-
# write size, record, and crc
|
32
|
-
write_int(@current_file, json.bytesize)
|
33
|
-
@current_file.write(json)
|
34
|
-
write_int(@current_file, Zlib::crc32(json))
|
35
|
-
end
|
36
|
-
end
|
37
|
-
|
38
|
-
def create_file
|
39
|
-
self.file_number += 1
|
40
|
-
|
41
|
-
filename = items_per_file == UNLIMITED_PER_FILE ? base_filename : "#{base_filename}.#{file_number}"
|
42
|
-
file = File.open(filename, "w")
|
43
|
-
file.set_encoding("ASCII-8BIT")
|
44
|
-
|
45
|
-
write_int(file, MAGIC)
|
46
|
-
write_int(file, CURRENT_VERSION)
|
5
|
+
module ActiveModelArchive
|
6
|
+
extend ActiveSupport::Autoload
|
47
7
|
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
if @current_file
|
53
|
-
write_int(@current_file, 0)
|
54
|
-
write_int(@current_file, item_count - ((file_number - 1) * items_per_file))
|
55
|
-
write_int(@current_file, MAGIC)
|
56
|
-
|
57
|
-
@current_file.close
|
58
|
-
end
|
59
|
-
end
|
60
|
-
|
61
|
-
def write_int(file, int)
|
62
|
-
file.write([int].pack("N"))
|
63
|
-
end
|
64
|
-
|
65
|
-
def encoder
|
66
|
-
@encoder ||= Yajl::Encoder.new
|
67
|
-
end
|
68
|
-
end
|
8
|
+
autoload :FileManager
|
9
|
+
autoload :Dump
|
10
|
+
autoload :Restore
|
11
|
+
end
|
69
12
|
|
13
|
+
module ActiveModel
|
14
|
+
module Archive
|
70
15
|
extend ActiveSupport::Concern
|
71
|
-
# dump file format:
|
72
|
-
# version 1:
|
73
|
-
#
|
74
|
-
# magic 4 bytes
|
75
|
-
# version 4 bytes
|
76
|
-
#
|
77
|
-
# record length 4 bytes
|
78
|
-
# record + \n
|
79
|
-
# crc 4 bytes
|
80
|
-
# record length 4 bytes
|
81
|
-
# record + \n
|
82
|
-
# crc 4 bytes
|
83
|
-
# ...
|
84
|
-
#
|
85
|
-
# null 4 bytes
|
86
|
-
# record count 4 bytes
|
87
|
-
# magic 4 bytes
|
88
|
-
#
|
89
|
-
|
90
|
-
# version 2:
|
91
|
-
# TODO add sync markers every N records
|
92
|
-
# OR keep the human-readable format simple, and use avro for splittable files
|
93
16
|
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
# uses paginated_each with query params
|
98
|
-
# executes the block with each dumped record if given
|
99
|
-
# the block can be used to log progress
|
100
|
-
def dump!(filename, options = {})
|
101
|
-
query = options[:query]
|
102
|
-
per_file = options[:per_file]
|
103
|
-
|
104
|
-
file_manager = FileManager.new(filename, per_file)
|
105
|
-
|
106
|
-
if query.blank?
|
107
|
-
find_each do |object|
|
108
|
-
yield(object) if block_given?
|
109
|
-
file_manager.add(object)
|
110
|
-
end
|
111
|
-
else
|
112
|
-
paginated_each(query) do |object|
|
113
|
-
yield(object) if block_given?
|
114
|
-
file_manager.add(object)
|
115
|
-
end
|
116
|
-
end
|
117
|
-
|
118
|
-
file_manager.close_current_file
|
119
|
-
file_manager.item_count
|
120
|
-
end
|
121
|
-
|
122
|
-
def restore!(stream, options={})
|
123
|
-
each_instance(stream) do |object|
|
124
|
-
object.save(:validate => !!options[:validate])
|
125
|
-
yield object if block_given?
|
126
|
-
end
|
127
|
-
end
|
128
|
-
|
129
|
-
def each_instance(stream)
|
130
|
-
each_archive(stream) do |hash|
|
131
|
-
yield from_archive(hash)
|
132
|
-
end
|
133
|
-
end
|
134
|
-
|
135
|
-
def each_record(stream)
|
136
|
-
each_archive(stream) do |hash|
|
137
|
-
yield find(hash.keys.first)
|
138
|
-
end
|
139
|
-
end
|
140
|
-
|
141
|
-
def each_archive(stream, &block)
|
142
|
-
# check magic byte
|
143
|
-
magic = archive_read_int(stream)
|
144
|
-
raise "Not an archive stream" unless magic == MAGIC
|
145
|
-
|
146
|
-
# check version byte
|
147
|
-
version = archive_read_int(stream)
|
148
|
-
raise "Unsupported version #{version}" unless SUPPORTED_VERSIONS.include?(version)
|
149
|
-
|
150
|
-
count = 0
|
151
|
-
parser = Yajl::Parser.new
|
152
|
-
parser.on_parse_complete = block if block_given?
|
153
|
-
|
154
|
-
# loop through records until a null is read
|
155
|
-
record_size = archive_read_int(stream)
|
156
|
-
while record_size != 0
|
157
|
-
json = stream.read(record_size)
|
158
|
-
crc = archive_read_int(stream)
|
159
|
-
raise "Invalid CRC at #{stream.pos-1}" unless crc == Zlib::crc32(json)
|
160
|
-
count += 1
|
161
|
-
parser << json
|
162
|
-
record_size = archive_read_int(stream)
|
163
|
-
end
|
164
|
-
|
165
|
-
record_count = archive_read_int(stream)
|
166
|
-
magic = archive_read_int(stream)
|
167
|
-
raise "Unexpected end of archive file" unless magic == MAGIC
|
168
|
-
raise "Record counts didn't match: #{record_count} in stream, #{count} read" unless record_count == count
|
169
|
-
|
170
|
-
count
|
171
|
-
end
|
172
|
-
|
173
|
-
# returns true if the archive is valid
|
174
|
-
def valid_archive?(stream)
|
175
|
-
begin
|
176
|
-
each_archive(stream)
|
177
|
-
rescue
|
178
|
-
return false
|
179
|
-
end
|
180
|
-
true
|
181
|
-
end
|
182
|
-
|
183
|
-
# generates an object from the archived hash
|
184
|
-
def from_archive(hash)
|
185
|
-
new.tap do |o|
|
186
|
-
o.id = hash.keys.first
|
187
|
-
o.attributes = hash.values.first[model_name.element]
|
188
|
-
end
|
189
|
-
end
|
190
|
-
|
191
|
-
private
|
192
|
-
|
193
|
-
def archive_read_int(stream)
|
194
|
-
stream.read(4).unpack("N").first
|
195
|
-
end
|
196
|
-
end
|
197
|
-
|
198
|
-
# instance method to return archived hash
|
199
|
-
def as_archive
|
200
|
-
# force include_root_in_json
|
201
|
-
# allows archives to include multiple object types
|
202
|
-
if include_root_in_json
|
203
|
-
{id => as_json}
|
204
|
-
else
|
205
|
-
{id => {self.class.model_name.element => as_json}}
|
206
|
-
end
|
17
|
+
included do
|
18
|
+
include ActiveModelArchive::Dump
|
19
|
+
include ActiveModelArchive::Restore
|
207
20
|
end
|
208
21
|
end
|
209
22
|
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
module ActiveModelArchive
|
2
|
+
module Dump
|
3
|
+
extend ActiveSupport::Concern
|
4
|
+
|
5
|
+
module ClassMethods
|
6
|
+
def dump!(filename, options = {})
|
7
|
+
query = options[:query]
|
8
|
+
per_file = options[:per_file]
|
9
|
+
|
10
|
+
file_manager = FileManager.new(filename, per_file)
|
11
|
+
|
12
|
+
if query.blank?
|
13
|
+
find_each do |object|
|
14
|
+
yield(object) if block_given?
|
15
|
+
file_manager.add(object)
|
16
|
+
end
|
17
|
+
else
|
18
|
+
paginated_each(query) do |object|
|
19
|
+
yield(object) if block_given?
|
20
|
+
file_manager.add(object)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
file_manager.flush
|
25
|
+
file_manager.item_count
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def as_archive
|
30
|
+
attributes.dup.update('id' => id)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
module ActiveModelArchive
|
2
|
+
class FileManager
|
3
|
+
UNLIMITED_PER_FILE = 999999999
|
4
|
+
attr_accessor :item_count, :file_number, :base_filename, :items_per_file
|
5
|
+
def initialize(base_filename, items_per_file)
|
6
|
+
@item_count = 0
|
7
|
+
@file_number = 0
|
8
|
+
@base_filename = base_filename
|
9
|
+
@items_per_file = (items_per_file.presence || UNLIMITED_PER_FILE).to_i
|
10
|
+
end
|
11
|
+
|
12
|
+
def add(object)
|
13
|
+
if item_count % items_per_file == 0
|
14
|
+
flush
|
15
|
+
@current_file = create_file
|
16
|
+
end
|
17
|
+
|
18
|
+
self.item_count += 1
|
19
|
+
|
20
|
+
encoder.encode(object.as_archive, nil) do |json|
|
21
|
+
@current_file.write(json)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def create_file
|
26
|
+
self.file_number += 1
|
27
|
+
|
28
|
+
filename = items_per_file == UNLIMITED_PER_FILE ? base_filename : "#{base_filename}.#{file_number}"
|
29
|
+
file = File.open(filename, "w")
|
30
|
+
|
31
|
+
file
|
32
|
+
end
|
33
|
+
|
34
|
+
def flush
|
35
|
+
if @current_file
|
36
|
+
@current_file.close
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def encoder
|
41
|
+
@encoder ||= Yajl::Encoder.new
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
module ActiveModelArchive
|
2
|
+
module Restore
|
3
|
+
extend ActiveSupport::Concern
|
4
|
+
|
5
|
+
module ClassMethods
|
6
|
+
def restore!(io, options={})
|
7
|
+
each_instance(io) do |object|
|
8
|
+
object.save(validate: !!options[:validate])
|
9
|
+
yield object if block_given?
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def each_instance(io)
|
14
|
+
each_archive(io) do |hash|
|
15
|
+
yield new(hash)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def each_record(io)
|
20
|
+
each_archive(io) do |hash|
|
21
|
+
yield find(hash['id'])
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def each_archive(io)
|
26
|
+
count = 0
|
27
|
+
parser = Yajl::Parser.new
|
28
|
+
|
29
|
+
parser.parse(io) do |hash|
|
30
|
+
count += 1
|
31
|
+
yield hash
|
32
|
+
end
|
33
|
+
|
34
|
+
count
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
data/test/dump_test.rb
ADDED
@@ -0,0 +1,55 @@
|
|
1
|
+
require 'test_helper'
|
2
|
+
|
3
|
+
class ActiveModelArchive::RestoreTest < MiniTest::Spec
|
4
|
+
def test_each_archive
|
5
|
+
json_io = StringIO.new %{
|
6
|
+
{
|
7
|
+
"id": 42,
|
8
|
+
"color": "green"
|
9
|
+
}
|
10
|
+
{
|
11
|
+
"id": 84,
|
12
|
+
"color": "yellow"
|
13
|
+
}
|
14
|
+
}
|
15
|
+
|
16
|
+
results = []
|
17
|
+
TestModel.each_archive(json_io) do |hash|
|
18
|
+
results << hash
|
19
|
+
end
|
20
|
+
|
21
|
+
expected = [
|
22
|
+
{
|
23
|
+
"id" => 42,
|
24
|
+
"color" => "green"
|
25
|
+
},
|
26
|
+
{
|
27
|
+
"id" => 84,
|
28
|
+
"color" => "yellow"
|
29
|
+
}
|
30
|
+
]
|
31
|
+
assert_equal expected, results
|
32
|
+
end
|
33
|
+
|
34
|
+
def test_each_instance
|
35
|
+
json_io = StringIO.new %{
|
36
|
+
{
|
37
|
+
"id": 42,
|
38
|
+
"color": "green"
|
39
|
+
}
|
40
|
+
}
|
41
|
+
|
42
|
+
results = []
|
43
|
+
TestModel.each_instance(json_io) do |object|
|
44
|
+
results << object
|
45
|
+
end
|
46
|
+
|
47
|
+
assert_equal 1, results.size
|
48
|
+
assert_equal 42, results.first.id
|
49
|
+
assert_equal 'green', results.first.color
|
50
|
+
end
|
51
|
+
|
52
|
+
def test_restore!
|
53
|
+
|
54
|
+
end
|
55
|
+
end
|
data/test/test_helper.rb
ADDED
@@ -0,0 +1,36 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'bundler/setup'
|
3
|
+
require 'minitest/autorun'
|
4
|
+
Bundler.require :default
|
5
|
+
|
6
|
+
class TestModel
|
7
|
+
include ActiveModel::Archive
|
8
|
+
|
9
|
+
class << self
|
10
|
+
def find(id)
|
11
|
+
new(id: id, color: 'blue')
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
def initialize(attrs = {})
|
16
|
+
@attributes = {}
|
17
|
+
self.attributes = attrs
|
18
|
+
end
|
19
|
+
|
20
|
+
def id
|
21
|
+
@attributes['id']
|
22
|
+
end
|
23
|
+
|
24
|
+
def color
|
25
|
+
@attributes['color']
|
26
|
+
end
|
27
|
+
|
28
|
+
def attributes=(attrs)
|
29
|
+
@attributes['color'] = attrs[:color] || attrs['color']
|
30
|
+
@attributes['id'] = attrs[:id] || attrs['id']
|
31
|
+
end
|
32
|
+
|
33
|
+
def attributes
|
34
|
+
@attributes
|
35
|
+
end
|
36
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: active_model_archive
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.9.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,11 +9,11 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2011-
|
12
|
+
date: 2011-11-03 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: activesupport
|
16
|
-
requirement: &
|
16
|
+
requirement: &70251111770820 !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - ~>
|
@@ -21,10 +21,10 @@ dependencies:
|
|
21
21
|
version: '3'
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
|
-
version_requirements: *
|
24
|
+
version_requirements: *70251111770820
|
25
25
|
- !ruby/object:Gem::Dependency
|
26
26
|
name: yajl-ruby
|
27
|
-
requirement: &
|
27
|
+
requirement: &70251111770120 !ruby/object:Gem::Requirement
|
28
28
|
none: false
|
29
29
|
requirements:
|
30
30
|
- - ! '>='
|
@@ -32,18 +32,7 @@ dependencies:
|
|
32
32
|
version: '0'
|
33
33
|
type: :runtime
|
34
34
|
prerelease: false
|
35
|
-
version_requirements: *
|
36
|
-
- !ruby/object:Gem::Dependency
|
37
|
-
name: rspec
|
38
|
-
requirement: &2164668500 !ruby/object:Gem::Requirement
|
39
|
-
none: false
|
40
|
-
requirements:
|
41
|
-
- - ~>
|
42
|
-
- !ruby/object:Gem::Version
|
43
|
-
version: '2'
|
44
|
-
type: :development
|
45
|
-
prerelease: false
|
46
|
-
version_requirements: *2164668500
|
35
|
+
version_requirements: *70251111770120
|
47
36
|
description: Adds dump and restore functionality to activemodel classes
|
48
37
|
email:
|
49
38
|
- grantr@gmail.com
|
@@ -57,9 +46,12 @@ files:
|
|
57
46
|
- Rakefile
|
58
47
|
- active_model_archive.gemspec
|
59
48
|
- lib/active_model_archive.rb
|
60
|
-
- lib/active_model_archive/
|
61
|
-
-
|
62
|
-
-
|
49
|
+
- lib/active_model_archive/dump.rb
|
50
|
+
- lib/active_model_archive/file_manager.rb
|
51
|
+
- lib/active_model_archive/restore.rb
|
52
|
+
- test/dump_test.rb
|
53
|
+
- test/restore_test.rb
|
54
|
+
- test/test_helper.rb
|
63
55
|
homepage: ''
|
64
56
|
licenses: []
|
65
57
|
post_install_message:
|
@@ -80,10 +72,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
80
72
|
version: '0'
|
81
73
|
requirements: []
|
82
74
|
rubyforge_project: active_model_archive
|
83
|
-
rubygems_version: 1.8.
|
75
|
+
rubygems_version: 1.8.10
|
84
76
|
signing_key:
|
85
77
|
specification_version: 3
|
86
78
|
summary: Dump and restore activemodel objects
|
87
79
|
test_files:
|
88
|
-
-
|
89
|
-
-
|
80
|
+
- test/dump_test.rb
|
81
|
+
- test/restore_test.rb
|
82
|
+
- test/test_helper.rb
|
data/spec/archive_spec.rb
DELETED
data/spec/spec_helper.rb
DELETED
@@ -1,12 +0,0 @@
|
|
1
|
-
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
2
|
-
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
3
|
-
require 'rspec'
|
4
|
-
require 'active_model_archive'
|
5
|
-
|
6
|
-
# Requires supporting files with custom matchers and macros, etc,
|
7
|
-
# in ./support/ and its subdirectories.
|
8
|
-
Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
|
9
|
-
|
10
|
-
RSpec.configure do |config|
|
11
|
-
config.mock_with :rspec
|
12
|
-
end
|