obuf 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.autotest +23 -0
- data/.gemtest +0 -0
- data/History.txt +6 -0
- data/Manifest.txt +7 -0
- data/README.rdoc +53 -0
- data/Rakefile +15 -0
- data/lib/obuf.rb +127 -0
- data/test/test_obuf.rb +85 -0
- metadata +97 -0
data/.autotest
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
# -*- ruby -*-
|
2
|
+
|
3
|
+
require 'autotest/restart'
|
4
|
+
|
5
|
+
# Autotest.add_hook :initialize do |at|
|
6
|
+
# at.extra_files << "../some/external/dependency.rb"
|
7
|
+
#
|
8
|
+
# at.libs << ":../some/external"
|
9
|
+
#
|
10
|
+
# at.add_exception 'vendor'
|
11
|
+
#
|
12
|
+
# at.add_mapping(/dependency.rb/) do |f, _|
|
13
|
+
# at.files_matching(/test_.*rb$/)
|
14
|
+
# end
|
15
|
+
#
|
16
|
+
# %w(TestA TestB).each do |klass|
|
17
|
+
# at.extra_class_map[klass] = "test/test_misc.rb"
|
18
|
+
# end
|
19
|
+
# end
|
20
|
+
|
21
|
+
# Autotest.add_hook :run_command do |at|
|
22
|
+
# system "rake build"
|
23
|
+
# end
|
data/.gemtest
ADDED
File without changes
|
data/History.txt
ADDED
data/Manifest.txt
ADDED
data/README.rdoc
ADDED
@@ -0,0 +1,53 @@
|
|
1
|
+
= obuf
|
2
|
+
|
3
|
+
* http://github.com/julik/obuf
|
4
|
+
|
5
|
+
== DESCRIPTION:
|
6
|
+
|
7
|
+
A simple Ruby object buffer. Use this if you need to temporarily store alot of serializable Ruby objects.
|
8
|
+
|
9
|
+
== FEATURES/PROBLEMS:
|
10
|
+
|
11
|
+
* Thread safe (for as far as I can gauge it)
|
12
|
+
* Gives both read and write access to the storage
|
13
|
+
|
14
|
+
== SYNOPSIS:
|
15
|
+
|
16
|
+
obuf = Obuf.new
|
17
|
+
5_000_000.times{ obuf.push(compute_some_object) } # no memory inflation
|
18
|
+
obuf.each do | stored_object | # objects are restored one by one
|
19
|
+
# do something with stored_object
|
20
|
+
end
|
21
|
+
|
22
|
+
== REQUIREMENTS:
|
23
|
+
|
24
|
+
* Ruby 1.8.6+
|
25
|
+
|
26
|
+
== INSTALL:
|
27
|
+
|
28
|
+
* gem install obuf
|
29
|
+
|
30
|
+
== LICENSE:
|
31
|
+
|
32
|
+
(The MIT License)
|
33
|
+
|
34
|
+
Copyright (c) 2011 Julik Tarkhanov <me@julik.nl>
|
35
|
+
|
36
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
37
|
+
a copy of this software and associated documentation files (the
|
38
|
+
'Software'), to deal in the Software without restriction, including
|
39
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
40
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
41
|
+
permit persons to whom the Software is furnished to do so, subject to
|
42
|
+
the following conditions:
|
43
|
+
|
44
|
+
The above copyright notice and this permission notice shall be
|
45
|
+
included in all copies or substantial portions of the Software.
|
46
|
+
|
47
|
+
THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
|
48
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
49
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
50
|
+
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
51
|
+
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
52
|
+
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
53
|
+
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/Rakefile
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
# -*- ruby -*-
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'hoe'
|
5
|
+
|
6
|
+
Hoe.spec 'obuf' do | p |
|
7
|
+
p.developer('Julik Tarkhanov', 'me@julik.nl')
|
8
|
+
|
9
|
+
p.readme_file = 'README.rdoc'
|
10
|
+
p.extra_rdoc_files = FileList['*.rdoc'] + FileList['*.txt']
|
11
|
+
p.extra_dev_deps = {"flexmock" => "~> 0.8", "cli_test" => "~>1.0"}
|
12
|
+
p.clean_globs = File.read(File.dirname(__FILE__) + "/.gitignore").split(/\s/).to_a
|
13
|
+
end
|
14
|
+
|
15
|
+
# vim: syntax=ruby
|
data/lib/obuf.rb
ADDED
@@ -0,0 +1,127 @@
|
|
1
|
+
require "tempfile"
|
2
|
+
|
3
|
+
# An object buffer for Ruby objects. Use it to sequentially store a shitload
|
4
|
+
# of objects on disk and then retreive them one by one. Make sure to call clear when done
|
5
|
+
# with it to discard the stored blob. It can be used like a disk-based object buffer.
|
6
|
+
# (Tracksperanto stores parsed trackers into it)
|
7
|
+
#
|
8
|
+
# a = Tracksperanto::Accumulator.new
|
9
|
+
# parse_big_file do | one_node |
|
10
|
+
# a.push(one_node)
|
11
|
+
# end
|
12
|
+
#
|
13
|
+
# a.size #=> 30932 # We've stored 30 thousand objects on disk without breaking a sweat
|
14
|
+
# a.each do | node_read_from_disk |
|
15
|
+
# # do something with node that has been recovered from disk
|
16
|
+
# end
|
17
|
+
#
|
18
|
+
# a.clear # ensure that the file is deleted
|
19
|
+
#
|
20
|
+
# Both reading and writing aim to be threadsafe (writing operations will be locked with a mutex, and )
|
21
|
+
class Obuf
|
22
|
+
VERSION = "1.0.0"
|
23
|
+
|
24
|
+
include Enumerable
|
25
|
+
|
26
|
+
DELIM = "\t"
|
27
|
+
END_RECORD = "\n"
|
28
|
+
|
29
|
+
# Returns the number of objects stored so far
|
30
|
+
attr_reader :size
|
31
|
+
|
32
|
+
def initialize
|
33
|
+
@sem = Mutex.new
|
34
|
+
@store = create_store
|
35
|
+
@store.set_encoding(Encoding::BINARY) if @store.respond_to?(:set_encoding)
|
36
|
+
@store.binmode
|
37
|
+
|
38
|
+
@size = 0
|
39
|
+
super
|
40
|
+
end
|
41
|
+
|
42
|
+
# Tells whether the buffer is empty
|
43
|
+
def empty?
|
44
|
+
@size.zero?
|
45
|
+
end
|
46
|
+
|
47
|
+
# Store an object
|
48
|
+
def push(object_to_store)
|
49
|
+
blob = marshal_object(object_to_store)
|
50
|
+
@sem.synchronize do
|
51
|
+
@store.write(blob.size)
|
52
|
+
@store.write(DELIM)
|
53
|
+
@store.write(blob)
|
54
|
+
@store.write(END_RECORD)
|
55
|
+
@size += 1
|
56
|
+
end
|
57
|
+
object_to_store
|
58
|
+
end
|
59
|
+
|
60
|
+
alias_method :<<, :push
|
61
|
+
|
62
|
+
# Retreive each stored object in succession. All other Enumerable
|
63
|
+
# methods are also available (but be careful with Enumerable#map and to_a)
|
64
|
+
def each
|
65
|
+
with_separate_read_io do | iterable |
|
66
|
+
@size.times { yield(recover_object_from(iterable)) }
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
# Calls close! on the datastore and deletes the objects in it
|
71
|
+
def clear
|
72
|
+
@store.close!
|
73
|
+
@size = 0
|
74
|
+
end
|
75
|
+
|
76
|
+
# Retreive a concrete object at index
|
77
|
+
def [](idx)
|
78
|
+
idx.respond_to?(:each) ? idx.map{|i| recover_at(i) } : recover_at(idx)
|
79
|
+
end
|
80
|
+
|
81
|
+
private
|
82
|
+
|
83
|
+
def recover_at(idx)
|
84
|
+
with_separate_read_io do | iterable |
|
85
|
+
iterable.seek(0)
|
86
|
+
|
87
|
+
# Do not unmarshal anything but wind the IO in fixed offsets
|
88
|
+
idx.times do
|
89
|
+
skip_bytes = iterable.gets("\t").to_i
|
90
|
+
iterable.seek(iterable.pos + skip_bytes)
|
91
|
+
end
|
92
|
+
|
93
|
+
recover_object_from(iterable)
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
# We first ensure that we have a disk-backed file, then reopen it as read-only
|
98
|
+
# and iterate through that (we will have one IO handle per loop nest)
|
99
|
+
def with_separate_read_io
|
100
|
+
# Ensure all data is written before we read it
|
101
|
+
@sem.synchronize { @store.flush }
|
102
|
+
|
103
|
+
iterable = File.open(@store.path, "r")
|
104
|
+
begin
|
105
|
+
yield(iterable)
|
106
|
+
ensure
|
107
|
+
iterable.close
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
def marshal_object(object_to_store)
|
112
|
+
d = Marshal.dump(object_to_store)
|
113
|
+
end
|
114
|
+
|
115
|
+
def recover_object_from(io)
|
116
|
+
# Up to the tab is the amount of bytes to read
|
117
|
+
demarshal_bytes = io.gets("\t").to_i
|
118
|
+
blob = io.read(demarshal_bytes)
|
119
|
+
|
120
|
+
Marshal.load(blob)
|
121
|
+
end
|
122
|
+
|
123
|
+
# In case you need to use something else (say, a StringIO)
|
124
|
+
def create_store
|
125
|
+
Tempfile.new("obuf")
|
126
|
+
end
|
127
|
+
end
|
data/test/test_obuf.rb
ADDED
@@ -0,0 +1,85 @@
|
|
1
|
+
require "test/unit"
|
2
|
+
require "obuf"
|
3
|
+
require "flexmock"
|
4
|
+
require "flexmock/test_unit"
|
5
|
+
|
6
|
+
# http://redmine.ruby-lang.org/issues/4882
|
7
|
+
# https://github.com/jimweirich/flexmock/issues/4
|
8
|
+
# https://github.com/julik/flexmock/commit/4acea00677e7b558bd564ec7c7630f0b27d368ca
|
9
|
+
class FlexMock::PartialMockProxy
|
10
|
+
def singleton?(method_name)
|
11
|
+
@obj.singleton_methods.include?(method_name.to_s)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
class TestObuf < Test::Unit::TestCase
|
16
|
+
def test_accumulator_saves_objs
|
17
|
+
a = Obuf.new
|
18
|
+
values = [3, {:foo => "bar"}, "foo"]
|
19
|
+
values.map(&a.method(:push))
|
20
|
+
|
21
|
+
assert_equal 3, a.size
|
22
|
+
assert_equal values, a.map{|e| e }, "Should return the same elements from the storage"
|
23
|
+
end
|
24
|
+
|
25
|
+
def test_accumulator_saves_shitload_of_objs
|
26
|
+
a = Obuf.new
|
27
|
+
50_000.times { a.push("A string" => rand) }
|
28
|
+
assert_equal 50_000, a.size
|
29
|
+
end
|
30
|
+
|
31
|
+
def test_accumulator_saves_few_strings_with_a_tab
|
32
|
+
a = Obuf.new
|
33
|
+
4.times { a.push("A \tstring") }
|
34
|
+
a.each {|e| assert_equal "A \tstring", e }
|
35
|
+
end
|
36
|
+
|
37
|
+
def test_accumulator_empty
|
38
|
+
a = Obuf.new
|
39
|
+
assert a.empty?
|
40
|
+
a.push(1)
|
41
|
+
assert !a.empty?
|
42
|
+
end
|
43
|
+
|
44
|
+
def test_accumulator_supports_nested_iteration
|
45
|
+
a = Obuf.new
|
46
|
+
("A".."Z").each{|e| a << e}
|
47
|
+
|
48
|
+
accumulated = []
|
49
|
+
seen_g = false
|
50
|
+
a.each do | first_level |
|
51
|
+
if first_level == "G"
|
52
|
+
seen_g = true
|
53
|
+
# Force a nested iteration and break it midway
|
54
|
+
a.each do | second_level |
|
55
|
+
accumulated.push(second_level)
|
56
|
+
break if second_level == "E"
|
57
|
+
end
|
58
|
+
elsif seen_g
|
59
|
+
assert_equal "H", first_level
|
60
|
+
return
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
def test_random_access
|
66
|
+
a = Obuf.new
|
67
|
+
letters = ("A".."Z").map{|e| "#{e}\r\nWow!"}.to_a
|
68
|
+
letters.map(&a.method(:push))
|
69
|
+
|
70
|
+
assert_equal "B\r\nWow!", a[1]
|
71
|
+
assert_equal "E\r\nWow!", a[4]
|
72
|
+
end
|
73
|
+
|
74
|
+
def test_clear_calls_close_on_buffer
|
75
|
+
io = Tempfile.new("testing")
|
76
|
+
flexmock(io).should_receive(:close!).once
|
77
|
+
flexmock(Tempfile).should_receive(:new).once.with("obuf").and_return(io)
|
78
|
+
|
79
|
+
a = Obuf.new
|
80
|
+
40.times { a.push("A string" => rand) }
|
81
|
+
assert_equal 40, a.size
|
82
|
+
a.clear
|
83
|
+
assert_equal 0, a.size
|
84
|
+
end
|
85
|
+
end
|
metadata
ADDED
@@ -0,0 +1,97 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: obuf
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
prerelease:
|
5
|
+
version: 1.0.0
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Julik Tarkhanov
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
|
13
|
+
date: 2011-08-02 00:00:00 Z
|
14
|
+
dependencies:
|
15
|
+
- !ruby/object:Gem::Dependency
|
16
|
+
name: flexmock
|
17
|
+
prerelease: false
|
18
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
19
|
+
none: false
|
20
|
+
requirements:
|
21
|
+
- - ~>
|
22
|
+
- !ruby/object:Gem::Version
|
23
|
+
version: "0.8"
|
24
|
+
type: :development
|
25
|
+
version_requirements: *id001
|
26
|
+
- !ruby/object:Gem::Dependency
|
27
|
+
name: cli_test
|
28
|
+
prerelease: false
|
29
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
30
|
+
none: false
|
31
|
+
requirements:
|
32
|
+
- - ~>
|
33
|
+
- !ruby/object:Gem::Version
|
34
|
+
version: "1.0"
|
35
|
+
type: :development
|
36
|
+
version_requirements: *id002
|
37
|
+
- !ruby/object:Gem::Dependency
|
38
|
+
name: hoe
|
39
|
+
prerelease: false
|
40
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ~>
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: "2.10"
|
46
|
+
type: :development
|
47
|
+
version_requirements: *id003
|
48
|
+
description: A simple Ruby object buffer. Use this if you need to temporarily store alot of serializable Ruby objects.
|
49
|
+
email:
|
50
|
+
- me@julik.nl
|
51
|
+
executables: []
|
52
|
+
|
53
|
+
extensions: []
|
54
|
+
|
55
|
+
extra_rdoc_files:
|
56
|
+
- History.txt
|
57
|
+
- Manifest.txt
|
58
|
+
- README.rdoc
|
59
|
+
files:
|
60
|
+
- .autotest
|
61
|
+
- History.txt
|
62
|
+
- Manifest.txt
|
63
|
+
- README.rdoc
|
64
|
+
- Rakefile
|
65
|
+
- lib/obuf.rb
|
66
|
+
- test/test_obuf.rb
|
67
|
+
- .gemtest
|
68
|
+
homepage: http://github.com/julik/obuf
|
69
|
+
licenses: []
|
70
|
+
|
71
|
+
post_install_message:
|
72
|
+
rdoc_options:
|
73
|
+
- --main
|
74
|
+
- README.rdoc
|
75
|
+
require_paths:
|
76
|
+
- lib
|
77
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
78
|
+
none: false
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: "0"
|
83
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
84
|
+
none: false
|
85
|
+
requirements:
|
86
|
+
- - ">="
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: "0"
|
89
|
+
requirements: []
|
90
|
+
|
91
|
+
rubyforge_project: obuf
|
92
|
+
rubygems_version: 1.8.5
|
93
|
+
signing_key:
|
94
|
+
specification_version: 3
|
95
|
+
summary: A simple Ruby object buffer
|
96
|
+
test_files:
|
97
|
+
- test/test_obuf.rb
|