obuf 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/.autotest ADDED
@@ -0,0 +1,23 @@
1
+ # -*- ruby -*-
2
+
3
+ require 'autotest/restart'
4
+
5
+ # Autotest.add_hook :initialize do |at|
6
+ # at.extra_files << "../some/external/dependency.rb"
7
+ #
8
+ # at.libs << ":../some/external"
9
+ #
10
+ # at.add_exception 'vendor'
11
+ #
12
+ # at.add_mapping(/dependency.rb/) do |f, _|
13
+ # at.files_matching(/test_.*rb$/)
14
+ # end
15
+ #
16
+ # %w(TestA TestB).each do |klass|
17
+ # at.extra_class_map[klass] = "test/test_misc.rb"
18
+ # end
19
+ # end
20
+
21
+ # Autotest.add_hook :run_command do |at|
22
+ # system "rake build"
23
+ # end
data/.gemtest ADDED
File without changes
data/History.txt ADDED
@@ -0,0 +1,6 @@
1
+ === 1.0.0 / 2011-08-02
2
+
3
+ * 1 major enhancement
4
+
5
+ * Birthday!
6
+
data/Manifest.txt ADDED
@@ -0,0 +1,7 @@
1
+ .autotest
2
+ History.txt
3
+ Manifest.txt
4
+ README.rdoc
5
+ Rakefile
6
+ lib/obuf.rb
7
+ test/test_obuf.rb
data/README.rdoc ADDED
@@ -0,0 +1,53 @@
1
+ = obuf
2
+
3
+ * http://github.com/julik/obuf
4
+
5
+ == DESCRIPTION:
6
+
7
+ A simple Ruby object buffer. Use this if you need to temporarily store alot of serializable Ruby objects.
8
+
9
+ == FEATURES/PROBLEMS:
10
+
11
+ * Thread safe (for as far as I can gauge it)
12
+ * Gives both read and write access to the storage
13
+
14
+ == SYNOPSIS:
15
+
16
+ obuf = Obuf.new
17
+ 5_000_000.times{ obuf.push(compute_some_object) } # no memory inflation
18
+ obuf.each do | stored_object | # objects are restored one by one
19
+ # do something with stored_object
20
+ end
21
+
22
+ == REQUIREMENTS:
23
+
24
+ * Ruby 1.8.6+
25
+
26
+ == INSTALL:
27
+
28
+ * gem install obuf
29
+
30
+ == LICENSE:
31
+
32
+ (The MIT License)
33
+
34
+ Copyright (c) 2011 Julik Tarkhanov <me@julik.nl>
35
+
36
+ Permission is hereby granted, free of charge, to any person obtaining
37
+ a copy of this software and associated documentation files (the
38
+ 'Software'), to deal in the Software without restriction, including
39
+ without limitation the rights to use, copy, modify, merge, publish,
40
+ distribute, sublicense, and/or sell copies of the Software, and to
41
+ permit persons to whom the Software is furnished to do so, subject to
42
+ the following conditions:
43
+
44
+ The above copyright notice and this permission notice shall be
45
+ included in all copies or substantial portions of the Software.
46
+
47
+ THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
48
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
49
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
50
+ IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
51
+ CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
52
+ TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
53
+ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/Rakefile ADDED
@@ -0,0 +1,15 @@
1
+ # -*- ruby -*-
2
+
3
+ require 'rubygems'
4
+ require 'hoe'
5
+
6
+ Hoe.spec 'obuf' do | p |
7
+ p.developer('Julik Tarkhanov', 'me@julik.nl')
8
+
9
+ p.readme_file = 'README.rdoc'
10
+ p.extra_rdoc_files = FileList['*.rdoc'] + FileList['*.txt']
11
+ p.extra_dev_deps = {"flexmock" => "~> 0.8", "cli_test" => "~>1.0"}
12
+ p.clean_globs = File.read(File.dirname(__FILE__) + "/.gitignore").split(/\s/).to_a
13
+ end
14
+
15
+ # vim: syntax=ruby
data/lib/obuf.rb ADDED
@@ -0,0 +1,127 @@
1
+ require "tempfile"
2
+
3
+ # An object buffer for Ruby objects. Use it to sequentially store a shitload
4
+ # of objects on disk and then retreive them one by one. Make sure to call clear when done
5
+ # with it to discard the stored blob. It can be used like a disk-based object buffer.
6
+ # (Tracksperanto stores parsed trackers into it)
7
+ #
8
+ # a = Tracksperanto::Accumulator.new
9
+ # parse_big_file do | one_node |
10
+ # a.push(one_node)
11
+ # end
12
+ #
13
+ # a.size #=> 30932 # We've stored 30 thousand objects on disk without breaking a sweat
14
+ # a.each do | node_read_from_disk |
15
+ # # do something with node that has been recovered from disk
16
+ # end
17
+ #
18
+ # a.clear # ensure that the file is deleted
19
+ #
20
+ # Both reading and writing aim to be threadsafe (writing operations will be locked with a mutex, and )
21
+ class Obuf
22
+ VERSION = "1.0.0"
23
+
24
+ include Enumerable
25
+
26
+ DELIM = "\t"
27
+ END_RECORD = "\n"
28
+
29
+ # Returns the number of objects stored so far
30
+ attr_reader :size
31
+
32
+ def initialize
33
+ @sem = Mutex.new
34
+ @store = create_store
35
+ @store.set_encoding(Encoding::BINARY) if @store.respond_to?(:set_encoding)
36
+ @store.binmode
37
+
38
+ @size = 0
39
+ super
40
+ end
41
+
42
+ # Tells whether the buffer is empty
43
+ def empty?
44
+ @size.zero?
45
+ end
46
+
47
+ # Store an object
48
+ def push(object_to_store)
49
+ blob = marshal_object(object_to_store)
50
+ @sem.synchronize do
51
+ @store.write(blob.size)
52
+ @store.write(DELIM)
53
+ @store.write(blob)
54
+ @store.write(END_RECORD)
55
+ @size += 1
56
+ end
57
+ object_to_store
58
+ end
59
+
60
+ alias_method :<<, :push
61
+
62
+ # Retreive each stored object in succession. All other Enumerable
63
+ # methods are also available (but be careful with Enumerable#map and to_a)
64
+ def each
65
+ with_separate_read_io do | iterable |
66
+ @size.times { yield(recover_object_from(iterable)) }
67
+ end
68
+ end
69
+
70
+ # Calls close! on the datastore and deletes the objects in it
71
+ def clear
72
+ @store.close!
73
+ @size = 0
74
+ end
75
+
76
+ # Retreive a concrete object at index
77
+ def [](idx)
78
+ idx.respond_to?(:each) ? idx.map{|i| recover_at(i) } : recover_at(idx)
79
+ end
80
+
81
+ private
82
+
83
+ def recover_at(idx)
84
+ with_separate_read_io do | iterable |
85
+ iterable.seek(0)
86
+
87
+ # Do not unmarshal anything but wind the IO in fixed offsets
88
+ idx.times do
89
+ skip_bytes = iterable.gets("\t").to_i
90
+ iterable.seek(iterable.pos + skip_bytes)
91
+ end
92
+
93
+ recover_object_from(iterable)
94
+ end
95
+ end
96
+
97
+ # We first ensure that we have a disk-backed file, then reopen it as read-only
98
+ # and iterate through that (we will have one IO handle per loop nest)
99
+ def with_separate_read_io
100
+ # Ensure all data is written before we read it
101
+ @sem.synchronize { @store.flush }
102
+
103
+ iterable = File.open(@store.path, "r")
104
+ begin
105
+ yield(iterable)
106
+ ensure
107
+ iterable.close
108
+ end
109
+ end
110
+
111
+ def marshal_object(object_to_store)
112
+ d = Marshal.dump(object_to_store)
113
+ end
114
+
115
+ def recover_object_from(io)
116
+ # Up to the tab is the amount of bytes to read
117
+ demarshal_bytes = io.gets("\t").to_i
118
+ blob = io.read(demarshal_bytes)
119
+
120
+ Marshal.load(blob)
121
+ end
122
+
123
+ # In case you need to use something else (say, a StringIO)
124
+ def create_store
125
+ Tempfile.new("obuf")
126
+ end
127
+ end
data/test/test_obuf.rb ADDED
@@ -0,0 +1,85 @@
1
+ require "test/unit"
2
+ require "obuf"
3
+ require "flexmock"
4
+ require "flexmock/test_unit"
5
+
6
+ # http://redmine.ruby-lang.org/issues/4882
7
+ # https://github.com/jimweirich/flexmock/issues/4
8
+ # https://github.com/julik/flexmock/commit/4acea00677e7b558bd564ec7c7630f0b27d368ca
9
+ class FlexMock::PartialMockProxy
10
+ def singleton?(method_name)
11
+ @obj.singleton_methods.include?(method_name.to_s)
12
+ end
13
+ end
14
+
15
+ class TestObuf < Test::Unit::TestCase
16
+ def test_accumulator_saves_objs
17
+ a = Obuf.new
18
+ values = [3, {:foo => "bar"}, "foo"]
19
+ values.map(&a.method(:push))
20
+
21
+ assert_equal 3, a.size
22
+ assert_equal values, a.map{|e| e }, "Should return the same elements from the storage"
23
+ end
24
+
25
+ def test_accumulator_saves_shitload_of_objs
26
+ a = Obuf.new
27
+ 50_000.times { a.push("A string" => rand) }
28
+ assert_equal 50_000, a.size
29
+ end
30
+
31
+ def test_accumulator_saves_few_strings_with_a_tab
32
+ a = Obuf.new
33
+ 4.times { a.push("A \tstring") }
34
+ a.each {|e| assert_equal "A \tstring", e }
35
+ end
36
+
37
+ def test_accumulator_empty
38
+ a = Obuf.new
39
+ assert a.empty?
40
+ a.push(1)
41
+ assert !a.empty?
42
+ end
43
+
44
+ def test_accumulator_supports_nested_iteration
45
+ a = Obuf.new
46
+ ("A".."Z").each{|e| a << e}
47
+
48
+ accumulated = []
49
+ seen_g = false
50
+ a.each do | first_level |
51
+ if first_level == "G"
52
+ seen_g = true
53
+ # Force a nested iteration and break it midway
54
+ a.each do | second_level |
55
+ accumulated.push(second_level)
56
+ break if second_level == "E"
57
+ end
58
+ elsif seen_g
59
+ assert_equal "H", first_level
60
+ return
61
+ end
62
+ end
63
+ end
64
+
65
+ def test_random_access
66
+ a = Obuf.new
67
+ letters = ("A".."Z").map{|e| "#{e}\r\nWow!"}.to_a
68
+ letters.map(&a.method(:push))
69
+
70
+ assert_equal "B\r\nWow!", a[1]
71
+ assert_equal "E\r\nWow!", a[4]
72
+ end
73
+
74
+ def test_clear_calls_close_on_buffer
75
+ io = Tempfile.new("testing")
76
+ flexmock(io).should_receive(:close!).once
77
+ flexmock(Tempfile).should_receive(:new).once.with("obuf").and_return(io)
78
+
79
+ a = Obuf.new
80
+ 40.times { a.push("A string" => rand) }
81
+ assert_equal 40, a.size
82
+ a.clear
83
+ assert_equal 0, a.size
84
+ end
85
+ end
metadata ADDED
@@ -0,0 +1,97 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: obuf
3
+ version: !ruby/object:Gem::Version
4
+ prerelease:
5
+ version: 1.0.0
6
+ platform: ruby
7
+ authors:
8
+ - Julik Tarkhanov
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+
13
+ date: 2011-08-02 00:00:00 Z
14
+ dependencies:
15
+ - !ruby/object:Gem::Dependency
16
+ name: flexmock
17
+ prerelease: false
18
+ requirement: &id001 !ruby/object:Gem::Requirement
19
+ none: false
20
+ requirements:
21
+ - - ~>
22
+ - !ruby/object:Gem::Version
23
+ version: "0.8"
24
+ type: :development
25
+ version_requirements: *id001
26
+ - !ruby/object:Gem::Dependency
27
+ name: cli_test
28
+ prerelease: false
29
+ requirement: &id002 !ruby/object:Gem::Requirement
30
+ none: false
31
+ requirements:
32
+ - - ~>
33
+ - !ruby/object:Gem::Version
34
+ version: "1.0"
35
+ type: :development
36
+ version_requirements: *id002
37
+ - !ruby/object:Gem::Dependency
38
+ name: hoe
39
+ prerelease: false
40
+ requirement: &id003 !ruby/object:Gem::Requirement
41
+ none: false
42
+ requirements:
43
+ - - ~>
44
+ - !ruby/object:Gem::Version
45
+ version: "2.10"
46
+ type: :development
47
+ version_requirements: *id003
48
+ description: A simple Ruby object buffer. Use this if you need to temporarily store alot of serializable Ruby objects.
49
+ email:
50
+ - me@julik.nl
51
+ executables: []
52
+
53
+ extensions: []
54
+
55
+ extra_rdoc_files:
56
+ - History.txt
57
+ - Manifest.txt
58
+ - README.rdoc
59
+ files:
60
+ - .autotest
61
+ - History.txt
62
+ - Manifest.txt
63
+ - README.rdoc
64
+ - Rakefile
65
+ - lib/obuf.rb
66
+ - test/test_obuf.rb
67
+ - .gemtest
68
+ homepage: http://github.com/julik/obuf
69
+ licenses: []
70
+
71
+ post_install_message:
72
+ rdoc_options:
73
+ - --main
74
+ - README.rdoc
75
+ require_paths:
76
+ - lib
77
+ required_ruby_version: !ruby/object:Gem::Requirement
78
+ none: false
79
+ requirements:
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: "0"
83
+ required_rubygems_version: !ruby/object:Gem::Requirement
84
+ none: false
85
+ requirements:
86
+ - - ">="
87
+ - !ruby/object:Gem::Version
88
+ version: "0"
89
+ requirements: []
90
+
91
+ rubyforge_project: obuf
92
+ rubygems_version: 1.8.5
93
+ signing_key:
94
+ specification_version: 3
95
+ summary: A simple Ruby object buffer
96
+ test_files:
97
+ - test/test_obuf.rb