hyper_record 0.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +3 -0
- data/CHANGELOG +83 -0
- data/LICENSE +20 -0
- data/README +49 -0
- data/Rakefile +43 -0
- data/VERSION.yml +4 -0
- data/benchmark/save.rb +58 -0
- data/hyper_record.gemspec +76 -0
- data/init.rb +1 -0
- data/lib/active_record/connection_adapters/hyper_table_definition.rb +26 -0
- data/lib/active_record/connection_adapters/hypertable_adapter.rb +680 -0
- data/lib/active_record/connection_adapters/qualified_column.rb +57 -0
- data/lib/associations/hyper_has_and_belongs_to_many_association_extension.rb +107 -0
- data/lib/associations/hyper_has_many_association_extension.rb +87 -0
- data/lib/hyper_record.rb +636 -0
- data/lib/hypertable/gen-rb/client_constants.rb +12 -0
- data/lib/hypertable/gen-rb/client_service.rb +1436 -0
- data/lib/hypertable/gen-rb/client_types.rb +253 -0
- data/lib/hypertable/gen-rb/hql_constants.rb +12 -0
- data/lib/hypertable/gen-rb/hql_service.rb +281 -0
- data/lib/hypertable/gen-rb/hql_types.rb +73 -0
- data/lib/hypertable/thrift_client.rb +94 -0
- data/lib/hypertable/thrift_transport_monkey_patch.rb +29 -0
- data/pkg/hyper_record-0.2.8.gem +0 -0
- data/spec/fixtures/pages.yml +8 -0
- data/spec/fixtures/qualified_pages.yml +1 -0
- data/spec/lib/associations_spec.rb +235 -0
- data/spec/lib/hyper_record_spec.rb +948 -0
- data/spec/lib/hypertable_adapter_spec.rb +121 -0
- data/spec/spec_helper.rb +130 -0
- data/test/test_helper.rb +10 -0
- data/test/thrift_client_test.rb +590 -0
- metadata +99 -0
data/.gitignore
ADDED
data/CHANGELOG
ADDED
@@ -0,0 +1,83 @@
|
|
1
|
+
0.2.8 (2010/03/01)
|
2
|
+
- support friendly migration-syntax declared using blocks
|
3
|
+
- new assemble_row_key_from_attributes that generates row keys in the same
|
4
|
+
order declared in row_key_attributes
|
5
|
+
- cache row_key_attributes results so that regex is not constantly
|
6
|
+
reevaluated for same value
|
7
|
+
- new :instantiate_only_requested_columns finder option that does not
|
8
|
+
instantiate object columns if they were not included in the :select
|
9
|
+
- makes large queries against large tables memory-friendly
|
10
|
+
- :select option allows column families to be identified using symbols
|
11
|
+
- tests and groundwork for mutator flush_interval (asynchronous writes)
|
12
|
+
- requires Hypertable 0.9.2.7
|
13
|
+
- fix bug in with_thrift_client invocation
|
14
|
+
- update tests to be compatible with Rails 2.3.4
|
15
|
+
- update tests to be compatible with RSpec 1.2.8
|
16
|
+
|
17
|
+
0.2.7 2009/10/01
|
18
|
+
- new row_key_attributes method: allows parts of the row key to be extracted
|
19
|
+
into attributes on the instantiated object
|
20
|
+
- retry block of write_cells was occasionally attemption to write to a
|
21
|
+
closed mutator (causing an exception)
|
22
|
+
- support mutators with auto-flush
|
23
|
+
- update Thrift-generated ruby code to code supplied in Hypertable 0.9.2.6
|
24
|
+
- some exceptions in write_cells method were being masked by exceptions in
|
25
|
+
ensure block, making it difficult to identify root cause of problem
|
26
|
+
- fix potential infinite loop in each_row
|
27
|
+
- handle deleted columns in schema operations
|
28
|
+
- hypertable deletes columns lazily, so they will still show up in the output
|
29
|
+
from schema commands.
|
30
|
+
|
31
|
+
0.2.6 2009/04/20
|
32
|
+
- fix bug interacting with new open_scanner API
|
33
|
+
|
34
|
+
0.2.5 2009/04/20
|
35
|
+
- added methods to raw ruby thrift client for iterating over large results
|
36
|
+
- each_cell_as_arrays
|
37
|
+
- each_row (still has bug in underlying code)
|
38
|
+
- each_row_as_arrays (still has bug in underlying code)
|
39
|
+
- new HyperRecord methods for iterating over results in small batches, doesn't
|
40
|
+
exhaust memory on million-row queries
|
41
|
+
- find_each_row (returns each row as HyperRecord object)
|
42
|
+
- find_each_row_as_arrays (returns each row in native array format - fast)
|
43
|
+
- find_to_scan_spec methods (converts finder options to scan spec)
|
44
|
+
- find_with_scanner (converts finder options to a scanner)
|
45
|
+
- new finder option (:row_intervals) that retrieves cells from multiple
|
46
|
+
row intervals in a single query
|
47
|
+
- now depends on Thrift SVN revision 765279 or later.
|
48
|
+
|
49
|
+
0.2.4 2009/04/06
|
50
|
+
- find_by_hql support
|
51
|
+
- support for :select option in calls to ActiveRecord.find
|
52
|
+
- expose open_mutator, flush_mutator and close_mutator methods to HyperRecord
|
53
|
+
to help speed up write-heavy applications
|
54
|
+
- don't duplicate cells unnecessarily while assembling rows
|
55
|
+
|
56
|
+
0.2.3 on 2009/03/18
|
57
|
+
- optimize writes by using set_cells_as_arrays to write data in cell native
|
58
|
+
array form
|
59
|
+
- monkey patch borrow method of Thrift::FramedTransport needed to return
|
60
|
+
substrings as well
|
61
|
+
|
62
|
+
0.2.2 on 2009/03/17
|
63
|
+
- monkey patch Thrift::FramedTransport to fix performance issue in thrift
|
64
|
+
transport ruby code (1000x faster than default implementation)
|
65
|
+
- import and utilize new thrift client moethods that implement native
|
66
|
+
array interface for cell retrieval (100x faster than creating
|
67
|
+
Hypertable::ThriftGen::Cell objects)
|
68
|
+
|
69
|
+
0.2.1 on 2009/03/06
|
70
|
+
- add rudimentary support for timestamps [kball]
|
71
|
+
|
72
|
+
0.2.0 on 2009/03/05
|
73
|
+
- bundle hypertable_adapter and hypertable_thrift_client code to
|
74
|
+
simplify installation
|
75
|
+
- support for simple migrations in raw HQL or ActiveRecord::Migration format
|
76
|
+
- attempt to reconnect to ThirftBroker on connection errors
|
77
|
+
- intercept Thrift exceptions to make them compatible with Rails
|
78
|
+
|
79
|
+
0.1.1 on 2009/03/03
|
80
|
+
- don't use quoted table name in find_by_options
|
81
|
+
|
82
|
+
0.1.0 on 2009/02/01
|
83
|
+
- initial release with basic ActiveRecord and association features
|
data/LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright (c) 2008 tylerkovacs
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README
ADDED
@@ -0,0 +1,49 @@
|
|
1
|
+
hyper_record
|
2
|
+
============
|
3
|
+
|
4
|
+
Hypertable is a high performance distributed data storage system designed
|
5
|
+
to support applications requiring maximum performance, scalability,
|
6
|
+
and reliability. Modeled after Google's well known Bigtable project,
|
7
|
+
Hypertable is designed to manage the storage and processing of information
|
8
|
+
on a large cluster of commodity servers, providing resilience to machine
|
9
|
+
and component failures.
|
10
|
+
|
11
|
+
HyperRecord exists to integrate Hypertable with ActiveRecord (the Ruby on
|
12
|
+
Rails ORM) allowing objects to be stored in Hypertable while maintaining
|
13
|
+
as many of the built-in ORM features as possible.
|
14
|
+
|
15
|
+
See:
|
16
|
+
- http://www.hypertable.org
|
17
|
+
- http://code.google.com/p/hypertable/wiki/HyperRecord
|
18
|
+
|
19
|
+
INSTALLATION
|
20
|
+
============
|
21
|
+
|
22
|
+
1. Add github as a gem source (if you haven't already)
|
23
|
+
|
24
|
+
gem sources -a http://gems.github.com
|
25
|
+
|
26
|
+
2. Install HyperRecord gem
|
27
|
+
|
28
|
+
gem install tylerkovacs-hyper_record
|
29
|
+
|
30
|
+
3. Configure Hypertable Adapter by adding an entry to config/database.yml
|
31
|
+
that points to an instance of the Hypertable Thrift Broker (uses port
|
32
|
+
38080 by default).
|
33
|
+
|
34
|
+
hypertable:
|
35
|
+
adapter: hypertable
|
36
|
+
host: localhost
|
37
|
+
port: 38080
|
38
|
+
|
39
|
+
NOTE: HyperRecord requires Thrift SVN revision 765279 or later.
|
40
|
+
|
41
|
+
4. Add the following to the bottom of config/environment.rb:
|
42
|
+
|
43
|
+
require 'hyper_record'
|
44
|
+
|
45
|
+
|
46
|
+
COPYRIGHT
|
47
|
+
=========
|
48
|
+
|
49
|
+
Copyright (c) 2008 tylerkovacs. See LICENSE for details.
|
data/Rakefile
ADDED
@@ -0,0 +1,43 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'rake'
|
3
|
+
require 'rake/testtask'
|
4
|
+
require 'rake/rdoctask'
|
5
|
+
require 'rcov/rcovtask'
|
6
|
+
|
7
|
+
begin
|
8
|
+
require 'jeweler'
|
9
|
+
Jeweler::Tasks.new do |s|
|
10
|
+
s.name = "hyper_record"
|
11
|
+
s.summary = %Q{Fully integrates ActiveRecord with Hypertable.}
|
12
|
+
s.email = "tyler.kovacs@gmail.com"
|
13
|
+
s.homepage = "http://github.com/tylerkovacs/hyper_record"
|
14
|
+
s.description = "See README"
|
15
|
+
s.authors = ["tylerkovacs"]
|
16
|
+
end
|
17
|
+
|
18
|
+
Jeweler::GemcutterTasks.new
|
19
|
+
rescue LoadError
|
20
|
+
puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
|
21
|
+
end
|
22
|
+
|
23
|
+
Rake::TestTask.new do |t|
|
24
|
+
t.libs << 'lib'
|
25
|
+
t.pattern = 'test/**/*_test.rb'
|
26
|
+
t.verbose = false
|
27
|
+
end
|
28
|
+
|
29
|
+
Rake::RDocTask.new do |rdoc|
|
30
|
+
rdoc.rdoc_dir = 'rdoc'
|
31
|
+
rdoc.title = 'hyper_record'
|
32
|
+
rdoc.options << '--line-numbers' << '--inline-source'
|
33
|
+
rdoc.rdoc_files.include('README*')
|
34
|
+
rdoc.rdoc_files.include('lib/**/*.rb')
|
35
|
+
end
|
36
|
+
|
37
|
+
Rcov::RcovTask.new do |t|
|
38
|
+
t.libs << 'spec'
|
39
|
+
t.test_files = FileList['spec/**/*_spec.rb']
|
40
|
+
t.verbose = true
|
41
|
+
end
|
42
|
+
|
43
|
+
task :default => :rcov
|
data/VERSION.yml
ADDED
data/benchmark/save.rb
ADDED
@@ -0,0 +1,58 @@
|
|
1
|
+
require File.join(File.dirname(__FILE__), '../spec/spec_helper.rb')
|
2
|
+
|
3
|
+
# Results with flush_interval=10
|
4
|
+
# 1000 save requests in 14.879574 sec: 67.2062251244559 r/s
|
5
|
+
# hypertable write latency was 13.632525
|
6
|
+
# 1000 save_with_mutator requests in 1.624659 sec: 615.513778583691 r/s
|
7
|
+
# hypertable write latency was 0.557777
|
8
|
+
#
|
9
|
+
# Results with flush_interval=100
|
10
|
+
# 1000 save requests in 13.127422 sec: 76.1764191019379 r/s
|
11
|
+
# hypertable write latency was 11.971095
|
12
|
+
# 1000 save_with_mutator requests in 0.886474 sec: 1128.06466969138 r/s
|
13
|
+
# hypertable write latency was 0.454088000000001
|
14
|
+
|
15
|
+
class Bench < ActiveRecord::HyperBase
|
16
|
+
def self.create_table
|
17
|
+
hql = "CREATE TABLE #{table_name} (
|
18
|
+
'value' MAX_VERSIONS=1
|
19
|
+
)"
|
20
|
+
connection.execute(hql)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
Bench.drop_table if Bench.table_exists?
|
25
|
+
Bench.create_table
|
26
|
+
|
27
|
+
n = 1000
|
28
|
+
|
29
|
+
t1 = Time.now
|
30
|
+
for i in 1..n
|
31
|
+
record = Bench.new({:ROW => i, :value => i})
|
32
|
+
record.save!
|
33
|
+
end
|
34
|
+
ts1 = Time.now - t1
|
35
|
+
|
36
|
+
read_latency, write_latency, cells_read = ActiveRecord::ConnectionAdapters::HypertableAdapter.get_timing
|
37
|
+
ActiveRecord::ConnectionAdapters::HypertableAdapter.reset_timing
|
38
|
+
puts "#{n} save requests in #{ts1} sec: #{n/ts1} r/s"
|
39
|
+
puts " hypertable write latency was #{write_latency}"
|
40
|
+
|
41
|
+
Bench.drop_table if Bench.table_exists?
|
42
|
+
Bench.create_table
|
43
|
+
|
44
|
+
flush_interval = 100
|
45
|
+
t2 = Time.now
|
46
|
+
m = Bench.open_mutator
|
47
|
+
for i in 1..n
|
48
|
+
record = Bench.new({:ROW => i, :value => i})
|
49
|
+
record.save_with_mutator!(m)
|
50
|
+
Bench.flush_mutator(m) if i % flush_interval == 0
|
51
|
+
end
|
52
|
+
Bench.close_mutator(m)
|
53
|
+
ts2 = Time.now - t2
|
54
|
+
|
55
|
+
read_latency, write_latency, cells_read = ActiveRecord::ConnectionAdapters::HypertableAdapter.get_timing
|
56
|
+
ActiveRecord::ConnectionAdapters::HypertableAdapter.reset_timing
|
57
|
+
puts "#{n} save_with_mutator requests in #{ts2} sec: #{n/ts2} r/s"
|
58
|
+
puts " hypertable write latency was #{write_latency}"
|
@@ -0,0 +1,76 @@
|
|
1
|
+
# Generated by jeweler
|
2
|
+
# DO NOT EDIT THIS FILE
|
3
|
+
# Instead, edit Jeweler::Tasks in Rakefile, and run `rake gemspec`
|
4
|
+
# -*- encoding: utf-8 -*-
|
5
|
+
|
6
|
+
Gem::Specification.new do |s|
|
7
|
+
s.name = %q{hyper_record}
|
8
|
+
s.version = "0.2.8"
|
9
|
+
|
10
|
+
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
|
+
s.authors = ["tylerkovacs"]
|
12
|
+
s.date = %q{2010-03-01}
|
13
|
+
s.description = %q{See README}
|
14
|
+
s.email = %q{tyler.kovacs@gmail.com}
|
15
|
+
s.extra_rdoc_files = [
|
16
|
+
"LICENSE",
|
17
|
+
"README"
|
18
|
+
]
|
19
|
+
s.files = [
|
20
|
+
".gitignore",
|
21
|
+
"CHANGELOG",
|
22
|
+
"LICENSE",
|
23
|
+
"README",
|
24
|
+
"Rakefile",
|
25
|
+
"VERSION.yml",
|
26
|
+
"benchmark/save.rb",
|
27
|
+
"hyper_record.gemspec",
|
28
|
+
"init.rb",
|
29
|
+
"lib/active_record/connection_adapters/hyper_table_definition.rb",
|
30
|
+
"lib/active_record/connection_adapters/hypertable_adapter.rb",
|
31
|
+
"lib/active_record/connection_adapters/qualified_column.rb",
|
32
|
+
"lib/associations/hyper_has_and_belongs_to_many_association_extension.rb",
|
33
|
+
"lib/associations/hyper_has_many_association_extension.rb",
|
34
|
+
"lib/hyper_record.rb",
|
35
|
+
"lib/hypertable/gen-rb/client_constants.rb",
|
36
|
+
"lib/hypertable/gen-rb/client_service.rb",
|
37
|
+
"lib/hypertable/gen-rb/client_types.rb",
|
38
|
+
"lib/hypertable/gen-rb/hql_constants.rb",
|
39
|
+
"lib/hypertable/gen-rb/hql_service.rb",
|
40
|
+
"lib/hypertable/gen-rb/hql_types.rb",
|
41
|
+
"lib/hypertable/thrift_client.rb",
|
42
|
+
"lib/hypertable/thrift_transport_monkey_patch.rb",
|
43
|
+
"pkg/hyper_record-0.2.8.gem",
|
44
|
+
"spec/fixtures/pages.yml",
|
45
|
+
"spec/fixtures/qualified_pages.yml",
|
46
|
+
"spec/lib/associations_spec.rb",
|
47
|
+
"spec/lib/hyper_record_spec.rb",
|
48
|
+
"spec/lib/hypertable_adapter_spec.rb",
|
49
|
+
"spec/spec_helper.rb",
|
50
|
+
"test/test_helper.rb",
|
51
|
+
"test/thrift_client_test.rb"
|
52
|
+
]
|
53
|
+
s.homepage = %q{http://github.com/tylerkovacs/hyper_record}
|
54
|
+
s.rdoc_options = ["--charset=UTF-8"]
|
55
|
+
s.require_paths = ["lib"]
|
56
|
+
s.rubygems_version = %q{1.3.6}
|
57
|
+
s.summary = %q{Fully integrates ActiveRecord with Hypertable.}
|
58
|
+
s.test_files = [
|
59
|
+
"spec/lib/hypertable_adapter_spec.rb",
|
60
|
+
"spec/lib/hyper_record_spec.rb",
|
61
|
+
"spec/lib/associations_spec.rb",
|
62
|
+
"spec/spec_helper.rb",
|
63
|
+
"test/thrift_client_test.rb",
|
64
|
+
"test/test_helper.rb"
|
65
|
+
]
|
66
|
+
|
67
|
+
if s.respond_to? :specification_version then
|
68
|
+
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
|
69
|
+
s.specification_version = 3
|
70
|
+
|
71
|
+
if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
|
72
|
+
else
|
73
|
+
end
|
74
|
+
else
|
75
|
+
end
|
76
|
+
end
|
data/init.rb
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require 'hyper_record'
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# HyperTableDefinition is a subclass of TableDefinition that provides
|
2
|
+
# support for Hypertable-specific features (e.g, max_versions) to
|
3
|
+
# Rails migrations.
|
4
|
+
|
5
|
+
module ActiveRecord
|
6
|
+
module ConnectionAdapters #:nodoc:
|
7
|
+
class HyperColumnDefinition < Struct.new(:base, :name, :type, :limit, :max_versions, :options) #:nodoc:
|
8
|
+
end
|
9
|
+
|
10
|
+
class HyperTableDefinition < TableDefinition
|
11
|
+
def column(name, type, options = {})
|
12
|
+
column = self[name] || HyperColumnDefinition.new(@base, name, type)
|
13
|
+
if options[:limit]
|
14
|
+
column.limit = options[:limit]
|
15
|
+
elsif native[type.to_sym].is_a?(Hash)
|
16
|
+
column.limit = native[type.to_sym][:limit]
|
17
|
+
end
|
18
|
+
column.max_versions = options[:max_versions]
|
19
|
+
column.options = options
|
20
|
+
@columns << column unless @columns.include? column
|
21
|
+
self
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|