quandl_cassandra 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +7 -0
- data/Gemfile +2 -0
- data/LICENSE +7 -0
- data/README.md +7 -0
- data/Rakefile +11 -0
- data/UPGRADE.md +3 -0
- data/lib/quandl/cassandra/base/attributes.rb +103 -0
- data/lib/quandl/cassandra/base/callbacks.rb +15 -0
- data/lib/quandl/cassandra/base/connection.rb +49 -0
- data/lib/quandl/cassandra/base/logging.rb +40 -0
- data/lib/quandl/cassandra/base/naming.rb +19 -0
- data/lib/quandl/cassandra/base/persistence.rb +67 -0
- data/lib/quandl/cassandra/base/sanitization.rb +38 -0
- data/lib/quandl/cassandra/base/schema.rb +79 -0
- data/lib/quandl/cassandra/base/scoping.rb +122 -0
- data/lib/quandl/cassandra/base.rb +51 -0
- data/lib/quandl/cassandra/configuration.rb +34 -0
- data/lib/quandl/cassandra/error.rb +10 -0
- data/lib/quandl/cassandra/types/abstract_type.rb +33 -0
- data/lib/quandl/cassandra/types/boolean_type.rb +10 -0
- data/lib/quandl/cassandra/types/decimal_type.rb +9 -0
- data/lib/quandl/cassandra/types/double_type.rb +9 -0
- data/lib/quandl/cassandra/types/float_type.rb +9 -0
- data/lib/quandl/cassandra/types/integer_type.rb +9 -0
- data/lib/quandl/cassandra/types/long_type.rb +9 -0
- data/lib/quandl/cassandra/types/timestamp_type.rb +15 -0
- data/lib/quandl/cassandra/types/utf8_type.rb +13 -0
- data/lib/quandl/cassandra/types/uuid_type.rb +21 -0
- data/lib/quandl/cassandra/types.rb +42 -0
- data/lib/quandl/cassandra/version.rb +5 -0
- data/lib/quandl/cassandra.rb +30 -0
- data/lib/quandl/cassandra_models/column/read/collapse.rb +64 -0
- data/lib/quandl/cassandra_models/column/read/column.rb +18 -0
- data/lib/quandl/cassandra_models/column/read/data_table.rb +57 -0
- data/lib/quandl/cassandra_models/column/read/offset.rb +114 -0
- data/lib/quandl/cassandra_models/column/read/query.rb +55 -0
- data/lib/quandl/cassandra_models/column/read/row.rb +20 -0
- data/lib/quandl/cassandra_models/column/read/transform.rb +53 -0
- data/lib/quandl/cassandra_models/column/read/type.rb +25 -0
- data/lib/quandl/cassandra_models/column/read.rb +28 -0
- data/lib/quandl/cassandra_models/column/write/group_data_by_column.rb +42 -0
- data/lib/quandl/cassandra_models/column/write/group_data_by_frequency.rb +24 -0
- data/lib/quandl/cassandra_models/column/write/insert_columns.rb +22 -0
- data/lib/quandl/cassandra_models/column/write/insert_data.rb +39 -0
- data/lib/quandl/cassandra_models/column/write.rb +22 -0
- data/lib/quandl/cassandra_models/column.rb +20 -0
- data/lib/quandl/cassandra_models/column_attribute.rb +11 -0
- data/lib/quandl/cassandra_models/data.rb +52 -0
- data/lib/quandl/cassandra_models/dataset.rb +83 -0
- data/lib/quandl/cassandra_models/dataset_attribute.rb +6 -0
- data/lib/quandl/cassandra_models/multiset.rb +50 -0
- data/lib/quandl/strategy.rb +59 -0
- data/quandl_cassandra.gemspec +35 -0
- data/spec/expectations/string.rb +5 -0
- data/spec/expectations/time.rb +5 -0
- data/spec/factories/dataset.rb +8 -0
- data/spec/lib/quandl/cassandra/base/scoping_spec.rb +40 -0
- data/spec/lib/quandl/cassandra_models/column/write/group_data_by_frequency_spec.rb +28 -0
- data/spec/lib/quandl/cassandra_models/column/write_spec.rb +15 -0
- data/spec/lib/quandl/cassandra_models/column_attribute_spec.rb +16 -0
- data/spec/lib/quandl/cassandra_models/column_spec.rb +17 -0
- data/spec/lib/quandl/cassandra_models/data_spec.rb +34 -0
- data/spec/lib/quandl/cassandra_models/dataset/collapse_spec.rb +41 -0
- data/spec/lib/quandl/cassandra_models/dataset/column_spec.rb +25 -0
- data/spec/lib/quandl/cassandra_models/dataset/persistence_spec.rb +24 -0
- data/spec/lib/quandl/cassandra_models/dataset/row_spec.rb +26 -0
- data/spec/lib/quandl/cassandra_models/dataset/transform_spec.rb +16 -0
- data/spec/lib/quandl/cassandra_models/dataset/trim_spec.rb +74 -0
- data/spec/lib/quandl/cassandra_models/dataset/update_spec.rb +37 -0
- data/spec/lib/quandl/cassandra_models/dataset_attribute_spec.rb +18 -0
- data/spec/lib/quandl/cassandra_models/dataset_spec.rb +63 -0
- data/spec/lib/quandl/cassandra_models/multiset/collapse_spec.rb +122 -0
- data/spec/lib/quandl/cassandra_models/multiset/columns_spec.rb +57 -0
- data/spec/lib/quandl/cassandra_models/multiset/data_spec.rb +25 -0
- data/spec/lib/quandl/cassandra_models/multiset/transform_spec.rb +68 -0
- data/spec/lib/quandl/cassandra_spec.rb +12 -0
- data/spec/spec_helper.rb +37 -0
- metadata +339 -0
@@ -0,0 +1,122 @@
|
|
1
|
+
module Quandl::Cassandra::Base::Scoping
|
2
|
+
|
3
|
+
extend ActiveSupport::Concern
|
4
|
+
|
5
|
+
module ClassMethods
|
6
|
+
|
7
|
+
def where(*args)
|
8
|
+
scope.new.where(*args)
|
9
|
+
end
|
10
|
+
|
11
|
+
end
|
12
|
+
|
13
|
+
included do
|
14
|
+
|
15
|
+
include ScopeComposer::Model
|
16
|
+
|
17
|
+
has_scope_composer
|
18
|
+
|
19
|
+
scope :select, :limit, :order, :count, :find, :first, :all, :pluck, :exists?
|
20
|
+
|
21
|
+
scope.class_eval do
|
22
|
+
|
23
|
+
delegate :to_a, to: :execute
|
24
|
+
|
25
|
+
delegate :primary_key, :table_name, :sanitize_attributes, to: :parent
|
26
|
+
|
27
|
+
include Enumerable
|
28
|
+
|
29
|
+
def each(&block)
|
30
|
+
all.each do |record|
|
31
|
+
if block_given?
|
32
|
+
block.call(record)
|
33
|
+
else
|
34
|
+
yield record
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def exists?
|
40
|
+
columns = primary_key.collect{|pk| attributes.keys.include?(pk) ? pk : nil }.compact
|
41
|
+
select(*columns).execute.present?
|
42
|
+
end
|
43
|
+
|
44
|
+
def pluck(*args)
|
45
|
+
r = select(*args).values
|
46
|
+
r = r.flatten if args.count <= 1
|
47
|
+
r
|
48
|
+
end
|
49
|
+
|
50
|
+
def values
|
51
|
+
to_a.collect(&:values)
|
52
|
+
end
|
53
|
+
|
54
|
+
def find(*args)
|
55
|
+
primary_key = parent.bind_primary_key(*args)
|
56
|
+
where(primary_key).first
|
57
|
+
end
|
58
|
+
|
59
|
+
def first
|
60
|
+
limit(1).execute_returning_record
|
61
|
+
end
|
62
|
+
|
63
|
+
def all
|
64
|
+
execute_returning_collection
|
65
|
+
end
|
66
|
+
|
67
|
+
def count
|
68
|
+
select("COUNT(*)").execute.first.try(:[], 'count').to_i
|
69
|
+
end
|
70
|
+
|
71
|
+
def to_cql
|
72
|
+
["SELECT #{cql_select} FROM", table_name, cql_where, cql_order, cql_limit].compact.join(' ')
|
73
|
+
end
|
74
|
+
|
75
|
+
protected
|
76
|
+
|
77
|
+
def cql_where
|
78
|
+
attrs = attributes.clone
|
79
|
+
attrs = sanitize_attributes(attrs) if parent.table_exists?
|
80
|
+
attrs = attrs.collect{|k,v| "#{k} = #{v}" }.join(' AND ')
|
81
|
+
cql = attrs.present? ? "WHERE #{attrs}" : nil
|
82
|
+
end
|
83
|
+
|
84
|
+
def cql_select
|
85
|
+
columns = Array(select).join(',') if select.present?
|
86
|
+
columns = '*' if columns.blank?
|
87
|
+
columns
|
88
|
+
end
|
89
|
+
|
90
|
+
def cql_order
|
91
|
+
"ORDER BY #{order}" if order.present?
|
92
|
+
end
|
93
|
+
|
94
|
+
def cql_limit
|
95
|
+
"LIMIT #{limit}" if limit.present?
|
96
|
+
end
|
97
|
+
|
98
|
+
def execute
|
99
|
+
@execute ||= parent.execute(to_cql)
|
100
|
+
end
|
101
|
+
|
102
|
+
def execute_async
|
103
|
+
@execute_async ||= parent.execute(to_cql)
|
104
|
+
end
|
105
|
+
|
106
|
+
def parent
|
107
|
+
self.class.parent
|
108
|
+
end
|
109
|
+
|
110
|
+
def execute_returning_collection
|
111
|
+
execute.collect{|r| parent.new_from_query_result(r) }
|
112
|
+
end
|
113
|
+
|
114
|
+
def execute_returning_record
|
115
|
+
parent.new_from_query_result( execute.first )
|
116
|
+
end
|
117
|
+
|
118
|
+
end
|
119
|
+
|
120
|
+
end
|
121
|
+
|
122
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
require 'active_model'
|
2
|
+
require 'active_model/attribute_methods'
|
3
|
+
require 'active_model/dirty'
|
4
|
+
require 'active_model/validations'
|
5
|
+
require 'active_model/conversion'
|
6
|
+
require 'active_model/naming'
|
7
|
+
require 'active_model/callbacks'
|
8
|
+
|
9
|
+
require 'quandl/cassandra/base/attributes'
|
10
|
+
require 'quandl/cassandra/base/callbacks'
|
11
|
+
require 'quandl/cassandra/base/connection'
|
12
|
+
require 'quandl/cassandra/base/naming'
|
13
|
+
require 'quandl/cassandra/base/persistence'
|
14
|
+
require 'quandl/cassandra/base/logging'
|
15
|
+
require 'quandl/cassandra/base/sanitization'
|
16
|
+
require 'quandl/cassandra/base/schema'
|
17
|
+
require 'quandl/cassandra/base/scoping'
|
18
|
+
|
19
|
+
class Quandl::Cassandra::Base
|
20
|
+
|
21
|
+
include Quandl::Cassandra::Base::Connection
|
22
|
+
include Quandl::Cassandra::Base::Logging if defined?(QUANDL_LOGGER) && QUANDL_LOGGER == true
|
23
|
+
|
24
|
+
class << self
|
25
|
+
|
26
|
+
def inherited(subclass)
|
27
|
+
# remember models that inherit from base
|
28
|
+
models << subclass unless models.include?(subclass)
|
29
|
+
# include model behaviour
|
30
|
+
subclass.class_eval do
|
31
|
+
|
32
|
+
include ActiveModel::Validations
|
33
|
+
|
34
|
+
include Quandl::Cassandra::Base::Scoping
|
35
|
+
include Quandl::Cassandra::Base::Naming
|
36
|
+
include Quandl::Cassandra::Base::Attributes
|
37
|
+
include Quandl::Cassandra::Base::Persistence
|
38
|
+
include Quandl::Cassandra::Base::Sanitization
|
39
|
+
include Quandl::Cassandra::Base::Schema
|
40
|
+
include Quandl::Cassandra::Base::Callbacks
|
41
|
+
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def models
|
46
|
+
@@models ||= []
|
47
|
+
end
|
48
|
+
|
49
|
+
end
|
50
|
+
|
51
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
module Quandl
|
2
|
+
module Cassandra
|
3
|
+
|
4
|
+
class << self
|
5
|
+
attr_accessor :configuration
|
6
|
+
end
|
7
|
+
|
8
|
+
def self.configuration
|
9
|
+
@configuration ||= Configuration.new
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.configure
|
13
|
+
yield(configuration)
|
14
|
+
true
|
15
|
+
end
|
16
|
+
|
17
|
+
class Configuration
|
18
|
+
|
19
|
+
attr_accessor :hosts, :consistency, :keyspace, :batch_size
|
20
|
+
|
21
|
+
def initialize
|
22
|
+
@hosts = ['localhost']
|
23
|
+
@consistency = :all
|
24
|
+
@batch_size = 35
|
25
|
+
end
|
26
|
+
|
27
|
+
def to_h
|
28
|
+
self.attributes
|
29
|
+
end
|
30
|
+
|
31
|
+
end
|
32
|
+
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
class Quandl::Cassandra::Types::AbstractType
|
2
|
+
|
3
|
+
class << self
|
4
|
+
|
5
|
+
def sanitize_for_cql(value)
|
6
|
+
sanitize(value)
|
7
|
+
end
|
8
|
+
|
9
|
+
def sanitize(value)
|
10
|
+
value
|
11
|
+
end
|
12
|
+
|
13
|
+
def quote_string(value)
|
14
|
+
["'",value.to_s.gsub(/[']/, '\\\\\''),"'"].join
|
15
|
+
end
|
16
|
+
|
17
|
+
end
|
18
|
+
|
19
|
+
attr_accessor :value
|
20
|
+
|
21
|
+
def initialize(v)
|
22
|
+
self.value = v
|
23
|
+
end
|
24
|
+
|
25
|
+
def sanitize_for_cql
|
26
|
+
self.class.sanitize_for_cql(value)
|
27
|
+
end
|
28
|
+
|
29
|
+
def sanitize
|
30
|
+
self.class.sanitize(value)
|
31
|
+
end
|
32
|
+
|
33
|
+
end
|
@@ -0,0 +1,10 @@
|
|
1
|
+
class Quandl::Cassandra::Types::BooleanType < Quandl::Cassandra::Types::AbstractType
|
2
|
+
|
3
|
+
def self.sanitize(value)
|
4
|
+
return true if value == true || value.to_s =~ (/(true|t|yes|y|1)$/i)
|
5
|
+
return false if value == false || value.blank? || value.to_s =~ (/(false|f|no|n|0)$/i)
|
6
|
+
rescue
|
7
|
+
raise Quandl::Cassandra::Error::CastException.new("Unable to convert value to #{self.class}", value)
|
8
|
+
end
|
9
|
+
|
10
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
class Quandl::Cassandra::Types::TimestampType < Quandl::Cassandra::Types::AbstractType
|
2
|
+
|
3
|
+
def self.sanitize_for_cql(value)
|
4
|
+
(value.to_f * 1000).to_i
|
5
|
+
end
|
6
|
+
|
7
|
+
def self.sanitize(value)
|
8
|
+
return value if value.is_a?(Time)
|
9
|
+
return Time.at(value.get_time / 1000.0) if value.respond_to?(:get_time)
|
10
|
+
Time.at(value.to_f) if value.present?
|
11
|
+
rescue => error
|
12
|
+
raise Quandl::Cassandra::Error::CastException.new("Unable to convert to Timestamp #{value} #{error}", value)
|
13
|
+
end
|
14
|
+
|
15
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
class Quandl::Cassandra::Types::UTF8Type < Quandl::Cassandra::Types::AbstractType
|
2
|
+
|
3
|
+
def self.sanitize_for_cql(value)
|
4
|
+
quote_string( sanitize(value) )
|
5
|
+
end
|
6
|
+
|
7
|
+
def self.sanitize(value)
|
8
|
+
RUBY_VERSION >= "1.9" ? value.to_s.dup.force_encoding('UTF-8') : value.to_s.dup
|
9
|
+
rescue => e
|
10
|
+
raise Quandl::Cassandra::Error::CastException.new("Unable to convert to UTF8", caller)
|
11
|
+
end
|
12
|
+
|
13
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module Quandl::Cassandra::Types
|
2
|
+
|
3
|
+
class UUIDType < Quandl::Cassandra::Types::AbstractType
|
4
|
+
|
5
|
+
def self.sanitize(value)
|
6
|
+
value = value.to_s
|
7
|
+
raise Quandl::Cassandra::Error::CastException, "Unable to convert to UUID", caller unless value =~ regex_validator
|
8
|
+
value
|
9
|
+
rescue => e
|
10
|
+
raise
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.regex_validator
|
14
|
+
/^[A-Za-z0-9]{8}-[A-Za-z0-9]{4}-[A-Za-z0-9]{4}-[A-Za-z0-9]{4}-[A-Za-z0-9]{12}$/
|
15
|
+
end
|
16
|
+
|
17
|
+
end
|
18
|
+
|
19
|
+
class TimeUUIDType < UUIDType; end
|
20
|
+
|
21
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
module Quandl::Cassandra::Types
|
2
|
+
|
3
|
+
require 'quandl/cassandra/types/abstract_type'
|
4
|
+
require 'quandl/cassandra/types/boolean_type'
|
5
|
+
require 'quandl/cassandra/types/timestamp_type'
|
6
|
+
require 'quandl/cassandra/types/decimal_type'
|
7
|
+
require 'quandl/cassandra/types/double_type'
|
8
|
+
require 'quandl/cassandra/types/float_type'
|
9
|
+
require 'quandl/cassandra/types/integer_type'
|
10
|
+
require 'quandl/cassandra/types/long_type'
|
11
|
+
require 'quandl/cassandra/types/utf8_type'
|
12
|
+
require 'quandl/cassandra/types/uuid_type'
|
13
|
+
|
14
|
+
class << self
|
15
|
+
def type(type)
|
16
|
+
self.types[type]
|
17
|
+
end
|
18
|
+
|
19
|
+
def types
|
20
|
+
@@types ||= {
|
21
|
+
bigint: Quandl::Cassandra::Types::DoubleType,
|
22
|
+
blob: Quandl::Cassandra::Types::UTF8Type,
|
23
|
+
boolean: Quandl::Cassandra::Types::BooleanType,
|
24
|
+
counter: Quandl::Cassandra::Types::IntegerType,
|
25
|
+
decimal: Quandl::Cassandra::Types::FloatType,
|
26
|
+
double: Quandl::Cassandra::Types::IntegerType,
|
27
|
+
float: Quandl::Cassandra::Types::FloatType,
|
28
|
+
int: Quandl::Cassandra::Types::IntegerType,
|
29
|
+
timestamp: Quandl::Cassandra::Types::TimestampType,
|
30
|
+
varchar: Quandl::Cassandra::Types::UTF8Type,
|
31
|
+
text: Quandl::Cassandra::Types::UTF8Type,
|
32
|
+
varint: Quandl::Cassandra::Types::IntegerType,
|
33
|
+
timeuuid: Quandl::Cassandra::Types::TimestampType,
|
34
|
+
uuid: Quandl::Cassandra::Types::UUIDType,
|
35
|
+
inet: Quandl::Cassandra::Types::UTF8Type,
|
36
|
+
}
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
41
|
+
|
42
|
+
|
@@ -0,0 +1,30 @@
|
|
1
|
+
require 'quandl/cassandra/version'
|
2
|
+
|
3
|
+
require "active_support"
|
4
|
+
require "active_support/inflector"
|
5
|
+
require "active_support/core_ext/hash"
|
6
|
+
require "active_support/core_ext/object"
|
7
|
+
|
8
|
+
require 'cql'
|
9
|
+
require 'scope_composer'
|
10
|
+
|
11
|
+
require 'quandl/logger'
|
12
|
+
require "quandl/data"
|
13
|
+
require 'quandl/strategy'
|
14
|
+
|
15
|
+
require 'quandl/cassandra/error'
|
16
|
+
require 'quandl/cassandra/types'
|
17
|
+
require 'quandl/cassandra/base'
|
18
|
+
require 'quandl/cassandra/configuration'
|
19
|
+
|
20
|
+
require 'quandl/cassandra_models/column'
|
21
|
+
require 'quandl/cassandra_models/column_attribute'
|
22
|
+
require 'quandl/cassandra_models/data'
|
23
|
+
require 'quandl/cassandra_models/dataset'
|
24
|
+
require 'quandl/cassandra_models/dataset_attribute'
|
25
|
+
require 'quandl/cassandra_models/multiset'
|
26
|
+
|
27
|
+
module Quandl
|
28
|
+
module Cassandra
|
29
|
+
end
|
30
|
+
end
|