quandl_cassandra 1.3.1 → 1.4.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +4 -1
- data/Berksfile +4 -0
- data/README.md +9 -3
- data/Rakefile +0 -8
- data/UPGRADE.md +6 -2
- data/Vagrantfile +43 -0
- data/examples/connect.rb +41 -3
- data/lib/quandl/cassandra.rb +2 -2
- data/lib/quandl/cassandra/base/attributes.rb +103 -102
- data/lib/quandl/cassandra/base/logging.rb +25 -29
- data/lib/quandl/cassandra/base/sanitization.rb +7 -11
- data/lib/quandl/cassandra/base/schema.rb +36 -31
- data/lib/quandl/cassandra/base/scoping.rb +41 -45
- data/lib/quandl/cassandra/batch/logging.rb +16 -16
- data/lib/quandl/cassandra/configuration.rb +30 -30
- data/lib/quandl/cassandra/logger.rb +10 -10
- data/lib/quandl/cassandra/version.rb +3 -3
- data/quandl_cassandra.gemspec +4 -6
- data/spec/config/cassandra.rb +3 -5
- data/spec/lib/quandl/cassandra/base/schema_spec.rb +47 -0
- metadata +57 -48
- checksums.yaml +0 -7
- data/VERSION +0 -1
@@ -1,89 +1,94 @@
|
|
1
1
|
module Quandl::Cassandra::Base::Schema
|
2
|
-
|
3
2
|
extend ActiveSupport::Concern
|
4
|
-
|
3
|
+
|
5
4
|
def primary_key?
|
6
5
|
!primary_key_attributes.values.include?(nil)
|
7
6
|
end
|
8
|
-
|
7
|
+
|
9
8
|
def primary_key_attributes
|
10
|
-
self.class.primary_key.inject({}){|m,key| m[key] = self.send(key); m }
|
9
|
+
self.class.primary_key.inject({}) { |m,key| m[key] = self.send(key); m }
|
11
10
|
end
|
12
|
-
|
11
|
+
|
13
12
|
module ClassMethods
|
14
|
-
|
15
13
|
def initialize_class_once
|
16
14
|
return if defined?(@initialize_class_once)
|
17
15
|
define_schema_attributes
|
18
16
|
@initialize_class_once = true
|
19
17
|
end
|
20
|
-
|
18
|
+
|
21
19
|
def column_type(type)
|
22
20
|
column_types[type.to_sym]
|
23
21
|
end
|
24
|
-
|
22
|
+
|
25
23
|
def column_types
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
24
|
+
unless @column_types && @column_types.is_a?(Hash) && !@column_types.empty?
|
25
|
+
@column_types = {}
|
26
|
+
columns.each do |c|
|
27
|
+
@column_types[c.column_name.to_sym] = Quandl::Cassandra::Types.type(c.type)
|
28
|
+
end
|
29
|
+
Quandl::Logger.info("Column types for #{self.to_s}: #{@column_types}")
|
30
30
|
end
|
31
31
|
@column_types
|
32
32
|
end
|
33
|
-
|
33
|
+
|
34
34
|
def column_names
|
35
|
-
@column_names
|
35
|
+
unless @column_names && @column_names.is_a?(Array) && !@column_names.empty?
|
36
|
+
@column_names = columns.map(&:column_name)
|
37
|
+
Quandl::Logger.info("Column names for #{self.to_s}: #{@column_names.join(', ')}")
|
38
|
+
end
|
39
|
+
@column_names
|
36
40
|
end
|
37
|
-
|
41
|
+
|
38
42
|
def columns
|
39
|
-
@columns
|
43
|
+
unless @columns && @columns.is_a?(Cql::Client::ResultMetadata)
|
44
|
+
@columns = execute("SELECT * FROM #{table_name} LIMIT 1").metadata
|
45
|
+
Quandl::Logger.info("Columns for #{self.to_s} #{@columns.inspect}")
|
46
|
+
end
|
47
|
+
@columns
|
40
48
|
end
|
41
|
-
|
49
|
+
|
42
50
|
def primary_key
|
43
51
|
@primary_key ||= key_aliases + column_aliases
|
44
52
|
end
|
45
|
-
|
53
|
+
|
46
54
|
def column_aliases
|
47
55
|
return [] if system_schema.blank?
|
48
|
-
@column_aliases ||= JSON.parse(system_schema['column_aliases']).
|
56
|
+
@column_aliases ||= JSON.parse(system_schema['column_aliases']).map(&:to_sym)
|
49
57
|
rescue JSON::ParserError
|
50
58
|
[]
|
51
59
|
end
|
52
|
-
|
60
|
+
|
53
61
|
def key_aliases
|
54
62
|
return [] if system_schema.blank?
|
55
|
-
@key_aliases ||= JSON.parse(system_schema['key_aliases']).
|
63
|
+
@key_aliases ||= JSON.parse(system_schema['key_aliases']).map(&:to_sym)
|
56
64
|
rescue JSON::ParserError
|
57
65
|
[]
|
58
66
|
end
|
59
|
-
|
67
|
+
|
60
68
|
def table_exists?
|
61
69
|
system_schema.present?
|
62
70
|
end
|
63
|
-
|
71
|
+
|
64
72
|
def system_schema
|
65
73
|
return @system_schema if defined?(@system_schema)
|
66
74
|
@system_schema = execute(%Q{
|
67
|
-
SELECT * FROM system.schema_columnfamilies WHERE
|
68
|
-
keyspace_name='#{Quandl::Cassandra.configuration.keyspace}'
|
75
|
+
SELECT * FROM system.schema_columnfamilies WHERE
|
76
|
+
keyspace_name='#{Quandl::Cassandra.configuration.keyspace}'
|
69
77
|
AND columnfamily_name = '#{table_name}'
|
70
78
|
}, 'ONE').first
|
71
79
|
end
|
72
|
-
|
80
|
+
|
73
81
|
private
|
74
|
-
|
82
|
+
|
75
83
|
def define_schema_attributes
|
76
84
|
self.class_eval do
|
77
85
|
define_attributes *column_names
|
78
86
|
end
|
79
87
|
end
|
80
|
-
|
81
88
|
end
|
82
|
-
|
89
|
+
|
83
90
|
def initialize(*args, &block)
|
84
91
|
self.class.initialize_class_once
|
85
92
|
super if defined?(super)
|
86
93
|
end
|
87
|
-
|
88
94
|
end
|
89
|
-
|
@@ -1,122 +1,118 @@
|
|
1
1
|
module Quandl::Cassandra::Base::Scoping
|
2
|
-
|
3
2
|
extend ActiveSupport::Concern
|
4
|
-
|
3
|
+
|
5
4
|
module ClassMethods
|
6
|
-
|
7
5
|
def scope_names
|
8
6
|
scope.scope_names
|
9
7
|
end
|
10
|
-
|
8
|
+
|
11
9
|
def where(*args)
|
12
10
|
scope.new.where(*args)
|
13
11
|
end
|
14
|
-
|
15
12
|
end
|
16
|
-
|
13
|
+
|
17
14
|
included do
|
18
|
-
|
19
15
|
include ScopeComposer::Model
|
20
|
-
|
16
|
+
|
21
17
|
has_scope_composer
|
22
|
-
|
18
|
+
|
23
19
|
scope :select, :delete_all, :limit,
|
24
20
|
:order, :count, :find, :first, :all, :pluck, :exists?, :to_h
|
25
|
-
|
21
|
+
|
26
22
|
scope.class_eval do
|
27
|
-
|
23
|
+
|
28
24
|
attr_accessor :statement_type
|
29
|
-
|
25
|
+
|
30
26
|
delegate :to_a, to: :execute
|
31
|
-
|
27
|
+
|
32
28
|
delegate :primary_key, :table_name, :sanitize_attributes, to: :parent
|
33
|
-
|
29
|
+
|
34
30
|
include Enumerable
|
35
|
-
|
31
|
+
|
36
32
|
def each(&block)
|
37
33
|
all.each do |record|
|
38
34
|
if block_given?
|
39
35
|
block.call(record)
|
40
|
-
else
|
36
|
+
else
|
41
37
|
yield record
|
42
38
|
end
|
43
|
-
end
|
39
|
+
end
|
44
40
|
end
|
45
|
-
|
41
|
+
|
46
42
|
def exists?
|
47
43
|
columns = primary_key.collect{|pk| attributes.keys.include?(pk) ? pk : nil }.compact
|
48
44
|
select(*columns).execute.present?
|
49
45
|
end
|
50
|
-
|
46
|
+
|
51
47
|
def to_h
|
52
48
|
execute
|
53
49
|
end
|
54
|
-
|
50
|
+
|
55
51
|
def pluck(*args)
|
56
52
|
r = select(*args).values
|
57
53
|
r = r.flatten if args.count <= 1
|
58
54
|
r
|
59
55
|
end
|
60
|
-
|
56
|
+
|
61
57
|
def select=(*names)
|
62
58
|
self.statement_type = :select
|
63
59
|
write_scope_attribute(:select, names)
|
64
60
|
end
|
65
|
-
|
61
|
+
|
66
62
|
def values
|
67
63
|
to_a.collect(&:values)
|
68
64
|
end
|
69
|
-
|
65
|
+
|
70
66
|
def find(*args)
|
71
67
|
primary_key = parent.bind_primary_key(*args)
|
72
68
|
return nil if primary_key.blank?
|
73
69
|
where(primary_key).first
|
74
70
|
end
|
75
|
-
|
71
|
+
|
76
72
|
def first
|
77
73
|
limit(1).execute_returning_record
|
78
74
|
end
|
79
|
-
|
75
|
+
|
80
76
|
def all
|
81
77
|
execute_returning_collection
|
82
78
|
end
|
83
|
-
|
79
|
+
|
84
80
|
def count
|
85
81
|
select("COUNT(*)").execute.first.try(:[], 'count').to_i
|
86
82
|
end
|
87
|
-
|
83
|
+
|
88
84
|
def to_cql
|
89
85
|
[cql_statement_type, table_name, cql_where, cql_order, cql_limit].compact.join(' ')
|
90
86
|
end
|
91
|
-
|
87
|
+
|
92
88
|
def delete_all
|
93
89
|
self.statement_type = :delete
|
94
90
|
execute
|
95
91
|
end
|
96
|
-
|
92
|
+
|
97
93
|
def insert
|
98
94
|
self.statement_type = :insert
|
99
95
|
self
|
100
96
|
end
|
101
|
-
|
97
|
+
|
102
98
|
protected
|
103
|
-
|
99
|
+
|
104
100
|
def cql_statement_type
|
105
101
|
case statement_type
|
106
102
|
when :insert then "INSERT INTO"
|
107
|
-
when :delete then "DELETE FROM"
|
103
|
+
when :delete then "DELETE FROM"
|
108
104
|
else
|
109
105
|
"SELECT #{cql_select} FROM"
|
110
106
|
end
|
111
107
|
end
|
112
|
-
|
108
|
+
|
113
109
|
def cql_where
|
114
110
|
attrs = attributes.clone
|
115
111
|
attrs = sanitize_attributes(attrs) if parent.table_exists?
|
116
112
|
attrs = attrs.collect{|key, value| cql_where_attribute(key, value) }.join(' AND ')
|
117
113
|
cql = attrs.present? ? "WHERE #{attrs}" : nil
|
118
114
|
end
|
119
|
-
|
115
|
+
|
120
116
|
def cql_where_attribute(key, value)
|
121
117
|
if value.is_a?(Array)
|
122
118
|
"#{key} IN (#{value.join(',')})"
|
@@ -124,43 +120,43 @@ module Quandl::Cassandra::Base::Scoping
|
|
124
120
|
"#{key} = #{value}"
|
125
121
|
end
|
126
122
|
end
|
127
|
-
|
123
|
+
|
128
124
|
def cql_select
|
129
125
|
columns = Array(select).join(',') if select.present?
|
130
126
|
columns = '*' if columns.blank?
|
131
127
|
columns
|
132
128
|
end
|
133
|
-
|
129
|
+
|
134
130
|
def cql_order
|
135
131
|
"ORDER BY #{order}" if order.present?
|
136
132
|
end
|
137
|
-
|
133
|
+
|
138
134
|
def cql_limit
|
139
135
|
"LIMIT #{limit}" if limit.present?
|
140
136
|
end
|
141
|
-
|
137
|
+
|
142
138
|
def execute
|
143
139
|
@execute ||= parent.execute(to_cql)
|
144
140
|
end
|
145
|
-
|
141
|
+
|
146
142
|
def execute_async
|
147
143
|
@execute_async ||= parent.execute(to_cql)
|
148
144
|
end
|
149
|
-
|
145
|
+
|
150
146
|
def parent
|
151
147
|
self.class.parent
|
152
148
|
end
|
153
|
-
|
149
|
+
|
154
150
|
def execute_returning_collection
|
155
151
|
execute.collect{|r| parent.new_from_query_result(r) }
|
156
152
|
end
|
157
|
-
|
153
|
+
|
158
154
|
def execute_returning_record
|
159
155
|
parent.new_from_query_result( execute.first )
|
160
156
|
end
|
161
|
-
|
157
|
+
|
162
158
|
end
|
163
|
-
|
159
|
+
|
164
160
|
end
|
165
|
-
|
161
|
+
|
166
162
|
end
|
@@ -1,22 +1,22 @@
|
|
1
1
|
class Quandl::Cassandra::Batch
|
2
|
-
module Logging
|
2
|
+
module Logging
|
3
3
|
|
4
|
-
|
4
|
+
extend ActiveSupport::Concern
|
5
|
+
|
6
|
+
module ClassMethods
|
7
|
+
|
8
|
+
def execute(rows, &block)
|
9
|
+
# log init
|
10
|
+
statement = block.call(rows[0].collect{'?'})
|
11
|
+
t1 = Time.now
|
12
|
+
# call method
|
13
|
+
r = super if defined?(super)
|
14
|
+
# log write
|
15
|
+
Quandl::Logger.debug("(#{t1.elapsed_ms}) ".blue + " [#{rows.count} rows] BATCH #{statement}")
|
16
|
+
r
|
17
|
+
end
|
5
18
|
|
6
|
-
module ClassMethods
|
7
|
-
|
8
|
-
def execute(rows, &block)
|
9
|
-
# log init
|
10
|
-
statement = block.call(rows[0].collect{'?'})
|
11
|
-
t1 = Time.now
|
12
|
-
# call method
|
13
|
-
r = super if defined?(super)
|
14
|
-
# log write
|
15
|
-
Quandl::Logger.debug("(#{t1.elapsed_ms}) ".blue + " [#{rows.count} rows] BATCH #{statement}")
|
16
|
-
r
|
17
19
|
end
|
18
|
-
|
19
|
-
end
|
20
20
|
|
21
|
-
end
|
21
|
+
end
|
22
22
|
end
|
@@ -1,40 +1,40 @@
|
|
1
1
|
module Quandl
|
2
|
-
module Cassandra
|
3
|
-
|
4
|
-
class << self
|
5
|
-
attr_accessor :configuration
|
6
|
-
end
|
7
|
-
|
8
|
-
def self.configuration
|
9
|
-
@configuration ||= Configuration.new
|
10
|
-
end
|
11
|
-
|
12
|
-
def self.configure
|
13
|
-
yield(configuration)
|
14
|
-
true
|
15
|
-
end
|
16
|
-
|
17
|
-
class Configuration
|
18
|
-
|
19
|
-
attr_accessor :hosts, :consistency, :keyspace, :batch_size, :read_consistency, :write_consistency
|
2
|
+
module Cassandra
|
20
3
|
|
21
|
-
|
22
|
-
|
23
|
-
@consistency = :all
|
24
|
-
@read_consistency = :one
|
25
|
-
@write_consistency = :one
|
26
|
-
@batch_size = 35 * 4
|
4
|
+
class << self
|
5
|
+
attr_accessor :configuration
|
27
6
|
end
|
28
7
|
|
29
|
-
def
|
30
|
-
|
8
|
+
def self.configuration
|
9
|
+
@configuration ||= Configuration.new
|
31
10
|
end
|
32
11
|
|
33
|
-
def
|
34
|
-
|
12
|
+
def self.configure
|
13
|
+
yield(configuration)
|
14
|
+
true
|
15
|
+
end
|
16
|
+
|
17
|
+
class Configuration
|
18
|
+
|
19
|
+
attr_accessor :hosts, :consistency, :keyspace, :batch_size, :read_consistency, :write_consistency
|
20
|
+
|
21
|
+
def initialize
|
22
|
+
@hosts = ['localhost']
|
23
|
+
@consistency = :all
|
24
|
+
@read_consistency = :one
|
25
|
+
@write_consistency = :one
|
26
|
+
@batch_size = 35 * 4
|
27
|
+
end
|
28
|
+
|
29
|
+
def attributes
|
30
|
+
{ hosts: hosts, consistency: consistency, keyspace: keyspace, batch_size: batch_size }
|
31
|
+
end
|
32
|
+
|
33
|
+
def to_h
|
34
|
+
attributes
|
35
|
+
end
|
36
|
+
|
35
37
|
end
|
36
38
|
|
37
39
|
end
|
38
|
-
|
39
|
-
end
|
40
40
|
end
|
@@ -1,14 +1,14 @@
|
|
1
1
|
module Quandl
|
2
|
-
module Cassandra
|
3
|
-
class Logger
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
2
|
+
module Cassandra
|
3
|
+
class Logger
|
4
|
+
class << self
|
5
|
+
|
6
|
+
def method_missing(method_sym, *args, &block)
|
7
|
+
args[0] = "CASSANDRA ".blue + args[0].to_s
|
8
|
+
Quandl::Logger.send(method_sym, *args)
|
9
|
+
end
|
10
|
+
|
11
|
+
end
|
9
12
|
end
|
10
|
-
|
11
13
|
end
|
12
|
-
end
|
13
|
-
end
|
14
14
|
end
|