quandl_cassandra 1.3.1 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +4 -1
- data/Berksfile +4 -0
- data/README.md +9 -3
- data/Rakefile +0 -8
- data/UPGRADE.md +6 -2
- data/Vagrantfile +43 -0
- data/examples/connect.rb +41 -3
- data/lib/quandl/cassandra.rb +2 -2
- data/lib/quandl/cassandra/base/attributes.rb +103 -102
- data/lib/quandl/cassandra/base/logging.rb +25 -29
- data/lib/quandl/cassandra/base/sanitization.rb +7 -11
- data/lib/quandl/cassandra/base/schema.rb +36 -31
- data/lib/quandl/cassandra/base/scoping.rb +41 -45
- data/lib/quandl/cassandra/batch/logging.rb +16 -16
- data/lib/quandl/cassandra/configuration.rb +30 -30
- data/lib/quandl/cassandra/logger.rb +10 -10
- data/lib/quandl/cassandra/version.rb +3 -3
- data/quandl_cassandra.gemspec +4 -6
- data/spec/config/cassandra.rb +3 -5
- data/spec/lib/quandl/cassandra/base/schema_spec.rb +47 -0
- metadata +57 -48
- checksums.yaml +0 -7
- data/VERSION +0 -1
@@ -1,89 +1,94 @@
|
|
1
1
|
module Quandl::Cassandra::Base::Schema
|
2
|
-
|
3
2
|
extend ActiveSupport::Concern
|
4
|
-
|
3
|
+
|
5
4
|
def primary_key?
|
6
5
|
!primary_key_attributes.values.include?(nil)
|
7
6
|
end
|
8
|
-
|
7
|
+
|
9
8
|
def primary_key_attributes
|
10
|
-
self.class.primary_key.inject({}){|m,key| m[key] = self.send(key); m }
|
9
|
+
self.class.primary_key.inject({}) { |m,key| m[key] = self.send(key); m }
|
11
10
|
end
|
12
|
-
|
11
|
+
|
13
12
|
module ClassMethods
|
14
|
-
|
15
13
|
def initialize_class_once
|
16
14
|
return if defined?(@initialize_class_once)
|
17
15
|
define_schema_attributes
|
18
16
|
@initialize_class_once = true
|
19
17
|
end
|
20
|
-
|
18
|
+
|
21
19
|
def column_type(type)
|
22
20
|
column_types[type.to_sym]
|
23
21
|
end
|
24
|
-
|
22
|
+
|
25
23
|
def column_types
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
24
|
+
unless @column_types && @column_types.is_a?(Hash) && !@column_types.empty?
|
25
|
+
@column_types = {}
|
26
|
+
columns.each do |c|
|
27
|
+
@column_types[c.column_name.to_sym] = Quandl::Cassandra::Types.type(c.type)
|
28
|
+
end
|
29
|
+
Quandl::Logger.info("Column types for #{self.to_s}: #{@column_types}")
|
30
30
|
end
|
31
31
|
@column_types
|
32
32
|
end
|
33
|
-
|
33
|
+
|
34
34
|
def column_names
|
35
|
-
@column_names
|
35
|
+
unless @column_names && @column_names.is_a?(Array) && !@column_names.empty?
|
36
|
+
@column_names = columns.map(&:column_name)
|
37
|
+
Quandl::Logger.info("Column names for #{self.to_s}: #{@column_names.join(', ')}")
|
38
|
+
end
|
39
|
+
@column_names
|
36
40
|
end
|
37
|
-
|
41
|
+
|
38
42
|
def columns
|
39
|
-
@columns
|
43
|
+
unless @columns && @columns.is_a?(Cql::Client::ResultMetadata)
|
44
|
+
@columns = execute("SELECT * FROM #{table_name} LIMIT 1").metadata
|
45
|
+
Quandl::Logger.info("Columns for #{self.to_s} #{@columns.inspect}")
|
46
|
+
end
|
47
|
+
@columns
|
40
48
|
end
|
41
|
-
|
49
|
+
|
42
50
|
def primary_key
|
43
51
|
@primary_key ||= key_aliases + column_aliases
|
44
52
|
end
|
45
|
-
|
53
|
+
|
46
54
|
def column_aliases
|
47
55
|
return [] if system_schema.blank?
|
48
|
-
@column_aliases ||= JSON.parse(system_schema['column_aliases']).
|
56
|
+
@column_aliases ||= JSON.parse(system_schema['column_aliases']).map(&:to_sym)
|
49
57
|
rescue JSON::ParserError
|
50
58
|
[]
|
51
59
|
end
|
52
|
-
|
60
|
+
|
53
61
|
def key_aliases
|
54
62
|
return [] if system_schema.blank?
|
55
|
-
@key_aliases ||= JSON.parse(system_schema['key_aliases']).
|
63
|
+
@key_aliases ||= JSON.parse(system_schema['key_aliases']).map(&:to_sym)
|
56
64
|
rescue JSON::ParserError
|
57
65
|
[]
|
58
66
|
end
|
59
|
-
|
67
|
+
|
60
68
|
def table_exists?
|
61
69
|
system_schema.present?
|
62
70
|
end
|
63
|
-
|
71
|
+
|
64
72
|
def system_schema
|
65
73
|
return @system_schema if defined?(@system_schema)
|
66
74
|
@system_schema = execute(%Q{
|
67
|
-
SELECT * FROM system.schema_columnfamilies WHERE
|
68
|
-
keyspace_name='#{Quandl::Cassandra.configuration.keyspace}'
|
75
|
+
SELECT * FROM system.schema_columnfamilies WHERE
|
76
|
+
keyspace_name='#{Quandl::Cassandra.configuration.keyspace}'
|
69
77
|
AND columnfamily_name = '#{table_name}'
|
70
78
|
}, 'ONE').first
|
71
79
|
end
|
72
|
-
|
80
|
+
|
73
81
|
private
|
74
|
-
|
82
|
+
|
75
83
|
def define_schema_attributes
|
76
84
|
self.class_eval do
|
77
85
|
define_attributes *column_names
|
78
86
|
end
|
79
87
|
end
|
80
|
-
|
81
88
|
end
|
82
|
-
|
89
|
+
|
83
90
|
def initialize(*args, &block)
|
84
91
|
self.class.initialize_class_once
|
85
92
|
super if defined?(super)
|
86
93
|
end
|
87
|
-
|
88
94
|
end
|
89
|
-
|
@@ -1,122 +1,118 @@
|
|
1
1
|
module Quandl::Cassandra::Base::Scoping
|
2
|
-
|
3
2
|
extend ActiveSupport::Concern
|
4
|
-
|
3
|
+
|
5
4
|
module ClassMethods
|
6
|
-
|
7
5
|
def scope_names
|
8
6
|
scope.scope_names
|
9
7
|
end
|
10
|
-
|
8
|
+
|
11
9
|
def where(*args)
|
12
10
|
scope.new.where(*args)
|
13
11
|
end
|
14
|
-
|
15
12
|
end
|
16
|
-
|
13
|
+
|
17
14
|
included do
|
18
|
-
|
19
15
|
include ScopeComposer::Model
|
20
|
-
|
16
|
+
|
21
17
|
has_scope_composer
|
22
|
-
|
18
|
+
|
23
19
|
scope :select, :delete_all, :limit,
|
24
20
|
:order, :count, :find, :first, :all, :pluck, :exists?, :to_h
|
25
|
-
|
21
|
+
|
26
22
|
scope.class_eval do
|
27
|
-
|
23
|
+
|
28
24
|
attr_accessor :statement_type
|
29
|
-
|
25
|
+
|
30
26
|
delegate :to_a, to: :execute
|
31
|
-
|
27
|
+
|
32
28
|
delegate :primary_key, :table_name, :sanitize_attributes, to: :parent
|
33
|
-
|
29
|
+
|
34
30
|
include Enumerable
|
35
|
-
|
31
|
+
|
36
32
|
def each(&block)
|
37
33
|
all.each do |record|
|
38
34
|
if block_given?
|
39
35
|
block.call(record)
|
40
|
-
else
|
36
|
+
else
|
41
37
|
yield record
|
42
38
|
end
|
43
|
-
end
|
39
|
+
end
|
44
40
|
end
|
45
|
-
|
41
|
+
|
46
42
|
def exists?
|
47
43
|
columns = primary_key.collect{|pk| attributes.keys.include?(pk) ? pk : nil }.compact
|
48
44
|
select(*columns).execute.present?
|
49
45
|
end
|
50
|
-
|
46
|
+
|
51
47
|
def to_h
|
52
48
|
execute
|
53
49
|
end
|
54
|
-
|
50
|
+
|
55
51
|
def pluck(*args)
|
56
52
|
r = select(*args).values
|
57
53
|
r = r.flatten if args.count <= 1
|
58
54
|
r
|
59
55
|
end
|
60
|
-
|
56
|
+
|
61
57
|
def select=(*names)
|
62
58
|
self.statement_type = :select
|
63
59
|
write_scope_attribute(:select, names)
|
64
60
|
end
|
65
|
-
|
61
|
+
|
66
62
|
def values
|
67
63
|
to_a.collect(&:values)
|
68
64
|
end
|
69
|
-
|
65
|
+
|
70
66
|
def find(*args)
|
71
67
|
primary_key = parent.bind_primary_key(*args)
|
72
68
|
return nil if primary_key.blank?
|
73
69
|
where(primary_key).first
|
74
70
|
end
|
75
|
-
|
71
|
+
|
76
72
|
def first
|
77
73
|
limit(1).execute_returning_record
|
78
74
|
end
|
79
|
-
|
75
|
+
|
80
76
|
def all
|
81
77
|
execute_returning_collection
|
82
78
|
end
|
83
|
-
|
79
|
+
|
84
80
|
def count
|
85
81
|
select("COUNT(*)").execute.first.try(:[], 'count').to_i
|
86
82
|
end
|
87
|
-
|
83
|
+
|
88
84
|
def to_cql
|
89
85
|
[cql_statement_type, table_name, cql_where, cql_order, cql_limit].compact.join(' ')
|
90
86
|
end
|
91
|
-
|
87
|
+
|
92
88
|
def delete_all
|
93
89
|
self.statement_type = :delete
|
94
90
|
execute
|
95
91
|
end
|
96
|
-
|
92
|
+
|
97
93
|
def insert
|
98
94
|
self.statement_type = :insert
|
99
95
|
self
|
100
96
|
end
|
101
|
-
|
97
|
+
|
102
98
|
protected
|
103
|
-
|
99
|
+
|
104
100
|
def cql_statement_type
|
105
101
|
case statement_type
|
106
102
|
when :insert then "INSERT INTO"
|
107
|
-
when :delete then "DELETE FROM"
|
103
|
+
when :delete then "DELETE FROM"
|
108
104
|
else
|
109
105
|
"SELECT #{cql_select} FROM"
|
110
106
|
end
|
111
107
|
end
|
112
|
-
|
108
|
+
|
113
109
|
def cql_where
|
114
110
|
attrs = attributes.clone
|
115
111
|
attrs = sanitize_attributes(attrs) if parent.table_exists?
|
116
112
|
attrs = attrs.collect{|key, value| cql_where_attribute(key, value) }.join(' AND ')
|
117
113
|
cql = attrs.present? ? "WHERE #{attrs}" : nil
|
118
114
|
end
|
119
|
-
|
115
|
+
|
120
116
|
def cql_where_attribute(key, value)
|
121
117
|
if value.is_a?(Array)
|
122
118
|
"#{key} IN (#{value.join(',')})"
|
@@ -124,43 +120,43 @@ module Quandl::Cassandra::Base::Scoping
|
|
124
120
|
"#{key} = #{value}"
|
125
121
|
end
|
126
122
|
end
|
127
|
-
|
123
|
+
|
128
124
|
def cql_select
|
129
125
|
columns = Array(select).join(',') if select.present?
|
130
126
|
columns = '*' if columns.blank?
|
131
127
|
columns
|
132
128
|
end
|
133
|
-
|
129
|
+
|
134
130
|
def cql_order
|
135
131
|
"ORDER BY #{order}" if order.present?
|
136
132
|
end
|
137
|
-
|
133
|
+
|
138
134
|
def cql_limit
|
139
135
|
"LIMIT #{limit}" if limit.present?
|
140
136
|
end
|
141
|
-
|
137
|
+
|
142
138
|
def execute
|
143
139
|
@execute ||= parent.execute(to_cql)
|
144
140
|
end
|
145
|
-
|
141
|
+
|
146
142
|
def execute_async
|
147
143
|
@execute_async ||= parent.execute(to_cql)
|
148
144
|
end
|
149
|
-
|
145
|
+
|
150
146
|
def parent
|
151
147
|
self.class.parent
|
152
148
|
end
|
153
|
-
|
149
|
+
|
154
150
|
def execute_returning_collection
|
155
151
|
execute.collect{|r| parent.new_from_query_result(r) }
|
156
152
|
end
|
157
|
-
|
153
|
+
|
158
154
|
def execute_returning_record
|
159
155
|
parent.new_from_query_result( execute.first )
|
160
156
|
end
|
161
|
-
|
157
|
+
|
162
158
|
end
|
163
|
-
|
159
|
+
|
164
160
|
end
|
165
|
-
|
161
|
+
|
166
162
|
end
|
@@ -1,22 +1,22 @@
|
|
1
1
|
class Quandl::Cassandra::Batch
|
2
|
-
module Logging
|
2
|
+
module Logging
|
3
3
|
|
4
|
-
|
4
|
+
extend ActiveSupport::Concern
|
5
|
+
|
6
|
+
module ClassMethods
|
7
|
+
|
8
|
+
def execute(rows, &block)
|
9
|
+
# log init
|
10
|
+
statement = block.call(rows[0].collect{'?'})
|
11
|
+
t1 = Time.now
|
12
|
+
# call method
|
13
|
+
r = super if defined?(super)
|
14
|
+
# log write
|
15
|
+
Quandl::Logger.debug("(#{t1.elapsed_ms}) ".blue + " [#{rows.count} rows] BATCH #{statement}")
|
16
|
+
r
|
17
|
+
end
|
5
18
|
|
6
|
-
module ClassMethods
|
7
|
-
|
8
|
-
def execute(rows, &block)
|
9
|
-
# log init
|
10
|
-
statement = block.call(rows[0].collect{'?'})
|
11
|
-
t1 = Time.now
|
12
|
-
# call method
|
13
|
-
r = super if defined?(super)
|
14
|
-
# log write
|
15
|
-
Quandl::Logger.debug("(#{t1.elapsed_ms}) ".blue + " [#{rows.count} rows] BATCH #{statement}")
|
16
|
-
r
|
17
19
|
end
|
18
|
-
|
19
|
-
end
|
20
20
|
|
21
|
-
end
|
21
|
+
end
|
22
22
|
end
|
@@ -1,40 +1,40 @@
|
|
1
1
|
module Quandl
|
2
|
-
module Cassandra
|
3
|
-
|
4
|
-
class << self
|
5
|
-
attr_accessor :configuration
|
6
|
-
end
|
7
|
-
|
8
|
-
def self.configuration
|
9
|
-
@configuration ||= Configuration.new
|
10
|
-
end
|
11
|
-
|
12
|
-
def self.configure
|
13
|
-
yield(configuration)
|
14
|
-
true
|
15
|
-
end
|
16
|
-
|
17
|
-
class Configuration
|
18
|
-
|
19
|
-
attr_accessor :hosts, :consistency, :keyspace, :batch_size, :read_consistency, :write_consistency
|
2
|
+
module Cassandra
|
20
3
|
|
21
|
-
|
22
|
-
|
23
|
-
@consistency = :all
|
24
|
-
@read_consistency = :one
|
25
|
-
@write_consistency = :one
|
26
|
-
@batch_size = 35 * 4
|
4
|
+
class << self
|
5
|
+
attr_accessor :configuration
|
27
6
|
end
|
28
7
|
|
29
|
-
def
|
30
|
-
|
8
|
+
def self.configuration
|
9
|
+
@configuration ||= Configuration.new
|
31
10
|
end
|
32
11
|
|
33
|
-
def
|
34
|
-
|
12
|
+
def self.configure
|
13
|
+
yield(configuration)
|
14
|
+
true
|
15
|
+
end
|
16
|
+
|
17
|
+
class Configuration
|
18
|
+
|
19
|
+
attr_accessor :hosts, :consistency, :keyspace, :batch_size, :read_consistency, :write_consistency
|
20
|
+
|
21
|
+
def initialize
|
22
|
+
@hosts = ['localhost']
|
23
|
+
@consistency = :all
|
24
|
+
@read_consistency = :one
|
25
|
+
@write_consistency = :one
|
26
|
+
@batch_size = 35 * 4
|
27
|
+
end
|
28
|
+
|
29
|
+
def attributes
|
30
|
+
{ hosts: hosts, consistency: consistency, keyspace: keyspace, batch_size: batch_size }
|
31
|
+
end
|
32
|
+
|
33
|
+
def to_h
|
34
|
+
attributes
|
35
|
+
end
|
36
|
+
|
35
37
|
end
|
36
38
|
|
37
39
|
end
|
38
|
-
|
39
|
-
end
|
40
40
|
end
|
@@ -1,14 +1,14 @@
|
|
1
1
|
module Quandl
|
2
|
-
module Cassandra
|
3
|
-
class Logger
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
2
|
+
module Cassandra
|
3
|
+
class Logger
|
4
|
+
class << self
|
5
|
+
|
6
|
+
def method_missing(method_sym, *args, &block)
|
7
|
+
args[0] = "CASSANDRA ".blue + args[0].to_s
|
8
|
+
Quandl::Logger.send(method_sym, *args)
|
9
|
+
end
|
10
|
+
|
11
|
+
end
|
9
12
|
end
|
10
|
-
|
11
13
|
end
|
12
|
-
end
|
13
|
-
end
|
14
14
|
end
|