sql_partitioner 0.6.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +61 -0
- data/.rspec +3 -0
- data/.travis.yml +10 -0
- data/CHANGELOG.md +30 -0
- data/Gemfile +4 -0
- data/LICENSE +19 -0
- data/README.md +119 -0
- data/Rakefile +6 -0
- data/lib/sql_partitioner/adapters/ar_adapter.rb +39 -0
- data/lib/sql_partitioner/adapters/base_adapter.rb +18 -0
- data/lib/sql_partitioner/adapters/dm_adapter.rb +24 -0
- data/lib/sql_partitioner/base_partitions_manager.rb +231 -0
- data/lib/sql_partitioner/loader.rb +16 -0
- data/lib/sql_partitioner/lock_wait_timeout_handler.rb +17 -0
- data/lib/sql_partitioner/partition.rb +158 -0
- data/lib/sql_partitioner/partitions_manager.rb +128 -0
- data/lib/sql_partitioner/sql_helper.rb +122 -0
- data/lib/sql_partitioner/time_unit_converter.rb +92 -0
- data/lib/sql_partitioner/version.rb +3 -0
- data/lib/sql_partitioner.rb +14 -0
- data/spec/db_conf.yml +7 -0
- data/spec/lib/spec_helper.rb +52 -0
- data/spec/lib/sql_partitioner/adapters/adapters_spec.rb +82 -0
- data/spec/lib/sql_partitioner/adapters/base_adapter_spec.rb +28 -0
- data/spec/lib/sql_partitioner/base_partitions_manager_spec.rb +442 -0
- data/spec/lib/sql_partitioner/loader_spec.rb +12 -0
- data/spec/lib/sql_partitioner/lock_wait_timeout_spec.rb +136 -0
- data/spec/lib/sql_partitioner/partition_spec.rb +268 -0
- data/spec/lib/sql_partitioner/partitions_manager_spec.rb +275 -0
- data/spec/lib/sql_partitioner/sql_helper_spec.rb +43 -0
- data/spec/lib/sql_partitioner/time_unit_converter_spec.rb +91 -0
- data/sql_partitioner.gemspec +47 -0
- metadata +234 -0
@@ -0,0 +1,158 @@
|
|
1
|
+
module SqlPartitioner
|
2
|
+
|
3
|
+
# Represents an array of `Partition` objects, with some extra helper methods.
|
4
|
+
class PartitionCollection < Array
|
5
|
+
|
6
|
+
# selects all partitions that hold records older than the timestamp provided
|
7
|
+
# @param [Fixnum] timestamp
|
8
|
+
# @return [Array<Partition>] partitions that hold data older than given timestamp
|
9
|
+
def older_than_timestamp(timestamp)
|
10
|
+
non_future_partitions.select do |p|
|
11
|
+
timestamp > p.timestamp
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
# selects all partitions that hold records newer than the timestamp provided
|
16
|
+
# @param [Fixnum] timestamp
|
17
|
+
# @return [Array<Partition>] partitions that hold data newer than given timestamp
|
18
|
+
def newer_than_timestamp(timestamp)
|
19
|
+
non_future_partitions.select do |p|
|
20
|
+
timestamp <= p.timestamp
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
# fetch the partition which is currently active. i.e. holds the records generated now
|
25
|
+
# @param [Fixnum] current_timestamp
|
26
|
+
# @return [Partition,NilClass]
|
27
|
+
def current_partition(current_timestamp)
|
28
|
+
non_future_partitions.select do |p|
|
29
|
+
p.timestamp > current_timestamp
|
30
|
+
end.min_by { |p| p.timestamp }
|
31
|
+
end
|
32
|
+
|
33
|
+
# @return [Array<Partition>] all partitions that do not have timestamp as `FUTURE_PARTITION_VALUE`
|
34
|
+
def non_future_partitions
|
35
|
+
self.reject { |p| p.future_partition? }
|
36
|
+
end
|
37
|
+
|
38
|
+
# fetch the latest partition that is not a future partition i.e. (value
|
39
|
+
# is not `FUTURE_PARTITION_VALUE`)
|
40
|
+
# @return [Partition,NilClass] partition with maximum timestamp value
|
41
|
+
def latest_partition
|
42
|
+
non_future_partitions.max_by{ |p| p.timestamp }
|
43
|
+
end
|
44
|
+
|
45
|
+
# @return [Partition,NilClass] the partition with oldest timestamp
|
46
|
+
def oldest_partition
|
47
|
+
non_future_partitions.min_by { |p| p.timestamp }
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
# Represents information for a single partition in the database
|
52
|
+
class Partition
|
53
|
+
FUTURE_PARTITION_NAME = 'future'
|
54
|
+
FUTURE_PARTITION_VALUE = 'MAXVALUE'
|
55
|
+
TO_LOG_ATTRIBUTES_SORT_ORDER = [
|
56
|
+
:ordinal_position, :name, :timestamp, :table_rows, :data_length, :index_length
|
57
|
+
]
|
58
|
+
|
59
|
+
# Likely only called by `Partition.all`
|
60
|
+
def initialize(partition_data)
|
61
|
+
@partition_data = partition_data
|
62
|
+
end
|
63
|
+
|
64
|
+
# Fetches info on all partitions for the given `table_name`, using the given `adapter`.
|
65
|
+
# @param [BaseAdapter] adapter
|
66
|
+
# @param [String] table_name
|
67
|
+
# @return [PartitionCollection]
|
68
|
+
def self.all(adapter, table_name)
|
69
|
+
select_sql = SqlPartitioner::SQL.partition_info
|
70
|
+
result = adapter.select(select_sql, adapter.schema_name, table_name).reject{|r| r.partition_description.nil? }
|
71
|
+
|
72
|
+
partition_collection = PartitionCollection.new
|
73
|
+
result.each{ |r| partition_collection << self.new(r) }
|
74
|
+
|
75
|
+
partition_collection
|
76
|
+
end
|
77
|
+
|
78
|
+
# @return [Fixnum]
|
79
|
+
def ordinal_position
|
80
|
+
@partition_data.partition_ordinal_position
|
81
|
+
end
|
82
|
+
|
83
|
+
# @return [String]
|
84
|
+
def name
|
85
|
+
@partition_data.partition_name
|
86
|
+
end
|
87
|
+
|
88
|
+
# @return [Fixnum,String] only a string for "future" partition
|
89
|
+
def timestamp
|
90
|
+
if @partition_data.partition_description == FUTURE_PARTITION_VALUE
|
91
|
+
FUTURE_PARTITION_VALUE
|
92
|
+
else
|
93
|
+
@partition_data.partition_description.to_i
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
# @return [Fixnum]
|
98
|
+
def table_rows
|
99
|
+
@partition_data.table_rows
|
100
|
+
end
|
101
|
+
|
102
|
+
# @return [Fixnum]
|
103
|
+
def data_length
|
104
|
+
@partition_data.data_length
|
105
|
+
end
|
106
|
+
|
107
|
+
# @return [Fixnum]
|
108
|
+
def index_length
|
109
|
+
@partition_data.index_length
|
110
|
+
end
|
111
|
+
|
112
|
+
# @return [Boolean]
|
113
|
+
def future_partition?
|
114
|
+
self.timestamp == FUTURE_PARTITION_VALUE
|
115
|
+
end
|
116
|
+
|
117
|
+
# @return [Hash]
|
118
|
+
def attributes
|
119
|
+
{
|
120
|
+
:ordinal_position => ordinal_position,
|
121
|
+
:name => name,
|
122
|
+
:timestamp => timestamp,
|
123
|
+
:table_rows => table_rows,
|
124
|
+
:data_length => data_length,
|
125
|
+
:index_length => index_length
|
126
|
+
}
|
127
|
+
end
|
128
|
+
|
129
|
+
# logs the formatted partition info from information schema
|
130
|
+
# @param [Array] array of partition objects
|
131
|
+
# @return [String] formatted partitions in tabular form
|
132
|
+
def self.to_log(partitions)
|
133
|
+
return "none" if partitions.empty?
|
134
|
+
|
135
|
+
padding = TO_LOG_ATTRIBUTES_SORT_ORDER.map do |attribute|
|
136
|
+
max_length = partitions.map do |partition|
|
137
|
+
partition.send(attribute).to_s.length
|
138
|
+
end.max
|
139
|
+
[attribute.to_s.length, max_length].max + 3
|
140
|
+
end
|
141
|
+
|
142
|
+
header = TO_LOG_ATTRIBUTES_SORT_ORDER.each_with_index.map do |attribute, index|
|
143
|
+
attribute.to_s.ljust(padding[index])
|
144
|
+
end.join
|
145
|
+
|
146
|
+
body = partitions.map do |partition|
|
147
|
+
TO_LOG_ATTRIBUTES_SORT_ORDER.each_with_index.map do |attribute, index|
|
148
|
+
partition.send(attribute).to_s.ljust(padding[index])
|
149
|
+
end.join
|
150
|
+
end.join("\n")
|
151
|
+
|
152
|
+
separator = ''.ljust(padding.inject(&:+),'-')
|
153
|
+
|
154
|
+
[separator, header, separator, body, separator].join("\n")
|
155
|
+
end
|
156
|
+
|
157
|
+
end
|
158
|
+
end
|
@@ -0,0 +1,128 @@
|
|
1
|
+
module SqlPartitioner
|
2
|
+
# Performs partitioning operations against a table.
|
3
|
+
class PartitionsManager < BasePartitionsManager
|
4
|
+
VALID_PARTITION_SIZE_UNITS = [:months, :days]
|
5
|
+
|
6
|
+
# Initialize the partitions based on intervals relative to current timestamp.
|
7
|
+
# Partition size specified by (partition_size, partition_size_unit), i.e. 1 month.
|
8
|
+
# Partitions will be created as needed to cover days_into_future.
|
9
|
+
#
|
10
|
+
# @param [Fixnum] days_into_future Number of days into the future from current_timestamp
|
11
|
+
# @param [Symbol] partition_size_unit one of: [:months, :days]
|
12
|
+
# @param [Fixnum] partition_size size of partition (in terms of partition_size_unit), i.e. 3 month
|
13
|
+
# @param [Boolean] dry_run Defaults to false. If true, query wont be executed.
|
14
|
+
# @return [Boolean] true if not dry run
|
15
|
+
# @return [String] sql to initialize partitions if dry run is true
|
16
|
+
# @raise [ArgumentError] if days is not array or if one of the
|
17
|
+
# days is not integer
|
18
|
+
def initialize_partitioning_in_intervals(days_into_future, partition_size_unit = :months, partition_size = 1, dry_run = false)
|
19
|
+
partition_data = partitions_to_append(@current_timestamp, partition_size_unit, partition_size, days_into_future)
|
20
|
+
initialize_partitioning(partition_data, dry_run)
|
21
|
+
end
|
22
|
+
|
23
|
+
# Get partition to add a partition to end with the given window size
|
24
|
+
#
|
25
|
+
# @param [Fixnum] partition_start_timestamp
|
26
|
+
# @param [Symbol] partition_size_unit: [:days, :months]
|
27
|
+
# @param [Fixnum] partition_size, size of partition (in terms of partition_size_unit)
|
28
|
+
# @param [Fixnum] partitions_into_future, how many partitions into the future should be covered
|
29
|
+
#
|
30
|
+
# @return [Hash] partition_data hash
|
31
|
+
def partitions_to_append(partition_start_timestamp, partition_size_unit, partition_size, days_into_future)
|
32
|
+
_validate_positive_fixnum(:days_into_future, days_into_future)
|
33
|
+
|
34
|
+
end_timestamp = @tuc.advance(current_timestamp, :days, days_into_future)
|
35
|
+
partitions_to_append_by_ts_range(partition_start_timestamp, end_timestamp, partition_size_unit, partition_size)
|
36
|
+
end
|
37
|
+
|
38
|
+
# Get partition_data hash based on the last partition's timestamp and covering end_timestamp
|
39
|
+
#
|
40
|
+
# @param [Fixnum] partition_start_timestamp, timestamp of last partition
|
41
|
+
# @param [Fixnum] end_timestamp, timestamp which the newest partition needs to include
|
42
|
+
# @param [Symbol] partition_size_unit: [:days, :months]
|
43
|
+
# @param [Fixnum] partition_size, intervals covered by the new partition
|
44
|
+
#
|
45
|
+
# @return [Hash] partition_data hash
|
46
|
+
def partitions_to_append_by_ts_range(partition_start_timestamp, end_timestamp, partition_size_unit, partition_size)
|
47
|
+
if partition_size_unit.nil? || !VALID_PARTITION_SIZE_UNITS.include?(partition_size_unit)
|
48
|
+
_raise_arg_err "partition_size_unit must be one of: #{VALID_PARTITION_SIZE_UNITS.inspect}"
|
49
|
+
end
|
50
|
+
|
51
|
+
_validate_positive_fixnum(:partition_size, partition_size)
|
52
|
+
_validate_positive_fixnum(:partition_start_timestamp, partition_start_timestamp)
|
53
|
+
_validate_positive_fixnum(:end_timestamp, end_timestamp)
|
54
|
+
|
55
|
+
timestamp = partition_start_timestamp
|
56
|
+
|
57
|
+
partitions_to_append = {}
|
58
|
+
while timestamp < end_timestamp
|
59
|
+
timestamp = @tuc.advance(timestamp, partition_size_unit, partition_size)
|
60
|
+
|
61
|
+
partition_name = name_from_timestamp(timestamp)
|
62
|
+
partitions_to_append[partition_name] = timestamp
|
63
|
+
end
|
64
|
+
|
65
|
+
partitions_to_append
|
66
|
+
end
|
67
|
+
|
68
|
+
# Wrapper around append partition to add a partition to end with the
|
69
|
+
# given window size
|
70
|
+
#
|
71
|
+
# @param [Symbol] partition_size_unit: [:days, :months]
|
72
|
+
# @param [Fixnum] partition_size, intervals covered by the new partition
|
73
|
+
# @param [Fixnum] days_into_future, how many days into the future need to be covered by partitions
|
74
|
+
# @param [Boolean] dry_run, Defaults to false. If true, query wont be executed.
|
75
|
+
# @return [Hash] partition_data hash of the partitions appended
|
76
|
+
# @raise [ArgumentError] if window size is nil or not greater than 0
|
77
|
+
def append_partition_intervals(partition_size_unit, partition_size, days_into_future = 30, dry_run = false)
|
78
|
+
partitions = Partition.all(adapter, table_name)
|
79
|
+
if partitions.blank? || partitions.non_future_partitions.blank?
|
80
|
+
raise "partitions must be properly initialized before appending"
|
81
|
+
end
|
82
|
+
latest_partition = partitions.latest_partition
|
83
|
+
|
84
|
+
new_partition_data = partitions_to_append(latest_partition.timestamp, partition_size_unit, partition_size, days_into_future)
|
85
|
+
|
86
|
+
if new_partition_data.empty?
|
87
|
+
msg = "Append: No-Op - Latest Partition Time of #{latest_partition.timestamp}, " +
|
88
|
+
"i.e. #{Time.at(@tuc.to_seconds(latest_partition.timestamp))} covers >= #{days_into_future} days_into_future"
|
89
|
+
else
|
90
|
+
msg = "Append: Appending the following new partitions: #{new_partition_data.inspect}"
|
91
|
+
reorg_future_partition(new_partition_data, dry_run)
|
92
|
+
end
|
93
|
+
|
94
|
+
log(msg)
|
95
|
+
|
96
|
+
new_partition_data
|
97
|
+
end
|
98
|
+
|
99
|
+
# drop partitions that are older than days(input) from now
|
100
|
+
# @param [Fixnum] days_from_now
|
101
|
+
# @param [Boolean] dry_run, Defaults to false. If true, query wont be executed.
|
102
|
+
def drop_partitions_older_than_in_days(days_from_now, dry_run = false)
|
103
|
+
timestamp = self.current_timestamp - @tuc.from_days(days_from_now)
|
104
|
+
drop_partitions_older_than(timestamp, dry_run)
|
105
|
+
end
|
106
|
+
|
107
|
+
# drop partitions that are older than the given timestamp
|
108
|
+
# @param [Fixnum] timestamp partitions older than this timestamp will be
|
109
|
+
# dropped
|
110
|
+
# @param [Boolean] dry_run, Defaults to false. If true, query wont be executed.
|
111
|
+
# @return [Array] an array of partition names that were dropped
|
112
|
+
def drop_partitions_older_than(timestamp, dry_run = false)
|
113
|
+
partitions = Partition.all(adapter, table_name).older_than_timestamp(timestamp)
|
114
|
+
partition_names = partitions.map(&:name)
|
115
|
+
|
116
|
+
if partition_names.empty?
|
117
|
+
msg = "Drop: No-Op - No partitions older than #{timestamp}, i.e. #{Time.at(@tuc.to_seconds(timestamp))} to drop"
|
118
|
+
else
|
119
|
+
msg = "Drop: Dropped partitions: #{partition_names.inspect}"
|
120
|
+
drop_partitions(partition_names, dry_run)
|
121
|
+
end
|
122
|
+
|
123
|
+
log(msg)
|
124
|
+
|
125
|
+
partition_names
|
126
|
+
end
|
127
|
+
end
|
128
|
+
end
|
@@ -0,0 +1,122 @@
|
|
1
|
+
module SqlPartitioner
|
2
|
+
class SQL
|
3
|
+
|
4
|
+
# SQL query will return rows having the following columns:
|
5
|
+
# - TABLE_CATALOG
|
6
|
+
# - TABLE_SCHEMA
|
7
|
+
# - TABLE_NAME
|
8
|
+
# - PARTITION_NAME
|
9
|
+
# - SUBPARTITION_NAME
|
10
|
+
# - PARTITION_ORDINAL_POSITION
|
11
|
+
# - SUBPARTITION_ORDINAL_POSITION
|
12
|
+
# - PARTITION_METHOD
|
13
|
+
# - SUBPARTITION_METHOD
|
14
|
+
# - PARTITION_EXPRESSION
|
15
|
+
# - SUBPARTITION_EXPRESSION
|
16
|
+
# - PARTITION_DESCRIPTION
|
17
|
+
# - TABLE_ROWS
|
18
|
+
# - AVG_ROW_LENGTH
|
19
|
+
# - DATA_LENGTH
|
20
|
+
# - MAX_DATA_LENGTH
|
21
|
+
# - INDEX_LENGTH
|
22
|
+
# - DATA_FREE
|
23
|
+
# - CREATE_TIME
|
24
|
+
# - UPDATE_TIME
|
25
|
+
# - CHECK_TIME
|
26
|
+
# - CHECKSUM
|
27
|
+
# - PARTITION_COMMENT
|
28
|
+
# - NODEGROUP
|
29
|
+
# - TABLESPACE_NAME
|
30
|
+
def self.partition_info
|
31
|
+
compress_lines(<<-SQL)
|
32
|
+
SELECT *
|
33
|
+
FROM information_schema.PARTITIONS
|
34
|
+
WHERE TABLE_SCHEMA = ?
|
35
|
+
AND TABLE_NAME = ?
|
36
|
+
SQL
|
37
|
+
end
|
38
|
+
|
39
|
+
def self.drop_partitions(table_name, partition_names)
|
40
|
+
return nil if partition_names.empty?
|
41
|
+
|
42
|
+
compress_lines(<<-SQL)
|
43
|
+
ALTER TABLE #{table_name}
|
44
|
+
DROP PARTITION #{partition_names.join(',')}
|
45
|
+
SQL
|
46
|
+
end
|
47
|
+
|
48
|
+
def self.create_partition(table_name, partition_name, until_timestamp)
|
49
|
+
compress_lines(<<-SQL)
|
50
|
+
ALTER TABLE #{table_name}
|
51
|
+
ADD PARTITION
|
52
|
+
(PARTITION #{partition_name}
|
53
|
+
VALUES LESS THAN (#{until_timestamp}))
|
54
|
+
SQL
|
55
|
+
end
|
56
|
+
|
57
|
+
def self.reorg_partitions(table_name, new_partition_data, reorg_partition_name)
|
58
|
+
return nil if new_partition_data.empty?
|
59
|
+
|
60
|
+
partition_suq_query = sort_partition_data(new_partition_data).map do |partition_name, until_timestamp|
|
61
|
+
"PARTITION #{partition_name} VALUES LESS THAN (#{until_timestamp})"
|
62
|
+
end.join(',')
|
63
|
+
|
64
|
+
compress_lines(<<-SQL)
|
65
|
+
ALTER TABLE #{table_name}
|
66
|
+
REORGANIZE PARTITION #{reorg_partition_name} INTO
|
67
|
+
(#{partition_suq_query})
|
68
|
+
SQL
|
69
|
+
end
|
70
|
+
|
71
|
+
def self.initialize_partitioning(table_name, partition_data)
|
72
|
+
partition_sub_query = sort_partition_data(partition_data).map do |partition_name, until_timestamp|
|
73
|
+
"PARTITION #{partition_name} VALUES LESS THAN (#{until_timestamp})"
|
74
|
+
end.join(',')
|
75
|
+
|
76
|
+
compress_lines(<<-SQL)
|
77
|
+
ALTER TABLE #{table_name}
|
78
|
+
PARTITION BY RANGE(timestamp)
|
79
|
+
(#{partition_sub_query})
|
80
|
+
SQL
|
81
|
+
end
|
82
|
+
|
83
|
+
|
84
|
+
# @param [Hash<String,Fixnum>] partition_data hash of name to timestamp
|
85
|
+
# @return [Array] array of partitions sorted by timestamp ascending, with the 'future' partition at the end
|
86
|
+
def self.sort_partition_data(partition_data)
|
87
|
+
partition_data.to_a.sort do |x,y|
|
88
|
+
if x[1] == "MAXVALUE"
|
89
|
+
1
|
90
|
+
elsif y[1] == "MAXVALUE"
|
91
|
+
-1
|
92
|
+
else
|
93
|
+
x[1] <=> y[1]
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
# Replace sequences of whitespace (including newlines) with either
|
99
|
+
# a single space or remove them entirely (according to param _spaced_).
|
100
|
+
#
|
101
|
+
# Copied from:
|
102
|
+
# https://github.com/datamapper/dm-core/blob/master/lib/dm-core/support/ext/string.rb
|
103
|
+
#
|
104
|
+
# compress_lines(<<QUERY)
|
105
|
+
# SELECT name
|
106
|
+
# FROM users
|
107
|
+
# QUERY => "SELECT name FROM users"
|
108
|
+
#
|
109
|
+
# @param [String] string
|
110
|
+
# The input string.
|
111
|
+
#
|
112
|
+
# @param [TrueClass, FalseClass] spaced (default=true)
|
113
|
+
# Determines whether returned string has whitespace collapsed or removed.
|
114
|
+
#
|
115
|
+
# @return [String] The input string with whitespace (including newlines) replaced.
|
116
|
+
#
|
117
|
+
def self.compress_lines(string, spaced = true)
|
118
|
+
string.split($/).map { |line| line.strip }.join(spaced ? ' ' : '')
|
119
|
+
end
|
120
|
+
|
121
|
+
end
|
122
|
+
end
|
@@ -0,0 +1,92 @@
|
|
1
|
+
module SqlPartitioner
|
2
|
+
class TimeUnitConverter
|
3
|
+
require 'date'
|
4
|
+
|
5
|
+
MINUTE_AS_SECONDS = 60
|
6
|
+
HOUR_AS_SECONDS = 60 * MINUTE_AS_SECONDS
|
7
|
+
DAY_AS_SECONDS = 24 * HOUR_AS_SECONDS
|
8
|
+
|
9
|
+
SUPPORTED_TIME_UNITS = [:seconds, :micro_seconds]
|
10
|
+
|
11
|
+
# @param [Symbol] time_unit one of SUPPORTED_TIME_UNITS
|
12
|
+
def initialize(time_unit)
|
13
|
+
raise ArgumentError.new("Invalid time unit #{time_unit} passed") if !SUPPORTED_TIME_UNITS.include?(time_unit)
|
14
|
+
|
15
|
+
@time_unit = time_unit
|
16
|
+
end
|
17
|
+
|
18
|
+
# @param [Fixnum] num_days
|
19
|
+
# @return [Fixnum] number of days represented in the configure time units
|
20
|
+
def from_days(num_days)
|
21
|
+
from_seconds(num_days * DAY_AS_SECONDS)
|
22
|
+
end
|
23
|
+
|
24
|
+
# @param [Fixnum] time_units_timestamp timestamp in configured time units
|
25
|
+
# @return [DateTime] representation of the given timestamp
|
26
|
+
def to_date_time(time_units_timestamp)
|
27
|
+
DateTime.strptime("#{to_seconds(time_units_timestamp)}", '%s')
|
28
|
+
end
|
29
|
+
|
30
|
+
# converts from seconds to the configured time unit
|
31
|
+
#
|
32
|
+
# @param [Fixnum] timestamp_seconds timestamp in seconds
|
33
|
+
#
|
34
|
+
# @return [Fixnum] timestamp in configured time units
|
35
|
+
def from_seconds(timestamp_seconds)
|
36
|
+
timestamp_seconds * time_units_per_second
|
37
|
+
end
|
38
|
+
|
39
|
+
# converts from the configured time unit to seconds
|
40
|
+
#
|
41
|
+
# @param [Fixnum] time_units_timestamp timestamp in the configured time units
|
42
|
+
#
|
43
|
+
# @return [Fixnum] timestamp in seconds
|
44
|
+
def to_seconds(time_units_timestamp)
|
45
|
+
time_units_timestamp / time_units_per_second
|
46
|
+
end
|
47
|
+
|
48
|
+
# @return [Fixnum] how many of the configured time_unit are in 1 second
|
49
|
+
def time_units_per_second
|
50
|
+
self.class.time_units_per_second(@time_unit)
|
51
|
+
end
|
52
|
+
|
53
|
+
# @param [Fixnum] time_units_timestamp
|
54
|
+
# @param [Symbol] calendar_unit unit for the given value, one of [:day(s), :month(s)]
|
55
|
+
# @param [Fixnum] value in terms of calendar_unit to add to the time_units_timestamp
|
56
|
+
#
|
57
|
+
# @return [Fixnum] new timestamp in configured time units
|
58
|
+
def advance(time_units_timestamp, calendar_unit, value)
|
59
|
+
date_time = to_date_time(time_units_timestamp)
|
60
|
+
date_time = self.class.advance_date_time(date_time, calendar_unit, value)
|
61
|
+
from_seconds(date_time.to_time.to_i)
|
62
|
+
end
|
63
|
+
|
64
|
+
# @param [DateTime] date_time to advance
|
65
|
+
# @param [Symbol] calendar_unit unit for the following `value`, one of [:day(s), :month(s)]
|
66
|
+
# @param [Fixnum] value in terms of calendar_unit to add to the date_time
|
67
|
+
# @return [DateTime] result of advancing the given date_time by the given value
|
68
|
+
def self.advance_date_time(date_time, calendar_unit, value)
|
69
|
+
new_time = case calendar_unit
|
70
|
+
when :days, :day
|
71
|
+
date_time + value
|
72
|
+
when :months, :month
|
73
|
+
date_time >> value
|
74
|
+
end
|
75
|
+
|
76
|
+
new_time
|
77
|
+
end
|
78
|
+
|
79
|
+
# @param [Symbol] time_unit one of `SUPPORTED_TIME_UNITS`
|
80
|
+
# @return [Fixnum] how many of the given time_unit are in 1 second
|
81
|
+
def self.time_units_per_second(time_unit)
|
82
|
+
case time_unit
|
83
|
+
when :micro_seconds
|
84
|
+
1_000_000
|
85
|
+
when :seconds
|
86
|
+
1
|
87
|
+
else
|
88
|
+
raise "unknown time_unit #{time_unit.inspect}"
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
# standard requires
|
2
|
+
require 'sql_partitioner/loader'
|
3
|
+
require 'sql_partitioner/lock_wait_timeout_handler'
|
4
|
+
require 'sql_partitioner/partition'
|
5
|
+
require 'sql_partitioner/time_unit_converter'
|
6
|
+
require 'sql_partitioner/sql_helper'
|
7
|
+
require 'sql_partitioner/base_partitions_manager'
|
8
|
+
require 'sql_partitioner/partitions_manager'
|
9
|
+
|
10
|
+
SqlPartitioner::Loader.require_or_skip('sql_partitioner/adapters/ar_adapter', 'ActiveRecord')
|
11
|
+
SqlPartitioner::Loader.require_or_skip('sql_partitioner/adapters/dm_adapter', 'DataMapper')
|
12
|
+
|
13
|
+
|
14
|
+
|
data/spec/db_conf.yml
ADDED
@@ -0,0 +1,52 @@
|
|
1
|
+
require 'simplecov'
|
2
|
+
SimpleCov.start
|
3
|
+
|
4
|
+
require 'active_record'
|
5
|
+
require 'data_mapper'
|
6
|
+
|
7
|
+
require 'sql_partitioner'
|
8
|
+
|
9
|
+
require 'logger'
|
10
|
+
|
11
|
+
#require 'ruby-debug' # enable debugger support
|
12
|
+
|
13
|
+
SPEC_LOGGER = Logger.new(nil)
|
14
|
+
|
15
|
+
RSpec.configure do |config|
|
16
|
+
# enable both should and expect syntax in rspec without deprecation warnings
|
17
|
+
config.expect_with :rspec do |c|
|
18
|
+
c.syntax = [:should, :expect]
|
19
|
+
end
|
20
|
+
config.mock_with :rspec do |c|
|
21
|
+
c.syntax = [:should, :expect]
|
22
|
+
end
|
23
|
+
|
24
|
+
config.before :suite do
|
25
|
+
require 'yaml'
|
26
|
+
db_conf = YAML.load_file('spec/db_conf.yml')
|
27
|
+
|
28
|
+
# establish both ActiveRecord and DataMapper connections
|
29
|
+
ActiveRecord::Base.establish_connection(db_conf["test"])
|
30
|
+
|
31
|
+
adapter,database,user,pass,host = db_conf["test"].values_at *%W(adapter database username password host)
|
32
|
+
connection_string = "#{adapter}://#{user}:#{pass}@#{host}/#{database}"
|
33
|
+
DataMapper.setup(:default, connection_string)
|
34
|
+
end
|
35
|
+
|
36
|
+
config.before :each do
|
37
|
+
sql = <<-SQL
|
38
|
+
DROP TABLE IF EXISTS `test_events`
|
39
|
+
SQL
|
40
|
+
DataMapper.repository.adapter.execute(sql)
|
41
|
+
|
42
|
+
sql = <<-SQL
|
43
|
+
CREATE TABLE IF NOT EXISTS `test_events` (
|
44
|
+
`id` bigint(20) NOT NULL AUTO_INCREMENT,
|
45
|
+
`timestamp` bigint(20) unsigned NOT NULL DEFAULT '0',
|
46
|
+
PRIMARY KEY (`id`,`timestamp`)
|
47
|
+
) ENGINE=InnoDB DEFAULT CHARSET=utf8
|
48
|
+
SQL
|
49
|
+
DataMapper.repository.adapter.execute(sql)
|
50
|
+
|
51
|
+
end
|
52
|
+
end
|
@@ -0,0 +1,82 @@
|
|
1
|
+
require File.expand_path("../../spec_helper", File.dirname(__FILE__))
|
2
|
+
|
3
|
+
shared_examples_for "an adapter" do
|
4
|
+
describe "#execute" do
|
5
|
+
it "should execute the SQL statement passed" do
|
6
|
+
adapter.execute("SET @sql_partitioner_test = 1")
|
7
|
+
|
8
|
+
adapter.select("SELECT @sql_partitioner_test").first.to_i.should == 1
|
9
|
+
|
10
|
+
adapter.execute("SET @sql_partitioner_test = NULL")
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
describe "#schema_name" do
|
15
|
+
it "should return the schema_name" do
|
16
|
+
adapter.schema_name.should == "sql_partitioner_test"
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
describe ".select" do
|
21
|
+
context "with multiple columns selected" do
|
22
|
+
context "and no data returned" do
|
23
|
+
it "should return an empty hash" do
|
24
|
+
adapter.select("SELECT * FROM test_events").should == []
|
25
|
+
end
|
26
|
+
end
|
27
|
+
context "and data returned" do
|
28
|
+
before(:each) do
|
29
|
+
adapter.execute("INSERT INTO test_events(timestamp) VALUES(1)")
|
30
|
+
adapter.execute("INSERT INTO test_events(timestamp) VALUES(2)")
|
31
|
+
end
|
32
|
+
it "should return a Struct with accessors for each column" do
|
33
|
+
result = adapter.select("SELECT * FROM test_events")
|
34
|
+
|
35
|
+
result.should be_a_kind_of(Array)
|
36
|
+
|
37
|
+
result.first.should be_a_kind_of(Struct)
|
38
|
+
result.first.timestamp.to_i.should == 1
|
39
|
+
|
40
|
+
result.last.should be_a_kind_of(Struct)
|
41
|
+
result.last.timestamp.to_i.should == 2
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
context "with a single column selected" do
|
46
|
+
before(:each) do
|
47
|
+
adapter.execute("INSERT INTO test_events(timestamp) VALUES(1)")
|
48
|
+
adapter.execute("INSERT INTO test_events(timestamp) VALUES(2)")
|
49
|
+
end
|
50
|
+
it "should return the values without Struct" do
|
51
|
+
result = adapter.select("SELECT timestamp FROM test_events")
|
52
|
+
|
53
|
+
result.should be_a_kind_of(Array)
|
54
|
+
|
55
|
+
result.first.to_i.should == 1
|
56
|
+
result.last.to_i.should == 2
|
57
|
+
end
|
58
|
+
it "should return an array even when only one row is selected" do
|
59
|
+
result = adapter.select("SELECT timestamp FROM test_events LIMIT 1")
|
60
|
+
|
61
|
+
result.should be_a_kind_of(Array)
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
describe SqlPartitioner::ARAdapter do
|
68
|
+
it_should_behave_like "an adapter" do
|
69
|
+
let(:adapter) do
|
70
|
+
SqlPartitioner::ARAdapter.new(ActiveRecord::Base.connection)
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
describe SqlPartitioner::DMAdapter do
|
76
|
+
it_should_behave_like "an adapter" do
|
77
|
+
let(:adapter) do
|
78
|
+
SqlPartitioner::DMAdapter.new(DataMapper.repository.adapter)
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
@@ -0,0 +1,28 @@
|
|
1
|
+
require File.expand_path("../../spec_helper", File.dirname(__FILE__))
|
2
|
+
|
3
|
+
describe "BaseAdapter" do
|
4
|
+
before(:each) do
|
5
|
+
@base_adapter = SqlPartitioner::BaseAdapter.new
|
6
|
+
end
|
7
|
+
describe "#select" do
|
8
|
+
it "should raise an RuntimeError" do
|
9
|
+
lambda{
|
10
|
+
@base_adapter.select("SELECT database()")
|
11
|
+
}.should raise_error(RuntimeError, /MUST BE IMPLEMENTED/)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
describe "#execute" do
|
15
|
+
it "should raise an RuntimeError" do
|
16
|
+
lambda{
|
17
|
+
@base_adapter.execute("SELECT database()")
|
18
|
+
}.should raise_error(RuntimeError, /MUST BE IMPLEMENTED/)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
describe "#schema_name" do
|
22
|
+
it "should raise an RuntimeError" do
|
23
|
+
lambda{
|
24
|
+
@base_adapter.schema_name
|
25
|
+
}.should raise_error(RuntimeError, /MUST BE IMPLEMENTED/)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|