sql_partitioner 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +61 -0
- data/.rspec +3 -0
- data/.travis.yml +10 -0
- data/CHANGELOG.md +30 -0
- data/Gemfile +4 -0
- data/LICENSE +19 -0
- data/README.md +119 -0
- data/Rakefile +6 -0
- data/lib/sql_partitioner/adapters/ar_adapter.rb +39 -0
- data/lib/sql_partitioner/adapters/base_adapter.rb +18 -0
- data/lib/sql_partitioner/adapters/dm_adapter.rb +24 -0
- data/lib/sql_partitioner/base_partitions_manager.rb +231 -0
- data/lib/sql_partitioner/loader.rb +16 -0
- data/lib/sql_partitioner/lock_wait_timeout_handler.rb +17 -0
- data/lib/sql_partitioner/partition.rb +158 -0
- data/lib/sql_partitioner/partitions_manager.rb +128 -0
- data/lib/sql_partitioner/sql_helper.rb +122 -0
- data/lib/sql_partitioner/time_unit_converter.rb +92 -0
- data/lib/sql_partitioner/version.rb +3 -0
- data/lib/sql_partitioner.rb +14 -0
- data/spec/db_conf.yml +7 -0
- data/spec/lib/spec_helper.rb +52 -0
- data/spec/lib/sql_partitioner/adapters/adapters_spec.rb +82 -0
- data/spec/lib/sql_partitioner/adapters/base_adapter_spec.rb +28 -0
- data/spec/lib/sql_partitioner/base_partitions_manager_spec.rb +442 -0
- data/spec/lib/sql_partitioner/loader_spec.rb +12 -0
- data/spec/lib/sql_partitioner/lock_wait_timeout_spec.rb +136 -0
- data/spec/lib/sql_partitioner/partition_spec.rb +268 -0
- data/spec/lib/sql_partitioner/partitions_manager_spec.rb +275 -0
- data/spec/lib/sql_partitioner/sql_helper_spec.rb +43 -0
- data/spec/lib/sql_partitioner/time_unit_converter_spec.rb +91 -0
- data/sql_partitioner.gemspec +47 -0
- metadata +234 -0
@@ -0,0 +1,158 @@
|
|
1
|
+
module SqlPartitioner
|
2
|
+
|
3
|
+
# Represents an array of `Partition` objects, with some extra helper methods.
|
4
|
+
class PartitionCollection < Array
|
5
|
+
|
6
|
+
# selects all partitions that hold records older than the timestamp provided
|
7
|
+
# @param [Fixnum] timestamp
|
8
|
+
# @return [Array<Partition>] partitions that hold data older than given timestamp
|
9
|
+
def older_than_timestamp(timestamp)
|
10
|
+
non_future_partitions.select do |p|
|
11
|
+
timestamp > p.timestamp
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
# selects all partitions that hold records newer than the timestamp provided
|
16
|
+
# @param [Fixnum] timestamp
|
17
|
+
# @return [Array<Partition>] partitions that hold data newer than given timestamp
|
18
|
+
def newer_than_timestamp(timestamp)
|
19
|
+
non_future_partitions.select do |p|
|
20
|
+
timestamp <= p.timestamp
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
# fetch the partition which is currently active. i.e. holds the records generated now
|
25
|
+
# @param [Fixnum] current_timestamp
|
26
|
+
# @return [Partition,NilClass]
|
27
|
+
def current_partition(current_timestamp)
|
28
|
+
non_future_partitions.select do |p|
|
29
|
+
p.timestamp > current_timestamp
|
30
|
+
end.min_by { |p| p.timestamp }
|
31
|
+
end
|
32
|
+
|
33
|
+
# @return [Array<Partition>] all partitions that do not have timestamp as `FUTURE_PARTITION_VALUE`
|
34
|
+
def non_future_partitions
|
35
|
+
self.reject { |p| p.future_partition? }
|
36
|
+
end
|
37
|
+
|
38
|
+
# fetch the latest partition that is not a future partition i.e. (value
|
39
|
+
# is not `FUTURE_PARTITION_VALUE`)
|
40
|
+
# @return [Partition,NilClass] partition with maximum timestamp value
|
41
|
+
def latest_partition
|
42
|
+
non_future_partitions.max_by{ |p| p.timestamp }
|
43
|
+
end
|
44
|
+
|
45
|
+
# @return [Partition,NilClass] the partition with oldest timestamp
|
46
|
+
def oldest_partition
|
47
|
+
non_future_partitions.min_by { |p| p.timestamp }
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
# Represents information for a single partition in the database
|
52
|
+
class Partition
|
53
|
+
FUTURE_PARTITION_NAME = 'future'
|
54
|
+
FUTURE_PARTITION_VALUE = 'MAXVALUE'
|
55
|
+
TO_LOG_ATTRIBUTES_SORT_ORDER = [
|
56
|
+
:ordinal_position, :name, :timestamp, :table_rows, :data_length, :index_length
|
57
|
+
]
|
58
|
+
|
59
|
+
# Likely only called by `Partition.all`
|
60
|
+
def initialize(partition_data)
|
61
|
+
@partition_data = partition_data
|
62
|
+
end
|
63
|
+
|
64
|
+
# Fetches info on all partitions for the given `table_name`, using the given `adapter`.
|
65
|
+
# @param [BaseAdapter] adapter
|
66
|
+
# @param [String] table_name
|
67
|
+
# @return [PartitionCollection]
|
68
|
+
def self.all(adapter, table_name)
|
69
|
+
select_sql = SqlPartitioner::SQL.partition_info
|
70
|
+
result = adapter.select(select_sql, adapter.schema_name, table_name).reject{|r| r.partition_description.nil? }
|
71
|
+
|
72
|
+
partition_collection = PartitionCollection.new
|
73
|
+
result.each{ |r| partition_collection << self.new(r) }
|
74
|
+
|
75
|
+
partition_collection
|
76
|
+
end
|
77
|
+
|
78
|
+
# @return [Fixnum]
|
79
|
+
def ordinal_position
|
80
|
+
@partition_data.partition_ordinal_position
|
81
|
+
end
|
82
|
+
|
83
|
+
# @return [String]
|
84
|
+
def name
|
85
|
+
@partition_data.partition_name
|
86
|
+
end
|
87
|
+
|
88
|
+
# @return [Fixnum,String] only a string for "future" partition
|
89
|
+
def timestamp
|
90
|
+
if @partition_data.partition_description == FUTURE_PARTITION_VALUE
|
91
|
+
FUTURE_PARTITION_VALUE
|
92
|
+
else
|
93
|
+
@partition_data.partition_description.to_i
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
# @return [Fixnum]
|
98
|
+
def table_rows
|
99
|
+
@partition_data.table_rows
|
100
|
+
end
|
101
|
+
|
102
|
+
# @return [Fixnum]
|
103
|
+
def data_length
|
104
|
+
@partition_data.data_length
|
105
|
+
end
|
106
|
+
|
107
|
+
# @return [Fixnum]
|
108
|
+
def index_length
|
109
|
+
@partition_data.index_length
|
110
|
+
end
|
111
|
+
|
112
|
+
# @return [Boolean]
|
113
|
+
def future_partition?
|
114
|
+
self.timestamp == FUTURE_PARTITION_VALUE
|
115
|
+
end
|
116
|
+
|
117
|
+
# @return [Hash]
|
118
|
+
def attributes
|
119
|
+
{
|
120
|
+
:ordinal_position => ordinal_position,
|
121
|
+
:name => name,
|
122
|
+
:timestamp => timestamp,
|
123
|
+
:table_rows => table_rows,
|
124
|
+
:data_length => data_length,
|
125
|
+
:index_length => index_length
|
126
|
+
}
|
127
|
+
end
|
128
|
+
|
129
|
+
# logs the formatted partition info from information schema
|
130
|
+
# @param [Array] array of partition objects
|
131
|
+
# @return [String] formatted partitions in tabular form
|
132
|
+
def self.to_log(partitions)
|
133
|
+
return "none" if partitions.empty?
|
134
|
+
|
135
|
+
padding = TO_LOG_ATTRIBUTES_SORT_ORDER.map do |attribute|
|
136
|
+
max_length = partitions.map do |partition|
|
137
|
+
partition.send(attribute).to_s.length
|
138
|
+
end.max
|
139
|
+
[attribute.to_s.length, max_length].max + 3
|
140
|
+
end
|
141
|
+
|
142
|
+
header = TO_LOG_ATTRIBUTES_SORT_ORDER.each_with_index.map do |attribute, index|
|
143
|
+
attribute.to_s.ljust(padding[index])
|
144
|
+
end.join
|
145
|
+
|
146
|
+
body = partitions.map do |partition|
|
147
|
+
TO_LOG_ATTRIBUTES_SORT_ORDER.each_with_index.map do |attribute, index|
|
148
|
+
partition.send(attribute).to_s.ljust(padding[index])
|
149
|
+
end.join
|
150
|
+
end.join("\n")
|
151
|
+
|
152
|
+
separator = ''.ljust(padding.inject(&:+),'-')
|
153
|
+
|
154
|
+
[separator, header, separator, body, separator].join("\n")
|
155
|
+
end
|
156
|
+
|
157
|
+
end
|
158
|
+
end
|
@@ -0,0 +1,128 @@
|
|
1
|
+
module SqlPartitioner
|
2
|
+
# Performs partitioning operations against a table.
|
3
|
+
class PartitionsManager < BasePartitionsManager
|
4
|
+
VALID_PARTITION_SIZE_UNITS = [:months, :days]
|
5
|
+
|
6
|
+
# Initialize the partitions based on intervals relative to current timestamp.
|
7
|
+
# Partition size specified by (partition_size, partition_size_unit), i.e. 1 month.
|
8
|
+
# Partitions will be created as needed to cover days_into_future.
|
9
|
+
#
|
10
|
+
# @param [Fixnum] days_into_future Number of days into the future from current_timestamp
|
11
|
+
# @param [Symbol] partition_size_unit one of: [:months, :days]
|
12
|
+
# @param [Fixnum] partition_size size of partition (in terms of partition_size_unit), i.e. 3 month
|
13
|
+
# @param [Boolean] dry_run Defaults to false. If true, query wont be executed.
|
14
|
+
# @return [Boolean] true if not dry run
|
15
|
+
# @return [String] sql to initialize partitions if dry run is true
|
16
|
+
# @raise [ArgumentError] if days is not array or if one of the
|
17
|
+
# days is not integer
|
18
|
+
def initialize_partitioning_in_intervals(days_into_future, partition_size_unit = :months, partition_size = 1, dry_run = false)
|
19
|
+
partition_data = partitions_to_append(@current_timestamp, partition_size_unit, partition_size, days_into_future)
|
20
|
+
initialize_partitioning(partition_data, dry_run)
|
21
|
+
end
|
22
|
+
|
23
|
+
# Get partition to add a partition to end with the given window size
|
24
|
+
#
|
25
|
+
# @param [Fixnum] partition_start_timestamp
|
26
|
+
# @param [Symbol] partition_size_unit: [:days, :months]
|
27
|
+
# @param [Fixnum] partition_size, size of partition (in terms of partition_size_unit)
|
28
|
+
# @param [Fixnum] partitions_into_future, how many partitions into the future should be covered
|
29
|
+
#
|
30
|
+
# @return [Hash] partition_data hash
|
31
|
+
def partitions_to_append(partition_start_timestamp, partition_size_unit, partition_size, days_into_future)
|
32
|
+
_validate_positive_fixnum(:days_into_future, days_into_future)
|
33
|
+
|
34
|
+
end_timestamp = @tuc.advance(current_timestamp, :days, days_into_future)
|
35
|
+
partitions_to_append_by_ts_range(partition_start_timestamp, end_timestamp, partition_size_unit, partition_size)
|
36
|
+
end
|
37
|
+
|
38
|
+
# Get partition_data hash based on the last partition's timestamp and covering end_timestamp
|
39
|
+
#
|
40
|
+
# @param [Fixnum] partition_start_timestamp, timestamp of last partition
|
41
|
+
# @param [Fixnum] end_timestamp, timestamp which the newest partition needs to include
|
42
|
+
# @param [Symbol] partition_size_unit: [:days, :months]
|
43
|
+
# @param [Fixnum] partition_size, intervals covered by the new partition
|
44
|
+
#
|
45
|
+
# @return [Hash] partition_data hash
|
46
|
+
def partitions_to_append_by_ts_range(partition_start_timestamp, end_timestamp, partition_size_unit, partition_size)
|
47
|
+
if partition_size_unit.nil? || !VALID_PARTITION_SIZE_UNITS.include?(partition_size_unit)
|
48
|
+
_raise_arg_err "partition_size_unit must be one of: #{VALID_PARTITION_SIZE_UNITS.inspect}"
|
49
|
+
end
|
50
|
+
|
51
|
+
_validate_positive_fixnum(:partition_size, partition_size)
|
52
|
+
_validate_positive_fixnum(:partition_start_timestamp, partition_start_timestamp)
|
53
|
+
_validate_positive_fixnum(:end_timestamp, end_timestamp)
|
54
|
+
|
55
|
+
timestamp = partition_start_timestamp
|
56
|
+
|
57
|
+
partitions_to_append = {}
|
58
|
+
while timestamp < end_timestamp
|
59
|
+
timestamp = @tuc.advance(timestamp, partition_size_unit, partition_size)
|
60
|
+
|
61
|
+
partition_name = name_from_timestamp(timestamp)
|
62
|
+
partitions_to_append[partition_name] = timestamp
|
63
|
+
end
|
64
|
+
|
65
|
+
partitions_to_append
|
66
|
+
end
|
67
|
+
|
68
|
+
# Wrapper around append partition to add a partition to end with the
|
69
|
+
# given window size
|
70
|
+
#
|
71
|
+
# @param [Symbol] partition_size_unit: [:days, :months]
|
72
|
+
# @param [Fixnum] partition_size, intervals covered by the new partition
|
73
|
+
# @param [Fixnum] days_into_future, how many days into the future need to be covered by partitions
|
74
|
+
# @param [Boolean] dry_run, Defaults to false. If true, query wont be executed.
|
75
|
+
# @return [Hash] partition_data hash of the partitions appended
|
76
|
+
# @raise [ArgumentError] if window size is nil or not greater than 0
|
77
|
+
def append_partition_intervals(partition_size_unit, partition_size, days_into_future = 30, dry_run = false)
|
78
|
+
partitions = Partition.all(adapter, table_name)
|
79
|
+
if partitions.blank? || partitions.non_future_partitions.blank?
|
80
|
+
raise "partitions must be properly initialized before appending"
|
81
|
+
end
|
82
|
+
latest_partition = partitions.latest_partition
|
83
|
+
|
84
|
+
new_partition_data = partitions_to_append(latest_partition.timestamp, partition_size_unit, partition_size, days_into_future)
|
85
|
+
|
86
|
+
if new_partition_data.empty?
|
87
|
+
msg = "Append: No-Op - Latest Partition Time of #{latest_partition.timestamp}, " +
|
88
|
+
"i.e. #{Time.at(@tuc.to_seconds(latest_partition.timestamp))} covers >= #{days_into_future} days_into_future"
|
89
|
+
else
|
90
|
+
msg = "Append: Appending the following new partitions: #{new_partition_data.inspect}"
|
91
|
+
reorg_future_partition(new_partition_data, dry_run)
|
92
|
+
end
|
93
|
+
|
94
|
+
log(msg)
|
95
|
+
|
96
|
+
new_partition_data
|
97
|
+
end
|
98
|
+
|
99
|
+
# drop partitions that are older than days(input) from now
|
100
|
+
# @param [Fixnum] days_from_now
|
101
|
+
# @param [Boolean] dry_run, Defaults to false. If true, query wont be executed.
|
102
|
+
def drop_partitions_older_than_in_days(days_from_now, dry_run = false)
|
103
|
+
timestamp = self.current_timestamp - @tuc.from_days(days_from_now)
|
104
|
+
drop_partitions_older_than(timestamp, dry_run)
|
105
|
+
end
|
106
|
+
|
107
|
+
# drop partitions that are older than the given timestamp
|
108
|
+
# @param [Fixnum] timestamp partitions older than this timestamp will be
|
109
|
+
# dropped
|
110
|
+
# @param [Boolean] dry_run, Defaults to false. If true, query wont be executed.
|
111
|
+
# @return [Array] an array of partition names that were dropped
|
112
|
+
def drop_partitions_older_than(timestamp, dry_run = false)
|
113
|
+
partitions = Partition.all(adapter, table_name).older_than_timestamp(timestamp)
|
114
|
+
partition_names = partitions.map(&:name)
|
115
|
+
|
116
|
+
if partition_names.empty?
|
117
|
+
msg = "Drop: No-Op - No partitions older than #{timestamp}, i.e. #{Time.at(@tuc.to_seconds(timestamp))} to drop"
|
118
|
+
else
|
119
|
+
msg = "Drop: Dropped partitions: #{partition_names.inspect}"
|
120
|
+
drop_partitions(partition_names, dry_run)
|
121
|
+
end
|
122
|
+
|
123
|
+
log(msg)
|
124
|
+
|
125
|
+
partition_names
|
126
|
+
end
|
127
|
+
end
|
128
|
+
end
|
@@ -0,0 +1,122 @@
|
|
1
|
+
module SqlPartitioner
|
2
|
+
class SQL
|
3
|
+
|
4
|
+
# SQL query will return rows having the following columns:
|
5
|
+
# - TABLE_CATALOG
|
6
|
+
# - TABLE_SCHEMA
|
7
|
+
# - TABLE_NAME
|
8
|
+
# - PARTITION_NAME
|
9
|
+
# - SUBPARTITION_NAME
|
10
|
+
# - PARTITION_ORDINAL_POSITION
|
11
|
+
# - SUBPARTITION_ORDINAL_POSITION
|
12
|
+
# - PARTITION_METHOD
|
13
|
+
# - SUBPARTITION_METHOD
|
14
|
+
# - PARTITION_EXPRESSION
|
15
|
+
# - SUBPARTITION_EXPRESSION
|
16
|
+
# - PARTITION_DESCRIPTION
|
17
|
+
# - TABLE_ROWS
|
18
|
+
# - AVG_ROW_LENGTH
|
19
|
+
# - DATA_LENGTH
|
20
|
+
# - MAX_DATA_LENGTH
|
21
|
+
# - INDEX_LENGTH
|
22
|
+
# - DATA_FREE
|
23
|
+
# - CREATE_TIME
|
24
|
+
# - UPDATE_TIME
|
25
|
+
# - CHECK_TIME
|
26
|
+
# - CHECKSUM
|
27
|
+
# - PARTITION_COMMENT
|
28
|
+
# - NODEGROUP
|
29
|
+
# - TABLESPACE_NAME
|
30
|
+
def self.partition_info
|
31
|
+
compress_lines(<<-SQL)
|
32
|
+
SELECT *
|
33
|
+
FROM information_schema.PARTITIONS
|
34
|
+
WHERE TABLE_SCHEMA = ?
|
35
|
+
AND TABLE_NAME = ?
|
36
|
+
SQL
|
37
|
+
end
|
38
|
+
|
39
|
+
def self.drop_partitions(table_name, partition_names)
|
40
|
+
return nil if partition_names.empty?
|
41
|
+
|
42
|
+
compress_lines(<<-SQL)
|
43
|
+
ALTER TABLE #{table_name}
|
44
|
+
DROP PARTITION #{partition_names.join(',')}
|
45
|
+
SQL
|
46
|
+
end
|
47
|
+
|
48
|
+
def self.create_partition(table_name, partition_name, until_timestamp)
|
49
|
+
compress_lines(<<-SQL)
|
50
|
+
ALTER TABLE #{table_name}
|
51
|
+
ADD PARTITION
|
52
|
+
(PARTITION #{partition_name}
|
53
|
+
VALUES LESS THAN (#{until_timestamp}))
|
54
|
+
SQL
|
55
|
+
end
|
56
|
+
|
57
|
+
def self.reorg_partitions(table_name, new_partition_data, reorg_partition_name)
|
58
|
+
return nil if new_partition_data.empty?
|
59
|
+
|
60
|
+
partition_suq_query = sort_partition_data(new_partition_data).map do |partition_name, until_timestamp|
|
61
|
+
"PARTITION #{partition_name} VALUES LESS THAN (#{until_timestamp})"
|
62
|
+
end.join(',')
|
63
|
+
|
64
|
+
compress_lines(<<-SQL)
|
65
|
+
ALTER TABLE #{table_name}
|
66
|
+
REORGANIZE PARTITION #{reorg_partition_name} INTO
|
67
|
+
(#{partition_suq_query})
|
68
|
+
SQL
|
69
|
+
end
|
70
|
+
|
71
|
+
def self.initialize_partitioning(table_name, partition_data)
|
72
|
+
partition_sub_query = sort_partition_data(partition_data).map do |partition_name, until_timestamp|
|
73
|
+
"PARTITION #{partition_name} VALUES LESS THAN (#{until_timestamp})"
|
74
|
+
end.join(',')
|
75
|
+
|
76
|
+
compress_lines(<<-SQL)
|
77
|
+
ALTER TABLE #{table_name}
|
78
|
+
PARTITION BY RANGE(timestamp)
|
79
|
+
(#{partition_sub_query})
|
80
|
+
SQL
|
81
|
+
end
|
82
|
+
|
83
|
+
|
84
|
+
# @param [Hash<String,Fixnum>] partition_data hash of name to timestamp
|
85
|
+
# @return [Array] array of partitions sorted by timestamp ascending, with the 'future' partition at the end
|
86
|
+
def self.sort_partition_data(partition_data)
|
87
|
+
partition_data.to_a.sort do |x,y|
|
88
|
+
if x[1] == "MAXVALUE"
|
89
|
+
1
|
90
|
+
elsif y[1] == "MAXVALUE"
|
91
|
+
-1
|
92
|
+
else
|
93
|
+
x[1] <=> y[1]
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
# Replace sequences of whitespace (including newlines) with either
|
99
|
+
# a single space or remove them entirely (according to param _spaced_).
|
100
|
+
#
|
101
|
+
# Copied from:
|
102
|
+
# https://github.com/datamapper/dm-core/blob/master/lib/dm-core/support/ext/string.rb
|
103
|
+
#
|
104
|
+
# compress_lines(<<QUERY)
|
105
|
+
# SELECT name
|
106
|
+
# FROM users
|
107
|
+
# QUERY => "SELECT name FROM users"
|
108
|
+
#
|
109
|
+
# @param [String] string
|
110
|
+
# The input string.
|
111
|
+
#
|
112
|
+
# @param [TrueClass, FalseClass] spaced (default=true)
|
113
|
+
# Determines whether returned string has whitespace collapsed or removed.
|
114
|
+
#
|
115
|
+
# @return [String] The input string with whitespace (including newlines) replaced.
|
116
|
+
#
|
117
|
+
def self.compress_lines(string, spaced = true)
|
118
|
+
string.split($/).map { |line| line.strip }.join(spaced ? ' ' : '')
|
119
|
+
end
|
120
|
+
|
121
|
+
end
|
122
|
+
end
|
@@ -0,0 +1,92 @@
|
|
1
|
+
module SqlPartitioner
|
2
|
+
class TimeUnitConverter
|
3
|
+
require 'date'
|
4
|
+
|
5
|
+
MINUTE_AS_SECONDS = 60
|
6
|
+
HOUR_AS_SECONDS = 60 * MINUTE_AS_SECONDS
|
7
|
+
DAY_AS_SECONDS = 24 * HOUR_AS_SECONDS
|
8
|
+
|
9
|
+
SUPPORTED_TIME_UNITS = [:seconds, :micro_seconds]
|
10
|
+
|
11
|
+
# @param [Symbol] time_unit one of SUPPORTED_TIME_UNITS
|
12
|
+
def initialize(time_unit)
|
13
|
+
raise ArgumentError.new("Invalid time unit #{time_unit} passed") if !SUPPORTED_TIME_UNITS.include?(time_unit)
|
14
|
+
|
15
|
+
@time_unit = time_unit
|
16
|
+
end
|
17
|
+
|
18
|
+
# @param [Fixnum] num_days
|
19
|
+
# @return [Fixnum] number of days represented in the configure time units
|
20
|
+
def from_days(num_days)
|
21
|
+
from_seconds(num_days * DAY_AS_SECONDS)
|
22
|
+
end
|
23
|
+
|
24
|
+
# @param [Fixnum] time_units_timestamp timestamp in configured time units
|
25
|
+
# @return [DateTime] representation of the given timestamp
|
26
|
+
def to_date_time(time_units_timestamp)
|
27
|
+
DateTime.strptime("#{to_seconds(time_units_timestamp)}", '%s')
|
28
|
+
end
|
29
|
+
|
30
|
+
# converts from seconds to the configured time unit
|
31
|
+
#
|
32
|
+
# @param [Fixnum] timestamp_seconds timestamp in seconds
|
33
|
+
#
|
34
|
+
# @return [Fixnum] timestamp in configured time units
|
35
|
+
def from_seconds(timestamp_seconds)
|
36
|
+
timestamp_seconds * time_units_per_second
|
37
|
+
end
|
38
|
+
|
39
|
+
# converts from the configured time unit to seconds
|
40
|
+
#
|
41
|
+
# @param [Fixnum] time_units_timestamp timestamp in the configured time units
|
42
|
+
#
|
43
|
+
# @return [Fixnum] timestamp in seconds
|
44
|
+
def to_seconds(time_units_timestamp)
|
45
|
+
time_units_timestamp / time_units_per_second
|
46
|
+
end
|
47
|
+
|
48
|
+
# @return [Fixnum] how many of the configured time_unit are in 1 second
|
49
|
+
def time_units_per_second
|
50
|
+
self.class.time_units_per_second(@time_unit)
|
51
|
+
end
|
52
|
+
|
53
|
+
# @param [Fixnum] time_units_timestamp
|
54
|
+
# @param [Symbol] calendar_unit unit for the given value, one of [:day(s), :month(s)]
|
55
|
+
# @param [Fixnum] value in terms of calendar_unit to add to the time_units_timestamp
|
56
|
+
#
|
57
|
+
# @return [Fixnum] new timestamp in configured time units
|
58
|
+
def advance(time_units_timestamp, calendar_unit, value)
|
59
|
+
date_time = to_date_time(time_units_timestamp)
|
60
|
+
date_time = self.class.advance_date_time(date_time, calendar_unit, value)
|
61
|
+
from_seconds(date_time.to_time.to_i)
|
62
|
+
end
|
63
|
+
|
64
|
+
# @param [DateTime] date_time to advance
|
65
|
+
# @param [Symbol] calendar_unit unit for the following `value`, one of [:day(s), :month(s)]
|
66
|
+
# @param [Fixnum] value in terms of calendar_unit to add to the date_time
|
67
|
+
# @return [DateTime] result of advancing the given date_time by the given value
|
68
|
+
def self.advance_date_time(date_time, calendar_unit, value)
|
69
|
+
new_time = case calendar_unit
|
70
|
+
when :days, :day
|
71
|
+
date_time + value
|
72
|
+
when :months, :month
|
73
|
+
date_time >> value
|
74
|
+
end
|
75
|
+
|
76
|
+
new_time
|
77
|
+
end
|
78
|
+
|
79
|
+
# @param [Symbol] time_unit one of `SUPPORTED_TIME_UNITS`
|
80
|
+
# @return [Fixnum] how many of the given time_unit are in 1 second
|
81
|
+
def self.time_units_per_second(time_unit)
|
82
|
+
case time_unit
|
83
|
+
when :micro_seconds
|
84
|
+
1_000_000
|
85
|
+
when :seconds
|
86
|
+
1
|
87
|
+
else
|
88
|
+
raise "unknown time_unit #{time_unit.inspect}"
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
# standard requires
|
2
|
+
require 'sql_partitioner/loader'
|
3
|
+
require 'sql_partitioner/lock_wait_timeout_handler'
|
4
|
+
require 'sql_partitioner/partition'
|
5
|
+
require 'sql_partitioner/time_unit_converter'
|
6
|
+
require 'sql_partitioner/sql_helper'
|
7
|
+
require 'sql_partitioner/base_partitions_manager'
|
8
|
+
require 'sql_partitioner/partitions_manager'
|
9
|
+
|
10
|
+
SqlPartitioner::Loader.require_or_skip('sql_partitioner/adapters/ar_adapter', 'ActiveRecord')
|
11
|
+
SqlPartitioner::Loader.require_or_skip('sql_partitioner/adapters/dm_adapter', 'DataMapper')
|
12
|
+
|
13
|
+
|
14
|
+
|
data/spec/db_conf.yml
ADDED
@@ -0,0 +1,52 @@
|
|
1
|
+
require 'simplecov'
|
2
|
+
SimpleCov.start
|
3
|
+
|
4
|
+
require 'active_record'
|
5
|
+
require 'data_mapper'
|
6
|
+
|
7
|
+
require 'sql_partitioner'
|
8
|
+
|
9
|
+
require 'logger'
|
10
|
+
|
11
|
+
#require 'ruby-debug' # enable debugger support
|
12
|
+
|
13
|
+
SPEC_LOGGER = Logger.new(nil)
|
14
|
+
|
15
|
+
RSpec.configure do |config|
|
16
|
+
# enable both should and expect syntax in rspec without deprecation warnings
|
17
|
+
config.expect_with :rspec do |c|
|
18
|
+
c.syntax = [:should, :expect]
|
19
|
+
end
|
20
|
+
config.mock_with :rspec do |c|
|
21
|
+
c.syntax = [:should, :expect]
|
22
|
+
end
|
23
|
+
|
24
|
+
config.before :suite do
|
25
|
+
require 'yaml'
|
26
|
+
db_conf = YAML.load_file('spec/db_conf.yml')
|
27
|
+
|
28
|
+
# establish both ActiveRecord and DataMapper connections
|
29
|
+
ActiveRecord::Base.establish_connection(db_conf["test"])
|
30
|
+
|
31
|
+
adapter,database,user,pass,host = db_conf["test"].values_at *%W(adapter database username password host)
|
32
|
+
connection_string = "#{adapter}://#{user}:#{pass}@#{host}/#{database}"
|
33
|
+
DataMapper.setup(:default, connection_string)
|
34
|
+
end
|
35
|
+
|
36
|
+
config.before :each do
|
37
|
+
sql = <<-SQL
|
38
|
+
DROP TABLE IF EXISTS `test_events`
|
39
|
+
SQL
|
40
|
+
DataMapper.repository.adapter.execute(sql)
|
41
|
+
|
42
|
+
sql = <<-SQL
|
43
|
+
CREATE TABLE IF NOT EXISTS `test_events` (
|
44
|
+
`id` bigint(20) NOT NULL AUTO_INCREMENT,
|
45
|
+
`timestamp` bigint(20) unsigned NOT NULL DEFAULT '0',
|
46
|
+
PRIMARY KEY (`id`,`timestamp`)
|
47
|
+
) ENGINE=InnoDB DEFAULT CHARSET=utf8
|
48
|
+
SQL
|
49
|
+
DataMapper.repository.adapter.execute(sql)
|
50
|
+
|
51
|
+
end
|
52
|
+
end
|
@@ -0,0 +1,82 @@
|
|
1
|
+
require File.expand_path("../../spec_helper", File.dirname(__FILE__))
|
2
|
+
|
3
|
+
shared_examples_for "an adapter" do
|
4
|
+
describe "#execute" do
|
5
|
+
it "should execute the SQL statement passed" do
|
6
|
+
adapter.execute("SET @sql_partitioner_test = 1")
|
7
|
+
|
8
|
+
adapter.select("SELECT @sql_partitioner_test").first.to_i.should == 1
|
9
|
+
|
10
|
+
adapter.execute("SET @sql_partitioner_test = NULL")
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
describe "#schema_name" do
|
15
|
+
it "should return the schema_name" do
|
16
|
+
adapter.schema_name.should == "sql_partitioner_test"
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
describe ".select" do
|
21
|
+
context "with multiple columns selected" do
|
22
|
+
context "and no data returned" do
|
23
|
+
it "should return an empty hash" do
|
24
|
+
adapter.select("SELECT * FROM test_events").should == []
|
25
|
+
end
|
26
|
+
end
|
27
|
+
context "and data returned" do
|
28
|
+
before(:each) do
|
29
|
+
adapter.execute("INSERT INTO test_events(timestamp) VALUES(1)")
|
30
|
+
adapter.execute("INSERT INTO test_events(timestamp) VALUES(2)")
|
31
|
+
end
|
32
|
+
it "should return a Struct with accessors for each column" do
|
33
|
+
result = adapter.select("SELECT * FROM test_events")
|
34
|
+
|
35
|
+
result.should be_a_kind_of(Array)
|
36
|
+
|
37
|
+
result.first.should be_a_kind_of(Struct)
|
38
|
+
result.first.timestamp.to_i.should == 1
|
39
|
+
|
40
|
+
result.last.should be_a_kind_of(Struct)
|
41
|
+
result.last.timestamp.to_i.should == 2
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
context "with a single column selected" do
|
46
|
+
before(:each) do
|
47
|
+
adapter.execute("INSERT INTO test_events(timestamp) VALUES(1)")
|
48
|
+
adapter.execute("INSERT INTO test_events(timestamp) VALUES(2)")
|
49
|
+
end
|
50
|
+
it "should return the values without Struct" do
|
51
|
+
result = adapter.select("SELECT timestamp FROM test_events")
|
52
|
+
|
53
|
+
result.should be_a_kind_of(Array)
|
54
|
+
|
55
|
+
result.first.to_i.should == 1
|
56
|
+
result.last.to_i.should == 2
|
57
|
+
end
|
58
|
+
it "should return an array even when only one row is selected" do
|
59
|
+
result = adapter.select("SELECT timestamp FROM test_events LIMIT 1")
|
60
|
+
|
61
|
+
result.should be_a_kind_of(Array)
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
describe SqlPartitioner::ARAdapter do
|
68
|
+
it_should_behave_like "an adapter" do
|
69
|
+
let(:adapter) do
|
70
|
+
SqlPartitioner::ARAdapter.new(ActiveRecord::Base.connection)
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
describe SqlPartitioner::DMAdapter do
|
76
|
+
it_should_behave_like "an adapter" do
|
77
|
+
let(:adapter) do
|
78
|
+
SqlPartitioner::DMAdapter.new(DataMapper.repository.adapter)
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
@@ -0,0 +1,28 @@
|
|
1
|
+
require File.expand_path("../../spec_helper", File.dirname(__FILE__))
|
2
|
+
|
3
|
+
describe "BaseAdapter" do
|
4
|
+
before(:each) do
|
5
|
+
@base_adapter = SqlPartitioner::BaseAdapter.new
|
6
|
+
end
|
7
|
+
describe "#select" do
|
8
|
+
it "should raise an RuntimeError" do
|
9
|
+
lambda{
|
10
|
+
@base_adapter.select("SELECT database()")
|
11
|
+
}.should raise_error(RuntimeError, /MUST BE IMPLEMENTED/)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
describe "#execute" do
|
15
|
+
it "should raise an RuntimeError" do
|
16
|
+
lambda{
|
17
|
+
@base_adapter.execute("SELECT database()")
|
18
|
+
}.should raise_error(RuntimeError, /MUST BE IMPLEMENTED/)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
describe "#schema_name" do
|
22
|
+
it "should raise an RuntimeError" do
|
23
|
+
lambda{
|
24
|
+
@base_adapter.schema_name
|
25
|
+
}.should raise_error(RuntimeError, /MUST BE IMPLEMENTED/)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|