highwatermark 0.1.1 → 0.1.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +17 -12
- data/lib/highwatermark.rb +94 -68
- data/lib/highwatermark/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 56e98393bae407005b7dfa6c128e590113541452
|
|
4
|
+
data.tar.gz: 6c3434a7e33d6abfde36230a5d04009c3e7398ef
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: b40829ba57481eccdc93d0baa3e4781c95809d395befa4b2dbf639019300faa6a1be7e8b93caabb9a9bb1618a9aeddab93fa42780edcf09639b47580ddcbf2c9
|
|
7
|
+
data.tar.gz: 3ecdab6db0d313c3f3bb7aa4738b41c8b8b907501d4bfa78ec87aeb297d9658dfaf47f02ed70159eba35c786f2a2985e2ddc2335caf4e1fb62769d198e77ec90
|
data/README.md
CHANGED
|
@@ -27,30 +27,35 @@ In your ruby code:
|
|
|
27
27
|
# To initilize high watermark
|
|
28
28
|
require 'highwatermark'
|
|
29
29
|
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
30
|
+
highwatermark_parameters={
|
|
31
|
+
"state_tag" => "your tag", # required, is the tag that will be used in state file or redis
|
|
32
|
+
"state_type" => "file", # required: could be <redis/file/memory>
|
|
33
|
+
"state_file" => "/path/to/state/file/or/redis/conf", # optional, the file path for store state, need state_type set to 'file'
|
|
34
|
+
"redis_host" => '127.0.0.1', #optional, to set remote redis, need state_type set to 'redis'
|
|
35
|
+
"redis_port" => '6379' #optional, to set remote redis, need state_type set to 'redis'
|
|
36
|
+
}
|
|
33
37
|
|
|
34
|
-
hwm = Highwatermark::HighWaterMark.new(
|
|
38
|
+
hwm = Highwatermark::HighWaterMark.new(highwatermark_parameters)
|
|
35
39
|
|
|
36
40
|
# To store time in high watermark
|
|
37
41
|
time = "what your want to store in high watermark"
|
|
38
|
-
hwm.update_records(time
|
|
42
|
+
hwm.update_records(time) #use tag in the configure 'state_tag'
|
|
43
|
+
|
|
44
|
+
tag = "some other specified tag"
|
|
45
|
+
hwm.update_records(time, tag) # use some other specified tag
|
|
46
|
+
|
|
39
47
|
|
|
40
48
|
# To get the high watermark
|
|
41
|
-
hwm.last_records(tag
|
|
49
|
+
hwm.last_records() #get high watermark with tag in the configure 'state_tag'
|
|
42
50
|
|
|
43
|
-
|
|
51
|
+
tag = "some other specified tag"
|
|
52
|
+
hwm.last_records(tag) #get high watermark with some other specified tag
|
|
44
53
|
|
|
45
|
-
Example of redis.conf (if set state_type = 'redis'):
|
|
46
54
|
|
|
47
|
-
```
|
|
48
|
-
# redis configure
|
|
49
|
-
host: 127.0.0.1
|
|
50
|
-
port: 6379
|
|
51
55
|
|
|
52
56
|
```
|
|
53
57
|
|
|
58
|
+
|
|
54
59
|
Output in the state file:
|
|
55
60
|
|
|
56
61
|
```
|
data/lib/highwatermark.rb
CHANGED
|
@@ -2,54 +2,84 @@ require "highwatermark/version"
|
|
|
2
2
|
|
|
3
3
|
module Highwatermark
|
|
4
4
|
class HighWaterMark
|
|
5
|
-
|
|
6
|
-
def initialize(
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
5
|
+
|
|
6
|
+
def initialize(parameters)
|
|
7
|
+
|
|
8
|
+
require 'yaml'
|
|
9
|
+
|
|
10
|
+
@state_type = parameters["state_type"] # require
|
|
11
|
+
@state_tag = parameters["state_tag"] # require
|
|
12
|
+
@path = parameters["state_file"] # optional, need state_type set to 'file'
|
|
13
|
+
@redis_host = parameters["redis_host"] # optional, need state_type set to 'redis'
|
|
14
|
+
@redis_port = parameters["redis_port"] # optional, need state_type set to 'redis'
|
|
15
|
+
|
|
16
|
+
# check the parameter configurartion
|
|
17
|
+
if @state_type == nil
|
|
18
|
+
raise "The parameter 'state_type' is required, please provide state_type (file, redis or memory)"
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
if @state_tag == nil
|
|
22
|
+
raise "The paramerter 'state_tag' is required, please provide state_tag for labelling high watermark info"
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
if @path != nil && @state_type != 'file'
|
|
26
|
+
raise "To use 'state_file' parameter, 'state_type' need to be set to 'file'"
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
if ((@redis_host != nil || @redis_port != nil ) && @state_type != 'redis')
|
|
30
|
+
raise "To use 'redis_host' or 'redis_port' parameters, 'state_type' need to be set to 'redis'"
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@data = {}
|
|
36
|
+
if @state_type == 'file'
|
|
37
|
+
|
|
38
|
+
if File.directory? (@path)
|
|
39
|
+
# create a new state file in the provided directory
|
|
40
|
+
# when recover from failure will also try to read from this file
|
|
41
|
+
@path = @path+"/"+@state_tag+".yaml"
|
|
42
|
+
puts "provided path is a valid derectory, created a new file on #{@path.inspect}"
|
|
43
|
+
@data = {}
|
|
44
|
+
else # not a directory, then check if it's a valid file
|
|
45
|
+
if File.exist?(@path)
|
|
46
|
+
@data = YAML.load_file(@path)
|
|
47
|
+
if @data == false || @data == []
|
|
48
|
+
# this happens if an users created an empty file accidentally, or the file is just initialized
|
|
49
|
+
puts "state file on #{@path.inspect} is empty "
|
|
50
|
+
@data = {}
|
|
51
|
+
elsif !@data.is_a?(Hash)
|
|
52
|
+
# if the file contains invalid data, that is not a hash
|
|
53
|
+
raise "state file on #{@path.inspect} contains invalid data, please use other file"
|
|
54
|
+
end
|
|
55
|
+
else
|
|
56
|
+
raise "#{@path.inspect} is not a valid directory or file, please provide valid state file path"
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
elsif @state_type =='memory'
|
|
64
|
+
@data = {}
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
elsif @state_type =='redis'
|
|
68
|
+
require 'redis'
|
|
69
|
+
if (@redis_host == nil || @redis_port == nil)
|
|
70
|
+
puts "No Redis host and port specified, use default local setting"
|
|
71
|
+
@redis = Redis.new
|
|
72
|
+
else # if has the redis host and port configure
|
|
73
|
+
begin
|
|
74
|
+
puts "Redis Host #{@redis_host} port #{@redis_port}"
|
|
75
|
+
@redis = Redis.new(host: @redis_host, port: @redis_port)
|
|
76
|
+
rescue Exception => e
|
|
77
|
+
puts e.message
|
|
78
|
+
puts e.backtrace.inspect
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
@data = {}
|
|
82
|
+
end # end of checking @state_type
|
|
53
83
|
|
|
54
84
|
if @data['last_records']==nil
|
|
55
85
|
@data['last_records'] = {}
|
|
@@ -57,52 +87,48 @@ module Highwatermark
|
|
|
57
87
|
|
|
58
88
|
end # end of intitialize
|
|
59
89
|
|
|
60
|
-
|
|
90
|
+
def last_records(tag=@state_tag)
|
|
61
91
|
if @state_type == 'file'
|
|
62
|
-
# return @data[@tag]
|
|
63
|
-
# pp @data['last_records'][tag]
|
|
64
92
|
return @data['last_records'][tag]
|
|
65
93
|
|
|
66
94
|
elsif @state_type =='memory'
|
|
67
95
|
return @data['last_records'][tag]
|
|
68
96
|
elsif @state_type =='redis'
|
|
69
97
|
begin
|
|
70
|
-
alertStart
|
|
98
|
+
alertStart=@redis.get(tag)
|
|
71
99
|
return alertStart
|
|
72
100
|
rescue Exception => e
|
|
73
|
-
|
|
74
|
-
|
|
101
|
+
puts e.message
|
|
102
|
+
puts e.backtrace.inspect
|
|
75
103
|
end
|
|
76
|
-
|
|
104
|
+
end
|
|
77
105
|
end
|
|
78
106
|
|
|
79
|
-
|
|
107
|
+
def update_records(time, tag=@state_tag)
|
|
80
108
|
if @state_type == 'file'
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
f.write YAML.dump(@data)
|
|
86
|
-
}
|
|
109
|
+
@data['last_records'][tag] = time
|
|
110
|
+
File.open(@path, 'w') {|f|
|
|
111
|
+
f.write YAML.dump(@data)
|
|
112
|
+
}
|
|
87
113
|
elsif @state_type =='memory'
|
|
88
114
|
@data['last_records'][tag] = time
|
|
89
115
|
|
|
90
116
|
elsif @state_type =='redis'
|
|
91
117
|
begin
|
|
92
|
-
alertStart
|
|
118
|
+
alertStart=@redis.set(tag,time)
|
|
93
119
|
rescue Exception => e
|
|
94
|
-
|
|
95
|
-
|
|
120
|
+
puts e.message
|
|
121
|
+
puts e.backtrace.inspect
|
|
96
122
|
end
|
|
97
123
|
end
|
|
98
124
|
|
|
99
|
-
|
|
125
|
+
end
|
|
100
126
|
|
|
101
|
-
|
|
127
|
+
end # end of class Highwatermark
|
|
102
128
|
|
|
103
129
|
|
|
104
130
|
|
|
105
|
-
|
|
131
|
+
|
|
106
132
|
|
|
107
133
|
|
|
108
134
|
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: highwatermark
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Ling Zhang
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2015-04
|
|
11
|
+
date: 2015-05-04 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: bundler
|