fluent-plugin-aliyun-odps 0.0.4 → 0.0.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -1
- data/README.cn.md +1 -1
- data/README.md +1 -1
- data/VERSION +1 -1
- data/lib/fluent/plugin/exceptions.rb +1 -0
- data/lib/fluent/plugin/odps/odps_table.rb +1 -1
- data/lib/fluent/plugin/out_aliyun_odps.rb +11 -28
- data/lib/fluent/plugin/stream_client.rb +28 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a83605973fae13e78ea73e422533ef6d613fe5fc
|
4
|
+
data.tar.gz: 8109f1fc93a0fce0869914893d073d25c911bf6d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c614f06a4063a1b78bb728ec48d0530a05b1f5951bd7c49932df8fb642fe300f979ca2acba507988bb5bee4b647a91c4557e51c1d181e257600b76daef0ad205
|
7
|
+
data.tar.gz: 6a0c6f9027fde9a0ebd23445d89b522b1d5f4ea5744875e94d20a2bd8d1b37c210bcf1d84f139579f7b151310022258d58f52d1b26d56ae851ef860955b0fe94
|
data/CHANGELOG.md
CHANGED
@@ -1,2 +1,4 @@
|
|
1
1
|
0.0.4
|
2
|
-
Fix datetime format bug, support String, DateTime, Time type when write to a datetime field.
|
2
|
+
Fix datetime format bug, support String, DateTime, Time type when write to a datetime field.
|
3
|
+
0.0.5
|
4
|
+
Add reload shard when import fails, and remove unload shard operation when shut down.
|
data/README.cn.md
CHANGED
@@ -91,7 +91,7 @@ $ cp aliyun-odps-fluentd-plugin/lib/fluent/plugin/* {YOUR_FLUENTD_DIRECTORY}/lib
|
|
91
91
|
- ʱ���ʽ�ؼ���: partition ctime=${datetime.strftime('%Y%m%d')} ������datetimeΪsource��ijʱ���ʽ�ֶΣ����Ϊ%Y%m%d��ʽ��Ϊ�������ƣ�
|
92
92
|
- time_format(Optional):
|
93
93
|
- ���ʹ��ʱ���ʽ�ؼ���Ϊ<partition>, �����ñ�����. ����: source[datetime]="29/Aug/2015:11:10:16 +0800",������<time_format>Ϊ"%d/%b/%Y:%H:%M:%S %z"
|
94
|
-
- shard_number(Optional):ָ��shard
|
94
|
+
- shard_number(Optional):ָ��shard���������������shard[0,shard_number-1]��Χ�ڵ�shard��д�����ݣ�����Ϊ����0��С��table��Ӧshard������������.
|
95
95
|
|
96
96
|
## �ٷ���վ
|
97
97
|
---
|
data/README.md
CHANGED
@@ -97,7 +97,7 @@ $ cp aliyun-odps-fluentd-plugin/lib/fluent/plugin/* {YOUR_FLUENTD_DIRECTORY}/lib
|
|
97
97
|
- key words int time format: partition ctime=${datetime.strftime('%Y%m%d')}
|
98
98
|
- time_format(Optional):
|
99
99
|
- if you are using the key words to set your <partition> and the key word is in time format, please set the param <time_format>. example: source[datetime] = "29/Aug/2015:11:10:16 +0800", and the param <time_format> is "%d/%b/%Y:%H:%M:%S %z"
|
100
|
-
- shard_number(Optional):less than the
|
100
|
+
- shard_number(Optional): will write data to shards between [0,shard_number-1], this config must more than 0 and less than the max shard number of your table.
|
101
101
|
|
102
102
|
## Useful Links
|
103
103
|
---
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.0.
|
1
|
+
0.0.5
|
@@ -20,6 +20,7 @@ module OdpsDatahub
|
|
20
20
|
$INVALID_ARGUMENT = "InvalidArgument"
|
21
21
|
$SCHEMA_NOT_MATCH = "SchemaNotMatch"
|
22
22
|
$PACK_SIZE_EXCEED = "PackSizeExceed"
|
23
|
+
$LOADSHARD_TIMEOUT = "LoadShardTimeOut"
|
23
24
|
$ACCESS_DENIED = "AccessDenied"
|
24
25
|
|
25
26
|
class OdpsDatahubException < StandardError
|
@@ -101,7 +101,7 @@ module OdpsDatahub
|
|
101
101
|
if value.is_a?String and value.length < $STRING_MAX_LENTH
|
102
102
|
setValue(idx, value)
|
103
103
|
else
|
104
|
-
raise "value show be String and len <
|
104
|
+
raise "value show be String and len < " + $STRING_MAX_LENTH.to_s
|
105
105
|
end
|
106
106
|
end
|
107
107
|
|
@@ -49,10 +49,9 @@ module Fluent
|
|
49
49
|
config_param :fields, :string, :default => nil
|
50
50
|
config_param :partition, :string, :default => nil
|
51
51
|
config_param :num_retries, :integer, :default => 5
|
52
|
-
config_param :shard_number, :integer, :default =>
|
52
|
+
config_param :shard_number, :integer, :default => 5
|
53
53
|
config_param :thread_number, :integer, :default => 1
|
54
|
-
config_param :time_format, :string, :default => nil
|
55
|
-
config_param :record_batch_size, :integer, :default => 10
|
54
|
+
config_param :time_format, :string, :default => nil
|
56
55
|
config_param :time_out, :integer, :default => 300
|
57
56
|
attr_accessor :partitionList
|
58
57
|
attr_reader :client
|
@@ -88,34 +87,13 @@ module Fluent
|
|
88
87
|
config[:aliyun_odps_endpoint],
|
89
88
|
config[:aliyun_odps_hub_endpoint],
|
90
89
|
config[:project])
|
91
|
-
if @
|
92
|
-
raise "
|
90
|
+
if @shard_number<=0
|
91
|
+
raise "shard number must more than 0"
|
93
92
|
end
|
94
93
|
begin
|
95
94
|
@client = OdpsDatahub::StreamClient.new(odpsConfig, config[:project], @table)
|
96
95
|
@client.loadShard(@shard_number)
|
97
|
-
|
98
|
-
loadtime=0
|
99
|
-
while !allLoaded do
|
100
|
-
count = 0
|
101
|
-
#get json like [{"ShardId": "0","State": "loaded"},{"ShardId": "1","State": "loaded"}]
|
102
|
-
@client.getShardStatus.each { |shard|
|
103
|
-
if shard["State"] != "loaded"
|
104
|
-
sleep(5)
|
105
|
-
loadtime+=5
|
106
|
-
break
|
107
|
-
else
|
108
|
-
count += 1
|
109
|
-
end
|
110
|
-
if count == @shard_number
|
111
|
-
allLoaded = true
|
112
|
-
@log.info "All shareds are loaded successfully"
|
113
|
-
end
|
114
|
-
if loadtime>=300
|
115
|
-
raise "Load shared timeout"
|
116
|
-
end
|
117
|
-
}
|
118
|
-
end
|
96
|
+
@client.waitForShardLoad
|
119
97
|
for i in 0..@thread_number-1
|
120
98
|
@writer[i] = @client.createStreamArrayWriter()
|
121
99
|
end
|
@@ -269,13 +247,18 @@ module Fluent
|
|
269
247
|
sendThread[thread].join
|
270
248
|
end
|
271
249
|
rescue => e
|
250
|
+
# reload shard
|
251
|
+
if e.message.include? "ShardNotReady" or e.message.include? "InvalidShardId"
|
252
|
+
@client.loadShard(@shard_number)
|
253
|
+
@client.waitForShardLoad
|
254
|
+
end
|
272
255
|
# ignore other exceptions to use Fluentd retry
|
273
256
|
raise "write records failed,"+e.message
|
274
257
|
end
|
275
258
|
end
|
276
259
|
|
277
260
|
def close()
|
278
|
-
|
261
|
+
#@client.loadShard(0)
|
279
262
|
end
|
280
263
|
|
281
264
|
end
|
@@ -87,6 +87,24 @@ module OdpsDatahub
|
|
87
87
|
return json_obj["ShardStatus"]
|
88
88
|
end
|
89
89
|
|
90
|
+
def waitForShardLoad(timeOut = 120) #seconds
|
91
|
+
if timeOut < 0
|
92
|
+
raise OdpsDatahubException.new($INVALID_ARGUMENT, "waitForShardLoad param invalid")
|
93
|
+
end
|
94
|
+
allLoaded = false
|
95
|
+
loadtime=0
|
96
|
+
while loadtime < timeOut do
|
97
|
+
if isShardLoadCompleted
|
98
|
+
return
|
99
|
+
end
|
100
|
+
sleep(5)
|
101
|
+
loadtime+=5
|
102
|
+
end
|
103
|
+
if !isShardLoadCompleted
|
104
|
+
raise OdpsDatahubException.new($LOADSHARD_TIMEOUT, "waitForShardLoad timeOut")
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
90
108
|
def loadShard(idx)
|
91
109
|
if idx < 0
|
92
110
|
raise OdpsDatahubException.new($INVALID_ARGUMENT, "loadShard num invalid")
|
@@ -107,5 +125,15 @@ module OdpsDatahub
|
|
107
125
|
def getResource
|
108
126
|
return "/projects/" + @mProject + "/tables/" + @mTable
|
109
127
|
end
|
128
|
+
|
129
|
+
def isShardLoadCompleted
|
130
|
+
#get json like [{"ShardId": "0","State": "loaded"},{"ShardId": "1","State": "loaded"}]
|
131
|
+
getShardStatus.each { |shard|
|
132
|
+
if shard["State"] != "loaded"
|
133
|
+
return false
|
134
|
+
end
|
135
|
+
}
|
136
|
+
return true
|
137
|
+
end
|
110
138
|
end
|
111
139
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-aliyun-odps
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Xiao Dong
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2015-
|
12
|
+
date: 2015-10-22 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|