fluent-plugin-aliyun-odps 0.1.1 → 0.1.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -1
- data/README.cn.md +22 -1
- data/VERSION +1 -1
- data/lib/fluent/plugin/http/http_connection.rb +1 -1
- data/lib/fluent/plugin/stream_writer.rb +10 -11
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f583712920902f51d0f4b2d50f1350e73ef43c37
|
4
|
+
data.tar.gz: 420ac14c5256bd3aae8be1693b856c7f18d6520d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: cd2b20d9faa6e79f3ea1d8f5e61f2aab0f3f2ccdb7e2ceee83974a70d64abe30dcad0e352ff42479e499bf43ae21f0553cd997ef4c7025797f3e1f9445740fd5
|
7
|
+
data.tar.gz: b7cea3a104ec7a849741738977f6f164621637a4e9e3196639eefb56a10ee5eb8b3519dcd5b00a07487d5ed5d5193c2e4b2e267482b4adac2aa2f93f60d7e309
|
data/CHANGELOG.md
CHANGED
data/README.cn.md
CHANGED
@@ -71,7 +71,7 @@ $ gem install fluent-plugin-aliyun-odps
|
|
71
71
|
- aliyun_access_key(Required):������access key.
|
72
72
|
- aliyun_odps_hub_endpoint(Required):�����ķ�������ECS�ϣ���ѱ�ֵ�趨Ϊ http://dh-ext.odps.aliyun-inc.com, ��������Ϊ http://dh.odps.aliyun.com.
|
73
73
|
- aliyunodps_endpoint(Required):�����ķ�������ECS�ϣ���ѱ�ֵ�趨Ϊ http://odps-ext.aiyun-inc.com/api, ��������Ϊ http://service.odps.aliyun.com/api .
|
74
|
-
- buffer_chunk_limit(Optional): ���С��֧�֡�k��(KB),��m��(MB)
|
74
|
+
- buffer_chunk_limit(Optional): ���С��֧�֡�k��(KB),��m��(MB)��λ��Ĭ�� 8MB������ֵ2MB, Ŀǰ���֧��20MB.
|
75
75
|
- buffer_queue_limit(Optional): ����д�С����ֵ��buffer_chunk_limit��ͬ����������������С��
|
76
76
|
- flush_interval(Optional): ǿ�Ʒ��ͼ�����ﵽʱ��������δ����ǿ�Ʒ���, Ĭ�� 60s.
|
77
77
|
- abandon_mode(Optional):�����������κ�������pack���ݡ�
|
@@ -108,3 +108,24 @@ $ gem install fluent-plugin-aliyun-odps
|
|
108
108
|
---
|
109
109
|
|
110
110
|
licensed under the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0.html)
|
111
|
+
|
112
|
+
## ����ʹ�������Լ��쳣����
|
113
|
+
---
|
114
|
+
* �����׳��쳣InvalidShardId/ShardNotReady��ʲôԭ���£�
|
115
|
+
- ����ϵͳ��������������ݳ���������⣬���ڶ�ʱ���ڻָ���
|
116
|
+
- fluentd������ڶ��������鿴������shard_num�Ƿ����ó���һ����ֵ�������ã���������ò�һ���ǻᵼ���������ģ�shard_number�ٵĽ��̻�Ѷ���shard Unload����
|
117
|
+
- ���ܴ��������ʹ��sdk�ȷ�ʽ������loadshard/unloadshard�Ȳ�����
|
118
|
+
* enable_fast_crc��μ���Ƿ���ݣ�
|
119
|
+
- ���������ú�������fluentd���̣�����ʱ����֤�����ʧ�ܻ��׳�����ԭ��reload���������֤�����������Ŀ¼������ldd�鿴aliyun-odps-fluentd-plugin/lib/fluent/plugin/crc/lib/linux/crc32c.so��
|
120
|
+
* retry_time/retry_interval��fluentd�Դ���retry�к�����
|
121
|
+
- fluentd�Դ�retryĬ�ϳ���36Сʱ���Ὣ����buffer_chunk�ط������ö�̬partition������ط�ȫ�����ݿ�����������ظ�������������ͻ�ʹ�ò���ڲ����ԣ��������ʧ�ܣ����ٸ���abandon_mode��ֵ�ж�������pack�����ݻ��ǽ���fluentd�ط�����buffer��
|
122
|
+
* Warning��ErrorCode: NoSuchPartition, Message: write failed because The specified partition does not exist.��ʲô��˼��
|
123
|
+
- ���������catch��Odps��NoSuchPartitionʱ��������������������������warn��ʾOdps���в��������ݶ�Ӧ���������Զ���������������ɹ�������Ϣ��ʾ��
|
124
|
+
* Fluent::BufferQueueLimitError error="queue size exceeds limit"��ʲôԭ��
|
125
|
+
- fluentd�ڶ�ȡ����-�������ݹ����У����ȶ�ȡ��һ��buffer�У������С����������buffer_chunk_limit��buffer_queue_limit��ͬ�������������������ܿ�������Ϊ�ѻ����ݵ���buffer���㣬���Գ�������buffer_queue_limit���������⡣
|
126
|
+
* ���config�ļ���ηֱ�����һ��fluentd���̣�
|
127
|
+
- ������ڶ��config�ļ�������ʹ��in_multiprocess������ͬʱ������ͬ�Ľ���������
|
128
|
+
* partition has no corresponding source key or the partition expression is wrong.����쳣��ʲôԭ��
|
129
|
+
- ����쳣��ʾ��source data���Ҳ���������partition�ֶ��е�ֵ������partition ctime=${remote}����remoteû�г�����source�У��������á�
|
130
|
+
* Failed to format the data.����쳣��ʲôԭ��
|
131
|
+
- ���������Ϣ�׳���������partition���̳������⣬����partition���ã���������д���������Ҳ��������������⡣
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.1.
|
1
|
+
0.1.2
|
@@ -65,7 +65,7 @@ module OdpsDatahub
|
|
65
65
|
@mHeader[$DATE] = Time.now.utc.strftime("%a, %d %b %Y %H:%M:%S GMT")
|
66
66
|
if not @mIsOdpsUrl
|
67
67
|
@mHeader[$TUNNEL_STREAM_VERSION] = "1"
|
68
|
-
@mHeader[$TUNNEL_VERSION] = "
|
68
|
+
@mHeader[$TUNNEL_VERSION] = "4"
|
69
69
|
end
|
70
70
|
@mHeader[$USER_AGENT] = $SDK_UA_STR + @mOdpsConfig.userAgent
|
71
71
|
@mHeader[$AUTHORIZATION] = signAuthorization
|
@@ -78,29 +78,28 @@ module OdpsDatahub
|
|
78
78
|
param[$PARAM_RECORD_COUNT] = @mRecordList.size.to_s
|
79
79
|
header[$CONTENT_ENCODING] = "deflate"
|
80
80
|
header[$CONTENT_TYPE] = "application/octet-stream"
|
81
|
-
|
81
|
+
# version 4
|
82
82
|
pack = OdpsDatahub::XStreamPack.new
|
83
83
|
pack.pack_data = Zlib::Deflate.deflate(@mUpStream.string)
|
84
84
|
pack.pack_meta = ""
|
85
|
-
|
86
|
-
pack.serialize_to(
|
87
|
-
header[$CONTENT_MD5] = Digest::MD5.hexdigest(
|
88
|
-
header[$CONTENT_LENGTH] =
|
85
|
+
upStream = ::StringIO.new
|
86
|
+
pack.serialize_to(upStream)
|
87
|
+
header[$CONTENT_MD5] = Digest::MD5.hexdigest(upStream.string)
|
88
|
+
header[$CONTENT_LENGTH] = upStream.string.length.to_s
|
89
89
|
|
90
|
-
|
91
|
-
=end
|
92
|
-
#version 3
|
90
|
+
=begin version 3
|
93
91
|
upStream = Zlib::Deflate.deflate(@mUpStream.string)
|
94
92
|
header[$CONTENT_MD5] = Digest::MD5.hexdigest(upStream)
|
95
93
|
header[$CONTENT_LENGTH] = upStream.length.to_s
|
94
|
+
=end
|
96
95
|
#MAX_LENGTH 2048*10KB
|
97
96
|
if upStream.length > $MAX_PACK_SIZE
|
98
|
-
raise OdpsDatahubException.new($PACK_SIZE_EXCEED, "pack size:" + upStream.length.to_s)
|
97
|
+
raise OdpsDatahubException.new($PACK_SIZE_EXCEED, "pack size:" + upStream.string.length.to_s)
|
99
98
|
end
|
100
99
|
if @mShardId != nil
|
101
|
-
conn = HttpConnection.new(@mOdpsConfig, header, param, @mPath + "/shards/" + @mShardId.to_s, "PUT", upStream)
|
100
|
+
conn = HttpConnection.new(@mOdpsConfig, header, param, @mPath + "/shards/" + @mShardId.to_s, "PUT", upStream.string)
|
102
101
|
else
|
103
|
-
conn = HttpConnection.new(@mOdpsConfig, header, param, @mPath + "/shards", "PUT", upStream)
|
102
|
+
conn = HttpConnection.new(@mOdpsConfig, header, param, @mPath + "/shards", "PUT", upStream.string)
|
104
103
|
end
|
105
104
|
|
106
105
|
reload
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-aliyun-odps
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Xiao Dong
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2016-
|
12
|
+
date: 2016-03-04 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|