fluent-plugin-aliyun-odps 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -1
- data/README.cn.md +22 -1
- data/VERSION +1 -1
- data/lib/fluent/plugin/http/http_connection.rb +1 -1
- data/lib/fluent/plugin/stream_writer.rb +10 -11
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: f583712920902f51d0f4b2d50f1350e73ef43c37
|
|
4
|
+
data.tar.gz: 420ac14c5256bd3aae8be1693b856c7f18d6520d
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: cd2b20d9faa6e79f3ea1d8f5e61f2aab0f3f2ccdb7e2ceee83974a70d64abe30dcad0e352ff42479e499bf43ae21f0553cd997ef4c7025797f3e1f9445740fd5
|
|
7
|
+
data.tar.gz: b7cea3a104ec7a849741738977f6f164621637a4e9e3196639eefb56a10ee5eb8b3519dcd5b00a07487d5ed5d5193c2e4b2e267482b4adac2aa2f93f60d7e309
|
data/CHANGELOG.md
CHANGED
data/README.cn.md
CHANGED
|
@@ -71,7 +71,7 @@ $ gem install fluent-plugin-aliyun-odps
|
|
|
71
71
|
- aliyun_access_key(Required):������access key.
|
|
72
72
|
- aliyun_odps_hub_endpoint(Required):�����ķ�������ECS�ϣ���ѱ�ֵ�趨Ϊ http://dh-ext.odps.aliyun-inc.com, ��������Ϊ http://dh.odps.aliyun.com.
|
|
73
73
|
- aliyunodps_endpoint(Required):�����ķ�������ECS�ϣ���ѱ�ֵ�趨Ϊ http://odps-ext.aiyun-inc.com/api, ��������Ϊ http://service.odps.aliyun.com/api .
|
|
74
|
-
- buffer_chunk_limit(Optional): ���С��֧�֡�k��(KB),��m��(MB)
|
|
74
|
+
- buffer_chunk_limit(Optional): ���С��֧�֡�k��(KB),��m��(MB)��λ��Ĭ�� 8MB������ֵ2MB, Ŀǰ���֧��20MB.
|
|
75
75
|
- buffer_queue_limit(Optional): ����д�С����ֵ��buffer_chunk_limit��ͬ����������������С��
|
|
76
76
|
- flush_interval(Optional): ǿ�Ʒ��ͼ�����ﵽʱ��������δ����ǿ�Ʒ���, Ĭ�� 60s.
|
|
77
77
|
- abandon_mode(Optional):�����������κ�������pack���ݡ�
|
|
@@ -108,3 +108,24 @@ $ gem install fluent-plugin-aliyun-odps
|
|
|
108
108
|
---
|
|
109
109
|
|
|
110
110
|
licensed under the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0.html)
|
|
111
|
+
|
|
112
|
+
## ����ʹ�������Լ��쳣����
|
|
113
|
+
---
|
|
114
|
+
* �����׳��쳣InvalidShardId/ShardNotReady��ʲôԭ���£�
|
|
115
|
+
- ����ϵͳ��������������ݳ���������⣬���ڶ�ʱ���ڻָ���
|
|
116
|
+
- fluentd������ڶ��������鿴������shard_num�Ƿ����ó���һ����ֵ�������ã���������ò�һ���ǻᵼ���������ģ�shard_number�ٵĽ��̻�Ѷ���shard Unload����
|
|
117
|
+
- ���ܴ��������ʹ��sdk�ȷ�ʽ������loadshard/unloadshard�Ȳ�����
|
|
118
|
+
* enable_fast_crc��μ���Ƿ���ݣ�
|
|
119
|
+
- ���������ú�������fluentd���̣�����ʱ����֤�����ʧ�ܻ��׳�����ԭ��reload���������֤�����������Ŀ¼������ldd�鿴aliyun-odps-fluentd-plugin/lib/fluent/plugin/crc/lib/linux/crc32c.so��
|
|
120
|
+
* retry_time/retry_interval��fluentd�Դ���retry�к�����
|
|
121
|
+
- fluentd�Դ�retryĬ�ϳ���36Сʱ���Ὣ����buffer_chunk�ط������ö�̬partition������ط�ȫ�����ݿ�����������ظ�������������ͻ�ʹ�ò���ڲ����ԣ��������ʧ�ܣ����ٸ���abandon_mode��ֵ�ж�������pack�����ݻ��ǽ���fluentd�ط�����buffer��
|
|
122
|
+
* Warning��ErrorCode: NoSuchPartition, Message: write failed because The specified partition does not exist.��ʲô��˼��
|
|
123
|
+
- ���������catch��Odps��NoSuchPartitionʱ��������������������������warn��ʾOdps���в��������ݶ�Ӧ���������Զ���������������ɹ�������Ϣ��ʾ��
|
|
124
|
+
* Fluent::BufferQueueLimitError error="queue size exceeds limit"��ʲôԭ��
|
|
125
|
+
- fluentd�ڶ�ȡ����-�������ݹ����У����ȶ�ȡ��һ��buffer�У������С����������buffer_chunk_limit��buffer_queue_limit��ͬ�������������������ܿ�������Ϊ�ѻ����ݵ���buffer���㣬���Գ�������buffer_queue_limit���������⡣
|
|
126
|
+
* ���config�ļ���ηֱ�����һ��fluentd���̣�
|
|
127
|
+
- ������ڶ��config�ļ�������ʹ��in_multiprocess������ͬʱ������ͬ�Ľ���������
|
|
128
|
+
* partition has no corresponding source key or the partition expression is wrong.����쳣��ʲôԭ��
|
|
129
|
+
- ����쳣��ʾ��source data���Ҳ���������partition�ֶ��е�ֵ������partition ctime=${remote}����remoteû�г�����source�У��������á�
|
|
130
|
+
* Failed to format the data.����쳣��ʲôԭ��
|
|
131
|
+
- ���������Ϣ�׳���������partition���̳������⣬����partition���ã���������д���������Ҳ��������������⡣
|
data/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
0.1.
|
|
1
|
+
0.1.2
|
|
@@ -65,7 +65,7 @@ module OdpsDatahub
|
|
|
65
65
|
@mHeader[$DATE] = Time.now.utc.strftime("%a, %d %b %Y %H:%M:%S GMT")
|
|
66
66
|
if not @mIsOdpsUrl
|
|
67
67
|
@mHeader[$TUNNEL_STREAM_VERSION] = "1"
|
|
68
|
-
@mHeader[$TUNNEL_VERSION] = "
|
|
68
|
+
@mHeader[$TUNNEL_VERSION] = "4"
|
|
69
69
|
end
|
|
70
70
|
@mHeader[$USER_AGENT] = $SDK_UA_STR + @mOdpsConfig.userAgent
|
|
71
71
|
@mHeader[$AUTHORIZATION] = signAuthorization
|
|
@@ -78,29 +78,28 @@ module OdpsDatahub
|
|
|
78
78
|
param[$PARAM_RECORD_COUNT] = @mRecordList.size.to_s
|
|
79
79
|
header[$CONTENT_ENCODING] = "deflate"
|
|
80
80
|
header[$CONTENT_TYPE] = "application/octet-stream"
|
|
81
|
-
|
|
81
|
+
# version 4
|
|
82
82
|
pack = OdpsDatahub::XStreamPack.new
|
|
83
83
|
pack.pack_data = Zlib::Deflate.deflate(@mUpStream.string)
|
|
84
84
|
pack.pack_meta = ""
|
|
85
|
-
|
|
86
|
-
pack.serialize_to(
|
|
87
|
-
header[$CONTENT_MD5] = Digest::MD5.hexdigest(
|
|
88
|
-
header[$CONTENT_LENGTH] =
|
|
85
|
+
upStream = ::StringIO.new
|
|
86
|
+
pack.serialize_to(upStream)
|
|
87
|
+
header[$CONTENT_MD5] = Digest::MD5.hexdigest(upStream.string)
|
|
88
|
+
header[$CONTENT_LENGTH] = upStream.string.length.to_s
|
|
89
89
|
|
|
90
|
-
|
|
91
|
-
=end
|
|
92
|
-
#version 3
|
|
90
|
+
=begin version 3
|
|
93
91
|
upStream = Zlib::Deflate.deflate(@mUpStream.string)
|
|
94
92
|
header[$CONTENT_MD5] = Digest::MD5.hexdigest(upStream)
|
|
95
93
|
header[$CONTENT_LENGTH] = upStream.length.to_s
|
|
94
|
+
=end
|
|
96
95
|
#MAX_LENGTH 2048*10KB
|
|
97
96
|
if upStream.length > $MAX_PACK_SIZE
|
|
98
|
-
raise OdpsDatahubException.new($PACK_SIZE_EXCEED, "pack size:" + upStream.length.to_s)
|
|
97
|
+
raise OdpsDatahubException.new($PACK_SIZE_EXCEED, "pack size:" + upStream.string.length.to_s)
|
|
99
98
|
end
|
|
100
99
|
if @mShardId != nil
|
|
101
|
-
conn = HttpConnection.new(@mOdpsConfig, header, param, @mPath + "/shards/" + @mShardId.to_s, "PUT", upStream)
|
|
100
|
+
conn = HttpConnection.new(@mOdpsConfig, header, param, @mPath + "/shards/" + @mShardId.to_s, "PUT", upStream.string)
|
|
102
101
|
else
|
|
103
|
-
conn = HttpConnection.new(@mOdpsConfig, header, param, @mPath + "/shards", "PUT", upStream)
|
|
102
|
+
conn = HttpConnection.new(@mOdpsConfig, header, param, @mPath + "/shards", "PUT", upStream.string)
|
|
104
103
|
end
|
|
105
104
|
|
|
106
105
|
reload
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-aliyun-odps
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Xiao Dong
|
|
@@ -9,7 +9,7 @@ authors:
|
|
|
9
9
|
autorequire:
|
|
10
10
|
bindir: bin
|
|
11
11
|
cert_chain: []
|
|
12
|
-
date: 2016-
|
|
12
|
+
date: 2016-03-04 00:00:00.000000000 Z
|
|
13
13
|
dependencies:
|
|
14
14
|
- !ruby/object:Gem::Dependency
|
|
15
15
|
name: fluentd
|