fluent-plugin-aliyun-odps 0.1.1 → 0.1.2

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 96472dd18671b2adc39de0934ea13aac995b81b9
4
- data.tar.gz: b9f1ffc43b9f1b81be6081201d811074e28a3bb5
3
+ metadata.gz: f583712920902f51d0f4b2d50f1350e73ef43c37
4
+ data.tar.gz: 420ac14c5256bd3aae8be1693b856c7f18d6520d
5
5
  SHA512:
6
- metadata.gz: e3d190f5bbc2f7efb676485541b593b7a5e58177aa9ac41a6c04ba4f958594e3540a57c3fdd04571e5c6ee4e765cfe8419fc8265302f2bb1e4c103ee8f0ec4e0
7
- data.tar.gz: f179cef524c9925924238c6660d05df73eaaa10833ac228b59d326f07cbd3b5b3be69885f71e5b4c1d3d3436997451a48db113b824ab3c953ff77004bba4e85e
6
+ metadata.gz: cd2b20d9faa6e79f3ea1d8f5e61f2aab0f3f2ccdb7e2ceee83974a70d64abe30dcad0e352ff42479e499bf43ae21f0553cd997ef4c7025797f3e1f9445740fd5
7
+ data.tar.gz: b7cea3a104ec7a849741738977f6f164621637a4e9e3196639eefb56a10ee5eb8b3519dcd5b00a07487d5ed5d5193c2e4b2e267482b4adac2aa2f93f60d7e309
data/CHANGELOG.md CHANGED
@@ -13,4 +13,6 @@ Hotfix retry log error bug.
13
13
  0.1.0
14
14
  Add partition when catch NoSuchPartition.
15
15
  0.1.1
16
- Fix some log format.
16
+ Fix some log format.
17
+ 0.1.2
18
+ Use XStreamPack.
data/README.cn.md CHANGED
@@ -71,7 +71,7 @@ $ gem install fluent-plugin-aliyun-odps
71
71
  - aliyun_access_key(Required):������access key.
72
72
  - aliyun_odps_hub_endpoint(Required):�����ķ�������ECS�ϣ���ѱ�ֵ�趨Ϊ http://dh-ext.odps.aliyun-inc.com, ��������Ϊ http://dh.odps.aliyun.com.
73
73
  - aliyunodps_endpoint(Required):�����ķ�������ECS�ϣ���ѱ�ֵ�趨Ϊ http://odps-ext.aiyun-inc.com/api, ��������Ϊ http://service.odps.aliyun.com/api .
74
- - buffer_chunk_limit(Optional): ���С��֧�֡�k��(KB),��m��(MB),��g��(GB)��λ��Ĭ�� 8MB������ֵ2MB, Ŀǰ���֧��20MB.
74
+ - buffer_chunk_limit(Optional): ���С��֧�֡�k��(KB),��m��(MB)��λ��Ĭ�� 8MB������ֵ2MB, Ŀǰ���֧��20MB.
75
75
  - buffer_queue_limit(Optional): ����д�С����ֵ��buffer_chunk_limit��ͬ����������������С��
76
76
  - flush_interval(Optional): ǿ�Ʒ��ͼ�����ﵽʱ��������δ����ǿ�Ʒ���, Ĭ�� 60s.
77
77
  - abandon_mode(Optional):�����������κ�������pack���ݡ�
@@ -108,3 +108,24 @@ $ gem install fluent-plugin-aliyun-odps
108
108
  ---
109
109
 
110
110
  licensed under the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0.html)
111
+
112
+ ## ����ʹ�������Լ��쳣����
113
+ ---
114
+ * �����׳��쳣InvalidShardId/ShardNotReady��ʲôԭ���£�
115
+ - ����ϵͳ��������������ݳ���������⣬���ڶ�ʱ���ڻָ���
116
+ - fluentd������ڶ��������鿴������shard_num�Ƿ����ó���һ����ֵ���򶼲����ã���������ò�һ���ǻᵼ���������ģ�shard_number�ٵĽ��̻�Ѷ���shard Unload����
117
+ - ���ܴ��������ʹ��sdk�ȷ�ʽ������loadshard/unloadshard�Ȳ�����
118
+ * enable_fast_crc��μ���Ƿ���ݣ�
119
+ - ���������ú�������fluentd���̣�����ʱ����֤�����ʧ�ܻ��׳�����ԭ��reload���������֤�����������Ŀ¼������ldd�鿴aliyun-odps-fluentd-plugin/lib/fluent/plugin/crc/lib/linux/crc32c.so��
120
+ * retry_time/retry_interval��fluentd�Դ���retry�к�����
121
+ - fluentd�Դ�retryĬ�ϳ���36Сʱ���Ὣ����buffer_chunk�ط������ö�̬partition������ط�ȫ�����ݿ�����������ظ�������������ͻ�ʹ�ò���ڲ����ԣ��������ʧ�ܣ����ٸ���abandon_mode��ֵ�ж�������pack�����ݻ��ǽ���fluentd�ط�����buffer��
122
+ * Warning��ErrorCode: NoSuchPartition, Message: write failed because The specified partition does not exist.��ʲô��˼��
123
+ - ���������catch��Odps��NoSuchPartitionʱ��������������������������warn��ʾOdps���в��������ݶ�Ӧ���������Զ���������������ɹ�������Ϣ��ʾ��
124
+ * Fluent::BufferQueueLimitError error="queue size exceeds limit"��ʲôԭ��
125
+ - fluentd�ڶ�ȡ����-�������ݹ����У����ȶ�ȡ��һ��buffer�У������С����������buffer_chunk_limit��buffer_queue_limit��ͬ�������������������󣬺ܿ�������Ϊ�ѻ����ݵ���buffer���㣬���Գ�������buffer_queue_limit���������⡣
126
+ * ���config�ļ���ηֱ�����һ��fluentd���̣�
127
+ - ������ڶ��config�ļ�������ʹ��in_multiprocess������ͬʱ������ͬ�Ľ���������
128
+ * partition has no corresponding source key or the partition expression is wrong.����쳣��ʲôԭ��
129
+ - ����쳣��ʾ��source data���Ҳ���������partition�ֶ��е�ֵ������partition ctime=${remote}����remoteû�г�����source�У��������á�
130
+ * Failed to format the data.����쳣��ʲôԭ��
131
+ - ���������Ϣ�׳���������partition���̳������⣬����partition���ã���������д���������Ҳ��������������⡣
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.1.1
1
+ 0.1.2
@@ -65,7 +65,7 @@ module OdpsDatahub
65
65
  @mHeader[$DATE] = Time.now.utc.strftime("%a, %d %b %Y %H:%M:%S GMT")
66
66
  if not @mIsOdpsUrl
67
67
  @mHeader[$TUNNEL_STREAM_VERSION] = "1"
68
- @mHeader[$TUNNEL_VERSION] = "3"
68
+ @mHeader[$TUNNEL_VERSION] = "4"
69
69
  end
70
70
  @mHeader[$USER_AGENT] = $SDK_UA_STR + @mOdpsConfig.userAgent
71
71
  @mHeader[$AUTHORIZATION] = signAuthorization
@@ -78,29 +78,28 @@ module OdpsDatahub
78
78
  param[$PARAM_RECORD_COUNT] = @mRecordList.size.to_s
79
79
  header[$CONTENT_ENCODING] = "deflate"
80
80
  header[$CONTENT_TYPE] = "application/octet-stream"
81
- =begin version 4
81
+ # version 4
82
82
  pack = OdpsDatahub::XStreamPack.new
83
83
  pack.pack_data = Zlib::Deflate.deflate(@mUpStream.string)
84
84
  pack.pack_meta = ""
85
- upstream = ::StringIO.new
86
- pack.serialize_to(upstream)
87
- header[$CONTENT_MD5] = Digest::MD5.hexdigest(upstream.string)
88
- header[$CONTENT_LENGTH] = upstream.length.to_s
85
+ upStream = ::StringIO.new
86
+ pack.serialize_to(upStream)
87
+ header[$CONTENT_MD5] = Digest::MD5.hexdigest(upStream.string)
88
+ header[$CONTENT_LENGTH] = upStream.string.length.to_s
89
89
 
90
- conn = HttpConnection.new(@mOdpsConfig, header, param, @mPath + "/shards/" + @mShardId.to_s, "PUT", upstream)
91
- =end
92
- #version 3
90
+ =begin version 3
93
91
  upStream = Zlib::Deflate.deflate(@mUpStream.string)
94
92
  header[$CONTENT_MD5] = Digest::MD5.hexdigest(upStream)
95
93
  header[$CONTENT_LENGTH] = upStream.length.to_s
94
+ =end
96
95
  #MAX_LENGTH 2048*10KB
97
96
  if upStream.length > $MAX_PACK_SIZE
98
- raise OdpsDatahubException.new($PACK_SIZE_EXCEED, "pack size:" + upStream.length.to_s)
97
+ raise OdpsDatahubException.new($PACK_SIZE_EXCEED, "pack size:" + upStream.string.length.to_s)
99
98
  end
100
99
  if @mShardId != nil
101
- conn = HttpConnection.new(@mOdpsConfig, header, param, @mPath + "/shards/" + @mShardId.to_s, "PUT", upStream)
100
+ conn = HttpConnection.new(@mOdpsConfig, header, param, @mPath + "/shards/" + @mShardId.to_s, "PUT", upStream.string)
102
101
  else
103
- conn = HttpConnection.new(@mOdpsConfig, header, param, @mPath + "/shards", "PUT", upStream)
102
+ conn = HttpConnection.new(@mOdpsConfig, header, param, @mPath + "/shards", "PUT", upStream.string)
104
103
  end
105
104
 
106
105
  reload
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-aliyun-odps
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.1
4
+ version: 0.1.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Xiao Dong
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2016-01-22 00:00:00.000000000 Z
12
+ date: 2016-03-04 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: fluentd