embulk-input-hdfs 0.1.5 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 304cdee9ca1c60697b3796bd4aab6a406da8aa1d
|
|
4
|
+
data.tar.gz: faa5cc173f74d4cdf5cafbc52d3fbeb77b1eface
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 868a12f39f550effcef880beec037d063c844459ae515bae0efd711d8ac9be1d103648679cfc5de49f53a0f68b0be16b60c8b979710384c0e59bdbe265b85f51
|
|
7
|
+
data.tar.gz: 63fa4e4e7cbde6297f652fbd87776e9d1153cc35a2329b255fd9003b66603ec8b1daaf7b416637e2d035332d10e7282a42a52d9c845c3f828fe9c701e6d32e72
|
data/build.gradle
CHANGED
|
Binary file
|
|
@@ -81,7 +81,7 @@ public class HdfsFileInputPlugin implements FileInputPlugin
|
|
|
81
81
|
}
|
|
82
82
|
|
|
83
83
|
task.setFiles(allocateHdfsFilesToTasks(task, getFs(task), originalFileList));
|
|
84
|
-
logger.info("Loading target files: {}", originalFileList);
|
|
84
|
+
logger.info("embulk-input-hdfs: Loading target files: {}", originalFileList);
|
|
85
85
|
}
|
|
86
86
|
catch (IOException e) {
|
|
87
87
|
logger.error(e.getMessage());
|
|
@@ -90,13 +90,13 @@ public class HdfsFileInputPlugin implements FileInputPlugin
|
|
|
90
90
|
|
|
91
91
|
// log the detail of partial files.
|
|
92
92
|
for (HdfsPartialFile partialFile : task.getFiles()) {
|
|
93
|
-
logger.
|
|
93
|
+
logger.debug("embulk-input-hdfs: target file: {}, start: {}, end: {}",
|
|
94
94
|
partialFile.getPath(), partialFile.getStart(), partialFile.getEnd());
|
|
95
95
|
}
|
|
96
96
|
|
|
97
97
|
// number of processors is same with number of targets
|
|
98
98
|
int taskCount = task.getFiles().size();
|
|
99
|
-
logger.info("task size: {}", taskCount);
|
|
99
|
+
logger.info("embulk-input-hdfs: task size: {}", taskCount);
|
|
100
100
|
|
|
101
101
|
return resume(task.dump(), taskCount, control);
|
|
102
102
|
}
|
|
@@ -259,7 +259,7 @@ public class HdfsFileInputPlugin implements FileInputPlugin
|
|
|
259
259
|
for (Path path : pathList) {
|
|
260
260
|
int fileLength = (int) fs.getFileStatus(path).getLen(); // declare `fileLength` here because this is used below.
|
|
261
261
|
if (fileLength <= 0) {
|
|
262
|
-
logger.info("Skip the 0 byte target file: {}", path);
|
|
262
|
+
logger.info("embulk-input-hdfs: Skip the 0 byte target file: {}", path);
|
|
263
263
|
continue;
|
|
264
264
|
}
|
|
265
265
|
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: embulk-input-hdfs
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.6
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- takahiro.nakayama
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2015-11-
|
|
11
|
+
date: 2015-11-17 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: bundler
|
|
@@ -82,7 +82,7 @@ files:
|
|
|
82
82
|
- classpath/curator-client-2.6.0.jar
|
|
83
83
|
- classpath/curator-framework-2.6.0.jar
|
|
84
84
|
- classpath/curator-recipes-2.6.0.jar
|
|
85
|
-
- classpath/embulk-input-hdfs-0.1.
|
|
85
|
+
- classpath/embulk-input-hdfs-0.1.6.jar
|
|
86
86
|
- classpath/gson-2.2.4.jar
|
|
87
87
|
- classpath/hadoop-annotations-2.6.0.jar
|
|
88
88
|
- classpath/hadoop-auth-2.6.0.jar
|