embulk-input-s3 0.2.7 → 0.2.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (6) hide show
  1. checksums.yaml +4 -4
  2. data/classpath/embulk-input-s3-0.2.8.jar +0 -0
  3. data/src/main/java/org/embulk/input/s3/AbstractS3FileInputPlugin.java +7 -1
  4. data/src/test/java/org/embulk/input/s3/TestS3FileInputPlugin.java +10 -0
  5. metadata +12 -12
  6. data/classpath/embulk-input-s3-0.2.7.jar +0 -0
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 3fde7e23b02972cc6b615c1845ab820c3e960770
4
- data.tar.gz: 7ab94c1a292321bbee5c4642be906c1404db35d2
3
+ metadata.gz: 4f8bc775d296e8ae5cfcaffe8d335f1f09de52cc
4
+ data.tar.gz: ddfb843558dc281243bc6925f9f5abfefe72f4bc
5
5
  SHA512:
6
- metadata.gz: ed3cfe60161e427fdffe19b0622feae7aa1e5138d2c3f10243125eb82349ee91efbda869ab21644fb7141125415c29172e36c77f20cf352128c85855222a8fc6
7
- data.tar.gz: 7235b4b49d0f7df056098084c70d8dea60be33dbe0335b589eac8e0478a835514caa4a82ab1ee1b5a45370a2b0353985b2f6b66a01ca9ce123202417917fcbbf
6
+ metadata.gz: 757deefb26f6d1ef1ca859d87d0f7aba7b347bfb3a98b845af112e0c786ca4c68775c52fad9dd79d4013e212f1975cbe6f507800dca9bbf04c1a2e02ed87f302
7
+ data.tar.gz: 435a0194c28b19152c631cc78bf616e3ee1a50aa701b8f913723ac45c5b0517968f516db0cd5f841822215524214bf6916cac36d071be944e01cefb64dcca7a7
data/classpath/embulk-input-s3-0.2.8.jar ADDED
Binary file
data/src/main/java/org/embulk/input/s3/AbstractS3FileInputPlugin.java CHANGED
@@ -65,6 +65,10 @@ public abstract class AbstractS3FileInputPlugin
65
65
  @ConfigDefault("null")
66
66
  public Optional<String> getAccessKeyId();
67
67
 
68
+ @Config("incremental")
69
+ @ConfigDefault("true")
70
+ public boolean getIncremental();
71
+
68
72
  // TODO timeout, ssl, etc
69
73
 
70
74
  public FileList getFiles();
@@ -104,7 +108,9 @@ public abstract class AbstractS3FileInputPlugin
104
108
  ConfigDiff configDiff = Exec.newConfigDiff();
105
109
 
106
110
  // last_path
107
- configDiff.set("last_path", task.getFiles().getLastPath(task.getLastPath()));
111
+ if (task.getIncremental()) {
112
+ configDiff.set("last_path", task.getFiles().getLastPath(task.getLastPath()));
113
+ }
108
114
 
109
115
  return configDiff;
110
116
  }
data/src/test/java/org/embulk/input/s3/TestS3FileInputPlugin.java CHANGED
@@ -23,6 +23,7 @@ import java.util.ArrayList;
23
23
  import java.util.List;
24
24
 
25
25
  import static org.junit.Assert.assertEquals;
26
+ import static org.junit.Assert.assertFalse;
26
27
  import static org.junit.Assert.assertNull;
27
28
  import static org.junit.Assume.assumeNotNull;
28
29
 
@@ -91,6 +92,15 @@ public class TestS3FileInputPlugin
91
92
  assertEquals(0, getRecords(config, output).size());
92
93
  }
93
94
 
95
+ @Test
96
+ public void useIncremental()
97
+ {
98
+ ConfigSource config = this.config.deepCopy().set("incremental", false);
99
+ ConfigDiff configDiff = runner.transaction(config, new Control(runner, output));
100
+
101
+ assertFalse(configDiff.has("last_path"));
102
+ }
103
+
94
104
  @Test
95
105
  public void emptyFilesWithLastPath()
96
106
  throws Exception
metadata CHANGED
@@ -1,43 +1,43 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-input-s3
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.7
4
+ version: 0.2.8
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sadayuki Furuhashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-12-21 00:00:00.000000000 Z
11
+ date: 2016-04-25 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- name: bundler
15
- version_requirements: !ruby/object:Gem::Requirement
16
- requirements:
17
- - - '>='
18
- - !ruby/object:Gem::Version
19
- version: '1.0'
20
14
  requirement: !ruby/object:Gem::Requirement
21
15
  requirements:
22
16
  - - '>='
23
17
  - !ruby/object:Gem::Version
24
18
  version: '1.0'
19
+ name: bundler
25
20
  prerelease: false
26
21
  type: :development
27
- - !ruby/object:Gem::Dependency
28
- name: rake
29
22
  version_requirements: !ruby/object:Gem::Requirement
30
23
  requirements:
31
24
  - - '>='
32
25
  - !ruby/object:Gem::Version
33
- version: 0.10.0
26
+ version: '1.0'
27
+ - !ruby/object:Gem::Dependency
34
28
  requirement: !ruby/object:Gem::Requirement
35
29
  requirements:
36
30
  - - '>='
37
31
  - !ruby/object:Gem::Version
38
32
  version: 0.10.0
33
+ name: rake
39
34
  prerelease: false
40
35
  type: :development
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - '>='
39
+ - !ruby/object:Gem::Version
40
+ version: 0.10.0
41
41
  description: Reads files stored on Amazon S3
42
42
  email:
43
43
  - frsyuki@gmail.com
@@ -61,7 +61,7 @@ files:
61
61
  - classpath/aws-java-sdk-kms-1.10.33.jar
62
62
  - classpath/aws-java-sdk-s3-1.10.33.jar
63
63
  - classpath/commons-codec-1.6.jar
64
- - classpath/embulk-input-s3-0.2.7.jar
64
+ - classpath/embulk-input-s3-0.2.8.jar
65
65
  - classpath/httpclient-4.3.6.jar
66
66
  - classpath/httpcore-4.3.3.jar
67
67
  - classpath/jcl-over-slf4j-1.7.12.jar
data/classpath/embulk-input-s3-0.2.7.jar DELETED
Binary file