embulk-input-marketo 0.6.2 → 0.6.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dcb412b1d17550359b38e4f4d342d7cc9c7e4620
-  data.tar.gz: 3ffc4c24ba389baa1d480e643b16ad0ee5c027c0
+  metadata.gz: cd1171d528fb826e6f5a55aa69e3006fea9e08e3
+  data.tar.gz: 800a0f43636d84a5bdf503d8ce7d10b17007f285
 SHA512:
-  metadata.gz: d0d455a95877cdf74feef66f94fc119d81585c889cfd92cfe4b1aacd031fdfc200b161f31af0eacf2198b5c9cb7d1bdb3af246caf602dd0458b261424e3b3ae5
-  data.tar.gz: 3b0b21c16697be01c404bd9df610935bc32fa99eaa564f16cd2a96f3708bee7c0cef73e668df8a1e5c6787de0dadc3a9cb703987521c5c9bb0e1ff0df758c8a9
+  metadata.gz: 28b96cceda90b61e429e9c76a01d1034852ec0348ccb33e45c4d1bba8e8cf0b5e47e1d0eb89d732519cd2d55b797629481a11e237b2c0da7eb11718533bb794b
+  data.tar.gz: 36e22aa23c8f08fb8c3731fce02fe9ef329f9db9c8fbb8702e0a4053f622b82286ff4959a025ee06c58a0eca9d0988264f11f15c7bf601bc5038874f287d0eaf

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.6.3 - 2017-11-13
+* [enhance] Ignore records with timestamp smaller or equal to latest_fetch_time  [#74](https://github.com/treasure-data/embulk-input-marketo/pull/74)
+
+## 0.6.2 - 2017-10-16
+* [fixed] NullPointerException when building config diff [#73](https://github.com/treasure-data/embulk-input-marketo/pull/73)
+
 ## 0.6.1 - 2017-10-12
 * [fixed] OutOfMemeory when run Lead by list, Lead by program   [#69](https://github.com/treasure-data/embulk-input-marketo/pull/69)
 * [fixed] SOAP only field cause NullPointerException [#69](https://github.com/treasure-data/embulk-input-marketo/pull/69)

data/build.gradle

@@ -16,7 +16,7 @@ repositories {
 configurations {
     provided
 }
-version = "0.6.2"
+version = "0.6.3"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 

data/src/main/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPlugin.java

@@ -204,7 +204,8 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
                         return new CsvRecordIterator(input, task);
                     }
                 }));
-                long currentTimestamp = task.getLatestFetchTime().or(0L);
+                Long latestFetchTime = task.getLatestFetchTime().or(0L);
+                long currentTimestamp = latestFetchTime;
                 Set<String> latestUids = task.getPreviousUids();
                 //Keep the preview code here when we can enable real preview
                 if (Exec.isPreview()) {
@@ -214,6 +215,12 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
                 while (csvRecords.hasNext()) {
                     Map<String, String> csvRecord = csvRecords.next();
                     if (task.getIncremental()) {
+                        String incrementalTimeStamp = csvRecord.get(incrementalColumn);
+                        long timestamp = ISO_DATETIME_FORMAT.parseDateTime(incrementalTimeStamp).getMillis();
+                        //Ignore records that have timestamp smaller or equal with latestFetchTime
+                        if (latestFetchTime >= timestamp) {
+                            continue;
+                        }
                         if (!csvRecord.containsKey(incrementalColumn)) {
                             throw new DataException("Extracted record doesn't have incremental column " + incrementalColumn);
                         }
@@ -224,8 +231,6 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
                                 continue;
                             }
                         }
-                        String incrementalTimeStamp = csvRecord.get(incrementalColumn);
-                        long timestamp = ISO_DATETIME_FORMAT.parseDateTime(incrementalTimeStamp).getMillis();
                         if (currentTimestamp < timestamp) {
                             currentTimestamp = timestamp;
                             //switch timestamp

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-marketo
 version: !ruby/object:Gem::Version
-  version: 0.6.2
+  version: 0.6.3
 platform: ruby
 authors:
 - uu59
@@ -10,36 +10,36 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-10-16 00:00:00.000000000 Z
+date: 2017-11-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: bundler
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '1.0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
+  name: bundler
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: rake
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '1.0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '10.0'
+  name: rake
   prerelease: false
   type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '10.0'
 description: Loads records from Marketo.
 email:
 - k@uu59.org
@@ -124,9 +124,9 @@ files:
 - src/test/resources/fixtures/lists_response.json
 - src/test/resources/fixtures/program_response.json
 - classpath/jetty-http-9.2.14.v20151106.jar
-- classpath/embulk-input-marketo-0.6.2.jar
 - classpath/embulk-base-restclient-0.5.3.jar
 - classpath/jetty-client-9.2.14.v20151106.jar
+- classpath/embulk-input-marketo-0.6.3.jar
 - classpath/jetty-util-9.2.14.v20151106.jar
 - classpath/embulk-util-retryhelper-jetty92-0.5.3.jar
 - classpath/jetty-io-9.2.14.v20151106.jar