embulk-output-analytics_cloud 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e56deed75f747698154a00fd6c09d78e10308d53
-  data.tar.gz: b9e83121ce91a01e7e4cf0d17fc31e104e5aa058
+  metadata.gz: f00ff3f2aad0af7593bf97525548b2dff9fab7e1
+  data.tar.gz: c4df8f7a6aaca06ddc21752e407ed72a2ce3d1a5
 SHA512:
-  metadata.gz: 535210cd11f90c91510855c49b2118b28dacada5e9a01a2d341f53d54e95236e8508d46541862ee8d3342e1c1d80a0d0f372b6dd0830c9d8d39dd2edae9958af
-  data.tar.gz: a54652f0de1cd6f433f9664f73eed1c02cbc966864f98306c81b3ff62b0efb49574e529df0cc7cc6858e020b126861d3cdb9453f0ed0cd34abc8542a525dad0d
+  metadata.gz: 0968200ce4dc1c48efb3d2ee8c0dccb71d39f60f138664f888d4d61ab75b7754a4b5260aeceb0c29937309f66d7a19cebd67402e301273822015d780f8065e8b
+  data.tar.gz: 85d08495056b061741f57d689a73199a9c02b61a54fc35f34c876802ba48bcc32421351933c63588446d73e3aa30fad2005b92ec506ef0a190c80ea7bbfff8f9

data/README.md

@@ -13,12 +13,18 @@ Embulk output plugin to load into Analytics Cloud.
 
 - **username**: analytics cloud username (string, required)
 - **password**: analytics cloud password (string, required)
-- **login_endpoint**: endpoint (string, default: `https://login.salesforce.com`)
+- **login_endpoint**: endpoint (string, default: `"https://login.salesforce.com"`)
 - **edgemart_alias**: edgemart alias (string, required)
 - **operation**: operation (string, default: `"Append"`)
 - **metadata_json**: MetadataJson (string, default: `null`)
-- **auto_metadata**: auto creation for MetadataJson (integer, required)
-- **batch_size**: data size for a InsightsExternalDataPart record (integer, default: `"3000"`)
+- **auto_metadata_settings**: auto creation for MetadataJson (object, default: `null`)
+  - **connector**: connector name (string, default: `"EmbulkOutputPluginConnector"`)
+  - **description**: description (string, default: `""`)
+  - **precision**: precision for Numeric columns (integer, default: `18`)
+  - **scale**: scale for Numeric columns (integer, default: `4`)
+  - **default_value**: default value for Numeric columns (integer)
+  - **format**: format for Timestamp columns (string, default: `"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"`)
+- **batch_size**: data size for a InsightsExternalDataPart record (integer, default: `3000`)
 - **version**: API version (string, default: `"34.0"`)
 
 ## Example
@@ -29,6 +35,7 @@ out:
   username: hoge
   password: fuga
   edgemart_alias: foobar
+  auto_metadata_settings: {}
 ```
 
 

data/build.gradle

@@ -12,7 +12,7 @@ configurations {
     provided
 }
 
-version = "0.1.0"
+version = "0.1.1"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.7.4"

data/src/main/java/org/embulk/output/analytics_cloud/AnalyticsCloudOutputPlugin.java

@@ -60,6 +60,34 @@ public class AnalyticsCloudOutputPlugin
         }
     };
 
+    public interface AutoMetadataTask extends Task
+    {
+        @Config("connector")
+        @ConfigDefault("\"EmbulkOutputPluginConnector\"")
+        public Optional<String> getConnectorName();
+
+        @Config("description")
+        @ConfigDefault("\"\"")
+        public Optional<String> getDescription();
+
+        @Config("scale")
+        @ConfigDefault("4")
+        public Optional<Integer> getScale();
+
+        @Config("precision")
+        @ConfigDefault("18")
+        public Optional<Integer> getPrecision();
+
+        @Config("default_value")
+        @ConfigDefault("0")
+        public Optional<Double> getDefaultValue();
+
+        @Config("format")
+        @ConfigDefault("\"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\"")
+        public Optional<String> getFormat();
+
+    }
+
     public interface PluginTask
             extends Task
     {
@@ -84,9 +112,9 @@ public class AnalyticsCloudOutputPlugin
         @ConfigDefault("null")
         public Optional<String> getMetadataJson();
 
-        @Config("auto_metadata")
-        @ConfigDefault("true")
-        public Optional<String> getAutoMetadata();
+        @Config("auto_metadata_settings")
+        @ConfigDefault("null")
+        public Optional<AutoMetadataTask> getAutoMetadataTask();
 
         @Config("batch_size")
         @ConfigDefault("3000")
@@ -213,24 +241,44 @@ public class AnalyticsCloudOutputPlugin
                     pageReader.getSchema().visitColumns(new ColumnVisitor() {
                         @Override
                         public void doubleColumn(Column column) {
-                            record.add(String.valueOf(pageReader.getDouble(column)));
+                            if (pageReader.isNull(column)) {
+                                record.add("");
+                            } else {
+                                record.add(String.valueOf(pageReader.getDouble(column)));
+                            }
                         }
                         @Override
                         public void timestampColumn(Column column) {
-                            DateTime dt = new DateTime(pageReader.getTimestamp(column).getEpochSecond()*1000);
-                            record.add(dt.toString("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"));
+                            if (pageReader.isNull(column)) {
+                                record.add("");
+                            } else {
+                                DateTime dt = new DateTime(pageReader.getTimestamp(column).getEpochSecond() * 1000);
+                                record.add(dt.toString("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"));
+                            }
                         }
                         @Override
                         public void stringColumn(Column column) {
-                            record.add(pageReader.getString(column));
+                            if (pageReader.isNull(column)) {
+                                record.add("");
+                            } else {
+                                record.add(pageReader.getString(column));
+                            }
                         }
                         @Override
                         public void longColumn(Column column) {
-                            record.add(String.valueOf(pageReader.getLong(column)));
+                            if (pageReader.isNull(column)) {
+                                record.add("");
+                            } else {
+                                record.add(String.valueOf(pageReader.getLong(column)));
+                            }
                         }
                         @Override
                         public void booleanColumn(Column column) {
-                            record.add(String.valueOf(pageReader.getBoolean(column)));
+                            if (pageReader.isNull(column)) {
+                                record.add("");
+                            } else {
+                                record.add(String.valueOf(pageReader.getBoolean(column)));
+                            }
                         }
 
                     });
@@ -328,7 +376,7 @@ public class AnalyticsCloudOutputPlugin
         insightsExdata.addField("Operation", task.getOperation().get());
         if (task.getMetadataJson().isPresent()) {
             insightsExdata.addField("MetadataJson", task.getMetadataJson().get().getBytes());
-        } else if (task.getAutoMetadata().get().toLowerCase().equals("true")){
+        } else if (task.getAutoMetadataTask().isPresent()){
             insightsExdata.addField("MetadataJson", this.createMetadataJSON(task, schema).getBytes());
         }
 
@@ -347,6 +395,15 @@ public class AnalyticsCloudOutputPlugin
     }
 
     private String createMetadataJSON(PluginTask task, Schema schema) {
+        if (task.getAutoMetadataTask().isPresent()) {
+            logger.info("***Is Present***");
+            logger.info("{}", task.getAutoMetadataTask().get().getConnectorName().get());
+            logger.info("{}", task.getAutoMetadataTask().get().getDefaultValue().get());
+            logger.info("{}", task.getAutoMetadataTask().get().getDescription().get());
+            logger.info("{}", task.getAutoMetadataTask().get().getFormat().get());
+            logger.info("{}", task.getAutoMetadataTask().get().getPrecision().get());
+            logger.info("{}", task.getAutoMetadataTask().get().getScale().get());
+        }
         HashMap<String, Object> metadata = new HashMap<>();
         metadata.put("fileFormat", new HashMap<String, Object>(){
             {
@@ -369,12 +426,12 @@ public class AnalyticsCloudOutputPlugin
                     put("type", DATATYPE_MAP.get(column.getType().toString()));
 
                     if (column.getType().getJavaType().equals(Timestamp.class)) {
-                        put("format", "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
+                        put("format", task.getAutoMetadataTask().get().getFormat().get());
                     }
                     if (column.getType().getJavaType().equals(long.class)) {
-                        put("scale", 6);
-                        put("precision", 18);
-                        put("defaultValue", 0);
+                        put("scale", task.getAutoMetadataTask().get().getScale().get());
+                        put("precision", task.getAutoMetadataTask().get().getPrecision().get());
+                        put("defaultValue", task.getAutoMetadataTask().get().getDefaultValue().get());
                     }
                 }
             });
@@ -383,8 +440,8 @@ public class AnalyticsCloudOutputPlugin
         ArrayList<HashMap<String, Object>> objects = new ArrayList<>();
         objects.add(new HashMap<String, Object>() {
             {
-                put("connector", "EmbulkOutputPluginConnector");
-                put("description", "");
+                put("connector", task.getAutoMetadataTask().get().getConnectorName().get());
+                put("description", task.getAutoMetadataTask().get().getDescription().get());
                 put("fullyQualifiedName", task.getEdgemartAlias());
                 put("label", task.getEdgemartAlias());
                 put("name", task.getEdgemartAlias());
@@ -395,6 +452,7 @@ public class AnalyticsCloudOutputPlugin
 
         ObjectMapper mapper = new ObjectMapper();
         try {
+            logger.info(mapper.writeValueAsString(metadata));
             return mapper.writeValueAsString(metadata);
         } catch (JsonProcessingException ex) {
             logger.error(ex.getMessage());

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-analytics_cloud
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Makoto Tajitsu
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-06-11 00:00:00.000000000 Z
+date: 2016-06-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -58,7 +58,7 @@ files:
 - lib/partner.jar
 - src/main/java/org/embulk/output/analytics_cloud/AnalyticsCloudOutputPlugin.java
 - src/test/java/org/embulk/output/analytics_cloud/TestAnalyticsCloudOutputPlugin.java
-- classpath/embulk-output-analytics_cloud-0.1.0.jar
+- classpath/embulk-output-analytics_cloud-0.1.1.jar
 - classpath/force-wsc-34.0.jar
 - classpath/partner.jar
 - classpath/super-csv-2.4.0.jar