embulk-formatter-fast_jsonl 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e484178b1ad2c59bcf8174c64e2ab5971ea66c4d
-  data.tar.gz: 750dafa0093696f5d5dd6dc70adb29ba75a25b43
+  metadata.gz: d506f3bbc729a3fbe07de916c993300443ed3f53
+  data.tar.gz: c93417de99924a0227d2db91652b16e690abcf1c
 SHA512:
-  metadata.gz: 7f87f64eefd47578fbf0f5d3c352ddc9a2297769b0579ae70d434ea107d51ac5f182044dfb7ec6c3791e531a94a569f88bdc492ef5c845d533170ac050a03ec7
-  data.tar.gz: 89ec108c4f83583bde29bcdd8be5efd4387ea04cabaa6e0a051b1c805740e253cdb7131bab2e8d2c39eaf1bd5738533b04ab79d8e6e56d36daa65c14561efd1a
+  metadata.gz: 838f1f2b97e9bec0345ca7b16c691e60e0a7fa43697b0bf4511d3fe17ff3ece745ebae4a0de784f1af416fe62accebce8e27c14f126480ffbddaea44a1b7b22a
+  data.tar.gz: 7d85448d372cb567bb494be1f4678733e2417d93ca3f48d6b4cb5d96debd60afd40fcae529191b9664628f249b5d3cea40fa2935712071b5c86664678f2ad011

data/README.md

@@ -16,6 +16,9 @@ Format json as 1 json in single line.
 - **explode_json_columns**: json column's explode to top fields. (array default:[])
 - **json_columns**:  json formatted as string column name. (array default:[])
 
+### additional mode
+- **suffix_key**: append suffix of key and value.(map Map[String,String]: {} )
+
 ## Example
 
 ```yaml
@@ -26,6 +29,8 @@ out:
     explode_json_columns: 
       - JSON_COLUMN_1
       - JSON_COLUMN_2
+    suffix_key:
+     append_key: append_value
 ```
 
 ## Run Examples

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.1.0"
+version = "0.1.1"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
@@ -24,6 +24,7 @@ dependencies {
     compile group: 'io.circe', name: 'circe-core_2.11', version: '0.8.0'
     compile group: 'io.circe', name: 'circe-generic_2.11', version: '0.8.0'
     compile group: 'io.circe', name: 'circe-parser_2.11', version: '0.8.0'
+    testCompile group: 'org.scalatest', name: 'scalatest_2.11', version: '3.0.1'
     provided "org.embulk:embulk-core:0.8.22"
 }
 

data/src/main/scala/org/embulk/formatter/fast_jsonl/PageOutput.scala

@@ -19,7 +19,8 @@ case class PageOutput(schema: Schema, task: PluginTask, output: FileOutput)
   val reader: PageReader = new PageReader(schema)
   val explodeColumns: Seq[String] = task.getExplodeJsonColumns().asScala
   val jsonColumns: Seq[String] = task.getJsonColumns().asScala
-  private var opened:Boolean = false
+  val suffixKey: Map[String, String] = task.getSuffixKey().asScala.toMap
+  private var opened: Boolean = false
 
   val timestampFormatter: TimestampFormatter =
     new TimestampFormatter(task, Optional.absent())
@@ -36,7 +37,8 @@ case class PageOutput(schema: Schema, task: PluginTask, output: FileOutput)
         ColumnVisitor(reader,
                       timestampFormatter,
                       explodeColumns,
-                      jsonColumns)
+                      jsonColumns,
+                      suffixKey)
       schema.visitColumns(visitor)
       encoder.addLine(visitor.getLine)
     }

data/src/main/scala/org/embulk/formatter/fast_jsonl/PluginTask.scala

@@ -21,4 +21,7 @@ trait PluginTask
   @ConfigDefault("[]")
   def getJsonColumns(): java.util.List[String]
 
+  @Config("suffix_key")
+  @ConfigDefault("{}")
+  def getSuffixKey(): java.util.Map[String, String]
 }

data/src/main/scala/org/embulk/formatter/fast_jsonl/json/ColumnVisitor.scala

@@ -11,7 +11,8 @@ import org.embulk.spi.{
 case class ColumnVisitor(reader: PageReader,
                          timestampFormatter: TimestampFormatter,
                          explodeColumns: Seq[String],
-                         jsonColumns: Seq[String])
+                         jsonColumns: Seq[String],
+                         suffixKey: Map[String, String])
     extends EmbulkColumnVisitor {
   import scala.collection.mutable
 
@@ -80,6 +81,10 @@ case class ColumnVisitor(reader: PageReader,
       case (key, json) =>
         recordMap.put(key, json)
     }
+    suffixKey.foreach {
+      case (key, value) =>
+        recordMap.put(key, Json.fromString(value))
+    }
     JsonEncoder(recordMap).noSpaces
   }
 

data/src/main/scala/org/embulk/formatter/fast_jsonl/json/JsonParser.scala

@@ -12,7 +12,7 @@ object JsonParser {
         sys.error(s"could not parse json. $value")
     }
 
-  def toJson(value: String):Json = {
+  def toJson(value: String): Json = {
     parse(value) match {
       case Right(v) =>
         v

data/src/test/scala/org/embulk/formatter/fast_jsonl/json/JsonEncoderSpec.scala

@@ -24,7 +24,8 @@ class JsonEncoderSpec extends FlatSpec with Matchers {
         map.put(string, json)
     }
     val jsonString = JsonEncoder(map).noSpaces
-    jsonString should be("{\"salutation\":\"Hey\",\"person\":{\"name\":\"Chris\"},\"exclamationMarks\":3}")
+    jsonString should be(
+      "{\"salutation\":\"Hey\",\"person\":{\"name\":\"Chris\"},\"exclamationMarks\":3}")
   }
 
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-formatter-fast_jsonl
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - smdmts
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-08-07 00:00:00.000000000 Z
+date: 2017-08-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -80,7 +80,7 @@ files:
 - classpath/circe-jawn_2.11-0.8.0.jar
 - classpath/circe-numbers_2.11-0.8.0.jar
 - classpath/circe-parser_2.11-0.8.0.jar
-- classpath/embulk-formatter-fast_jsonl-0.1.0.jar
+- classpath/embulk-formatter-fast_jsonl-0.1.1.jar
 - classpath/jawn-parser_2.11-0.10.4.jar
 - classpath/machinist_2.11-0.6.1.jar
 - classpath/macro-compat_2.11-1.1.1.jar