embulk 0.7.0-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (393) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +15 -0
  3. data/.ruby-version +1 -0
  4. data/.travis.yml +18 -0
  5. data/COPYING +14 -0
  6. data/Gemfile +2 -0
  7. data/Gemfile.lock +31 -0
  8. data/README.md +206 -0
  9. data/Rakefile +26 -0
  10. data/appveyor.yml +20 -0
  11. data/bin/embulk +106 -0
  12. data/build.gradle +338 -0
  13. data/embulk-cli/build.gradle +6 -0
  14. data/embulk-cli/src/main/java/org/embulk/cli/Main.java +22 -0
  15. data/embulk-cli/src/main/sh/selfrun.sh +158 -0
  16. data/embulk-cli/src/test/java/org/embulk/cli/DummyMain.java +23 -0
  17. data/embulk-cli/src/test/java/org/embulk/cli/SelfrunTest.java +281 -0
  18. data/embulk-core/build.gradle +59 -0
  19. data/embulk-core/src/main/java/org/embulk/EmbulkEmbed.java +315 -0
  20. data/embulk-core/src/main/java/org/embulk/EmbulkService.java +76 -0
  21. data/embulk-core/src/main/java/org/embulk/command/PreviewPrinter.java +84 -0
  22. data/embulk-core/src/main/java/org/embulk/command/TablePreviewPrinter.java +107 -0
  23. data/embulk-core/src/main/java/org/embulk/command/VerticalPreviewPrinter.java +47 -0
  24. data/embulk-core/src/main/java/org/embulk/config/CommitReport.java +33 -0
  25. data/embulk-core/src/main/java/org/embulk/config/Config.java +15 -0
  26. data/embulk-core/src/main/java/org/embulk/config/ConfigDefault.java +15 -0
  27. data/embulk-core/src/main/java/org/embulk/config/ConfigDiff.java +29 -0
  28. data/embulk-core/src/main/java/org/embulk/config/ConfigException.java +20 -0
  29. data/embulk-core/src/main/java/org/embulk/config/ConfigInject.java +14 -0
  30. data/embulk-core/src/main/java/org/embulk/config/ConfigLoader.java +141 -0
  31. data/embulk-core/src/main/java/org/embulk/config/ConfigSource.java +31 -0
  32. data/embulk-core/src/main/java/org/embulk/config/DataSource.java +39 -0
  33. data/embulk-core/src/main/java/org/embulk/config/DataSourceImpl.java +231 -0
  34. data/embulk-core/src/main/java/org/embulk/config/DataSourceSerDe.java +84 -0
  35. data/embulk-core/src/main/java/org/embulk/config/GenericTypeReference.java +20 -0
  36. data/embulk-core/src/main/java/org/embulk/config/ModelManager.java +123 -0
  37. data/embulk-core/src/main/java/org/embulk/config/Task.java +10 -0
  38. data/embulk-core/src/main/java/org/embulk/config/TaskInvocationHandler.java +180 -0
  39. data/embulk-core/src/main/java/org/embulk/config/TaskReport.java +29 -0
  40. data/embulk-core/src/main/java/org/embulk/config/TaskSerDe.java +345 -0
  41. data/embulk-core/src/main/java/org/embulk/config/TaskSource.java +31 -0
  42. data/embulk-core/src/main/java/org/embulk/config/TaskValidationException.java +38 -0
  43. data/embulk-core/src/main/java/org/embulk/config/TaskValidator.java +24 -0
  44. data/embulk-core/src/main/java/org/embulk/exec/BulkLoader.java +652 -0
  45. data/embulk-core/src/main/java/org/embulk/exec/ExecModule.java +52 -0
  46. data/embulk-core/src/main/java/org/embulk/exec/ExecutionInterruptedException.java +10 -0
  47. data/embulk-core/src/main/java/org/embulk/exec/ExecutionResult.java +26 -0
  48. data/embulk-core/src/main/java/org/embulk/exec/ExtensionServiceLoaderModule.java +43 -0
  49. data/embulk-core/src/main/java/org/embulk/exec/ForGuess.java +16 -0
  50. data/embulk-core/src/main/java/org/embulk/exec/ForSystemConfig.java +16 -0
  51. data/embulk-core/src/main/java/org/embulk/exec/GuessExecutor.java +373 -0
  52. data/embulk-core/src/main/java/org/embulk/exec/LocalExecutorPlugin.java +129 -0
  53. data/embulk-core/src/main/java/org/embulk/exec/LocalThreadExecutor.java +34 -0
  54. data/embulk-core/src/main/java/org/embulk/exec/LoggerProvider.java +60 -0
  55. data/embulk-core/src/main/java/org/embulk/exec/NoSampleException.java +10 -0
  56. data/embulk-core/src/main/java/org/embulk/exec/PartialExecutionException.java +18 -0
  57. data/embulk-core/src/main/java/org/embulk/exec/PooledBufferAllocator.java +77 -0
  58. data/embulk-core/src/main/java/org/embulk/exec/PreviewExecutor.java +183 -0
  59. data/embulk-core/src/main/java/org/embulk/exec/PreviewResult.java +27 -0
  60. data/embulk-core/src/main/java/org/embulk/exec/PreviewedNoticeError.java +17 -0
  61. data/embulk-core/src/main/java/org/embulk/exec/ResumeState.java +100 -0
  62. data/embulk-core/src/main/java/org/embulk/exec/SamplingParserPlugin.java +136 -0
  63. data/embulk-core/src/main/java/org/embulk/exec/SetCurrentThreadName.java +19 -0
  64. data/embulk-core/src/main/java/org/embulk/exec/SystemConfigModule.java +24 -0
  65. data/embulk-core/src/main/java/org/embulk/exec/TempFileAllocator.java +35 -0
  66. data/embulk-core/src/main/java/org/embulk/guice/Bootstrap.java +157 -0
  67. data/embulk-core/src/main/java/org/embulk/guice/CloseableInjector.java +22 -0
  68. data/embulk-core/src/main/java/org/embulk/guice/InjectorProxy.java +145 -0
  69. data/embulk-core/src/main/java/org/embulk/guice/LifeCycleInjector.java +26 -0
  70. data/embulk-core/src/main/java/org/embulk/guice/LifeCycleInjectorProxy.java +61 -0
  71. data/embulk-core/src/main/java/org/embulk/guice/LifeCycleManager.java +187 -0
  72. data/embulk-core/src/main/java/org/embulk/guice/LifeCycleMethods.java +89 -0
  73. data/embulk-core/src/main/java/org/embulk/guice/LifeCycleMethodsMap.java +38 -0
  74. data/embulk-core/src/main/java/org/embulk/guice/LifeCycleModule.java +97 -0
  75. data/embulk-core/src/main/java/org/embulk/jruby/JRubyPluginSource.java +72 -0
  76. data/embulk-core/src/main/java/org/embulk/jruby/JRubyScriptingModule.java +119 -0
  77. data/embulk-core/src/main/java/org/embulk/plugin/BuiltinPluginSourceModule.java +17 -0
  78. data/embulk-core/src/main/java/org/embulk/plugin/InjectedPluginSource.java +96 -0
  79. data/embulk-core/src/main/java/org/embulk/plugin/PluginClassLoader.java +168 -0
  80. data/embulk-core/src/main/java/org/embulk/plugin/PluginClassLoaderFactory.java +9 -0
  81. data/embulk-core/src/main/java/org/embulk/plugin/PluginClassLoaderModule.java +71 -0
  82. data/embulk-core/src/main/java/org/embulk/plugin/PluginManager.java +78 -0
  83. data/embulk-core/src/main/java/org/embulk/plugin/PluginSource.java +6 -0
  84. data/embulk-core/src/main/java/org/embulk/plugin/PluginSourceNotMatchException.java +19 -0
  85. data/embulk-core/src/main/java/org/embulk/plugin/PluginType.java +47 -0
  86. data/embulk-core/src/main/java/org/embulk/plugin/compat/InputPluginWrapper.java +102 -0
  87. data/embulk-core/src/main/java/org/embulk/plugin/compat/PluginWrappers.java +30 -0
  88. data/embulk-core/src/main/java/org/embulk/plugin/compat/TransactionalFileInputWrapper.java +96 -0
  89. data/embulk-core/src/main/java/org/embulk/plugin/compat/TransactionalFileOutputWrapper.java +102 -0
  90. data/embulk-core/src/main/java/org/embulk/plugin/compat/TransactionalPageOutputWrapper.java +95 -0
  91. data/embulk-core/src/main/java/org/embulk/spi/Buffer.java +148 -0
  92. data/embulk-core/src/main/java/org/embulk/spi/BufferAllocator.java +8 -0
  93. data/embulk-core/src/main/java/org/embulk/spi/Column.java +92 -0
  94. data/embulk-core/src/main/java/org/embulk/spi/ColumnConfig.java +112 -0
  95. data/embulk-core/src/main/java/org/embulk/spi/ColumnVisitor.java +14 -0
  96. data/embulk-core/src/main/java/org/embulk/spi/DecoderPlugin.java +16 -0
  97. data/embulk-core/src/main/java/org/embulk/spi/EncoderPlugin.java +16 -0
  98. data/embulk-core/src/main/java/org/embulk/spi/Exec.java +113 -0
  99. data/embulk-core/src/main/java/org/embulk/spi/ExecAction.java +6 -0
  100. data/embulk-core/src/main/java/org/embulk/spi/ExecSession.java +217 -0
  101. data/embulk-core/src/main/java/org/embulk/spi/ExecutorPlugin.java +19 -0
  102. data/embulk-core/src/main/java/org/embulk/spi/Extension.java +44 -0
  103. data/embulk-core/src/main/java/org/embulk/spi/FileInput.java +11 -0
  104. data/embulk-core/src/main/java/org/embulk/spi/FileInputPlugin.java +30 -0
  105. data/embulk-core/src/main/java/org/embulk/spi/FileInputRunner.java +162 -0
  106. data/embulk-core/src/main/java/org/embulk/spi/FileOutput.java +13 -0
  107. data/embulk-core/src/main/java/org/embulk/spi/FileOutputPlugin.java +28 -0
  108. data/embulk-core/src/main/java/org/embulk/spi/FileOutputRunner.java +202 -0
  109. data/embulk-core/src/main/java/org/embulk/spi/FilterPlugin.java +18 -0
  110. data/embulk-core/src/main/java/org/embulk/spi/FormatterPlugin.java +18 -0
  111. data/embulk-core/src/main/java/org/embulk/spi/GuessPlugin.java +9 -0
  112. data/embulk-core/src/main/java/org/embulk/spi/InputPlugin.java +33 -0
  113. data/embulk-core/src/main/java/org/embulk/spi/OutputPlugin.java +29 -0
  114. data/embulk-core/src/main/java/org/embulk/spi/Page.java +51 -0
  115. data/embulk-core/src/main/java/org/embulk/spi/PageBuilder.java +338 -0
  116. data/embulk-core/src/main/java/org/embulk/spi/PageFormat.java +47 -0
  117. data/embulk-core/src/main/java/org/embulk/spi/PageOutput.java +11 -0
  118. data/embulk-core/src/main/java/org/embulk/spi/PageReader.java +226 -0
  119. data/embulk-core/src/main/java/org/embulk/spi/ParserPlugin.java +17 -0
  120. data/embulk-core/src/main/java/org/embulk/spi/ProcessState.java +10 -0
  121. data/embulk-core/src/main/java/org/embulk/spi/ProcessTask.java +117 -0
  122. data/embulk-core/src/main/java/org/embulk/spi/Schema.java +134 -0
  123. data/embulk-core/src/main/java/org/embulk/spi/SchemaConfig.java +93 -0
  124. data/embulk-core/src/main/java/org/embulk/spi/SchemaConfigException.java +22 -0
  125. data/embulk-core/src/main/java/org/embulk/spi/TaskState.java +81 -0
  126. data/embulk-core/src/main/java/org/embulk/spi/TempFileException.java +19 -0
  127. data/embulk-core/src/main/java/org/embulk/spi/TempFileSpace.java +87 -0
  128. data/embulk-core/src/main/java/org/embulk/spi/Transactional.java +10 -0
  129. data/embulk-core/src/main/java/org/embulk/spi/TransactionalFileInput.java +17 -0
  130. data/embulk-core/src/main/java/org/embulk/spi/TransactionalFileOutput.java +19 -0
  131. data/embulk-core/src/main/java/org/embulk/spi/TransactionalPageOutput.java +17 -0
  132. data/embulk-core/src/main/java/org/embulk/spi/time/DateTimeZoneSerDe.java +55 -0
  133. data/embulk-core/src/main/java/org/embulk/spi/time/JRubyTimeParserHelper.java +8 -0
  134. data/embulk-core/src/main/java/org/embulk/spi/time/JRubyTimeParserHelperFactory.java +6 -0
  135. data/embulk-core/src/main/java/org/embulk/spi/time/Timestamp.java +159 -0
  136. data/embulk-core/src/main/java/org/embulk/spi/time/TimestampFormat.java +100 -0
  137. data/embulk-core/src/main/java/org/embulk/spi/time/TimestampFormatter.java +97 -0
  138. data/embulk-core/src/main/java/org/embulk/spi/time/TimestampParseException.java +10 -0
  139. data/embulk-core/src/main/java/org/embulk/spi/time/TimestampParser.java +104 -0
  140. data/embulk-core/src/main/java/org/embulk/spi/time/TimestampSerDe.java +49 -0
  141. data/embulk-core/src/main/java/org/embulk/spi/type/AbstractType.java +58 -0
  142. data/embulk-core/src/main/java/org/embulk/spi/type/BooleanType.java +12 -0
  143. data/embulk-core/src/main/java/org/embulk/spi/type/DoubleType.java +12 -0
  144. data/embulk-core/src/main/java/org/embulk/spi/type/LongType.java +12 -0
  145. data/embulk-core/src/main/java/org/embulk/spi/type/StringType.java +12 -0
  146. data/embulk-core/src/main/java/org/embulk/spi/type/TimestampType.java +41 -0
  147. data/embulk-core/src/main/java/org/embulk/spi/type/Type.java +15 -0
  148. data/embulk-core/src/main/java/org/embulk/spi/type/TypeDeserializer.java +44 -0
  149. data/embulk-core/src/main/java/org/embulk/spi/type/Types.java +14 -0
  150. data/embulk-core/src/main/java/org/embulk/spi/unit/ByteSize.java +156 -0
  151. data/embulk-core/src/main/java/org/embulk/spi/unit/LocalFile.java +106 -0
  152. data/embulk-core/src/main/java/org/embulk/spi/unit/LocalFileSerDe.java +113 -0
  153. data/embulk-core/src/main/java/org/embulk/spi/unit/ToString.java +54 -0
  154. data/embulk-core/src/main/java/org/embulk/spi/unit/ToStringMap.java +34 -0
  155. data/embulk-core/src/main/java/org/embulk/spi/util/CharsetSerDe.java +55 -0
  156. data/embulk-core/src/main/java/org/embulk/spi/util/Decoders.java +81 -0
  157. data/embulk-core/src/main/java/org/embulk/spi/util/DynamicColumnNotFoundException.java +10 -0
  158. data/embulk-core/src/main/java/org/embulk/spi/util/DynamicColumnSetter.java +18 -0
  159. data/embulk-core/src/main/java/org/embulk/spi/util/DynamicColumnSetterFactory.java +94 -0
  160. data/embulk-core/src/main/java/org/embulk/spi/util/DynamicPageBuilder.java +161 -0
  161. data/embulk-core/src/main/java/org/embulk/spi/util/Encoders.java +81 -0
  162. data/embulk-core/src/main/java/org/embulk/spi/util/Executors.java +95 -0
  163. data/embulk-core/src/main/java/org/embulk/spi/util/FileInputInputStream.java +111 -0
  164. data/embulk-core/src/main/java/org/embulk/spi/util/FileOutputOutputStream.java +119 -0
  165. data/embulk-core/src/main/java/org/embulk/spi/util/Filters.java +100 -0
  166. data/embulk-core/src/main/java/org/embulk/spi/util/InputStreamFileInput.java +190 -0
  167. data/embulk-core/src/main/java/org/embulk/spi/util/InputStreamTransactionalFileInput.java +25 -0
  168. data/embulk-core/src/main/java/org/embulk/spi/util/Inputs.java +65 -0
  169. data/embulk-core/src/main/java/org/embulk/spi/util/LineDecoder.java +118 -0
  170. data/embulk-core/src/main/java/org/embulk/spi/util/LineEncoder.java +123 -0
  171. data/embulk-core/src/main/java/org/embulk/spi/util/ListFileInput.java +52 -0
  172. data/embulk-core/src/main/java/org/embulk/spi/util/Newline.java +38 -0
  173. data/embulk-core/src/main/java/org/embulk/spi/util/OutputStreamFileOutput.java +88 -0
  174. data/embulk-core/src/main/java/org/embulk/spi/util/PagePrinter.java +102 -0
  175. data/embulk-core/src/main/java/org/embulk/spi/util/Pages.java +139 -0
  176. data/embulk-core/src/main/java/org/embulk/spi/util/ResumableInputStream.java +128 -0
  177. data/embulk-core/src/main/java/org/embulk/spi/util/RetryExecutor.java +130 -0
  178. data/embulk-core/src/main/java/org/embulk/spi/util/Timestamps.java +53 -0
  179. data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/AbstractDynamicColumnSetter.java +79 -0
  180. data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/BooleanColumnSetter.java +64 -0
  181. data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/DefaultValueSetter.java +18 -0
  182. data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/DoubleColumnSetter.java +61 -0
  183. data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/LongColumnSetter.java +69 -0
  184. data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/NullDefaultValueSetter.java +34 -0
  185. data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/SkipColumnSetter.java +52 -0
  186. data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/StringColumnSetter.java +56 -0
  187. data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/TimestampColumnSetter.java +64 -0
  188. data/embulk-core/src/main/resources/embulk/logback-color.xml +72 -0
  189. data/embulk-core/src/main/resources/embulk/logback-console.xml +14 -0
  190. data/embulk-core/src/main/resources/embulk/logback-file.xml +25 -0
  191. data/embulk-core/src/main/resources/embulk/parent_first_packages.properties +70 -0
  192. data/embulk-core/src/main/resources/embulk/parent_first_resources.properties +28 -0
  193. data/embulk-core/src/test/java/org/embulk/EmbulkTestRuntime.java +114 -0
  194. data/embulk-core/src/test/java/org/embulk/GuiceBinder.java +72 -0
  195. data/embulk-core/src/test/java/org/embulk/RandomManager.java +53 -0
  196. data/embulk-core/src/test/java/org/embulk/TestPluginSourceModule.java +23 -0
  197. data/embulk-core/src/test/java/org/embulk/TestUtilityModule.java +17 -0
  198. data/embulk-core/src/test/java/org/embulk/config/TestConfigLoader.java +66 -0
  199. data/embulk-core/src/test/java/org/embulk/config/TestConfigSource.java +114 -0
  200. data/embulk-core/src/test/java/org/embulk/config/TestTaskSource.java +70 -0
  201. data/embulk-core/src/test/java/org/embulk/plugin/MockPluginSource.java +58 -0
  202. data/embulk-core/src/test/java/org/embulk/plugin/TestPluginType.java +18 -0
  203. data/embulk-core/src/test/java/org/embulk/spi/MockFileOutput.java +63 -0
  204. data/embulk-core/src/test/java/org/embulk/spi/MockFormatterPlugin.java +101 -0
  205. data/embulk-core/src/test/java/org/embulk/spi/MockParserPlugin.java +73 -0
  206. data/embulk-core/src/test/java/org/embulk/spi/PageTestUtils.java +57 -0
  207. data/embulk-core/src/test/java/org/embulk/spi/TestBuffer.java +24 -0
  208. data/embulk-core/src/test/java/org/embulk/spi/TestFileInputInputStream.java +89 -0
  209. data/embulk-core/src/test/java/org/embulk/spi/TestFileInputRunner.java +196 -0
  210. data/embulk-core/src/test/java/org/embulk/spi/TestFileOutputRunner.java +207 -0
  211. data/embulk-core/src/test/java/org/embulk/spi/TestInputStreamFileInput.java +188 -0
  212. data/embulk-core/src/test/java/org/embulk/spi/TestPageBuilderReader.java +319 -0
  213. data/embulk-core/src/test/java/org/embulk/spi/time/TestTimestamp.java +116 -0
  214. data/embulk-core/src/test/java/org/embulk/spi/time/TestTimestampFormatterParser.java +63 -0
  215. data/embulk-core/src/test/java/org/embulk/spi/time/TestTimestampFormatterParserDeprecated.java +67 -0
  216. data/embulk-core/src/test/java/org/embulk/spi/type/TestTypeSerDe.java +45 -0
  217. data/embulk-core/src/test/java/org/embulk/spi/unit/TestByteSize.java +79 -0
  218. data/embulk-core/src/test/java/org/embulk/spi/util/TestLineDecoder.java +132 -0
  219. data/embulk-core/src/test/java/org/embulk/spi/util/TestLineEncoder.java +123 -0
  220. data/embulk-docs/Makefile +178 -0
  221. data/embulk-docs/build.gradle +32 -0
  222. data/embulk-docs/make.bat +243 -0
  223. data/embulk-docs/push-gh-pages.sh +49 -0
  224. data/embulk-docs/src/_static/embulk-architecture.png +0 -0
  225. data/embulk-docs/src/_static/embulk-logo.png +0 -0
  226. data/embulk-docs/src/_static/embulk-logo.svg +133 -0
  227. data/embulk-docs/src/built-in.rst +440 -0
  228. data/embulk-docs/src/conf.py +260 -0
  229. data/embulk-docs/src/customization.rst +184 -0
  230. data/embulk-docs/src/index.rst +84 -0
  231. data/embulk-docs/src/recipe.rst +8 -0
  232. data/embulk-docs/src/recipe/scheduled-csv-load-to-elasticsearch-kibana4.rst +153 -0
  233. data/embulk-docs/src/release.rst +57 -0
  234. data/embulk-docs/src/release/release-0.1.0.rst +8 -0
  235. data/embulk-docs/src/release/release-0.2.0.rst +16 -0
  236. data/embulk-docs/src/release/release-0.2.1.rst +19 -0
  237. data/embulk-docs/src/release/release-0.3.0.rst +34 -0
  238. data/embulk-docs/src/release/release-0.3.1.rst +11 -0
  239. data/embulk-docs/src/release/release-0.3.2.rst +15 -0
  240. data/embulk-docs/src/release/release-0.4.0.rst +74 -0
  241. data/embulk-docs/src/release/release-0.4.1.rst +18 -0
  242. data/embulk-docs/src/release/release-0.4.10.rst +17 -0
  243. data/embulk-docs/src/release/release-0.4.2.rst +18 -0
  244. data/embulk-docs/src/release/release-0.4.3.rst +34 -0
  245. data/embulk-docs/src/release/release-0.4.4.rst +39 -0
  246. data/embulk-docs/src/release/release-0.4.5.rst +24 -0
  247. data/embulk-docs/src/release/release-0.4.6.rst +30 -0
  248. data/embulk-docs/src/release/release-0.4.7.rst +16 -0
  249. data/embulk-docs/src/release/release-0.4.8.rst +15 -0
  250. data/embulk-docs/src/release/release-0.4.9.rst +23 -0
  251. data/embulk-docs/src/release/release-0.5.0.rst +89 -0
  252. data/embulk-docs/src/release/release-0.5.1.rst +13 -0
  253. data/embulk-docs/src/release/release-0.5.2.rst +30 -0
  254. data/embulk-docs/src/release/release-0.5.3.rst +22 -0
  255. data/embulk-docs/src/release/release-0.5.4.rst +24 -0
  256. data/embulk-docs/src/release/release-0.5.5.rst +18 -0
  257. data/embulk-docs/src/release/release-0.6.0.rst +34 -0
  258. data/embulk-docs/src/release/release-0.6.1.rst +11 -0
  259. data/embulk-docs/src/release/release-0.6.10.rst +15 -0
  260. data/embulk-docs/src/release/release-0.6.11.rst +19 -0
  261. data/embulk-docs/src/release/release-0.6.12.rst +31 -0
  262. data/embulk-docs/src/release/release-0.6.13.rst +23 -0
  263. data/embulk-docs/src/release/release-0.6.14.rst +47 -0
  264. data/embulk-docs/src/release/release-0.6.15.rst +26 -0
  265. data/embulk-docs/src/release/release-0.6.16.rst +26 -0
  266. data/embulk-docs/src/release/release-0.6.17.rst +39 -0
  267. data/embulk-docs/src/release/release-0.6.18.rst +14 -0
  268. data/embulk-docs/src/release/release-0.6.19.rst +18 -0
  269. data/embulk-docs/src/release/release-0.6.2.rst +17 -0
  270. data/embulk-docs/src/release/release-0.6.20.rst +19 -0
  271. data/embulk-docs/src/release/release-0.6.21.rst +20 -0
  272. data/embulk-docs/src/release/release-0.6.22.rst +26 -0
  273. data/embulk-docs/src/release/release-0.6.23.rst +17 -0
  274. data/embulk-docs/src/release/release-0.6.24.rst +13 -0
  275. data/embulk-docs/src/release/release-0.6.25.rst +12 -0
  276. data/embulk-docs/src/release/release-0.6.3.rst +23 -0
  277. data/embulk-docs/src/release/release-0.6.4.rst +13 -0
  278. data/embulk-docs/src/release/release-0.6.5.rst +17 -0
  279. data/embulk-docs/src/release/release-0.6.6.rst +17 -0
  280. data/embulk-docs/src/release/release-0.6.7.rst +17 -0
  281. data/embulk-docs/src/release/release-0.6.8.rst +24 -0
  282. data/embulk-docs/src/release/release-0.6.9.rst +24 -0
  283. data/embulk-docs/src/release/release-0.7.0.rst +96 -0
  284. data/embulk-standards/build.gradle +5 -0
  285. data/embulk-standards/src/main/java/org/embulk/standards/CsvFormatterPlugin.java +284 -0
  286. data/embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java +379 -0
  287. data/embulk-standards/src/main/java/org/embulk/standards/CsvTokenizer.java +411 -0
  288. data/embulk-standards/src/main/java/org/embulk/standards/GzipFileDecoderPlugin.java +55 -0
  289. data/embulk-standards/src/main/java/org/embulk/standards/GzipFileEncoderPlugin.java +71 -0
  290. data/embulk-standards/src/main/java/org/embulk/standards/LocalFileInputPlugin.java +203 -0
  291. data/embulk-standards/src/main/java/org/embulk/standards/LocalFileOutputPlugin.java +148 -0
  292. data/embulk-standards/src/main/java/org/embulk/standards/NullOutputPlugin.java +59 -0
  293. data/embulk-standards/src/main/java/org/embulk/standards/RenameFilterPlugin.java +56 -0
  294. data/embulk-standards/src/main/java/org/embulk/standards/StandardPluginExtension.java +16 -0
  295. data/embulk-standards/src/main/java/org/embulk/standards/StandardPluginModule.java +53 -0
  296. data/embulk-standards/src/main/java/org/embulk/standards/StdoutOutputPlugin.java +85 -0
  297. data/embulk-standards/src/main/resources/META-INF/services/org.embulk.spi.Extension +1 -0
  298. data/embulk-standards/src/test/java/org/embulk/standards/TestCsvFormatterPlugin.java +312 -0
  299. data/embulk-standards/src/test/java/org/embulk/standards/TestCsvParserPlugin.java +75 -0
  300. data/embulk-standards/src/test/java/org/embulk/standards/TestCsvTokenizer.java +360 -0
  301. data/embulk-standards/src/test/java/org/embulk/standards/TestRenameFilterPlugin.java +88 -0
  302. data/embulk.gemspec +39 -0
  303. data/gradle/wrapper/gradle-wrapper.jar +0 -0
  304. data/gradle/wrapper/gradle-wrapper.properties +6 -0
  305. data/gradlew +164 -0
  306. data/gradlew.bat +90 -0
  307. data/lib/embulk.rb +72 -0
  308. data/lib/embulk/buffer.rb +22 -0
  309. data/lib/embulk/column.rb +70 -0
  310. data/lib/embulk/command/embulk_bundle.rb +56 -0
  311. data/lib/embulk/command/embulk_example.rb +32 -0
  312. data/lib/embulk/command/embulk_generate_bin.rb +62 -0
  313. data/lib/embulk/command/embulk_main.rb +2 -0
  314. data/lib/embulk/command/embulk_migrate_plugin.rb +170 -0
  315. data/lib/embulk/command/embulk_new_plugin.rb +124 -0
  316. data/lib/embulk/command/embulk_run.rb +470 -0
  317. data/lib/embulk/command/embulk_selfupdate.rb +84 -0
  318. data/lib/embulk/data/bundle/.bundle/config +3 -0
  319. data/lib/embulk/data/bundle/.ruby-version +1 -0
  320. data/lib/embulk/data/bundle/Gemfile +26 -0
  321. data/lib/embulk/data/bundle/embulk/filter/example.rb +42 -0
  322. data/lib/embulk/data/bundle/embulk/input/example.rb +54 -0
  323. data/lib/embulk/data/bundle/embulk/output/example.rb +58 -0
  324. data/lib/embulk/data/new/LICENSE.txt +21 -0
  325. data/lib/embulk/data/new/README.md.erb +111 -0
  326. data/lib/embulk/data/new/gitignore.erb +13 -0
  327. data/lib/embulk/data/new/java/build.gradle.erb +73 -0
  328. data/lib/embulk/data/new/java/decoder.java.erb +84 -0
  329. data/lib/embulk/data/new/java/encoder.java.erb +86 -0
  330. data/lib/embulk/data/new/java/file_input.java.erb +143 -0
  331. data/lib/embulk/data/new/java/file_output.java.erb +93 -0
  332. data/lib/embulk/data/new/java/filter.java.erb +56 -0
  333. data/lib/embulk/data/new/java/formatter.java.erb +54 -0
  334. data/lib/embulk/data/new/java/gradle/wrapper/gradle-wrapper.jar +0 -0
  335. data/lib/embulk/data/new/java/gradle/wrapper/gradle-wrapper.properties +6 -0
  336. data/lib/embulk/data/new/java/gradlew +164 -0
  337. data/lib/embulk/data/new/java/gradlew.bat +90 -0
  338. data/lib/embulk/data/new/java/input.java.erb +87 -0
  339. data/lib/embulk/data/new/java/output.java.erb +77 -0
  340. data/lib/embulk/data/new/java/parser.java.erb +60 -0
  341. data/lib/embulk/data/new/java/plugin_loader.rb.erb +3 -0
  342. data/lib/embulk/data/new/java/test.java.erb +5 -0
  343. data/lib/embulk/data/new/ruby/.ruby-version +1 -0
  344. data/lib/embulk/data/new/ruby/Gemfile +2 -0
  345. data/lib/embulk/data/new/ruby/Rakefile +3 -0
  346. data/lib/embulk/data/new/ruby/decoder_guess.rb.erb +25 -0
  347. data/lib/embulk/data/new/ruby/filter.rb.erb +41 -0
  348. data/lib/embulk/data/new/ruby/formatter.rb.erb +49 -0
  349. data/lib/embulk/data/new/ruby/gemspec.erb +20 -0
  350. data/lib/embulk/data/new/ruby/input.rb.erb +59 -0
  351. data/lib/embulk/data/new/ruby/output.rb.erb +61 -0
  352. data/lib/embulk/data/new/ruby/parser.rb.erb +44 -0
  353. data/lib/embulk/data/new/ruby/parser_guess.rb.erb +65 -0
  354. data/lib/embulk/data/package_data.rb +50 -0
  355. data/lib/embulk/data_source.rb +220 -0
  356. data/lib/embulk/decoder_plugin.rb +27 -0
  357. data/lib/embulk/encoder_plugin.rb +27 -0
  358. data/lib/embulk/error.rb +8 -0
  359. data/lib/embulk/executor_plugin.rb +23 -0
  360. data/lib/embulk/file_input.rb +87 -0
  361. data/lib/embulk/file_input_plugin.rb +27 -0
  362. data/lib/embulk/file_output.rb +56 -0
  363. data/lib/embulk/file_output_plugin.rb +27 -0
  364. data/lib/embulk/filter_plugin.rb +105 -0
  365. data/lib/embulk/formatter_plugin.rb +105 -0
  366. data/lib/embulk/guess/charset.rb +44 -0
  367. data/lib/embulk/guess/csv.rb +327 -0
  368. data/lib/embulk/guess/gzip.rb +18 -0
  369. data/lib/embulk/guess/newline.rb +22 -0
  370. data/lib/embulk/guess/schema_guess.rb +118 -0
  371. data/lib/embulk/guess/time_format_guess.rb +394 -0
  372. data/lib/embulk/guess_plugin.rb +129 -0
  373. data/lib/embulk/input_plugin.rb +121 -0
  374. data/lib/embulk/java/bootstrap.rb +24 -0
  375. data/lib/embulk/java/imports.rb +69 -0
  376. data/lib/embulk/java/time_helper.rb +79 -0
  377. data/lib/embulk/java_plugin.rb +90 -0
  378. data/lib/embulk/logger.rb +154 -0
  379. data/lib/embulk/output_plugin.rb +150 -0
  380. data/lib/embulk/page.rb +30 -0
  381. data/lib/embulk/page_builder.rb +76 -0
  382. data/lib/embulk/parser_plugin.rb +78 -0
  383. data/lib/embulk/plugin.rb +239 -0
  384. data/lib/embulk/plugin_registry.rb +96 -0
  385. data/lib/embulk/runner.rb +184 -0
  386. data/lib/embulk/schema.rb +103 -0
  387. data/lib/embulk/version.rb +3 -0
  388. data/settings.gradle +6 -0
  389. data/test/guess/test_schema_guess.rb +11 -0
  390. data/test/guess/test_time_format_guess.rb +133 -0
  391. data/test/helper.rb +21 -0
  392. data/test/run-test.rb +14 -0
  393. metadata +566 -0
@@ -0,0 +1,59 @@
1
+ package org.embulk.standards;
2
+
3
+ import java.util.List;
4
+ import org.embulk.config.ConfigSource;
5
+ import org.embulk.config.TaskSource;
6
+ import org.embulk.config.ConfigDiff;
7
+ import org.embulk.config.TaskReport;
8
+ import org.embulk.spi.Schema;
9
+ import org.embulk.spi.Page;
10
+ import org.embulk.spi.Exec;
11
+ import org.embulk.spi.OutputPlugin;
12
+ import org.embulk.spi.TransactionalPageOutput;
13
+
14
+ public class NullOutputPlugin
15
+ implements OutputPlugin
16
+ {
17
+ @Override
18
+ public ConfigDiff transaction(ConfigSource config,
19
+ Schema schema, int taskCount,
20
+ OutputPlugin.Control control)
21
+ {
22
+ return resume(Exec.newTaskSource(), schema, taskCount, control);
23
+ }
24
+
25
+ public ConfigDiff resume(TaskSource taskSource,
26
+ Schema schema, int taskCount,
27
+ OutputPlugin.Control control)
28
+ {
29
+ control.run(taskSource);
30
+ return Exec.newConfigDiff();
31
+ }
32
+
33
+ public void cleanup(TaskSource taskSource,
34
+ Schema schema, int taskCount,
35
+ List<TaskReport> successTaskReports)
36
+ { }
37
+
38
+ @Override
39
+ public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int taskIndex)
40
+ {
41
+ return new TransactionalPageOutput() {
42
+ public void add(Page page)
43
+ {
44
+ page.release();
45
+ }
46
+
47
+ public void finish() { }
48
+
49
+ public void close() { }
50
+
51
+ public void abort() { }
52
+
53
+ public TaskReport commit()
54
+ {
55
+ return Exec.newTaskReport();
56
+ }
57
+ };
58
+ }
59
+ }
@@ -0,0 +1,56 @@
1
+ package org.embulk.standards;
2
+
3
+ import org.embulk.config.Config;
4
+ import org.embulk.config.ConfigDefault;
5
+ import org.embulk.config.ConfigSource;
6
+ import org.embulk.config.Task;
7
+ import org.embulk.config.TaskSource;
8
+ import org.embulk.spi.Column;
9
+ import org.embulk.spi.FilterPlugin;
10
+ import org.embulk.spi.PageOutput;
11
+ import org.embulk.spi.Schema;
12
+
13
+ import java.util.Map;
14
+
15
+ public class RenameFilterPlugin
16
+ implements FilterPlugin
17
+ {
18
+ public interface PluginTask
19
+ extends Task
20
+ {
21
+ @Config("columns")
22
+ @ConfigDefault("{}")
23
+ Map<String, String> getRenameMap();
24
+ }
25
+
26
+ @Override
27
+ public void transaction(ConfigSource config, Schema inputSchema,
28
+ FilterPlugin.Control control)
29
+ {
30
+ PluginTask task = config.loadConfig(PluginTask.class);
31
+ Map<String, String> renameMap = task.getRenameMap();
32
+
33
+ // check column_options is valid or not
34
+ for (String columnName : renameMap.keySet()) {
35
+ inputSchema.lookupColumn(columnName); // throws SchemaConfigException
36
+ }
37
+
38
+ Schema.Builder builder = Schema.builder();
39
+ for (Column column : inputSchema.getColumns()) {
40
+ String name = column.getName();
41
+ if (renameMap.containsKey(name)) {
42
+ name = renameMap.get(name);
43
+ }
44
+ builder.add(name, column.getType());
45
+ }
46
+
47
+ control.run(task.dump(), builder.build());
48
+ }
49
+
50
+ @Override
51
+ public PageOutput open(TaskSource taskSource, Schema inputSchema,
52
+ Schema outputSchema, PageOutput output)
53
+ {
54
+ return output;
55
+ }
56
+ }
@@ -0,0 +1,16 @@
1
+ package org.embulk.standards;
2
+
3
+ import java.util.List;
4
+ import com.google.common.collect.ImmutableList;
5
+ import com.google.inject.Module;
6
+ import org.embulk.spi.Extension;
7
+ import org.embulk.config.ConfigSource;
8
+
9
+ public class StandardPluginExtension
10
+ implements Extension
11
+ {
12
+ public List<Module> getModules(ConfigSource systemConfig)
13
+ {
14
+ return ImmutableList.<Module>of(new StandardPluginModule());
15
+ }
16
+ }
@@ -0,0 +1,53 @@
1
+ package org.embulk.standards;
2
+
3
+ import com.google.common.base.Preconditions;
4
+ import com.google.inject.Binder;
5
+ import com.google.inject.Module;
6
+ import org.embulk.spi.FilterPlugin;
7
+ import org.embulk.spi.FormatterPlugin;
8
+ import org.embulk.spi.InputPlugin;
9
+ import org.embulk.spi.OutputPlugin;
10
+ import org.embulk.spi.ParserPlugin;
11
+ import org.embulk.spi.DecoderPlugin;
12
+ import org.embulk.spi.EncoderPlugin;
13
+ import org.embulk.plugin.PluginType;
14
+ import static org.embulk.plugin.InjectedPluginSource.registerPluginTo;
15
+ import static org.embulk.exec.GuessExecutor.registerDefaultGuessPluginTo;
16
+
17
+ public class StandardPluginModule
18
+ implements Module
19
+ {
20
+ @Override
21
+ public void configure(Binder binder)
22
+ {
23
+ Preconditions.checkNotNull(binder, "binder is null.");
24
+
25
+ // input plugins
26
+ registerPluginTo(binder, InputPlugin.class, "file", LocalFileInputPlugin.class);
27
+
28
+ // parser plugins
29
+ registerPluginTo(binder, ParserPlugin.class, "csv", CsvParserPlugin.class);
30
+
31
+ // file decoder plugins
32
+ registerPluginTo(binder, DecoderPlugin.class, "gzip", GzipFileDecoderPlugin.class);
33
+
34
+ // output plugins
35
+ registerPluginTo(binder, OutputPlugin.class, "file", LocalFileOutputPlugin.class);
36
+ registerPluginTo(binder, OutputPlugin.class, "null", NullOutputPlugin.class);
37
+ registerPluginTo(binder, OutputPlugin.class, "stdout", StdoutOutputPlugin.class);
38
+
39
+ // formatter plugins
40
+ registerPluginTo(binder, FormatterPlugin.class, "csv", CsvFormatterPlugin.class);
41
+
42
+ // file encoder plugins
43
+ registerPluginTo(binder, EncoderPlugin.class, "gzip", GzipFileEncoderPlugin.class);
44
+
45
+ // filter plugins
46
+ registerPluginTo(binder, FilterPlugin.class, "rename", RenameFilterPlugin.class);
47
+
48
+ // default guess plugins
49
+ registerDefaultGuessPluginTo(binder, new PluginType("gzip"));
50
+ registerDefaultGuessPluginTo(binder, new PluginType("csv"));
51
+ // charset and newline guess plugins are loaded and invoked by CsvGuessPlugin
52
+ }
53
+ }
@@ -0,0 +1,85 @@
1
+ package org.embulk.standards;
2
+
3
+ import java.util.List;
4
+ import org.embulk.config.ConfigSource;
5
+ import org.embulk.config.TaskSource;
6
+ import org.embulk.config.ConfigDiff;
7
+ import org.embulk.config.TaskReport;
8
+ import org.embulk.config.Task;
9
+ import org.embulk.spi.time.TimestampFormatter;
10
+ import org.embulk.spi.Schema;
11
+ import org.embulk.spi.Page;
12
+ import org.embulk.spi.Exec;
13
+ import org.embulk.spi.OutputPlugin;
14
+ import org.embulk.spi.TransactionalPageOutput;
15
+ import org.embulk.spi.PageReader;
16
+ import org.embulk.spi.util.PagePrinter;
17
+
18
+ public class StdoutOutputPlugin
19
+ implements OutputPlugin
20
+ {
21
+ public interface PluginTask
22
+ extends Task, TimestampFormatter.FormatterTask
23
+ {
24
+ }
25
+
26
+ @Override
27
+ public ConfigDiff transaction(ConfigSource config,
28
+ Schema schema, int taskCount,
29
+ OutputPlugin.Control control)
30
+ {
31
+ final PluginTask task = config.loadConfig(PluginTask.class);
32
+ return resume(task.dump(), schema, taskCount, control);
33
+ }
34
+
35
+ @Override
36
+ public ConfigDiff resume(TaskSource taskSource,
37
+ Schema schema, int taskCount,
38
+ OutputPlugin.Control control)
39
+ {
40
+ control.run(taskSource);
41
+ return Exec.newConfigDiff();
42
+ }
43
+
44
+ public void cleanup(TaskSource taskSource,
45
+ Schema schema, int taskCount,
46
+ List<TaskReport> successTaskReports)
47
+ { }
48
+
49
+ @Override
50
+ public TransactionalPageOutput open(TaskSource taskSource, final Schema schema,
51
+ int taskIndex)
52
+ {
53
+ final PluginTask task = taskSource.loadTask(PluginTask.class);
54
+
55
+ return new TransactionalPageOutput() {
56
+ private final PageReader reader = new PageReader(schema);
57
+ private final PagePrinter printer = new PagePrinter(schema, task);
58
+
59
+ public void add(Page page)
60
+ {
61
+ reader.setPage(page);
62
+ while (reader.nextRecord()) {
63
+ System.out.println(printer.printRecord(reader, ","));
64
+ }
65
+ }
66
+
67
+ public void finish()
68
+ {
69
+ System.out.flush();
70
+ }
71
+
72
+ public void close()
73
+ {
74
+ reader.close();
75
+ }
76
+
77
+ public void abort() { }
78
+
79
+ public TaskReport commit()
80
+ {
81
+ return Exec.newTaskReport();
82
+ }
83
+ };
84
+ }
85
+ }
@@ -0,0 +1 @@
1
+ org.embulk.standards.StandardPluginExtension
@@ -0,0 +1,312 @@
1
+ package org.embulk.standards;
2
+
3
+ import com.google.common.collect.ImmutableList;
4
+ import com.google.common.collect.ImmutableMap;
5
+ import org.joda.time.DateTimeZone;
6
+ import org.junit.Rule;
7
+ import org.junit.Test;
8
+ import java.lang.reflect.InvocationTargetException;
9
+ import java.lang.reflect.Method;
10
+ import static org.junit.Assert.assertEquals;
11
+ import java.nio.charset.Charset;
12
+ import org.embulk.EmbulkTestRuntime;
13
+ import org.embulk.config.ConfigSource;
14
+ import org.embulk.spi.Exec;
15
+ import org.embulk.spi.util.Newline;
16
+
17
+ public class TestCsvFormatterPlugin
18
+ {
19
+ @Rule
20
+ public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
21
+
22
+ @Test
23
+ public void checkDefaultValues()
24
+ {
25
+ ConfigSource config = Exec.newConfigSource();
26
+
27
+ CsvFormatterPlugin.PluginTask task = config.loadConfig(CsvFormatterPlugin.PluginTask.class);
28
+ assertEquals(Charset.forName("utf-8"), task.getCharset());
29
+ assertEquals(Newline.CRLF, task.getNewline());
30
+ assertEquals(true, task.getHeaderLine());
31
+ assertEquals(',', task.getDelimiterChar());
32
+ assertEquals('\"', task.getQuoteChar());
33
+ assertEquals(CsvFormatterPlugin.QuotePolicy.MINIMAL, task.getQuotePolicy());
34
+ assertEquals(false, task.getEscapeChar().isPresent());
35
+ assertEquals("", task.getNullString());
36
+ assertEquals(DateTimeZone.UTC, task.getDefaultTimeZone());
37
+ assertEquals("%Y-%m-%d %H:%M:%S.%6N %z", task.getDefaultTimestampFormat());
38
+ assertEquals(Newline.LF, task.getNewlineInField());
39
+ }
40
+
41
+ @Test
42
+ public void checkLoadConfig()
43
+ {
44
+ ConfigSource config = Exec.newConfigSource()
45
+ .set("charset", "utf-16")
46
+ .set("newline", "LF")
47
+ .set("header_line", false)
48
+ .set("delimiter", "\t")
49
+ .set("quote", "\\")
50
+ .set("quote_policy", "ALL")
51
+ .set("escape", "\"")
52
+ .set("null_string", "\\N")
53
+ .set("newline_in_field", "CRLF");
54
+
55
+ CsvFormatterPlugin.PluginTask task = config.loadConfig(CsvFormatterPlugin.PluginTask.class);
56
+ assertEquals(Charset.forName("utf-16"), task.getCharset());
57
+ assertEquals(Newline.LF, task.getNewline());
58
+ assertEquals(false, task.getHeaderLine());
59
+ assertEquals('\t', task.getDelimiterChar());
60
+ assertEquals('\\', task.getQuoteChar());
61
+ assertEquals(CsvFormatterPlugin.QuotePolicy.ALL, task.getQuotePolicy());
62
+ assertEquals('\"', (char) task.getEscapeChar().get());
63
+ assertEquals("\\N", task.getNullString());
64
+ assertEquals(Newline.CRLF, task.getNewlineInField());
65
+ }
66
+
67
+ @Test
68
+ public void testQuoteValue()
69
+ throws NoSuchMethodException, SecurityException, IllegalAccessException, IllegalArgumentException, InvocationTargetException
70
+ {
71
+ Method method = CsvFormatterPlugin.class.getDeclaredMethod("setQuoteValue", String.class, char.class);
72
+ method.setAccessible(true);
73
+ CsvFormatterPlugin formatter = new CsvFormatterPlugin();
74
+
75
+ assertEquals("\"ABCD\"", method.invoke(formatter, "ABCD", '"'));
76
+ assertEquals("\"\"", method.invoke(formatter, "", '"'));
77
+ assertEquals("'ABCD'", method.invoke(formatter, "ABCD", '\''));
78
+ assertEquals("''", method.invoke(formatter, "", '\''));
79
+ }
80
+
81
+ @Test
82
+ public void testEscapeQuote()
83
+ throws NoSuchMethodException, SecurityException, IllegalAccessException, IllegalArgumentException, InvocationTargetException
84
+ {
85
+ Method method = CsvFormatterPlugin.class.getDeclaredMethod("setEscapeAndQuoteValue", String.class, char.class,
86
+ CsvFormatterPlugin.QuotePolicy.class, char.class, char.class, String.class, String.class);
87
+ method.setAccessible(true);
88
+ CsvFormatterPlugin formatter = new CsvFormatterPlugin();
89
+
90
+ char delimiter = ',';
91
+ CsvFormatterPlugin.QuotePolicy policy = CsvFormatterPlugin.QuotePolicy.MINIMAL;
92
+ String newline = Newline.LF.getString();
93
+
94
+ assertEquals("\"AB\\\"CD\"", method.invoke(formatter, "AB\"CD", delimiter, policy, '"', '\\', newline, ""));
95
+ assertEquals("\"AB\"\"CD\"", method.invoke(formatter, "AB\"CD", delimiter, policy, '"', '"', newline, ""));
96
+ }
97
+
98
+ @Test
99
+ public void testQuotePolicyAll()
100
+ throws NoSuchMethodException, SecurityException, IllegalAccessException, IllegalArgumentException, InvocationTargetException
101
+ {
102
+ Method method = CsvFormatterPlugin.class.getDeclaredMethod("setEscapeAndQuoteValue", String.class, char.class,
103
+ CsvFormatterPlugin.QuotePolicy.class, char.class, char.class, String.class, String.class);
104
+ method.setAccessible(true);
105
+ CsvFormatterPlugin formatter = new CsvFormatterPlugin();
106
+
107
+ char delimiter = ',';
108
+ char quote = '"';
109
+ char escape = '"';
110
+ CsvFormatterPlugin.QuotePolicy policy = CsvFormatterPlugin.QuotePolicy.ALL;
111
+ String newline = Newline.LF.getString();
112
+ String nullString = "";
113
+
114
+ @SuppressWarnings("unchecked")
115
+ ImmutableList<ImmutableMap<String, String>> testCases = ImmutableList.of(
116
+ ImmutableMap.of("expected", "\"true\"", "actual", "true"),
117
+ ImmutableMap.of("expected", "\"false\"", "actual", "false"),
118
+ ImmutableMap.of("expected", "\"0\"", "actual", "0"),
119
+ ImmutableMap.of("expected", "\"1\"", "actual", "1"),
120
+ ImmutableMap.of("expected", "\"1234\"", "actual", "1234"),
121
+ ImmutableMap.of("expected", "\"-1234\"", "actual", "-1234"),
122
+ ImmutableMap.of("expected", "\"+1234\"", "actual", "+1234"),
123
+ ImmutableMap.of("expected", "\"0x4d2\"", "actual", "0x4d2"),
124
+ ImmutableMap.of("expected", "\"123L\"", "actual", "123L"),
125
+ ImmutableMap.of("expected", "\"3.141592\"", "actual", "3.141592"),
126
+ ImmutableMap.of("expected", "\"1,000\"", "actual", "1,000"),
127
+ ImmutableMap.of("expected", "\"ABC\"", "actual", "ABC"),
128
+ ImmutableMap.of("expected", "\"ABC\"\"DEF\"", "actual", "ABC\"DEF"),
129
+ ImmutableMap.of("expected", "\"ABC\nDEF\"", "actual", "ABC\nDEF"),
130
+ ImmutableMap.of("expected", "\"\"", "actual", ""),
131
+ ImmutableMap.of("expected", "\"NULL\"", "actual", "NULL"),
132
+ ImmutableMap.of("expected", "\"2015-01-01 12:01:01\"", "actual", "2015-01-01 12:01:01"),
133
+ ImmutableMap.of("expected", "\"20150101\"", "actual", "20150101"));
134
+
135
+ for (ImmutableMap testCase : testCases) {
136
+ String expected = (String) testCase.get("expected");
137
+ String actual = (String) testCase.get("actual");
138
+ assertEquals(expected, method.invoke(formatter, actual, delimiter, policy, quote, escape, newline, nullString));
139
+ }
140
+ }
141
+
142
+ @Test
143
+ public void testQuotePolicyMinimal()
144
+ throws NoSuchMethodException, SecurityException, IllegalAccessException, IllegalArgumentException, InvocationTargetException
145
+ {
146
+ Method method = CsvFormatterPlugin.class.getDeclaredMethod("setEscapeAndQuoteValue", String.class, char.class,
147
+ CsvFormatterPlugin.QuotePolicy.class, char.class, char.class, String.class, String.class);
148
+ method.setAccessible(true);
149
+ CsvFormatterPlugin formatter = new CsvFormatterPlugin();
150
+
151
+ char delimiter = ',';
152
+ char quote = '"';
153
+ char escape = '"';
154
+ CsvFormatterPlugin.QuotePolicy policy = CsvFormatterPlugin.QuotePolicy.MINIMAL;
155
+ String newline = Newline.LF.getString();
156
+ String nullString = "";
157
+
158
+ @SuppressWarnings("unchecked")
159
+ ImmutableList<ImmutableMap<String, String>> testCases = ImmutableList.of(
160
+ ImmutableMap.of("expected", "true", "actual", "true"),
161
+ ImmutableMap.of("expected", "false", "actual", "false"),
162
+ ImmutableMap.of("expected", "0", "actual", "0"),
163
+ ImmutableMap.of("expected", "1", "actual", "1"),
164
+ ImmutableMap.of("expected", "1234", "actual", "1234"),
165
+ ImmutableMap.of("expected", "-1234", "actual", "-1234"),
166
+ ImmutableMap.of("expected", "+1234", "actual", "+1234"),
167
+ ImmutableMap.of("expected", "0x4d2", "actual", "0x4d2"),
168
+ ImmutableMap.of("expected", "123L", "actual", "123L"),
169
+ ImmutableMap.of("expected", "3.141592", "actual", "3.141592"),
170
+ ImmutableMap.of("expected", "\"1,000\"", "actual", "1,000"),
171
+ ImmutableMap.of("expected", "ABC", "actual", "ABC"),
172
+ ImmutableMap.of("expected", "\"ABC\"\"DEF\"", "actual", "ABC\"DEF"),
173
+ ImmutableMap.of("expected", "\"ABC\nDEF\"", "actual", "ABC\nDEF"),
174
+ ImmutableMap.of("expected", "\"\"", "actual", ""),
175
+ ImmutableMap.of("expected", "NULL", "actual", "NULL"),
176
+ ImmutableMap.of("expected", "2015-01-01 12:01:01", "actual", "2015-01-01 12:01:01"),
177
+ ImmutableMap.of("expected", "20150101", "actual", "20150101"));
178
+
179
+ for (ImmutableMap testCase : testCases) {
180
+ String expected = (String) testCase.get("expected");
181
+ String actual = (String) testCase.get("actual");
182
+ assertEquals(expected, method.invoke(formatter, actual, delimiter, policy, quote, escape, newline, nullString));
183
+ }
184
+ }
185
+
186
+ @Test
187
+ public void testQuotePolicyNone()
188
+ throws NoSuchMethodException, SecurityException, IllegalAccessException, IllegalArgumentException, InvocationTargetException
189
+ {
190
+ Method method = CsvFormatterPlugin.class.getDeclaredMethod("setEscapeAndQuoteValue", String.class, char.class,
191
+ CsvFormatterPlugin.QuotePolicy.class, char.class, char.class, String.class, String.class);
192
+ method.setAccessible(true);
193
+ CsvFormatterPlugin formatter = new CsvFormatterPlugin();
194
+
195
+ char delimiter = ',';
196
+ char quote = '"';
197
+ char escape = '"';
198
+ CsvFormatterPlugin.QuotePolicy policy = CsvFormatterPlugin.QuotePolicy.NONE;
199
+ String newline = Newline.LF.getString();
200
+ String nullString = "";
201
+
202
+ @SuppressWarnings("unchecked")
203
+ ImmutableList<ImmutableMap<String, String>> testCases = ImmutableList.of(
204
+ ImmutableMap.of("expected", "true", "actual", "true"),
205
+ ImmutableMap.of("expected", "false", "actual", "false"),
206
+ ImmutableMap.of("expected", "0", "actual", "0"),
207
+ ImmutableMap.of("expected", "1", "actual", "1"),
208
+ ImmutableMap.of("expected", "1234", "actual", "1234"),
209
+ ImmutableMap.of("expected", "-1234", "actual", "-1234"),
210
+ ImmutableMap.of("expected", "+1234", "actual", "+1234"),
211
+ ImmutableMap.of("expected", "0x4d2", "actual", "0x4d2"),
212
+ ImmutableMap.of("expected", "123L", "actual", "123L"),
213
+ ImmutableMap.of("expected", "3.141592", "actual", "3.141592"),
214
+ ImmutableMap.of("expected", "1\",000", "actual", "1,000"),
215
+ ImmutableMap.of("expected", "ABC", "actual", "ABC"),
216
+ ImmutableMap.of("expected", "ABC\"\"DEF", "actual", "ABC\"DEF"),
217
+ ImmutableMap.of("expected", "ABC\"\nDEF", "actual", "ABC\nDEF"),
218
+ ImmutableMap.of("expected", "", "actual", ""),
219
+ ImmutableMap.of("expected", "NULL", "actual", "NULL"),
220
+ ImmutableMap.of("expected", "2015-01-01 12:01:01", "actual", "2015-01-01 12:01:01"),
221
+ ImmutableMap.of("expected", "20150101", "actual", "20150101"));
222
+
223
+ for (ImmutableMap testCase : testCases) {
224
+ String expected = (String) testCase.get("expected");
225
+ String actual = (String) testCase.get("actual");
226
+ assertEquals(expected, method.invoke(formatter, actual, delimiter, policy, quote, escape, newline, nullString));
227
+ }
228
+ }
229
+
230
+ @Test
231
+ public void testNewlineInField()
232
+ throws NoSuchMethodException, SecurityException, IllegalAccessException, IllegalArgumentException, InvocationTargetException
233
+ {
234
+ Method method = CsvFormatterPlugin.class.getDeclaredMethod("setEscapeAndQuoteValue", String.class, char.class,
235
+ CsvFormatterPlugin.QuotePolicy.class, char.class, char.class, String.class, String.class);
236
+ method.setAccessible(true);
237
+ CsvFormatterPlugin formatter = new CsvFormatterPlugin();
238
+
239
+ char delimiter = ',';
240
+ char quote = '"';
241
+ char escape = '"';
242
+ String newline;
243
+ CsvFormatterPlugin.QuotePolicy policy = CsvFormatterPlugin.QuotePolicy.MINIMAL;
244
+ String nullString = "";
245
+
246
+ ImmutableList<ImmutableMap<String, String>> testCases;
247
+
248
+ newline = Newline.LF.getString();
249
+ testCases = ImmutableList.of(
250
+ ImmutableMap.of("expected", "\"ABC\nDEF\"", "actual", "ABC\r\nDEF"),
251
+ ImmutableMap.of("expected", "\"ABC\nDEF\"", "actual", "ABC\rDEF"),
252
+ ImmutableMap.of("expected", "\"ABC\nDEF\"", "actual", "ABC\nDEF"));
253
+
254
+ for (ImmutableMap testCase : testCases) {
255
+ String expected = (String) testCase.get("expected");
256
+ String actual = (String) testCase.get("actual");
257
+ assertEquals(expected, method.invoke(formatter, actual, delimiter, policy, quote, escape, newline, nullString));
258
+ }
259
+
260
+
261
+ newline = Newline.CRLF.getString();
262
+ testCases = ImmutableList.of(
263
+ ImmutableMap.of("expected", "\"ABC\r\nDEF\"", "actual", "ABC\r\nDEF"),
264
+ ImmutableMap.of("expected", "\"ABC\r\nDEF\"", "actual", "ABC\rDEF"),
265
+ ImmutableMap.of("expected", "\"ABC\r\nDEF\"", "actual", "ABC\nDEF"));
266
+
267
+ for (ImmutableMap testCase : testCases) {
268
+ String expected = (String) testCase.get("expected");
269
+ String actual = (String) testCase.get("actual");
270
+ assertEquals(expected, method.invoke(formatter, actual, delimiter, policy, quote, escape, newline, nullString));
271
+ }
272
+
273
+
274
+ newline = Newline.CR.getString();
275
+ testCases = ImmutableList.of(
276
+ ImmutableMap.of("expected", "\"ABC\rDEF\"", "actual", "ABC\r\nDEF"),
277
+ ImmutableMap.of("expected", "\"ABC\rDEF\"", "actual", "ABC\rDEF"),
278
+ ImmutableMap.of("expected", "\"ABC\rDEF\"", "actual", "ABC\nDEF"));
279
+
280
+ for (ImmutableMap testCase : testCases) {
281
+ String expected = (String) testCase.get("expected");
282
+ String actual = (String) testCase.get("actual");
283
+ assertEquals(expected, method.invoke(formatter, actual, delimiter, policy, quote, escape, newline, nullString));
284
+ }
285
+ }
286
+
287
+ @Test
288
+ public void testNullString()
289
+ throws NoSuchMethodException, SecurityException, IllegalAccessException, IllegalArgumentException, InvocationTargetException
290
+ {
291
+ Method method = CsvFormatterPlugin.class.getDeclaredMethod("setEscapeAndQuoteValue", String.class, char.class,
292
+ CsvFormatterPlugin.QuotePolicy.class, char.class, char.class, String.class, String.class);
293
+ method.setAccessible(true);
294
+ CsvFormatterPlugin formatter = new CsvFormatterPlugin();
295
+
296
+ char delimiter = ',';
297
+ char quote = '"';
298
+ char escape = '"';
299
+ CsvFormatterPlugin.QuotePolicy policy = CsvFormatterPlugin.QuotePolicy.MINIMAL;
300
+ String newline = Newline.LF.getString();
301
+
302
+ assertEquals("\"\"", method.invoke(formatter, "", delimiter, CsvFormatterPlugin.QuotePolicy.MINIMAL, quote, escape, newline, ""));
303
+ assertEquals("N/A", method.invoke(formatter, "N/A", delimiter, CsvFormatterPlugin.QuotePolicy.MINIMAL, quote, escape, newline, ""));
304
+ assertEquals("", method.invoke(formatter, "", delimiter, CsvFormatterPlugin.QuotePolicy.NONE, quote, escape, newline, ""));
305
+ assertEquals("N/A", method.invoke(formatter, "N/A", delimiter, CsvFormatterPlugin.QuotePolicy.NONE, quote, escape, newline, ""));
306
+
307
+ assertEquals("", method.invoke(formatter, "", delimiter, CsvFormatterPlugin.QuotePolicy.MINIMAL, quote, escape, newline, "N/A"));
308
+ assertEquals("\"N/A\"", method.invoke(formatter, "N/A", delimiter, CsvFormatterPlugin.QuotePolicy.MINIMAL, quote, escape, newline, "N/A"));
309
+ assertEquals("", method.invoke(formatter, "", delimiter, CsvFormatterPlugin.QuotePolicy.NONE, quote, escape, newline, "N/A"));
310
+ assertEquals("N/A", method.invoke(formatter, "N/A", delimiter, CsvFormatterPlugin.QuotePolicy.NONE, quote, escape, newline, "N/A"));
311
+ }
312
+ }