embulk 0.8.35-java → 0.10.24-java

Sign up to get free protection for your applications and to get access to all the features.
Files changed (595) hide show
  1. checksums.yaml +5 -5
  2. data/Gemfile +1 -1
  3. data/LICENSE +202 -0
  4. data/embulk.gemspec +34 -33
  5. data/lib/embulk.rb +5 -61
  6. data/lib/embulk/buffer.rb +1 -2
  7. data/lib/embulk/guess/charset.rb +2 -11
  8. data/lib/embulk/guess/schema_guess.rb +1 -1
  9. data/lib/embulk/input_plugin.rb +8 -1
  10. data/lib/embulk/java/bootstrap.rb +1 -2
  11. data/lib/embulk/java_plugin.rb +9 -1
  12. data/lib/embulk/page_builder.rb +38 -5
  13. data/lib/embulk/plugin.rb +2 -2
  14. data/lib/embulk/schema.rb +5 -7
  15. data/lib/embulk/version.rb +3 -22
  16. metadata +23 -752
  17. data/.gitignore +0 -15
  18. data/.ruby-version +0 -1
  19. data/.travis.yml +0 -28
  20. data/COPYING +0 -14
  21. data/Gemfile.lock +0 -30
  22. data/README.md +0 -225
  23. data/Rakefile +0 -26
  24. data/appveyor.yml +0 -28
  25. data/bin/embulk +0 -139
  26. data/build.gradle +0 -415
  27. data/embulk-cli/build.gradle +0 -9
  28. data/embulk-cli/src/main/bat/selfrun.bat +0 -107
  29. data/embulk-cli/src/main/java/org/embulk/cli/EmbulkArguments.java +0 -54
  30. data/embulk-cli/src/main/java/org/embulk/cli/EmbulkCommandLine.java +0 -227
  31. data/embulk-cli/src/main/java/org/embulk/cli/EmbulkCommandLineException.java +0 -25
  32. data/embulk-cli/src/main/java/org/embulk/cli/EmbulkExample.java +0 -82
  33. data/embulk-cli/src/main/java/org/embulk/cli/EmbulkMigrate.java +0 -480
  34. data/embulk-cli/src/main/java/org/embulk/cli/EmbulkNew.java +0 -419
  35. data/embulk-cli/src/main/java/org/embulk/cli/EmbulkRun.java +0 -774
  36. data/embulk-cli/src/main/java/org/embulk/cli/EmbulkSelfUpdate.java +0 -235
  37. data/embulk-cli/src/main/java/org/embulk/cli/EmbulkSubcommand.java +0 -47
  38. data/embulk-cli/src/main/java/org/embulk/cli/Main.java +0 -31
  39. data/embulk-cli/src/main/java/org/embulk/cli/parse/AbstractHelpLineDefinition.java +0 -15
  40. data/embulk-cli/src/main/java/org/embulk/cli/parse/CliHelpFormatterWithHelpMessages.java +0 -141
  41. data/embulk-cli/src/main/java/org/embulk/cli/parse/CliOptionsWithHelpMessages.java +0 -45
  42. data/embulk-cli/src/main/java/org/embulk/cli/parse/EmbulkCommandLineHelpRequired.java +0 -10
  43. data/embulk-cli/src/main/java/org/embulk/cli/parse/EmbulkCommandLineParseException.java +0 -25
  44. data/embulk-cli/src/main/java/org/embulk/cli/parse/EmbulkCommandLineParser.java +0 -187
  45. data/embulk-cli/src/main/java/org/embulk/cli/parse/HelpMessageAsCliOption.java +0 -36
  46. data/embulk-cli/src/main/java/org/embulk/cli/parse/HelpMessageLineDefinition.java +0 -20
  47. data/embulk-cli/src/main/java/org/embulk/cli/parse/OptionBehavior.java +0 -39
  48. data/embulk-cli/src/main/java/org/embulk/cli/parse/OptionDefinition.java +0 -120
  49. data/embulk-cli/src/main/sh/selfrun.sh +0 -60
  50. data/embulk-cli/src/test/java/org/embulk/cli/DummyMain.java +0 -23
  51. data/embulk-cli/src/test/java/org/embulk/cli/SelfrunTest.java +0 -296
  52. data/embulk-core/build.gradle +0 -83
  53. data/embulk-core/src/main/java/org/embulk/EmbulkEmbed.java +0 -321
  54. data/embulk-core/src/main/java/org/embulk/EmbulkRunner.java +0 -582
  55. data/embulk-core/src/main/java/org/embulk/EmbulkService.java +0 -78
  56. data/embulk-core/src/main/java/org/embulk/EmbulkSetup.java +0 -49
  57. data/embulk-core/src/main/java/org/embulk/EmbulkVersion.java +0 -109
  58. data/embulk-core/src/main/java/org/embulk/command/PreviewPrinter.java +0 -87
  59. data/embulk-core/src/main/java/org/embulk/command/TablePreviewPrinter.java +0 -107
  60. data/embulk-core/src/main/java/org/embulk/command/VerticalPreviewPrinter.java +0 -47
  61. data/embulk-core/src/main/java/org/embulk/config/CommitReport.java +0 -36
  62. data/embulk-core/src/main/java/org/embulk/config/Config.java +0 -15
  63. data/embulk-core/src/main/java/org/embulk/config/ConfigDefault.java +0 -15
  64. data/embulk-core/src/main/java/org/embulk/config/ConfigDiff.java +0 -32
  65. data/embulk-core/src/main/java/org/embulk/config/ConfigException.java +0 -26
  66. data/embulk-core/src/main/java/org/embulk/config/ConfigInject.java +0 -14
  67. data/embulk-core/src/main/java/org/embulk/config/ConfigLoader.java +0 -149
  68. data/embulk-core/src/main/java/org/embulk/config/ConfigSource.java +0 -34
  69. data/embulk-core/src/main/java/org/embulk/config/DataSource.java +0 -41
  70. data/embulk-core/src/main/java/org/embulk/config/DataSourceImpl.java +0 -243
  71. data/embulk-core/src/main/java/org/embulk/config/DataSourceSerDe.java +0 -84
  72. data/embulk-core/src/main/java/org/embulk/config/GenericTypeReference.java +0 -20
  73. data/embulk-core/src/main/java/org/embulk/config/ModelManager.java +0 -123
  74. data/embulk-core/src/main/java/org/embulk/config/Task.java +0 -10
  75. data/embulk-core/src/main/java/org/embulk/config/TaskInvocationHandler.java +0 -180
  76. data/embulk-core/src/main/java/org/embulk/config/TaskReport.java +0 -32
  77. data/embulk-core/src/main/java/org/embulk/config/TaskSerDe.java +0 -349
  78. data/embulk-core/src/main/java/org/embulk/config/TaskSource.java +0 -34
  79. data/embulk-core/src/main/java/org/embulk/config/TaskValidationException.java +0 -38
  80. data/embulk-core/src/main/java/org/embulk/config/TaskValidator.java +0 -24
  81. data/embulk-core/src/main/java/org/embulk/config/UserDataException.java +0 -4
  82. data/embulk-core/src/main/java/org/embulk/config/UserDataExceptions.java +0 -17
  83. data/embulk-core/src/main/java/org/embulk/config/YamlTagResolver.java +0 -53
  84. data/embulk-core/src/main/java/org/embulk/exec/BufferFileInputPlugin.java +0 -88
  85. data/embulk-core/src/main/java/org/embulk/exec/BulkLoader.java +0 -754
  86. data/embulk-core/src/main/java/org/embulk/exec/ConfigurableGuessInputPlugin.java +0 -9
  87. data/embulk-core/src/main/java/org/embulk/exec/ExecModule.java +0 -53
  88. data/embulk-core/src/main/java/org/embulk/exec/ExecutionInterruptedException.java +0 -10
  89. data/embulk-core/src/main/java/org/embulk/exec/ExecutionResult.java +0 -33
  90. data/embulk-core/src/main/java/org/embulk/exec/ExtensionServiceLoaderModule.java +0 -43
  91. data/embulk-core/src/main/java/org/embulk/exec/ForGuess.java +0 -16
  92. data/embulk-core/src/main/java/org/embulk/exec/ForSystemConfig.java +0 -16
  93. data/embulk-core/src/main/java/org/embulk/exec/GuessExecutor.java +0 -309
  94. data/embulk-core/src/main/java/org/embulk/exec/LocalExecutorPlugin.java +0 -563
  95. data/embulk-core/src/main/java/org/embulk/exec/LoggerProvider.java +0 -68
  96. data/embulk-core/src/main/java/org/embulk/exec/NoSampleException.java +0 -10
  97. data/embulk-core/src/main/java/org/embulk/exec/PartialExecutionException.java +0 -26
  98. data/embulk-core/src/main/java/org/embulk/exec/PooledBufferAllocator.java +0 -77
  99. data/embulk-core/src/main/java/org/embulk/exec/PreviewExecutor.java +0 -222
  100. data/embulk-core/src/main/java/org/embulk/exec/PreviewResult.java +0 -27
  101. data/embulk-core/src/main/java/org/embulk/exec/PreviewedNoticeError.java +0 -17
  102. data/embulk-core/src/main/java/org/embulk/exec/ResumeState.java +0 -100
  103. data/embulk-core/src/main/java/org/embulk/exec/SamplingParserPlugin.java +0 -215
  104. data/embulk-core/src/main/java/org/embulk/exec/SetCurrentThreadName.java +0 -22
  105. data/embulk-core/src/main/java/org/embulk/exec/SkipTransactionException.java +0 -23
  106. data/embulk-core/src/main/java/org/embulk/exec/SystemConfigModule.java +0 -24
  107. data/embulk-core/src/main/java/org/embulk/exec/TempFileAllocator.java +0 -35
  108. data/embulk-core/src/main/java/org/embulk/exec/TransactionStage.java +0 -27
  109. data/embulk-core/src/main/java/org/embulk/jruby/JRubyPluginSource.java +0 -76
  110. data/embulk-core/src/main/java/org/embulk/jruby/JRubyScriptingModule.java +0 -395
  111. data/embulk-core/src/main/java/org/embulk/plugin/BuiltinPluginSourceModule.java +0 -17
  112. data/embulk-core/src/main/java/org/embulk/plugin/DefaultPluginType.java +0 -50
  113. data/embulk-core/src/main/java/org/embulk/plugin/InjectedPluginSource.java +0 -100
  114. data/embulk-core/src/main/java/org/embulk/plugin/MavenPluginType.java +0 -112
  115. data/embulk-core/src/main/java/org/embulk/plugin/PluginClassLoader.java +0 -208
  116. data/embulk-core/src/main/java/org/embulk/plugin/PluginClassLoaderFactory.java +0 -9
  117. data/embulk-core/src/main/java/org/embulk/plugin/PluginClassLoaderModule.java +0 -71
  118. data/embulk-core/src/main/java/org/embulk/plugin/PluginManager.java +0 -78
  119. data/embulk-core/src/main/java/org/embulk/plugin/PluginSource.java +0 -49
  120. data/embulk-core/src/main/java/org/embulk/plugin/PluginSourceNotMatchException.java +0 -25
  121. data/embulk-core/src/main/java/org/embulk/plugin/PluginType.java +0 -122
  122. data/embulk-core/src/main/java/org/embulk/plugin/compat/InputPluginWrapper.java +0 -102
  123. data/embulk-core/src/main/java/org/embulk/plugin/compat/PluginWrappers.java +0 -30
  124. data/embulk-core/src/main/java/org/embulk/plugin/compat/TransactionalFileInputWrapper.java +0 -96
  125. data/embulk-core/src/main/java/org/embulk/plugin/compat/TransactionalFileOutputWrapper.java +0 -102
  126. data/embulk-core/src/main/java/org/embulk/plugin/compat/TransactionalPageOutputWrapper.java +0 -95
  127. data/embulk-core/src/main/java/org/embulk/plugin/jar/InvalidJarPluginException.java +0 -14
  128. data/embulk-core/src/main/java/org/embulk/plugin/jar/JarPluginLoader.java +0 -205
  129. data/embulk-core/src/main/java/org/embulk/plugin/maven/MavenArtifactFinder.java +0 -134
  130. data/embulk-core/src/main/java/org/embulk/plugin/maven/MavenArtifactNotFoundException.java +0 -20
  131. data/embulk-core/src/main/java/org/embulk/plugin/maven/MavenPluginSource.java +0 -187
  132. data/embulk-core/src/main/java/org/embulk/plugin/maven/MavenPluginSourceModule.java +0 -22
  133. data/embulk-core/src/main/java/org/embulk/plugin/maven/MavenRepositoryNotFoundException.java +0 -31
  134. data/embulk-core/src/main/java/org/embulk/spi/AbortTransactionResource.java +0 -36
  135. data/embulk-core/src/main/java/org/embulk/spi/Buffer.java +0 -148
  136. data/embulk-core/src/main/java/org/embulk/spi/BufferAllocator.java +0 -8
  137. data/embulk-core/src/main/java/org/embulk/spi/CloseResource.java +0 -42
  138. data/embulk-core/src/main/java/org/embulk/spi/Column.java +0 -95
  139. data/embulk-core/src/main/java/org/embulk/spi/ColumnConfig.java +0 -112
  140. data/embulk-core/src/main/java/org/embulk/spi/ColumnVisitor.java +0 -16
  141. data/embulk-core/src/main/java/org/embulk/spi/DataException.java +0 -23
  142. data/embulk-core/src/main/java/org/embulk/spi/DecoderPlugin.java +0 -16
  143. data/embulk-core/src/main/java/org/embulk/spi/EncoderPlugin.java +0 -16
  144. data/embulk-core/src/main/java/org/embulk/spi/Exec.java +0 -113
  145. data/embulk-core/src/main/java/org/embulk/spi/ExecAction.java +0 -6
  146. data/embulk-core/src/main/java/org/embulk/spi/ExecSession.java +0 -225
  147. data/embulk-core/src/main/java/org/embulk/spi/ExecutorPlugin.java +0 -19
  148. data/embulk-core/src/main/java/org/embulk/spi/Extension.java +0 -44
  149. data/embulk-core/src/main/java/org/embulk/spi/FileInput.java +0 -11
  150. data/embulk-core/src/main/java/org/embulk/spi/FileInputPlugin.java +0 -30
  151. data/embulk-core/src/main/java/org/embulk/spi/FileInputRunner.java +0 -169
  152. data/embulk-core/src/main/java/org/embulk/spi/FileOutput.java +0 -13
  153. data/embulk-core/src/main/java/org/embulk/spi/FileOutputPlugin.java +0 -28
  154. data/embulk-core/src/main/java/org/embulk/spi/FileOutputRunner.java +0 -199
  155. data/embulk-core/src/main/java/org/embulk/spi/FilterPlugin.java +0 -18
  156. data/embulk-core/src/main/java/org/embulk/spi/FormatterPlugin.java +0 -18
  157. data/embulk-core/src/main/java/org/embulk/spi/GuessPlugin.java +0 -9
  158. data/embulk-core/src/main/java/org/embulk/spi/InputPlugin.java +0 -33
  159. data/embulk-core/src/main/java/org/embulk/spi/OutputPlugin.java +0 -29
  160. data/embulk-core/src/main/java/org/embulk/spi/Page.java +0 -86
  161. data/embulk-core/src/main/java/org/embulk/spi/PageBuilder.java +0 -696
  162. data/embulk-core/src/main/java/org/embulk/spi/PageFormat.java +0 -47
  163. data/embulk-core/src/main/java/org/embulk/spi/PageOutput.java +0 -11
  164. data/embulk-core/src/main/java/org/embulk/spi/PageReader.java +0 -248
  165. data/embulk-core/src/main/java/org/embulk/spi/ParserPlugin.java +0 -17
  166. data/embulk-core/src/main/java/org/embulk/spi/ProcessState.java +0 -10
  167. data/embulk-core/src/main/java/org/embulk/spi/ProcessTask.java +0 -117
  168. data/embulk-core/src/main/java/org/embulk/spi/Schema.java +0 -139
  169. data/embulk-core/src/main/java/org/embulk/spi/SchemaConfig.java +0 -93
  170. data/embulk-core/src/main/java/org/embulk/spi/SchemaConfigException.java +0 -22
  171. data/embulk-core/src/main/java/org/embulk/spi/TaskState.java +0 -81
  172. data/embulk-core/src/main/java/org/embulk/spi/TempFileException.java +0 -19
  173. data/embulk-core/src/main/java/org/embulk/spi/TempFileSpace.java +0 -88
  174. data/embulk-core/src/main/java/org/embulk/spi/Transactional.java +0 -10
  175. data/embulk-core/src/main/java/org/embulk/spi/TransactionalFileInput.java +0 -17
  176. data/embulk-core/src/main/java/org/embulk/spi/TransactionalFileOutput.java +0 -19
  177. data/embulk-core/src/main/java/org/embulk/spi/TransactionalPageOutput.java +0 -17
  178. data/embulk-core/src/main/java/org/embulk/spi/json/JsonParseException.java +0 -17
  179. data/embulk-core/src/main/java/org/embulk/spi/json/JsonParser.java +0 -233
  180. data/embulk-core/src/main/java/org/embulk/spi/json/RubyValueApi.java +0 -62
  181. data/embulk-core/src/main/java/org/embulk/spi/time/DateTimeZoneSerDe.java +0 -55
  182. data/embulk-core/src/main/java/org/embulk/spi/time/JRubyTimeParserHelper.java +0 -8
  183. data/embulk-core/src/main/java/org/embulk/spi/time/JRubyTimeParserHelperFactory.java +0 -6
  184. data/embulk-core/src/main/java/org/embulk/spi/time/Timestamp.java +0 -159
  185. data/embulk-core/src/main/java/org/embulk/spi/time/TimestampFormat.java +0 -100
  186. data/embulk-core/src/main/java/org/embulk/spi/time/TimestampFormatter.java +0 -125
  187. data/embulk-core/src/main/java/org/embulk/spi/time/TimestampParseException.java +0 -12
  188. data/embulk-core/src/main/java/org/embulk/spi/time/TimestampParser.java +0 -283
  189. data/embulk-core/src/main/java/org/embulk/spi/time/TimestampSerDe.java +0 -49
  190. data/embulk-core/src/main/java/org/embulk/spi/type/AbstractType.java +0 -58
  191. data/embulk-core/src/main/java/org/embulk/spi/type/BooleanType.java +0 -12
  192. data/embulk-core/src/main/java/org/embulk/spi/type/DoubleType.java +0 -12
  193. data/embulk-core/src/main/java/org/embulk/spi/type/JsonType.java +0 -14
  194. data/embulk-core/src/main/java/org/embulk/spi/type/LongType.java +0 -12
  195. data/embulk-core/src/main/java/org/embulk/spi/type/StringType.java +0 -12
  196. data/embulk-core/src/main/java/org/embulk/spi/type/TimestampType.java +0 -41
  197. data/embulk-core/src/main/java/org/embulk/spi/type/Type.java +0 -15
  198. data/embulk-core/src/main/java/org/embulk/spi/type/TypeDeserializer.java +0 -45
  199. data/embulk-core/src/main/java/org/embulk/spi/type/Types.java +0 -16
  200. data/embulk-core/src/main/java/org/embulk/spi/unit/ByteSize.java +0 -156
  201. data/embulk-core/src/main/java/org/embulk/spi/unit/LocalFile.java +0 -106
  202. data/embulk-core/src/main/java/org/embulk/spi/unit/LocalFileSerDe.java +0 -113
  203. data/embulk-core/src/main/java/org/embulk/spi/unit/ToString.java +0 -54
  204. data/embulk-core/src/main/java/org/embulk/spi/unit/ToStringMap.java +0 -34
  205. data/embulk-core/src/main/java/org/embulk/spi/util/CharsetSerDe.java +0 -55
  206. data/embulk-core/src/main/java/org/embulk/spi/util/Decoders.java +0 -81
  207. data/embulk-core/src/main/java/org/embulk/spi/util/DynamicColumnNotFoundException.java +0 -10
  208. data/embulk-core/src/main/java/org/embulk/spi/util/DynamicColumnSetter.java +0 -21
  209. data/embulk-core/src/main/java/org/embulk/spi/util/DynamicColumnSetterFactory.java +0 -100
  210. data/embulk-core/src/main/java/org/embulk/spi/util/DynamicPageBuilder.java +0 -162
  211. data/embulk-core/src/main/java/org/embulk/spi/util/Encoders.java +0 -81
  212. data/embulk-core/src/main/java/org/embulk/spi/util/Executors.java +0 -93
  213. data/embulk-core/src/main/java/org/embulk/spi/util/FileInputInputStream.java +0 -111
  214. data/embulk-core/src/main/java/org/embulk/spi/util/FileOutputOutputStream.java +0 -119
  215. data/embulk-core/src/main/java/org/embulk/spi/util/Filters.java +0 -100
  216. data/embulk-core/src/main/java/org/embulk/spi/util/InputStreamFileInput.java +0 -189
  217. data/embulk-core/src/main/java/org/embulk/spi/util/InputStreamTransactionalFileInput.java +0 -25
  218. data/embulk-core/src/main/java/org/embulk/spi/util/Inputs.java +0 -65
  219. data/embulk-core/src/main/java/org/embulk/spi/util/LineDecoder.java +0 -157
  220. data/embulk-core/src/main/java/org/embulk/spi/util/LineEncoder.java +0 -123
  221. data/embulk-core/src/main/java/org/embulk/spi/util/ListFileInput.java +0 -52
  222. data/embulk-core/src/main/java/org/embulk/spi/util/Newline.java +0 -38
  223. data/embulk-core/src/main/java/org/embulk/spi/util/OutputStreamFileOutput.java +0 -88
  224. data/embulk-core/src/main/java/org/embulk/spi/util/PagePrinter.java +0 -108
  225. data/embulk-core/src/main/java/org/embulk/spi/util/Pages.java +0 -149
  226. data/embulk-core/src/main/java/org/embulk/spi/util/ResumableInputStream.java +0 -168
  227. data/embulk-core/src/main/java/org/embulk/spi/util/RetryExecutor.java +0 -130
  228. data/embulk-core/src/main/java/org/embulk/spi/util/Timestamps.java +0 -53
  229. data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/AbstractDynamicColumnSetter.java +0 -83
  230. data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/BooleanColumnSetter.java +0 -71
  231. data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/DefaultValueSetter.java +0 -20
  232. data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/DoubleColumnSetter.java +0 -68
  233. data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/JsonColumnSetter.java +0 -64
  234. data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/LongColumnSetter.java +0 -78
  235. data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/NullDefaultValueSetter.java +0 -39
  236. data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/SkipColumnSetter.java +0 -57
  237. data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/StringColumnSetter.java +0 -63
  238. data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/TimestampColumnSetter.java +0 -72
  239. data/embulk-core/src/main/resources/embulk/logback-color.xml +0 -72
  240. data/embulk-core/src/main/resources/embulk/logback-console.xml +0 -14
  241. data/embulk-core/src/main/resources/embulk/logback-file.xml +0 -23
  242. data/embulk-core/src/main/resources/embulk/parent_first_packages.properties +0 -72
  243. data/embulk-core/src/main/resources/embulk/parent_first_resources.properties +0 -29
  244. data/embulk-core/src/test/java/org/embulk/EmbulkTestRuntime.java +0 -122
  245. data/embulk-core/src/test/java/org/embulk/GuiceBinder.java +0 -72
  246. data/embulk-core/src/test/java/org/embulk/RandomManager.java +0 -53
  247. data/embulk-core/src/test/java/org/embulk/TestPluginSourceModule.java +0 -23
  248. data/embulk-core/src/test/java/org/embulk/TestUtilityModule.java +0 -17
  249. data/embulk-core/src/test/java/org/embulk/config/TestConfigLoader.java +0 -66
  250. data/embulk-core/src/test/java/org/embulk/config/TestConfigSource.java +0 -114
  251. data/embulk-core/src/test/java/org/embulk/config/TestTaskSource.java +0 -70
  252. data/embulk-core/src/test/java/org/embulk/plugin/MockPluginSource.java +0 -58
  253. data/embulk-core/src/test/java/org/embulk/plugin/TestPluginType.java +0 -83
  254. data/embulk-core/src/test/java/org/embulk/plugin/TestPluginTypeSerDe.java +0 -67
  255. data/embulk-core/src/test/java/org/embulk/plugin/jar/ExampleJarSpiV0.java +0 -9
  256. data/embulk-core/src/test/java/org/embulk/plugin/jar/JarBuilder.java +0 -101
  257. data/embulk-core/src/test/java/org/embulk/plugin/jar/TestJarPluginLoader.java +0 -60
  258. data/embulk-core/src/test/java/org/embulk/plugin/maven/TestMavenArtifactFinder.java +0 -41
  259. data/embulk-core/src/test/java/org/embulk/spi/MockFileOutput.java +0 -63
  260. data/embulk-core/src/test/java/org/embulk/spi/MockFormatterPlugin.java +0 -108
  261. data/embulk-core/src/test/java/org/embulk/spi/MockParserPlugin.java +0 -80
  262. data/embulk-core/src/test/java/org/embulk/spi/PageTestUtils.java +0 -59
  263. data/embulk-core/src/test/java/org/embulk/spi/TestBuffer.java +0 -24
  264. data/embulk-core/src/test/java/org/embulk/spi/TestFileInputInputStream.java +0 -89
  265. data/embulk-core/src/test/java/org/embulk/spi/TestFileInputRunner.java +0 -199
  266. data/embulk-core/src/test/java/org/embulk/spi/TestFileOutputRunner.java +0 -221
  267. data/embulk-core/src/test/java/org/embulk/spi/TestInputStreamFileInput.java +0 -188
  268. data/embulk-core/src/test/java/org/embulk/spi/TestPageBuilderReader.java +0 -411
  269. data/embulk-core/src/test/java/org/embulk/spi/json/TestJsonParser.java +0 -102
  270. data/embulk-core/src/test/java/org/embulk/spi/time/TestTimestamp.java +0 -116
  271. data/embulk-core/src/test/java/org/embulk/spi/time/TestTimestampFormatterParser.java +0 -75
  272. data/embulk-core/src/test/java/org/embulk/spi/time/TestTimestampFormatterParserDeprecated.java +0 -67
  273. data/embulk-core/src/test/java/org/embulk/spi/type/TestTypeSerDe.java +0 -45
  274. data/embulk-core/src/test/java/org/embulk/spi/unit/TestByteSize.java +0 -79
  275. data/embulk-core/src/test/java/org/embulk/spi/util/TestLineDecoder.java +0 -237
  276. data/embulk-core/src/test/java/org/embulk/spi/util/TestLineEncoder.java +0 -123
  277. data/embulk-core/src/test/resources/m2.test/.gitignore +0 -1
  278. data/embulk-core/src/test/resources/m2.test/org/embulk/example/embulk-example-maven-artifact/0.1.2/embulk-example-maven-artifact-0.1.2.jar +0 -0
  279. data/embulk-core/src/test/resources/m2.test/org/embulk/example/embulk-example-maven-artifact/0.1.2/embulk-example-maven-artifact-0.1.2.jar.sha1 +0 -1
  280. data/embulk-core/src/test/resources/m2.test/org/embulk/example/embulk-example-maven-artifact/0.1.2/embulk-example-maven-artifact-0.1.2.pom +0 -9
  281. data/embulk-core/src/test/resources/m2.test/org/embulk/example/embulk-example-maven-artifact/0.1.2/embulk-example-maven-artifact-0.1.2.pom.sha1 +0 -1
  282. data/embulk-docs/Makefile +0 -178
  283. data/embulk-docs/build.gradle +0 -33
  284. data/embulk-docs/make.bat +0 -243
  285. data/embulk-docs/push-gh-pages.sh +0 -49
  286. data/embulk-docs/src/_static/embulk-architecture.png +0 -0
  287. data/embulk-docs/src/_static/embulk-logo-v2/embulk-logo-v2-oneline-tr.png +0 -0
  288. data/embulk-docs/src/_static/embulk-logo-v2/embulk-logo-v2-oneline-wt.png +0 -0
  289. data/embulk-docs/src/_static/embulk-logo-v2/embulk-logo-v2-oneline.ai +0 -396
  290. data/embulk-docs/src/_static/embulk-logo-v2/embulk-logo-v2-sq-tr-small.png +0 -0
  291. data/embulk-docs/src/_static/embulk-logo-v2/embulk-logo-v2-sq-tr.png +0 -0
  292. data/embulk-docs/src/_static/embulk-logo-v2/embulk-logo-v2-sq-wt.png +0 -0
  293. data/embulk-docs/src/_static/embulk-logo-v2/embulk-logo-v2-sq.ai +0 -417
  294. data/embulk-docs/src/_static/embulk-logo-v2/embulk-logo-v2-sq.svg +0 -1
  295. data/embulk-docs/src/_static/embulk-logo-v2/embulk-logo-v2-symbol-tr.png +0 -0
  296. data/embulk-docs/src/_static/embulk-logo-v2/embulk-logo-v2-symbol-wt.png +0 -0
  297. data/embulk-docs/src/_static/embulk-logo-v2/embulk-logo-v2-symbol.ai +1 -394
  298. data/embulk-docs/src/_static/embulk-logo.svg +0 -133
  299. data/embulk-docs/src/built-in.rst +0 -1005
  300. data/embulk-docs/src/conf.py +0 -260
  301. data/embulk-docs/src/customization.rst +0 -184
  302. data/embulk-docs/src/developers/index.rst +0 -45
  303. data/embulk-docs/src/index.rst +0 -94
  304. data/embulk-docs/src/logo.rst +0 -27
  305. data/embulk-docs/src/recipe.rst +0 -8
  306. data/embulk-docs/src/recipe/scheduled-csv-load-to-elasticsearch-kibana5.rst +0 -163
  307. data/embulk-docs/src/release.rst +0 -105
  308. data/embulk-docs/src/release/release-0.1.0.rst +0 -8
  309. data/embulk-docs/src/release/release-0.2.0.rst +0 -16
  310. data/embulk-docs/src/release/release-0.2.1.rst +0 -19
  311. data/embulk-docs/src/release/release-0.3.0.rst +0 -34
  312. data/embulk-docs/src/release/release-0.3.1.rst +0 -11
  313. data/embulk-docs/src/release/release-0.3.2.rst +0 -15
  314. data/embulk-docs/src/release/release-0.4.0.rst +0 -74
  315. data/embulk-docs/src/release/release-0.4.1.rst +0 -18
  316. data/embulk-docs/src/release/release-0.4.10.rst +0 -17
  317. data/embulk-docs/src/release/release-0.4.2.rst +0 -18
  318. data/embulk-docs/src/release/release-0.4.3.rst +0 -34
  319. data/embulk-docs/src/release/release-0.4.4.rst +0 -39
  320. data/embulk-docs/src/release/release-0.4.5.rst +0 -24
  321. data/embulk-docs/src/release/release-0.4.6.rst +0 -30
  322. data/embulk-docs/src/release/release-0.4.7.rst +0 -16
  323. data/embulk-docs/src/release/release-0.4.8.rst +0 -15
  324. data/embulk-docs/src/release/release-0.4.9.rst +0 -23
  325. data/embulk-docs/src/release/release-0.5.0.rst +0 -89
  326. data/embulk-docs/src/release/release-0.5.1.rst +0 -13
  327. data/embulk-docs/src/release/release-0.5.2.rst +0 -30
  328. data/embulk-docs/src/release/release-0.5.3.rst +0 -22
  329. data/embulk-docs/src/release/release-0.5.4.rst +0 -24
  330. data/embulk-docs/src/release/release-0.5.5.rst +0 -18
  331. data/embulk-docs/src/release/release-0.6.0.rst +0 -34
  332. data/embulk-docs/src/release/release-0.6.1.rst +0 -11
  333. data/embulk-docs/src/release/release-0.6.10.rst +0 -15
  334. data/embulk-docs/src/release/release-0.6.11.rst +0 -19
  335. data/embulk-docs/src/release/release-0.6.12.rst +0 -31
  336. data/embulk-docs/src/release/release-0.6.13.rst +0 -23
  337. data/embulk-docs/src/release/release-0.6.14.rst +0 -47
  338. data/embulk-docs/src/release/release-0.6.15.rst +0 -26
  339. data/embulk-docs/src/release/release-0.6.16.rst +0 -26
  340. data/embulk-docs/src/release/release-0.6.17.rst +0 -39
  341. data/embulk-docs/src/release/release-0.6.18.rst +0 -14
  342. data/embulk-docs/src/release/release-0.6.19.rst +0 -18
  343. data/embulk-docs/src/release/release-0.6.2.rst +0 -17
  344. data/embulk-docs/src/release/release-0.6.20.rst +0 -19
  345. data/embulk-docs/src/release/release-0.6.21.rst +0 -20
  346. data/embulk-docs/src/release/release-0.6.22.rst +0 -26
  347. data/embulk-docs/src/release/release-0.6.23.rst +0 -17
  348. data/embulk-docs/src/release/release-0.6.24.rst +0 -13
  349. data/embulk-docs/src/release/release-0.6.25.rst +0 -12
  350. data/embulk-docs/src/release/release-0.6.26.rst +0 -17
  351. data/embulk-docs/src/release/release-0.6.27.rst +0 -11
  352. data/embulk-docs/src/release/release-0.6.3.rst +0 -23
  353. data/embulk-docs/src/release/release-0.6.4.rst +0 -13
  354. data/embulk-docs/src/release/release-0.6.5.rst +0 -17
  355. data/embulk-docs/src/release/release-0.6.6.rst +0 -17
  356. data/embulk-docs/src/release/release-0.6.7.rst +0 -17
  357. data/embulk-docs/src/release/release-0.6.8.rst +0 -24
  358. data/embulk-docs/src/release/release-0.6.9.rst +0 -24
  359. data/embulk-docs/src/release/release-0.7.0.rst +0 -96
  360. data/embulk-docs/src/release/release-0.7.1.rst +0 -22
  361. data/embulk-docs/src/release/release-0.7.10.rst +0 -13
  362. data/embulk-docs/src/release/release-0.7.11.rst +0 -12
  363. data/embulk-docs/src/release/release-0.7.2.rst +0 -25
  364. data/embulk-docs/src/release/release-0.7.3.rst +0 -21
  365. data/embulk-docs/src/release/release-0.7.4.rst +0 -14
  366. data/embulk-docs/src/release/release-0.7.5.rst +0 -22
  367. data/embulk-docs/src/release/release-0.7.6.rst +0 -18
  368. data/embulk-docs/src/release/release-0.7.7.rst +0 -13
  369. data/embulk-docs/src/release/release-0.7.8.rst +0 -14
  370. data/embulk-docs/src/release/release-0.7.9.rst +0 -14
  371. data/embulk-docs/src/release/release-0.8.0.rst +0 -74
  372. data/embulk-docs/src/release/release-0.8.1.rst +0 -18
  373. data/embulk-docs/src/release/release-0.8.10.rst +0 -35
  374. data/embulk-docs/src/release/release-0.8.11.rst +0 -12
  375. data/embulk-docs/src/release/release-0.8.12.rst +0 -12
  376. data/embulk-docs/src/release/release-0.8.13.rst +0 -12
  377. data/embulk-docs/src/release/release-0.8.14.rst +0 -31
  378. data/embulk-docs/src/release/release-0.8.15.rst +0 -17
  379. data/embulk-docs/src/release/release-0.8.16.rst +0 -43
  380. data/embulk-docs/src/release/release-0.8.17.rst +0 -11
  381. data/embulk-docs/src/release/release-0.8.18.rst +0 -27
  382. data/embulk-docs/src/release/release-0.8.19.rst +0 -43
  383. data/embulk-docs/src/release/release-0.8.2.rst +0 -19
  384. data/embulk-docs/src/release/release-0.8.20.rst +0 -11
  385. data/embulk-docs/src/release/release-0.8.21.rst +0 -17
  386. data/embulk-docs/src/release/release-0.8.22.rst +0 -15
  387. data/embulk-docs/src/release/release-0.8.23.rst +0 -14
  388. data/embulk-docs/src/release/release-0.8.24.rst +0 -15
  389. data/embulk-docs/src/release/release-0.8.25.rst +0 -14
  390. data/embulk-docs/src/release/release-0.8.26.rst +0 -16
  391. data/embulk-docs/src/release/release-0.8.27.rst +0 -15
  392. data/embulk-docs/src/release/release-0.8.28.rst +0 -14
  393. data/embulk-docs/src/release/release-0.8.29.rst +0 -14
  394. data/embulk-docs/src/release/release-0.8.3.rst +0 -15
  395. data/embulk-docs/src/release/release-0.8.30.rst +0 -14
  396. data/embulk-docs/src/release/release-0.8.31.rst +0 -12
  397. data/embulk-docs/src/release/release-0.8.32.rst +0 -14
  398. data/embulk-docs/src/release/release-0.8.33.rst +0 -13
  399. data/embulk-docs/src/release/release-0.8.34.rst +0 -12
  400. data/embulk-docs/src/release/release-0.8.35.rst +0 -12
  401. data/embulk-docs/src/release/release-0.8.4.rst +0 -18
  402. data/embulk-docs/src/release/release-0.8.5.rst +0 -11
  403. data/embulk-docs/src/release/release-0.8.6.rst +0 -14
  404. data/embulk-docs/src/release/release-0.8.7.rst +0 -18
  405. data/embulk-docs/src/release/release-0.8.8.rst +0 -18
  406. data/embulk-docs/src/release/release-0.8.9.rst +0 -14
  407. data/embulk-jruby-strptime/build.gradle +0 -3
  408. data/embulk-jruby-strptime/src/main/java/org/embulk/spi/time/RubyDateParser.java +0 -121
  409. data/embulk-jruby-strptime/src/main/java/org/embulk/spi/time/StrptimeFormat.java +0 -53
  410. data/embulk-jruby-strptime/src/main/java/org/embulk/spi/time/StrptimeParser.java +0 -884
  411. data/embulk-jruby-strptime/src/main/java/org/embulk/spi/time/StrptimeToken.java +0 -111
  412. data/embulk-jruby-strptime/src/main/java/org/embulk/spi/time/TimeZoneConverter.java +0 -466
  413. data/embulk-jruby-strptime/src/main/java/org/embulk/spi/time/lexer/StrptimeLexer.flex +0 -62
  414. data/embulk-jruby-strptime/src/main/java/org/embulk/spi/time/lexer/StrptimeLexer.java +0 -577
  415. data/embulk-standards/build.gradle +0 -7
  416. data/embulk-standards/src/main/java/org/embulk/standards/Bzip2FileDecoderPlugin.java +0 -55
  417. data/embulk-standards/src/main/java/org/embulk/standards/Bzip2FileEncoderPlugin.java +0 -67
  418. data/embulk-standards/src/main/java/org/embulk/standards/ConfigInputPlugin.java +0 -170
  419. data/embulk-standards/src/main/java/org/embulk/standards/CsvFormatterPlugin.java +0 -295
  420. data/embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java +0 -394
  421. data/embulk-standards/src/main/java/org/embulk/standards/CsvTokenizer.java +0 -512
  422. data/embulk-standards/src/main/java/org/embulk/standards/GzipFileDecoderPlugin.java +0 -55
  423. data/embulk-standards/src/main/java/org/embulk/standards/GzipFileEncoderPlugin.java +0 -71
  424. data/embulk-standards/src/main/java/org/embulk/standards/JsonParserPlugin.java +0 -235
  425. data/embulk-standards/src/main/java/org/embulk/standards/LocalFileInputPlugin.java +0 -232
  426. data/embulk-standards/src/main/java/org/embulk/standards/LocalFileOutputPlugin.java +0 -148
  427. data/embulk-standards/src/main/java/org/embulk/standards/NullOutputPlugin.java +0 -59
  428. data/embulk-standards/src/main/java/org/embulk/standards/RemoveColumnsFilterPlugin.java +0 -268
  429. data/embulk-standards/src/main/java/org/embulk/standards/RenameFilterPlugin.java +0 -479
  430. data/embulk-standards/src/main/java/org/embulk/standards/StandardPluginExtension.java +0 -16
  431. data/embulk-standards/src/main/java/org/embulk/standards/StandardPluginModule.java +0 -60
  432. data/embulk-standards/src/main/java/org/embulk/standards/StdoutOutputPlugin.java +0 -85
  433. data/embulk-standards/src/main/resources/META-INF/services/org.embulk.spi.Extension +0 -1
  434. data/embulk-standards/src/test/java/org/embulk/standards/TestCsvFormatterPlugin.java +0 -312
  435. data/embulk-standards/src/test/java/org/embulk/standards/TestCsvParserPlugin.java +0 -75
  436. data/embulk-standards/src/test/java/org/embulk/standards/TestCsvTokenizer.java +0 -381
  437. data/embulk-standards/src/test/java/org/embulk/standards/TestJsonParserPlugin.java +0 -351
  438. data/embulk-standards/src/test/java/org/embulk/standards/TestRemoveColumnsFilterPlugin.java +0 -121
  439. data/embulk-standards/src/test/java/org/embulk/standards/TestRenameFilterPlugin.java +0 -1020
  440. data/embulk-standards/src/test/java/org/embulk/standards/guess/TestCsvAllStringsGuessPlugin.java +0 -38
  441. data/embulk-standards/src/test/java/org/embulk/standards/guess/TestCsvGuessPlugin.java +0 -248
  442. data/embulk-standards/src/test/java/org/embulk/standards/preview/TestFilePreview.java +0 -73
  443. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_int_single_column_row.csv +0 -1
  444. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_int_single_column_row_and_header.csv +0 -2
  445. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_int_single_column_row_and_header_guessed.yml +0 -12
  446. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_int_single_column_row_and_header_seed.yml +0 -1
  447. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_int_single_column_row_guessed.yml +0 -12
  448. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_int_single_column_row_seed.yml +0 -1
  449. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_rows.csv +0 -1
  450. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_rows_and_header.csv +0 -2
  451. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_rows_and_header_guessed.yml +0 -16
  452. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_rows_and_header_seed.yml +0 -1
  453. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_rows_and_header_with_trim_needed.csv +0 -2
  454. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_rows_and_header_with_trim_needed_guessed.yml +0 -16
  455. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_rows_and_header_with_trim_needed_seed.yml +0 -1
  456. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_rows_guessed.yml +0 -16
  457. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_rows_seed.yml +0 -1
  458. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_rows_with_trim_needed.csv +0 -1
  459. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_rows_with_trim_needed_guessed.yml +0 -16
  460. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_rows_with_trim_needed_seed.yml +0 -1
  461. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_string_single_column_row.csv +0 -1
  462. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_string_single_column_row_and_header.csv +0 -2
  463. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_string_single_column_row_and_header_guessed.yml +0 -12
  464. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_string_single_column_row_and_header_seed.yml +0 -1
  465. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_string_single_column_row_guessed.yml +0 -12
  466. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_string_single_column_row_seed.yml +0 -1
  467. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_2_int_single_column_rows.csv +0 -2
  468. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_2_int_single_column_rows_guessed.yml +0 -12
  469. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_2_int_single_column_rows_seed.yml +0 -1
  470. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_2_rows.csv +0 -2
  471. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_2_rows_and_header.csv +0 -3
  472. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_2_rows_and_header_guessed.yml +0 -16
  473. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_2_rows_and_header_seed.yml +0 -1
  474. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_2_rows_guessed.yml +0 -16
  475. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_2_rows_seed.yml +0 -1
  476. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_2_string_single_column_rows.csv +0 -2
  477. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_2_string_single_column_rows_guessed.yml +0 -12
  478. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_2_string_single_column_rows_seed.yml +0 -1
  479. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_backslash_escape.csv +0 -5
  480. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_backslash_escape_guessed.yml +0 -17
  481. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_backslash_escape_seed.yml +0 -1
  482. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_double_single_column.csv +0 -12
  483. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_double_single_column_guessed.yml +0 -12
  484. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_double_single_column_seed.yml +0 -1
  485. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_int_single_column.csv +0 -4
  486. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_int_single_column_guessed.yml +0 -12
  487. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_int_single_column_seed.yml +0 -1
  488. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_int_single_column_with_header.csv +0 -5
  489. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_int_single_column_with_header_guessed.yml +0 -12
  490. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_int_single_column_with_header_seed.yml +0 -1
  491. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_semicolon_delimiter.csv +0 -5
  492. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_semicolon_delimiter_guessed.yml +0 -17
  493. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_semicolon_delimiter_seed.yml +0 -1
  494. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_simple.csv +0 -5
  495. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_simple_guessed.yml +0 -17
  496. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_simple_seed.yml +0 -1
  497. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_single_quote.csv +0 -5
  498. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_single_quote_guessed.yml +0 -17
  499. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_single_quote_seed.yml +0 -1
  500. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_skip_suggest_if_empty_sample_records.csv +0 -5
  501. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_skip_suggest_if_empty_sample_records_guessed.yml +0 -2
  502. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_skip_suggest_if_empty_sample_records_seed.yml +0 -1
  503. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_string_single_column.csv +0 -4
  504. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_string_single_column_guessed.yml +0 -12
  505. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_string_single_column_seed.yml +0 -1
  506. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_string_single_column_with_header.csv +0 -5
  507. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_string_single_column_with_header_guessed.yml +0 -12
  508. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_string_single_column_with_header_seed.yml +0 -1
  509. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_tab_delimiter.csv +0 -4
  510. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_tab_delimiter_guessed.yml +0 -16
  511. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_tab_delimiter_seed.yml +0 -1
  512. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv_all_strings/test/test_simple.csv +0 -5
  513. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv_all_strings/test/test_simple_guessed.yml +0 -17
  514. data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv_all_strings/test/test_simple_seed.yml +0 -1
  515. data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes.csv +0 -5
  516. data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes_exec.yml +0 -1
  517. data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes_load.yml +0 -19
  518. data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes_previewed.csv +0 -1
  519. data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_simple.csv +0 -5
  520. data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_simple_load.yml +0 -19
  521. data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_simple_previewed.csv +0 -4
  522. data/embulk-standards/src/test/resources/org/embulk/standards/remove_columns/test/test_keep.csv +0 -5
  523. data/embulk-standards/src/test/resources/org/embulk/standards/remove_columns/test/test_keep_expected.csv +0 -4
  524. data/embulk-standards/src/test/resources/org/embulk/standards/remove_columns/test/test_keep_filter.yml +0 -2
  525. data/embulk-standards/src/test/resources/org/embulk/standards/remove_columns/test/test_keep_in.yml +0 -18
  526. data/embulk-standards/src/test/resources/org/embulk/standards/remove_columns/test/test_keep_with_duplicated_column_names.csv +0 -5
  527. data/embulk-standards/src/test/resources/org/embulk/standards/remove_columns/test/test_keep_with_duplicated_column_names.yml +0 -2
  528. data/embulk-standards/src/test/resources/org/embulk/standards/remove_columns/test/test_keep_with_duplicated_column_names_expected.csv +0 -4
  529. data/embulk-standards/src/test/resources/org/embulk/standards/remove_columns/test/test_keep_with_duplicated_column_names_in.yml +0 -17
  530. data/embulk-standards/src/test/resources/org/embulk/standards/remove_columns/test/test_keep_with_unmatched_filter.yml +0 -3
  531. data/embulk-standards/src/test/resources/org/embulk/standards/remove_columns/test/test_keep_without_unmatched_filter.yml +0 -2
  532. data/embulk-standards/src/test/resources/org/embulk/standards/remove_columns/test/test_remove.csv +0 -5
  533. data/embulk-standards/src/test/resources/org/embulk/standards/remove_columns/test/test_remove_expected.csv +0 -4
  534. data/embulk-standards/src/test/resources/org/embulk/standards/remove_columns/test/test_remove_filter.yml +0 -2
  535. data/embulk-standards/src/test/resources/org/embulk/standards/remove_columns/test/test_remove_in.yml +0 -18
  536. data/embulk-standards/src/test/resources/org/embulk/standards/remove_columns/test/test_remove_with_unmatched_filter.yml +0 -3
  537. data/embulk-standards/src/test/resources/org/embulk/standards/remove_columns/test/test_remove_without_unmatched_filter.yml +0 -2
  538. data/embulk-test/build.gradle +0 -6
  539. data/embulk-test/src/main/java/org/embulk/test/EmbulkTests.java +0 -75
  540. data/embulk-test/src/main/java/org/embulk/test/PreviewResultInputPlugin.java +0 -65
  541. data/embulk-test/src/main/java/org/embulk/test/TestingBulkLoader.java +0 -129
  542. data/embulk-test/src/main/java/org/embulk/test/TestingEmbulk.java +0 -710
  543. data/gradle/wrapper/gradle-wrapper.jar +0 -0
  544. data/gradle/wrapper/gradle-wrapper.properties +0 -5
  545. data/gradlew +0 -172
  546. data/gradlew.bat +0 -84
  547. data/lib/embulk/data/bundle/.bundle/config +0 -3
  548. data/lib/embulk/data/bundle/.ruby-version +0 -1
  549. data/lib/embulk/data/bundle/Gemfile +0 -31
  550. data/lib/embulk/data/bundle/embulk/filter/example.rb +0 -42
  551. data/lib/embulk/data/bundle/embulk/input/example.rb +0 -54
  552. data/lib/embulk/data/bundle/embulk/output/example.rb +0 -58
  553. data/lib/embulk/data/new/LICENSE.txt +0 -21
  554. data/lib/embulk/data/new/README.md.vm +0 -106
  555. data/lib/embulk/data/new/gitignore.vm +0 -17
  556. data/lib/embulk/data/new/java/build.gradle.vm +0 -96
  557. data/lib/embulk/data/new/java/config/checkstyle/checkstyle.xml +0 -128
  558. data/lib/embulk/data/new/java/config/checkstyle/default.xml +0 -108
  559. data/lib/embulk/data/new/java/decoder.java.vm +0 -86
  560. data/lib/embulk/data/new/java/encoder.java.vm +0 -88
  561. data/lib/embulk/data/new/java/file_input.java.vm +0 -145
  562. data/lib/embulk/data/new/java/file_output.java.vm +0 -95
  563. data/lib/embulk/data/new/java/filter.java.vm +0 -57
  564. data/lib/embulk/data/new/java/formatter.java.vm +0 -55
  565. data/lib/embulk/data/new/java/gradle/wrapper/gradle-wrapper.jar +0 -0
  566. data/lib/embulk/data/new/java/gradle/wrapper/gradle-wrapper.properties +0 -5
  567. data/lib/embulk/data/new/java/gradlew +0 -172
  568. data/lib/embulk/data/new/java/gradlew.bat +0 -84
  569. data/lib/embulk/data/new/java/input.java.vm +0 -89
  570. data/lib/embulk/data/new/java/output.java.vm +0 -79
  571. data/lib/embulk/data/new/java/parser.java.vm +0 -61
  572. data/lib/embulk/data/new/java/plugin_loader.rb.vm +0 -3
  573. data/lib/embulk/data/new/java/test.java.vm +0 -5
  574. data/lib/embulk/data/new/ruby/.ruby-version +0 -1
  575. data/lib/embulk/data/new/ruby/Gemfile +0 -2
  576. data/lib/embulk/data/new/ruby/Rakefile +0 -3
  577. data/lib/embulk/data/new/ruby/decoder_guess.rb.vm +0 -25
  578. data/lib/embulk/data/new/ruby/filter.rb.vm +0 -50
  579. data/lib/embulk/data/new/ruby/formatter.rb.vm +0 -49
  580. data/lib/embulk/data/new/ruby/gemspec.vm +0 -20
  581. data/lib/embulk/data/new/ruby/input.rb.vm +0 -59
  582. data/lib/embulk/data/new/ruby/output.rb.vm +0 -61
  583. data/lib/embulk/data/new/ruby/parser.rb.vm +0 -44
  584. data/lib/embulk/data/new/ruby/parser_guess.rb.vm +0 -65
  585. data/lib/embulk/data/package_data.rb +0 -59
  586. data/lib/embulk/java/time_helper.rb +0 -79
  587. data/lib/embulk/runner.rb +0 -53
  588. data/settings.gradle +0 -8
  589. data/test/guess/test_csv_all_strings.rb +0 -43
  590. data/test/guess/test_csv_guess.rb +0 -170
  591. data/test/guess/test_schema_guess.rb +0 -29
  592. data/test/guess/test_time_format_guess.rb +0 -135
  593. data/test/helper.rb +0 -23
  594. data/test/mri/date/test_date_strptime.rb +0 -515
  595. data/test/run-test.rb +0 -28
data/embulk-docs/src/_static/embulk-logo.svg DELETED
@@ -1,133 +0,0 @@
1
- <?xml version="1.0" encoding="utf-8"?>
2
- <!-- Generator: Adobe Illustrator 15.0.2, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
3
- <!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
4
- <svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
5
- width="466px" height="268px" viewBox="0 0 466 268" enable-background="new 0 0 466 268" xml:space="preserve">
6
- <g>
7
- <path fill="#EF4F39" d="M124.753,173.479c0,0,7.781,2.789,8.031,6.721c0.57,8.859,5.539,30.307-0.93,30.686
8
- c-8.516,0.502-20.797-12.975-24.602-22.738c-5.82-14.951-4.18-19.992-4.18-19.992L124.753,173.479z"/>
9
- <g>
10
- <g>
11
- <path fill="#1F1F1F" d="M38.015,225.368c1.477,4.021,3.523,7.008,6.141,8.953s5.891,2.918,9.828,2.918
12
- c2.859,0,5.742-0.58,8.648-1.744c0.762-0.312,3.422-1.586,7.984-3.822c1.383-0.672,2.93-1.006,4.625-1.006
13
- c2.328,0,4.297,1.051,5.906,3.152c1.34,1.699,2.012,3.398,2.012,5.096c0,3.803-2.707,7.246-8.113,10.33
14
- c-6.488,3.713-13.844,5.57-22.07,5.57c-12.297,0-21.98-4.273-29.047-12.814c-5.094-6.17-7.645-13.592-7.645-22.268
15
- c0-11.762,3.777-21.152,11.336-28.172c6.527-6.037,14.266-9.057,23.207-9.057c7.828,0,14.781,2.416,20.859,7.246
16
- c5.859,4.695,9.75,11.066,11.672,19.115c0.672,2.863,1.008,5.725,1.008,8.586c0,5.277-2.93,7.916-8.789,7.916H38.015z
17
- M64.511,211.616c-0.848-3.264-1.832-5.701-2.949-7.312c-2.414-3.531-5.812-5.299-10.195-5.299c-3.758,0-6.953,1.365-9.594,4.094
18
- c-1.699,1.787-3.195,4.627-4.492,8.518H64.511z"/>
19
- <path fill="#1F1F1F" d="M112.202,189.481c2.367-2.102,4.539-3.623,6.508-4.561c3.352-1.609,7.133-2.416,11.336-2.416
20
- c5.23,0,9.812,1.117,13.75,3.354c1.742,0.984,3.598,2.797,5.566,5.434c3.355-2.727,6.215-4.693,8.586-5.902
21
- c3.934-1.922,8.16-2.885,12.676-2.885c6.977,0,12.742,2.215,17.305,6.641c4.609,4.518,6.91,10.061,6.91,16.635v35.146
22
- c0,3.848-0.449,6.666-1.34,8.451c-0.762,1.432-1.957,2.627-3.59,3.59c-1.633,0.961-3.344,1.443-5.129,1.443
23
- c-1.793,0-3.5-0.469-5.133-1.41c-1.633-0.939-2.828-2.146-3.59-3.623c-0.895-1.785-1.34-4.604-1.34-8.451v-30.652
24
- c0-2.951-0.895-5.342-2.684-7.176c-2.012-2.146-4.559-3.221-7.645-3.221c-3.133,0-5.75,1.051-7.852,3.154
25
- c-1.875,1.832-2.816,4.246-2.816,7.242v30.652c0,3.805-0.449,6.621-1.34,8.451c-0.762,1.477-1.957,2.684-3.59,3.623
26
- c-1.633,0.941-3.344,1.41-5.129,1.41c-1.789,0-3.5-0.469-5.133-1.41c-1.633-0.939-2.828-2.146-3.59-3.623
27
- c-0.895-1.785-1.34-4.604-1.34-8.451v-28.641c0-4.066-0.961-7.152-2.887-9.254c-1.922-2.104-4.449-3.154-7.578-3.154
28
- c-3.086,0-5.699,1.098-7.848,3.287c-1.789,1.879-2.68,4.248-2.68,7.109v30.652c0,3.805-0.453,6.621-1.344,8.451
29
- c-0.762,1.477-1.957,2.684-3.59,3.623c-1.633,0.941-3.34,1.41-5.133,1.41c-1.785,0-3.496-0.469-5.129-1.41
30
- c-1.633-0.939-2.828-2.146-3.59-3.623c-0.895-1.785-1.34-4.604-1.34-8.451v-46.414c0-3.398,0.848-6.127,2.547-8.186
31
- c1.879-2.279,4.336-3.42,7.379-3.42c2.594,0,4.809,0.873,6.641,2.617C109.897,186.284,110.952,187.603,112.202,189.481z"/>
32
- <path fill="#1F1F1F" d="M112.202,189.481c2.367-2.102,4.539-3.623,6.508-4.561c3.352-1.609,7.133-2.416,11.336-2.416
33
- c5.23,0,9.812,1.117,13.75,3.354c1.742,0.984,3.598,2.797,5.566,5.434c3.355-2.727,6.215-4.693,8.586-5.902
34
- c3.934-1.922,8.16-2.885,12.676-2.885c6.977,0,12.742,2.215,17.305,6.641c4.609,4.518,6.91,10.061,6.91,16.635v35.146
35
- c0,3.848-0.449,6.666-1.34,8.451c-0.762,1.432-1.957,2.627-3.59,3.59c-1.633,0.961-3.344,1.443-5.129,1.443
36
- c-1.793,0-3.5-0.469-5.133-1.41c-1.633-0.939-2.828-2.146-3.59-3.623c-0.895-1.785-1.34-4.604-1.34-8.451v-30.652
37
- c0-2.951-0.895-5.342-2.684-7.176c-2.012-2.146-4.559-3.221-7.645-3.221c-3.133,0-5.75,1.051-7.852,3.154
38
- c-1.875,1.832-2.816,4.246-2.816,7.242v30.652c0,3.805-0.449,6.621-1.34,8.451c-0.762,1.477-1.957,2.684-3.59,3.623
39
- c-1.633,0.941-3.344,1.41-5.129,1.41c-1.789,0-3.5-0.469-5.133-1.41c-1.633-0.939-2.828-2.146-3.59-3.623
40
- c-0.895-1.785-1.34-4.604-1.34-8.451v-28.641c0-4.066-0.961-7.152-2.887-9.254c-1.922-2.104-4.449-3.154-7.578-3.154
41
- c-3.086,0-5.699,1.098-7.848,3.287c-1.789,1.879-2.68,4.248-2.68,7.109v30.652c0,3.805-0.453,6.621-1.344,8.451
42
- c-0.762,1.477-1.957,2.684-3.59,3.623c-1.633,0.941-3.34,1.41-5.133,1.41c-1.785,0-3.496-0.469-5.129-1.41
43
- c-1.633-0.939-2.828-2.146-3.59-3.623c-0.895-1.785-1.34-4.604-1.34-8.451v-46.414c0-3.398,0.848-6.127,2.547-8.186
44
- c1.879-2.279,4.336-3.42,7.379-3.42c2.594,0,4.809,0.873,6.641,2.617C109.897,186.284,110.952,187.603,112.202,189.481z"/>
45
- </g>
46
- </g>
47
- <g>
48
- <path fill="#FFFFFF" d="M32.706,101.839c0,0,12.794,37.536,28.406,50.635c23.352,19.592,42.266,24.008,63.57,26.375
49
- c27.992,3.109,68.055,0.375,76.672-3.381c19.914-8.666,36.523-27.725,41.93-43.279c5.414-15.557-82.844-32.125-82.844-32.125
50
- l-30.766-13.865c0,0-19.617-10.143-26.719-6.76c-7.102,3.381-13.859,5.748-19.273,7.1c-5.406,1.354-13.523,0.338-22.656,2.029
51
- C51.901,90.257,32.706,101.839,32.706,101.839z"/>
52
- <path fill="#EF4F39" d="M32.331,101.245c0,0,0.391,10.795,3.898,14.678c4.727,5.23,3.203,11.189,13.461,25.664
53
- c13.281,18.732,38.328,36.789,89.406,38.908c50.391,2.094,50.086-1.395,50.086-1.395s-6.891,1.152-44.25-0.252
54
- c-23.625-0.889-56.344-5.354-75.016-20.842c-17.734-14.709-24.203-24.42-29.125-39.309c-2.094-6.34-5.797-6.422-5.992-15.996
55
- C34.776,101.249,32.331,101.245,32.331,101.245z"/>
56
- <g>
57
- <path fill="#F16523" d="M195.331,176.63c0,0,5.219-1.023,11.352-4.504c8.711-4.943,14.773-11.02,15.852-11.918
58
- c4.883-4.057,15.219-16.484,15.219-16.484c-0.414,0.613-5.562,9.764-12.305,15.977c-4.484,4.133-1.898,2.156-9.25,7.861
59
- c-9.078,7.043-22,11.92-22,11.92"/>
60
- <path fill="#F16523" d="M284.198,12.147c0,0-7.016,13.777-24.008,15.553c-11.57,1.211-13.188,11.16-13.188,11.16
61
- s-4.055-10.652-14.453-12.174c-15.477-2.264-20.375-13.525-20.375-13.525c-4.391,16.23,0.68,24.346,3.383,28.404
62
- c5.188,7.781,18.258,9.469,21.305,16.23c3.039,6.764,0.672,11.498-2.703,27.391c-3.383,15.893-19.281,12.848-32.125,11.156
63
- c-7.117-0.934-18.672-8.105-41.508-16.402c0.242-4.438-2.695-9.434-1.016-16.23c6.086-24.6,24.094-34.361,12.938-32.715
64
- c-8.805,1.301-50.031,32.121-56.523,36.102c-8.344-1.617-17.086-2.822-26.156-3.387c-31.359-1.947-39.305,14.033-48.945,20.627
65
- c-17.148,11.734-8.07,17.84-8.07,17.84s28.812,1.553,40.109,3.975c9.469,2.027,32.125,8.453,33.812,11.496
66
- c1.695,3.043,4.398,11.834,12.516,10.145c8.109-1.691,8.75-7.969,8.75-7.969c-0.344,9.131,6.805,21.156,14.242,31.639
67
- s19.273,13.188,19.273,13.188c2.031,25.359-11.562,49.064-8.031,49.195c23.844,0.889,38.156-23.922,40.453-28.275
68
- c4.82-9.131,3.297-19.656,3.297-19.656c42.266-8.113,49.461-34.107,53.203-45.637c6.344-19.529,3.055-59.438,3.055-59.438
69
- s8.789-0.676,20.961-9.467C286.565,42.579,284.198,12.147,284.198,12.147z"/>
70
- </g>
71
- <path fill="#FFFFFF" d="M107.753,97.312c0,0-0.617-14.115,18.625-10.881s19.953,13.1,4.945,17.168
72
- C116.323,107.669,108.065,103.565,107.753,97.312z"/>
73
- <g>
74
- <circle fill="#1F1F1F" cx="101.776" cy="92.2" r="4.312"/>
75
- <path fill="#FFFFFF" d="M101.776,96.761c-2.516,0-4.562-2.046-4.562-4.561c0-2.516,2.047-4.562,4.562-4.562
76
- s4.562,2.047,4.562,4.562C106.339,94.715,104.292,96.761,101.776,96.761z M101.776,88.138c-2.24,0-4.062,1.822-4.062,4.062
77
- c0,2.239,1.822,4.061,4.062,4.061s4.062-1.822,4.062-4.061C105.839,89.96,104.017,88.138,101.776,88.138z"/>
78
- <path fill="#FFFFFF" d="M104.394,90.595c0,0.512-0.414,0.928-0.93,0.928c-0.508,0-0.93-0.416-0.93-0.928
79
- c0-0.516,0.422-0.932,0.93-0.932C103.979,89.663,104.394,90.079,104.394,90.595z"/>
80
- </g>
81
- <g>
82
- <path fill="#EF4F39" d="M164.237,161.2c-1.523,0.254-4.141,0.254-9.383-1.184c-4.789-1.311-12.648-9.082-13.984-10.43
83
- c0.438,0.631,0.875,1.254,1.312,1.875c7.438,10.482,19.273,13.188,19.273,13.188c1.969,24.617-10.773,47.672-8.297,49.121
84
- c0.008-0.064,0.094-0.949,0.258-1.764c0.164-0.846,0.594-2.283,1.266-3.805c0.68-1.521,10.062-18.682,12.172-26.543
85
- c2.117-7.861,1.188-21.727,1.188-21.727S165.761,160.948,164.237,161.2z"/>
86
- <path fill="#EF4F39" d="M122.995,115.067c0,0,2.531-7.607,10.648-6.086s5.328,10.271,5.328,10.271s0.031-6.586-5.711-7.734
87
- C128.19,110.503,122.995,115.067,122.995,115.067z"/>
88
- <path fill="#EF4F39" d="M117.237,126.362c-5.328-0.635-5.711-8.242-9.891-11.287c-3.484-2.529-25.367-9.129-34.242-11.031
89
- s-36.266-3.803-38.547-3.93s-2.914-3.043-3.297-3.553c-0.242-0.326,0.25-2.391,0.633-3.816c-3.898,6.225,0.859,9.432,0.859,9.432
90
- s28.812,1.553,40.109,3.975c9.469,2.027,32.125,8.453,33.812,11.496c1.695,3.043,4.398,11.834,12.516,10.145
91
- c8.109-1.691,8.75-7.969,8.75-7.969S122.144,126.946,117.237,126.362z"/>
92
- <path fill="#EF4F39" d="M251.394,111.146c-1.352,7.1-6.594,22.994-16.398,32.799c-6.102,6.098-15.898,10.652-23.672,14.203
93
- c-6.266,2.859-15.219,3.887-15.219,3.887l1.07,3.879c42.266-8.113,49.461-34.107,53.203-45.637
94
- c2.305-7.115,3.344-16.934,3.719-26.541C253.448,98.396,252.308,106.351,251.394,111.146z"/>
95
- </g>
96
- </g>
97
- <rect x="133.667" y="203.567" fill="#1F1F1F" width="20.055" height="21.766"/>
98
- <g>
99
- <g>
100
- <path fill="#1F1F1F" d="M226.835,188.409c2.816-2.104,5.188-3.512,7.109-4.227c2.992-1.162,6.059-1.744,9.188-1.744
101
- c8.32,0,15.586,3.379,21.801,10.127c6.348,6.889,9.527,15.496,9.527,25.826c0,11.402-3.555,20.658-10.668,27.77
102
- c-5.723,5.768-12.543,8.654-20.457,8.654c-4.609,0-8.562-1.031-11.875-3.086c-1.516-0.984-3.148-2.619-4.895-4.896
103
- c-2.059,5.051-5.23,7.578-9.527,7.578c-2.637,0-4.961-0.85-6.973-2.549c-2.238-1.967-3.355-4.697-3.355-8.182v-77.543
104
- c0-3.799,0.449-6.594,1.344-8.383c0.758-1.477,1.953-2.684,3.59-3.623c1.629-0.939,3.34-1.408,5.129-1.408s3.5,0.469,5.133,1.408
105
- s2.828,2.146,3.586,3.623c0.895,1.789,1.344,4.584,1.344,8.383V188.409z M240.519,201.22c-3.758,0-7.02,1.631-9.793,4.896
106
- c-2.684,3.219-4.023,7.379-4.023,12.477c0,5.141,1.34,9.324,4.023,12.543c2.773,3.264,6.078,4.896,9.926,4.896
107
- c4.16,0,7.488-1.746,9.996-5.23c2.457-3.49,3.688-7.535,3.688-12.143c0-4.695-1.23-8.742-3.688-12.143
108
- C248.097,202.987,244.718,201.22,240.519,201.22z"/>
109
- <path fill="#1F1F1F" d="M345.358,196.39v29.916c0,6.842-1.637,12.387-4.898,16.635c-2.773,3.578-6.352,6.396-10.734,8.451
110
- c-4.918,2.324-10.148,3.488-15.695,3.488c-7.781,0-14.598-2.078-20.457-6.238c-4.559-3.221-7.645-7.064-9.258-11.537
111
- c-1.074-3.037-1.605-6.641-1.605-10.799V196.39c0-3.801,0.445-6.617,1.336-8.451c0.762-1.477,1.949-2.684,3.559-3.623
112
- c1.605-0.938,3.332-1.408,5.164-1.408c1.789,0,3.496,0.471,5.129,1.408c1.633,0.939,2.828,2.146,3.59,3.623
113
- c0.895,1.789,1.34,4.607,1.34,8.451v27.434c0,2.818,0.449,4.945,1.344,6.373c2.281,3.668,5.57,5.502,9.859,5.502
114
- c3.129,0,5.793-1.074,7.984-3.221c2.145-2.059,3.219-4.941,3.219-8.654V196.39c0-3.844,0.445-6.662,1.34-8.451
115
- c0.762-1.477,1.957-2.684,3.59-3.623c1.633-0.938,3.344-1.408,5.133-1.408c1.785,0,3.5,0.471,5.129,1.408
116
- c1.633,0.939,2.828,2.146,3.59,3.623C344.909,189.728,345.358,192.546,345.358,196.39z"/>
117
- <path fill="#1F1F1F" d="M377.351,166.138v74.789c0,3.848-0.445,6.666-1.344,8.451c-0.758,1.477-1.953,2.684-3.586,3.623
118
- c-1.633,0.941-3.344,1.41-5.133,1.41s-3.5-0.469-5.129-1.41c-1.633-0.939-2.832-2.146-3.59-3.623
119
- c-0.895-1.785-1.344-4.604-1.344-8.451v-74.789c0-3.799,0.449-6.594,1.344-8.383c0.758-1.477,1.957-2.684,3.59-3.623
120
- c1.629-0.939,3.34-1.408,5.129-1.408s3.5,0.469,5.133,1.408s2.828,2.146,3.586,3.623
121
- C376.905,159.544,377.351,162.339,377.351,166.138z"/>
122
- <path fill="#1F1F1F" d="M409.343,220.403v20.523c0,3.805-0.445,6.621-1.34,8.451c-0.762,1.477-1.957,2.684-3.586,3.623
123
- c-1.633,0.941-3.344,1.41-5.133,1.41s-3.496-0.469-5.129-1.41c-1.633-0.939-2.828-2.146-3.594-3.623
124
- c-0.891-1.785-1.34-4.604-1.34-8.451v-74.789c0-3.799,0.449-6.594,1.34-8.383c0.766-1.477,1.961-2.684,3.594-3.623
125
- s3.34-1.408,5.129-1.408s3.5,0.469,5.133,1.408c1.629,0.939,2.824,2.146,3.586,3.623c0.895,1.789,1.34,4.584,1.34,8.383v40.582
126
- l18.516-18.982c3.445-3.533,6.527-5.299,9.258-5.299c2.727,0,5.051,0.984,6.977,2.951c1.922,1.924,2.883,4.227,2.883,6.908
127
- c0,1.969-0.738,3.936-2.215,5.904c-0.488,0.625-1.566,1.744-3.219,3.354l-12.609,12.072l17.641,21.936
128
- c2.902,3.578,4.355,6.551,4.355,8.918c0,3.178-1.27,5.793-3.82,7.85c-1.836,1.432-3.957,2.146-6.375,2.146
129
- c-2.41,0-4.672-0.961-6.77-2.885c-0.629-0.58-1.656-1.787-3.09-3.621L409.343,220.403z"/>
130
- </g>
131
- </g>
132
- </g>
133
- </svg>
data/embulk-docs/src/built-in.rst DELETED
@@ -1,1005 +0,0 @@
1
- Configuration
2
- ==============
3
-
4
- .. contents::
5
- :local:
6
- :depth: 2
7
-
8
- Embulk configuration file format
9
- ---------------------------------
10
-
11
- Embulk uses a YAML file to define a bulk data loading. Here is an example of the file:
12
-
13
- .. code-block:: yaml
14
-
15
- in:
16
- type: file
17
- path_prefix: ./mydata/csv/
18
- decoders:
19
- - {type: gzip}
20
- parser:
21
- charset: UTF-8
22
- newline: CRLF
23
- type: csv
24
- delimiter: ','
25
- quote: '"'
26
- escape: '"'
27
- null_string: 'NULL'
28
- skip_header_lines: 1
29
- columns:
30
- - {name: id, type: long}
31
- - {name: account, type: long}
32
- - {name: time, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
33
- - {name: purchase, type: timestamp, format: '%Y%m%d'}
34
- - {name: comment, type: string}
35
- filters:
36
- - type: speedometer
37
- speed_limit: 250000
38
- out:
39
- type: stdout
40
-
41
- A configuration file consists of following sections:
42
-
43
- * **in:** Input plugin options. An input plugin is either record-based (`MySQL <https://github.com/embulk/embulk-input-jdbc>`_, `DynamoDB <https://github.com/lulichn/embulk-input-dynamodb>`_, etc) or file-based (`S3 <https://github.com/embulk/embulk-input-s3>`_, `HTTP <https://github.com/takumakanari/embulk-input-http>`_, etc).
44
-
45
- * **parser:** If the input is file-based, parser plugin parses a file format (built-in csv, `json <https://github.com/takumakanari/embulk-parser-json>`_, etc).
46
-
47
- * **decoder:** If the input is file-based, decoder plugin decodes compression or encryption (built-in gzip, bzip2, `zip <https://github.com/hata/embulk-decoder-commons-compress>`_, `tar.gz <https://github.com/hata/embulk-decoder-commons-compress>`_, etc).
48
-
49
- * **out:** Output plugin options. An output plugin is either record-based (`Oracle <https://github.com/embulk/embulk-output-jdbc>`_, `Elasticsearch <https://github.com/muga/embulk-output-elasticsearch>`_, etc) or file-based (`Google Cloud Storage <https://github.com/hakobera/embulk-output-gcs>`_, `Command <https://github.com/embulk/embulk-output-command>`_, etc)
50
-
51
- * **formatter:** If the output is file-based, formatter plugin formats a file format (such as built-in csv, `jsonl <https://github.com/takei-yuya/embulk-formatter-jsonl>`_)
52
-
53
- * **encoder:** If the output is file-based, encoder plugin encodes compression or encryption (such as built-in gzip or bzip2)
54
-
55
- * **filters:** Filter plugins options (optional).
56
-
57
- * **exec:** Executor plugin options. An executor plugin control parallel processing (such as built-in thread executor, `Hadoop MapReduce executor <https://github.com/embulk/embulk-executor-mapreduce>`_)
58
-
59
- In many cases, what you need to write is **in:**, **out**: and **formatter** sections only because ``guess`` command guesses **parser** and **decoder** options for you. See also the `Quick Start <https://github.com/embulk/embulk#quick-start>`_.
60
-
61
-
62
- Using variables
63
- ~~~~~~~~~~~~~~~~
64
-
65
- You can embed environment variables in configuration file using `Liquid template engine <http://liquidmarkup.org/>`_ (This is experimental feature. Behavior might change or be removed in future releases).
66
-
67
- To use template engine, configuration file name must end with ``.yml.liquid``.
68
-
69
- Environment variables are set to ``env`` variable.
70
-
71
- .. code-block:: yaml
72
-
73
- in:
74
- type: file
75
- path_prefix: {{ env.path_prefix }}
76
- decoders:
77
- - {type: gzip}
78
- parser:
79
- ...
80
- out:
81
- type: postgresql
82
- host: {{ env.pg_host }}
83
- port: {{ env.pg_port }}
84
- user: {{ env.pg_user }}
85
- password: "{{ env.pg_password }}"
86
- database: embulk_load
87
- mode: insert
88
- table: {{ env.pg_table }}
89
-
90
-
91
- Including files
92
- ~~~~~~~~~~~~~~~~
93
-
94
- Configuration file can include another configuration file. To use it, configuration file name must end with ``.yml.liquid``.
95
-
96
- File will be searched from the relative path of the input configuration file. And file name will be ``_<name>.yml.liquid``. For example, if you add ``{% include 'subdir/inc' %}`` tag to ``myconfig/config.yml.liquid`` file, it includes ``myconfig/subdir/_inc.yml.liquid`` file.
97
-
98
- .. code-block:: liquid
99
-
100
- # config.yml.liquid
101
- {% include 'in_mysql' %}
102
- out:
103
- type: stdout
104
-
105
- .. code-block:: yaml
106
-
107
- # _in_mysql.yml.liquid
108
- in:
109
- type: mysql
110
-
111
- With above 2 files, actual configuration file will be:
112
-
113
- .. code-block:: yaml
114
-
115
- # $ embulk run config.yml.liquid
116
- in:
117
- type: mysql
118
- out:
119
- type: stdout
120
-
121
-
122
-
123
- Local file input plugin
124
- ------------------------
125
-
126
- The ``file`` input plugin reads files from local file system.
127
-
128
- Options
129
- ~~~~~~~~
130
-
131
- +------------------+----------+------------------------------------------------+-----------------------+
132
- | name | type | description | required? |
133
- +==================+==========+================================================+=======================+
134
- | path\_prefix | string | Path prefix of input files | required |
135
- +------------------+----------+------------------------------------------------+-----------------------+
136
- | parsers | hash | Parsers configurations (see below) | required |
137
- +------------------+----------+------------------------------------------------+-----------------------+
138
- | decoders | array | Decoder configuration (see below) | |
139
- +------------------+----------+------------------------------------------------+-----------------------+
140
- | last\_path | string | Name of last read file in previous operation | |
141
- +------------------+----------+------------------------------------------------+-----------------------+
142
- | follow\_symlinks | boolean | If `true`, follow symbolic link directories | ``false`` by default |
143
- +------------------+----------+------------------------------------------------+-----------------------+
144
-
145
- The ``path_prefix`` option is required. If you have files as following, you may set ``path_prefix: /path/to/files/sample_``:
146
-
147
- ::
148
-
149
- .
150
- `-- path
151
- `-- to
152
- `-- files
153
- |-- sample_01.csv -> read
154
- |-- sample_02.csv -> read
155
- |-- sample_03.csv -> read
156
- |-- sample_04.csv -> read
157
-
158
- The ``last_path`` option is used to skip files older than or same with the file in dictionary order.
159
- For example, if you set ``last_path: /path/to/files/sample_02.csv``, Embulk reads following files:
160
-
161
- ::
162
-
163
- .
164
- `-- path
165
- `-- to
166
- `-- files
167
- |-- sample_01.csv -> skip
168
- |-- sample_02.csv -> skip
169
- |-- sample_03.csv -> read
170
- |-- sample_04.csv -> read
171
-
172
- Example
173
- ~~~~~~~~
174
-
175
- .. code-block:: yaml
176
-
177
- in:
178
- type: file
179
- path_prefix: /path/to/files/sample_
180
- last_path: /path/to/files/sample_02.csv
181
- parser:
182
- ...
183
-
184
- In most of cases, you'll use guess to configure the parsers and decoders. See also `Quick Start <https://github.com/embulk/embulk#quick-start>`_.
185
-
186
- CSV parser plugin
187
- ------------------
188
-
189
- The ``csv`` parser plugin parses CSV and TSV files.
190
-
191
- Options
192
- ~~~~~~~~
193
-
194
- +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+---------------------------+
195
- | name | type | description | required? |
196
- +============================+==========+================================================================================================================+===========================+
197
- | delimiter | string | Delimiter character such as ``,`` for CSV, ``"\t"`` for TSV, ``"|"`` | ``,`` by default |
198
- +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+---------------------------+
199
- | quote | string | The character surrounding a quoted value. Setting ``null`` disables quoting. | ``"`` by default |
200
- +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+---------------------------+
201
- | escape | string | Escape character to escape a special character. Setting ``null`` disables escaping. | ``\\`` by default |
202
- +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+---------------------------+
203
- | skip\_header\_lines | integer | Skip this number of lines first. Set 1 if the file has header line. | ``0`` by default |
204
- +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+---------------------------+
205
- | null\_string | string | If a value is this string, converts it to NULL. For example, set ``\N`` for CSV files created by mysqldump | |
206
- +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+---------------------------+
207
- | trim\_if\_not\_quoted | boolean | If true, remove spaces of a value if the value is not surrounded by the quote character | ``false`` by default |
208
- +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+---------------------------+
209
- | comment\_line\_marker | string | Skip a line if the line begins with this string | null by default |
210
- +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+---------------------------+
211
- | allow\_optional\_columns | boolean | If true, set null to insufficient columns. Otherwise, skip the row in case of insufficient number of columns | ``false`` by default |
212
- +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+---------------------------+
213
- | allow\_extra\_columns | boolean | If true, ignore too many columns. Otherwise, skip the row in case of too many columns | ``false`` by default |
214
- +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+---------------------------+
215
- | max\_quoted\_size\_limit | integer | Maximum number of bytes of a quoted value. If a value exceeds the limit, the row will be skipped | ``131072`` by default |
216
- +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+---------------------------+
217
- | stop\_on\_invalid\_record | boolean | Stop bulk load transaction if a file includes invalid record (such as invalid timestamp) | ``false`` by default |
218
- +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+---------------------------+
219
- | default\_timezone | string | Time zone of timestamp columns if the value itself doesn't include time zone description (eg. Asia/Tokyo) | ``UTC`` by default |
220
- +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+---------------------------+
221
- | default\_date | string | Set date part if the format doesn’t include date part. | ``1970-01-01`` by default |
222
- +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+---------------------------+
223
- | newline | enum | Newline character (CRLF, LF or CR) | ``CRLF`` by default |
224
- +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+---------------------------+
225
- | charset | enum | Character encoding (eg. ISO-8859-1, UTF-8) | ``UTF-8`` by default |
226
- +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+---------------------------+
227
- | columns | hash | Columns (see below) | required |
228
- +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+---------------------------+
229
-
230
- The ``columns`` option declares the list of columns. This CSV parser plugin ignores the header line.
231
-
232
- +----------+--------------------------------------------------------+
233
- | name | description |
234
- +==========+========================================================+
235
- | name | Name of the column |
236
- +----------+--------------------------------------------------------+
237
- | type | Type of the column (see below) |
238
- +----------+--------------------------------------------------------+
239
- | format | Format of the timestamp if type is timestamp |
240
- +----------+--------------------------------------------------------+
241
- | date | Set date part if the format doesn’t include date part |
242
- +----------+--------------------------------------------------------+
243
-
244
- .. note::
245
-
246
- The Timestamp format refers to `Ruby strftime format <https://docs.ruby-lang.org/en/2.4.0/Date.html#method-i-strftime>`_
247
-
248
- List of types:
249
-
250
- +-------------+----------------------------------------------+
251
- | name | description |
252
- +=============+==============================================+
253
- | boolean | true or false |
254
- +-------------+----------------------------------------------+
255
- | long | 64-bit signed integers |
256
- +-------------+----------------------------------------------+
257
- | timestamp | Date and time with nano-seconds precision |
258
- +-------------+----------------------------------------------+
259
- | double | 64-bit floating point numbers |
260
- +-------------+----------------------------------------------+
261
- | string | Strings |
262
- +-------------+----------------------------------------------+
263
-
264
- The ``null_string`` option converts certain values to NULL. Values will be converted as following:
265
-
266
- +---------------------------------+-------------------------+--------------------------+----------------+--------------------+
267
- | | non-quoted empty string | quoted empty string ("") | non-quoted \\N | quoted \\N ("\\N") |
268
- +=================================+=========================+==========================+================+====================+
269
- | ``null_string: ""`` | NULL | NULL | ``\N`` | ``\N`` |
270
- +---------------------------------+-------------------------+--------------------------+----------------+--------------------+
271
- | ``null_string: \N`` | (empty string) | (empty string) | NULL | NULL |
272
- +---------------------------------+-------------------------+--------------------------+----------------+--------------------+
273
- | ``null_string: null`` (default) | NULL | (empty string) | ``\N`` | ``\N`` |
274
- +---------------------------------+-------------------------+--------------------------+----------------+--------------------+
275
-
276
- You can use ``guess`` to automatically generate the column settings. See also `Quick Start <https://github.com/embulk/embulk#quick-start>`_.
277
-
278
- Example
279
- ~~~~~~~~
280
-
281
- .. code-block:: yaml
282
-
283
- in:
284
- ...
285
- parser:
286
- type: csv
287
- charset: UTF-8
288
- newline: CRLF
289
- delimiter: "\t"
290
- quote: '"'
291
- escape: '"'
292
- null_string: 'NULL'
293
- skip_header_lines: 1
294
- comment_line_marker: '#'
295
- columns:
296
- - {name: id, type: long}
297
- - {name: account, type: long}
298
- - {name: time, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
299
- - {name: purchase, type: timestamp, format: '%Y%m%d'}
300
- - {name: comment, type: string}
301
-
302
-
303
- .. note::
304
-
305
- CSV parser supports ``format: '%s'`` to parse UNIX timestamp in seconds (e.g. 1470148959) as timestamp.
306
-
307
- However, CSV parser itself can't parse UNIX timestamp in millisecond (e.g. 1470148959542) as timestamp. You can still parse the column as ``long`` type first, then apply `timestamp_format <https://github.com/sonots/embulk-filter-timestamp_format>`_ filter plugin to convert long to timestamp. Here is an example:
308
-
309
- .. code-block:: yaml
310
-
311
- in:
312
- type: file
313
- path_prefix: /my_csv_files
314
- parser:
315
- ...
316
- columns:
317
- - {name: timestamp_in_seconds, type: timestamp, format: '%s'}
318
- - {name: timestamp_in_millis, type: long}
319
- filters:
320
- - type: timestamp_format
321
- columns:
322
- - {name: timestamp_in_millis, from_unit: ms}
323
-
324
-
325
- JSON parser plugin
326
- -------------------
327
-
328
- The ``json`` parser plugin parses a JSON file that contains a sequence of JSON objects. Example:
329
-
330
- .. code-block:: javascript
331
-
332
- {"time":1455829282,"ip":"93.184.216.34","name":"frsyuki"}
333
- {"time":1455829282,"ip":"172.36.8.109", "name":"sadayuki"}
334
- {"time":1455829284,"ip":"example.com","name":"Treasure Data"}
335
- {"time":1455829282,"ip":"10.98.43.1","name":"MessagePack"}
336
-
337
- ``json`` parser plugin outputs a single record named "record" (type is json).
338
-
339
- Options
340
- ~~~~~~~~
341
-
342
- +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------------+
343
- | name | type | description | required? |
344
- +============================+==========+================================================================================================================+==============================+
345
- | stop\_on\_invalid\_record | boolean | Stop bulk load transaction if a file includes invalid record (such as invalid json) | ``false`` by default |
346
- +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------------+
347
- | invalid\_string\_escapes | enum | Escape strategy of invalid json string such as using invalid ``\`` like ``\a``. (PASSTHROUGH, SKIP, UNESCAPE) | ``PASSTHROUGH`` by default |
348
- +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------------+
349
-
350
-
351
- if you set invalid\_string\_escapes and appear invalid JSON string (such as ``\a``), it makes following the action.
352
-
353
- +----------------------------+------------------+
354
- | invalid\_string\_escapes | convert to |
355
- +============================+==================+
356
- | PASSTHROUGH *1 | ``\a`` |
357
- +----------------------------+------------------+
358
- | SKIP | empty string |
359
- +----------------------------+------------------+
360
- | UNESCAPE | ``a`` |
361
- +----------------------------+------------------+
362
-
363
- (\*1): Throwing an exception.
364
-
365
-
366
- Example
367
- ~~~~~~~~
368
-
369
- .. code-block:: yaml
370
-
371
- in:
372
- parser:
373
- type: json
374
-
375
- Gzip decoder plugin
376
- --------------------
377
-
378
- The ``gzip`` decoder plugin decompresses gzip files before input plugins read them.
379
-
380
- Options
381
- ~~~~~~~~
382
-
383
- This plugin doesn't have any options.
384
-
385
- Example
386
- ~~~~~~~~
387
-
388
- .. code-block:: yaml
389
-
390
- in:
391
- ...
392
- decoders:
393
- - {type: gzip}
394
-
395
-
396
- BZip2 decoder plugin
397
- ---------------------
398
-
399
- The ``bzip2`` decoder plugin decompresses bzip2 files before input plugins read them.
400
-
401
- Options
402
- ~~~~~~~~
403
-
404
- This plugin doesn't have any options.
405
-
406
- Example
407
- ~~~~~~~~
408
-
409
- .. code-block:: yaml
410
-
411
- in:
412
- ...
413
- decoders:
414
- - {type: bzip2}
415
-
416
-
417
- File output plugin
418
- -------------------
419
-
420
- The ``file`` output plugin writes records to local file system.
421
-
422
- Options
423
- ~~~~~~~~
424
-
425
- +--------------------+----------+---------------------------------------------------+----------------------------+
426
- | name | type | description | required? |
427
- +====================+==========+===================================================+============================+
428
- | path\_prefix | string | Path prefix of the output files | required |
429
- +--------------------+----------+---------------------------------------------------+----------------------------+
430
- | sequence\_format | string | Format of the sequence number of the output files | ``%03d.%02d.`` by default |
431
- +--------------------+----------+---------------------------------------------------+----------------------------+
432
- | file\_ext | string | Path suffix of the output files (e.g. ``"csv"``) | required |
433
- +--------------------+----------+---------------------------------------------------+----------------------------+
434
-
435
- For example, if you set ``path_prefix: /path/to/output/sample_``, ``sequence_format: "%03d.%02d."``, and ``file_ext: csv``, name of the output files will be as following:
436
-
437
- ::
438
-
439
- .
440
- `-- path
441
- `-- to
442
- `-- output
443
- |-- sample_01.000.csv
444
- |-- sample_02.000.csv
445
- |-- sample_03.000.csv
446
- |-- sample_04.000.csv
447
-
448
- ``sequence_format`` formats task index and sequence number in a task.
449
-
450
- Example
451
- ~~~~~~~~
452
-
453
- .. code-block:: yaml
454
-
455
- out:
456
- type: file
457
- path_prefix: /path/to/output/sample_
458
- file_ext: csv
459
- formatter:
460
- ...
461
-
462
- CSV formatter plugin
463
- ---------------------
464
-
465
- The ``csv`` formatter plugin formats records using CSV or TSV format.
466
-
467
- Options
468
- ~~~~~~~~
469
-
470
- +----------------------+---------+-------------------------------------------------------------------------------------------------------+-------------------------------+
471
- | name | type | description | required? |
472
- +======================+=========+=======================================================================================================+===============================+
473
- | delimiter | string | Delimiter character such as ``,`` for CSV, ``"\t"`` for TSV, ``"|"`` or any single-byte character | ``,`` by default |
474
- +----------------------+---------+-------------------------------------------------------------------------------------------------------+-------------------------------+
475
- | quote | string | The character surrounding a quoted value | ``"`` by default |
476
- +----------------------+---------+-------------------------------------------------------------------------------------------------------+-------------------------------+
477
- | quote\_policy | enum | Policy for quote (ALL, MINIMAL, NONE) (see below) | ``MINIMAL`` by default |
478
- +----------------------+---------+-------------------------------------------------------------------------------------------------------+-------------------------------+
479
- | escape | string | Escape character to escape quote character | same with quote default (\*1) |
480
- +----------------------+---------+-------------------------------------------------------------------------------------------------------+-------------------------------+
481
- | header\_line | boolean | If true, write the header line with column name at the first line | ``true`` by default |
482
- +----------------------+---------+-------------------------------------------------------------------------------------------------------+-------------------------------+
483
- | null_string | string | Expression of NULL values | empty by default |
484
- +----------------------+---------+-------------------------------------------------------------------------------------------------------+-------------------------------+
485
- | newline | enum | Newline character (CRLF, LF or CR) | ``CRLF`` by default |
486
- +----------------------+---------+-------------------------------------------------------------------------------------------------------+-------------------------------+
487
- | newline\_in\_field | enum | Newline character in each field (CRLF, LF, CR) | ``LF`` by default |
488
- +----------------------+---------+-------------------------------------------------------------------------------------------------------+-------------------------------+
489
- | charset | enum | Character encoding (eg. ISO-8859-1, UTF-8) | ``UTF-8`` by default |
490
- +----------------------+---------+-------------------------------------------------------------------------------------------------------+-------------------------------+
491
- | default\_timezone | string | Time zone of timestamp columns. This can be overwritten for each column using ``column_options`` | ``UTC`` by default |
492
- +----------------------+---------+-------------------------------------------------------------------------------------------------------+-------------------------------+
493
- | column\_options | hash | See bellow | optional |
494
- +----------------------+---------+-------------------------------------------------------------------------------------------------------+-------------------------------+
495
-
496
- (\*1): if quote\_policy is NONE, ``quote`` option is ignored, and default ``escape`` is ``\``.
497
-
498
- The ``quote_policy`` option is used to determine field type to quote.
499
-
500
- +------------+--------------------------------------------------------------------------------------------------------+
501
- | name | description |
502
- +============+========================================================================================================+
503
- | ALL | Quote all fields |
504
- +------------+--------------------------------------------------------------------------------------------------------+
505
- | MINIMAL | Only quote those fields which contain delimiter, quote or any of the characters in lineterminator |
506
- +------------+--------------------------------------------------------------------------------------------------------+
507
- | NONE | Never quote fields. When the delimiter occurs in field, escape with escape char |
508
- +------------+--------------------------------------------------------------------------------------------------------+
509
-
510
- The ``column_options`` option is a map whose keys are name of columns, and values are configuration with following parameters:
511
-
512
- +----------------------+---------+-------------------------------------------------------------------------------------------------------+-----------------------------------------+
513
- | name | type | description | required? |
514
- +======================+=========+=======================================================================================================+=========================================+
515
- | timezone | string | Time zone if type of this column is timestamp. If not set, ``default\_timezone`` is used. | optional |
516
- +----------------------+---------+-------------------------------------------------------------------------------------------------------+-----------------------------------------+
517
- | format | string | Timestamp format if type of this column is timestamp. | ``%Y-%m-%d %H:%M:%S.%6N %z`` by default |
518
- +----------------------+---------+-------------------------------------------------------------------------------------------------------+-----------------------------------------+
519
-
520
- .. note::
521
-
522
- The Timestamp format refers to `Ruby strftime format <https://docs.ruby-lang.org/en/2.4.0/Date.html#method-i-strftime>`_
523
-
524
- Example
525
- ~~~~~~~~
526
-
527
- .. code-block:: yaml
528
-
529
- out:
530
- ...
531
- formatter:
532
- type: csv
533
- delimiter: "\t"
534
- newline: CRLF
535
- newline_in_field: LF
536
- charset: UTF-8
537
- quote_policy: MINIMAL
538
- quote: '"'
539
- escape: "\\"
540
- null_string: "\\N"
541
- default_timezone: 'UTC'
542
- column_options:
543
- mycol1: {format: '%Y-%m-%d %H:%M:%S'}
544
- mycol2: {format: '%Y-%m-%d %H:%M:%S', timezone: 'America/Los_Angeles'}
545
-
546
- Gzip encoder plugin
547
- --------------------
548
-
549
- The ``gzip`` encoder plugin compresses output files using gzip.
550
-
551
- Options
552
- ~~~~~~~~
553
-
554
- +---------+----------+----------------------------------------------------------------------+--------------------+
555
- | name | type | description | required? |
556
- +=========+==========+======================================================================+====================+
557
- | level | integer | Compression level. From 0 (no compression) to 9 (best compression). | ``6`` by default |
558
- +---------+----------+----------------------------------------------------------------------+--------------------+
559
-
560
- Example
561
- ~~~~~~~~
562
-
563
- .. code-block:: yaml
564
-
565
- out:
566
- ...
567
- encoders:
568
- - type: gzip
569
- level: 1
570
-
571
-
572
- BZip2 encoder plugin
573
- ---------------------
574
-
575
- The ``bzip2`` encoder plugin compresses output files using bzip2.
576
-
577
- Options
578
- ~~~~~~~~
579
-
580
- +---------+----------+----------------------------------------------------------------------+--------------------+
581
- | name | type | description | required? |
582
- +=========+==========+======================================================================+====================+
583
- | level | integer | Compression level. From 1 to 9 (best compression). | ``9`` by default |
584
- +---------+----------+----------------------------------------------------------------------+--------------------+
585
-
586
- Example
587
- ~~~~~~~~
588
-
589
- .. code-block:: yaml
590
-
591
- out:
592
- ...
593
- encoders:
594
- - type: bzip2
595
- level: 6
596
-
597
-
598
- Rename filter plugin
599
- ---------------------
600
-
601
- The ``rename`` filter plugin changes column names. This plugin has no impact on performance.
602
-
603
- Options
604
- ~~~~~~~~
605
-
606
- +---------+----------+----------------------------------------------------------------------+--------------------+
607
- | name | type | description | required? |
608
- +=========+==========+======================================================================+====================+
609
- | rules | array | An array of rule-based renaming operations. (See below for rules.) | ``[]`` by default |
610
- +---------+----------+----------------------------------------------------------------------+--------------------+
611
- | columns | hash | A map whose keys are existing column names. values are new names. | ``{}`` by default |
612
- +---------+----------+----------------------------------------------------------------------+--------------------+
613
-
614
- Renaming rules
615
- ~~~~~~~~~~~~~~~
616
-
617
- The ``rules`` is an array of rules as below applied top-down for all the columns.
618
-
619
- +-------------------------+----------------------------------------------------------------------------------------+
620
- | rule | description |
621
- +=========================+========================================================================================+
622
- | character\_types | Restrict characters by types. Replace restricted characteres. |
623
- +-------------------------+----------------------------------------------------------------------------------------+
624
- | first\_character\_types | Restrict the first character by types. Prefix or replace first restricted characters. |
625
- +-------------------------+----------------------------------------------------------------------------------------+
626
- | lower\_to\_upper | Convert lower-case alphabets to upper-case. |
627
- +-------------------------+----------------------------------------------------------------------------------------+
628
- | regex\_replace | Replace with a regular expressions. |
629
- +-------------------------+----------------------------------------------------------------------------------------+
630
- | truncate | Truncate. |
631
- +-------------------------+----------------------------------------------------------------------------------------+
632
- | upper\_to\_lower | Convert upper-case alphabets to lower-case |
633
- +-------------------------+----------------------------------------------------------------------------------------+
634
- | unique\_number\_suffix | Make column names unique in the schema. |
635
- +-------------------------+----------------------------------------------------------------------------------------+
636
-
637
- Renaming rule: character\_types
638
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
639
-
640
- The rule ``character_types`` replaces restricted characters.
641
-
642
- +-------------------+--------------------------------------------------------------------------------------------------------------------------------------------+--------------------+
643
- | option | description | required? |
644
- +===================+============================================================================================================================================+====================+
645
- | pass\_characteres | Characters to be allowed. | ``""`` by default |
646
- +-------------------+--------------------------------------------------------------------------------------------------------------------------------------------+--------------------+
647
- | pass\_types | Sets of characters to be allowed. The array must consist of "a-z" (lower-case alphabets), "A-Z" (upper-case alphabets), or "0-9" (digits). | ``[]`` by default |
648
- +-------------------+--------------------------------------------------------------------------------------------------------------------------------------------+--------------------+
649
- | replace | A character that disallowed characters are replaced with. It must consist of just 1 character. | ``"_"`` by default |
650
- +-------------------+--------------------------------------------------------------------------------------------------------------------------------------------+--------------------+
651
-
652
- Example
653
- """"""""
654
-
655
- .. code-block:: yaml
656
-
657
- # This configuration replaces characters into "_" except for "_", lower-case alphabets, and digits.
658
- filters:
659
- ...
660
- - type: rename
661
- rules:
662
- - rule: character_types
663
- pass_characters: "_"
664
- pass_types: [ "a-z", "0-9" ]
665
-
666
-
667
- Renaming rule: first\_character\_types
668
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
669
-
670
- The rule ``first_character_types`` prefixes or replaces a restricted character at the beginning.
671
-
672
- +-------------------+--------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------+
673
- | option | description | required? |
674
- +===================+============================================================================================================================================+==============================================+
675
- | pass\_characteres | Characters to be allowed. | ``""`` by default |
676
- +-------------------+--------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------+
677
- | pass\_types | Sets of characters to be allowed. The array must consist of "a-z" (lower-case alphabets), "A-Z" (upper-case alphabets), or "0-9" (digits). | ``[]`` by default |
678
- +-------------------+--------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------+
679
- | prefix | A character that a disallowed first character is replaced with. | one of ``prefix`` or ``replace`` is required |
680
- +-------------------+--------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------+
681
- | replace | A character that a disallowed first character is prefixed with. | one of ``prefix`` or ``replace`` is required |
682
- +-------------------+--------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------+
683
-
684
- Example
685
- """"""""
686
-
687
- .. code-block:: yaml
688
-
689
- # This configuration prefixes a column name with "_" unless the name starts from "_" or a lower-case alphabet.
690
- filters:
691
- ...
692
- - type: rename
693
- rules:
694
- - rule: first_character_types
695
- pass_characters: "_"
696
- pass_types: [ "a-z" ]
697
- prefix: "_"
698
-
699
- Renaming rule: lower\_to\_upper
700
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
701
-
702
- The rule ``lower_to_upper`` converts lower-case alphabets to upper-case.
703
-
704
- Example
705
- """"""""
706
-
707
- .. code-block:: yaml
708
-
709
- # This configuration converts all lower-case alphabets to upper-case.
710
- filters:
711
- ...
712
- - type: rename
713
- rules:
714
- - rule: lower_to_upper
715
-
716
-
717
- Renaming rule: regex\_replace
718
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
719
-
720
- The rule ``regex_replace`` replaces column names based on a regular expression.
721
-
722
- +---------+--------------------------------------------------------------------------------------------------------------------------------------+-----------+
723
- | option | description | required? |
724
- +=========+======================================================================================================================================+===========+
725
- | match | A `Java-style regular expression <https://docs.oracle.com/javase/tutorial/essential/regex/>`_ to which this string is to be matched. | required |
726
- +---------+--------------------------------------------------------------------------------------------------------------------------------------+-----------+
727
- | replace | A string to be substibuted for each match in Java-style. | required |
728
- +---------+--------------------------------------------------------------------------------------------------------------------------------------+-----------+
729
-
730
- Example
731
- """"""""
732
-
733
- .. code-block:: yaml
734
-
735
- # This configuration replaces all patterns
736
- filters:
737
- ...
738
- - type: rename
739
- rules:
740
- - rule: regex_replace
741
- match: "([0-9]+)_dollars"
742
- replace: "USD$1"
743
-
744
-
745
- Renaming rule: truncate
746
- ^^^^^^^^^^^^^^^^^^^^^^^^
747
-
748
- The rule ``truncate`` truncates column names.
749
-
750
- +------------+-----------------------------------------------------+--------------------+
751
- | option | description | required? |
752
- +============+=====================================================+====================+
753
- | max_length | The length to which the column names are truncated. | ``128`` by default |
754
- +------------+-----------------------------------------------------+--------------------+
755
-
756
- Example
757
- """"""""
758
-
759
- .. code-block:: yaml
760
-
761
- # This configuration drops all characters after the 20th character.
762
- filters:
763
- ...
764
- - type: rename
765
- rules:
766
- - rule: truncate
767
- max_length: 20
768
-
769
- Renaming rule: upper\_to\_lower
770
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
771
-
772
- The rule ``upper_to_lower`` converts upper-case alphabets to lower-case.
773
-
774
- Example
775
- """"""""
776
-
777
- .. code-block:: yaml
778
-
779
- # This configuration converts all upper-case alphabets to lower-case.
780
- filters:
781
- ...
782
- - type: rename
783
- rules:
784
- - rule: upper_to_lower
785
-
786
- Renaming rule: unique\_number\_suffix
787
- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
788
-
789
- The rule ``unique_number_suffix`` makes column names unique in the schema by suffixing numbers.
790
-
791
- +------------+-----------------------------------------------------------------------------------------------------------------------------+--------------------+
792
- | option | description | required? |
793
- +============+=============================================================================================================================+====================+
794
- | delimiter | A delimiter character inserted before a suffix number. It must be just 1 non-digit character. | ``"_"`` by default |
795
- +------------+-----------------------------------------------------------------------------------------------------------------------------+--------------------+
796
- | digits | An integer that specifies the number of zero-filled digits of a suffix number. The suffix number zero-filled to the digits. | optional |
797
- +------------+-----------------------------------------------------------------------------------------------------------------------------+--------------------+
798
- | max_length | The length to which the column names are truncated. The column name is truncated before the suffix number. | optional |
799
- +------------+-----------------------------------------------------------------------------------------------------------------------------+--------------------+
800
- | offset | An integer where the suffix number starts. The first duplicative column name is suffixed by (```offset``` + 1). | ``1`` by default |
801
- +------------+-----------------------------------------------------------------------------------------------------------------------------+--------------------+
802
-
803
- .. hint::
804
- The procedure to make column names unique is not very trivial. There are many feasible ways. This renaming rule works as follows:
805
-
806
- Basic policies:
807
-
808
- * Suffix numbers are counted per original column name.
809
- * Column names are fixed from the first column to the last column.
810
-
811
- Actual procedure applied from the first (leftmost) column to the last (rightmost) column:
812
-
813
- 1. Fix the column name as-is with truncating if the truncated name is not duplicated with left columns.
814
- 2. Suffix the column name otherwise.
815
-
816
- a. Try to append the suffix number for the original column name with truncating.
817
- b. Fix it if the suffixed name is not duplicated with left columns nor original columns.
818
- c. Retry (a) with the suffix number increased otherwise.
819
-
820
- Example
821
- """"""""
822
-
823
- .. code-block:: yaml
824
-
825
- # This configuration suffixes numbers to duplicative column names. (Ex. ["column", "column", "column"] goes to ["column", "column_2", "column_3"].)
826
- filters:
827
- ...
828
- - type: rename
829
- rules:
830
- - rule: unique_number_suffix
831
-
832
- Example of renaming rules
833
- ~~~~~~~~~~~~~~~~~~~~~~~~~~
834
-
835
- .. code-block:: yaml
836
-
837
- filters:
838
- ...
839
- - type: rename
840
- rules:
841
- - rule: upper_to_lower # All upper-case are converted to lower-case.
842
- - rule: character_types # Only lower-case, digits and "_" are allowed. (No upper-case by the rule ahove.)
843
- pass_types: [ "a-z", "0-9" ]
844
- pass_characters: "_"
845
- - rule: unique_number_suffix # Ensure all column names are unique.
846
-
847
- Columns: not recommended
848
- ~~~~~~~~~~~~~~~~~~~~~~~~~
849
-
850
- ``columns`` is not recommended to use anymore. Consider using ``rules`` instead.
851
-
852
- .. code-block:: yaml
853
-
854
- filters:
855
- ...
856
- - type: rename
857
- columns:
858
- my_existing_column1: new_column1
859
- my_existing_column2: new_column2
860
-
861
- .. hint::
862
- ``columns`` are applied before ``rules`` if ``columns`` and ``rules`` are specified together. (It is discouraged to specify them together, though.)
863
-
864
-
865
- Remove columns filter plugin
866
- -----------------------------
867
-
868
- The ``remove_columns`` filter plugin removes columns from schema.
869
-
870
- Options
871
- ~~~~~~~~
872
-
873
- +--------------------------+----------+------------------------------------------------------------+-----------------------+
874
- | name | type | description | required? |
875
- +==========================+==========+============================================================+=======================+
876
- | remove | array | An array of names of columns that it removes from schema. | ``[]`` by default |
877
- +--------------------------+----------+------------------------------------------------------------+-----------------------+
878
- | keep | array | An array of names of columns that it keeps in schema. | ``[]`` by default |
879
- +--------------------------+----------+------------------------------------------------------------+-----------------------+
880
- | accept_unmatched_columns | boolean | If true, skip columns that aren't included in schemas. | ``false`` by default |
881
- +--------------------------+----------+------------------------------------------------------------+-----------------------+
882
-
883
-
884
- remove: and keep: options are not multi-select.
885
-
886
- Example
887
- ~~~~~~~~
888
-
889
- .. code-block:: yaml
890
-
891
- # This configuration removes "_c0" and "_c1" named columns from schema.
892
- filters:
893
- ...
894
- - type: remove_columns
895
- remove: ["_c0", "_c1"]
896
-
897
-
898
- Local executor plugin
899
- ----------------------
900
-
901
- The ``local`` executor plugin runs tasks using local threads. This is the only built-in executor plugin.
902
-
903
- Options
904
- ~~~~~~~~
905
-
906
- +------------------+----------+----------------------------------------------------------------------+--------------------------------------+
907
- | name | type | description | required? |
908
- +==================+==========+======================================================================+======================================+
909
- | max_threads | integer | Maximum number of threads to run concurrently. | 2x of available CPU cores by default |
910
- +------------------+----------+----------------------------------------------------------------------+--------------------------------------+
911
- | min_output_tasks | integer | Mimimum number of output tasks to enable page scattering. | 1x of available CPU cores by default |
912
- +------------------+----------+----------------------------------------------------------------------+--------------------------------------+
913
-
914
-
915
- The ``max_threads`` option controls maximum concurrency. Setting smaller number here is useful if too many threads make the destination or source storage overloaded. Setting larger number here is useful if CPU utilization is too low due to high latency.
916
-
917
- The ``min_output_tasks`` option enables "page scattering". The feature is enabled if number of input tasks is less than ``min_output_tasks``. It uses multiple filter & output threads for each input task so that one input task can use multiple threads. Setting larger number here is useful if embulk doesn't use multi-threading with enough concurrency due to too few number of input tasks. Setting 1 here disables page scattering completely.
918
-
919
- Example
920
- ~~~~~~~~
921
-
922
- .. code-block:: yaml
923
-
924
- exec:
925
- max_threads: 8 # run at most 8 tasks concurrently
926
- min_output_tasks: 1 # disable page scattering
927
- in:
928
- type: ...
929
- ...
930
- out:
931
- type: ...
932
- ...
933
-
934
- Guess executor
935
- ---------------
936
-
937
- The guess executor is called by ``guess`` command. It executes default guess plugins in a sequential order and suggests Embulk config by appropriate guess plugin. The default guess plugins and the order are ``gzip``, ``'bzip2``, ``json`` and ``csv``.
938
-
939
- Options
940
- ~~~~~~~~
941
-
942
- +---------------------------+----------+----------------------------------------------------------------------+--------------------------------------+
943
- | name | type | description | required? |
944
- +===========================+==========+======================================================================+======================================+
945
- | guess_plugins | array | ``guess`` command uses specified guess plugins. | ``[]`` by default |
946
- +---------------------------+----------+----------------------------------------------------------------------+--------------------------------------+
947
- | exclude_guess_plugins | array | ``guess`` command doesn't use specified plugins. | ``[]`` by default |
948
- +---------------------------+----------+----------------------------------------------------------------------+--------------------------------------+
949
- | guess_sample_buffer_bytes | int | Bytes of sample buffer that it tries to read from input source. | 32768 (32KB) by default |
950
- +-------------------------------+----------+----------------------------------------------------------------------+----------------------------------+
951
-
952
- The ``guess_plugins`` option includes specified guess plugin in the bottom of the list of default guess plugins.
953
-
954
- The ``exclude_guess_plugins`` option exclude specified guess plugins from the list of default guess plugins that the guess executor uses.
955
-
956
- The ``guess_sample_buffer_bytes`` option controls the bytes of sample buffer that GuessExecutor tries to read from specified input source.
957
-
958
- This example shows how to use ``csv_all_strings`` guess plugin, which suggests column types within CSV files as string types. It needs to be explicitly specified by users when it's used instead of ``csv`` guess plugin because the plugin is not included in default guess plugins. We also can exclude default ``csv`` guess plugin.
959
-
960
- Example
961
- ~~~~~~~~
962
-
963
- .. code-block:: yaml
964
-
965
- exec:
966
- guess_plugins: ['csv_all_strings']
967
- exclude_guess_plugins: ['csv']
968
- in:
969
- type: ...
970
- ...
971
- out:
972
- type: ...
973
- ...
974
-
975
- Preview executor
976
- ----------------
977
-
978
- The preview executor is called by ``preview`` command. It tries to read sample buffer from a specified input source and writes them to Page objects. ``preview`` outputs the Page objects to console.
979
-
980
- Options
981
- ~~~~~~~~
982
-
983
- +-------------------------------+----------+----------------------------------------------------------------------+--------------------------------------+
984
- | name | type | description | required? |
985
- +===============================+==========+======================================================================+======================================+
986
- | preview_sample_buffer_bytes | int | Bytes of sample buffer that it tries to read from input source. | 32768 (32KB) by default |
987
- +-------------------------------+----------+----------------------------------------------------------------------+--------------------------------------+
988
-
989
- The ``preview_sample_buffer_bytes`` option controls the bytes of sample buffer that PreviewExecutor tries to read from specified input source.
990
-
991
- This example shows how to change the bytes of sample buffer.
992
-
993
- Example
994
- ~~~~~~~~
995
-
996
- .. code-block:: yaml
997
-
998
- exec:
999
- preview_sample_buffer_bytes: 65536 # 64KB
1000
- in:
1001
- type: ...
1002
- ...
1003
- out:
1004
- type: ...
1005
- ...