embulk 0.7.0-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +15 -0
- data/.ruby-version +1 -0
- data/.travis.yml +18 -0
- data/COPYING +14 -0
- data/Gemfile +2 -0
- data/Gemfile.lock +31 -0
- data/README.md +206 -0
- data/Rakefile +26 -0
- data/appveyor.yml +20 -0
- data/bin/embulk +106 -0
- data/build.gradle +338 -0
- data/embulk-cli/build.gradle +6 -0
- data/embulk-cli/src/main/java/org/embulk/cli/Main.java +22 -0
- data/embulk-cli/src/main/sh/selfrun.sh +158 -0
- data/embulk-cli/src/test/java/org/embulk/cli/DummyMain.java +23 -0
- data/embulk-cli/src/test/java/org/embulk/cli/SelfrunTest.java +281 -0
- data/embulk-core/build.gradle +59 -0
- data/embulk-core/src/main/java/org/embulk/EmbulkEmbed.java +315 -0
- data/embulk-core/src/main/java/org/embulk/EmbulkService.java +76 -0
- data/embulk-core/src/main/java/org/embulk/command/PreviewPrinter.java +84 -0
- data/embulk-core/src/main/java/org/embulk/command/TablePreviewPrinter.java +107 -0
- data/embulk-core/src/main/java/org/embulk/command/VerticalPreviewPrinter.java +47 -0
- data/embulk-core/src/main/java/org/embulk/config/CommitReport.java +33 -0
- data/embulk-core/src/main/java/org/embulk/config/Config.java +15 -0
- data/embulk-core/src/main/java/org/embulk/config/ConfigDefault.java +15 -0
- data/embulk-core/src/main/java/org/embulk/config/ConfigDiff.java +29 -0
- data/embulk-core/src/main/java/org/embulk/config/ConfigException.java +20 -0
- data/embulk-core/src/main/java/org/embulk/config/ConfigInject.java +14 -0
- data/embulk-core/src/main/java/org/embulk/config/ConfigLoader.java +141 -0
- data/embulk-core/src/main/java/org/embulk/config/ConfigSource.java +31 -0
- data/embulk-core/src/main/java/org/embulk/config/DataSource.java +39 -0
- data/embulk-core/src/main/java/org/embulk/config/DataSourceImpl.java +231 -0
- data/embulk-core/src/main/java/org/embulk/config/DataSourceSerDe.java +84 -0
- data/embulk-core/src/main/java/org/embulk/config/GenericTypeReference.java +20 -0
- data/embulk-core/src/main/java/org/embulk/config/ModelManager.java +123 -0
- data/embulk-core/src/main/java/org/embulk/config/Task.java +10 -0
- data/embulk-core/src/main/java/org/embulk/config/TaskInvocationHandler.java +180 -0
- data/embulk-core/src/main/java/org/embulk/config/TaskReport.java +29 -0
- data/embulk-core/src/main/java/org/embulk/config/TaskSerDe.java +345 -0
- data/embulk-core/src/main/java/org/embulk/config/TaskSource.java +31 -0
- data/embulk-core/src/main/java/org/embulk/config/TaskValidationException.java +38 -0
- data/embulk-core/src/main/java/org/embulk/config/TaskValidator.java +24 -0
- data/embulk-core/src/main/java/org/embulk/exec/BulkLoader.java +652 -0
- data/embulk-core/src/main/java/org/embulk/exec/ExecModule.java +52 -0
- data/embulk-core/src/main/java/org/embulk/exec/ExecutionInterruptedException.java +10 -0
- data/embulk-core/src/main/java/org/embulk/exec/ExecutionResult.java +26 -0
- data/embulk-core/src/main/java/org/embulk/exec/ExtensionServiceLoaderModule.java +43 -0
- data/embulk-core/src/main/java/org/embulk/exec/ForGuess.java +16 -0
- data/embulk-core/src/main/java/org/embulk/exec/ForSystemConfig.java +16 -0
- data/embulk-core/src/main/java/org/embulk/exec/GuessExecutor.java +373 -0
- data/embulk-core/src/main/java/org/embulk/exec/LocalExecutorPlugin.java +129 -0
- data/embulk-core/src/main/java/org/embulk/exec/LocalThreadExecutor.java +34 -0
- data/embulk-core/src/main/java/org/embulk/exec/LoggerProvider.java +60 -0
- data/embulk-core/src/main/java/org/embulk/exec/NoSampleException.java +10 -0
- data/embulk-core/src/main/java/org/embulk/exec/PartialExecutionException.java +18 -0
- data/embulk-core/src/main/java/org/embulk/exec/PooledBufferAllocator.java +77 -0
- data/embulk-core/src/main/java/org/embulk/exec/PreviewExecutor.java +183 -0
- data/embulk-core/src/main/java/org/embulk/exec/PreviewResult.java +27 -0
- data/embulk-core/src/main/java/org/embulk/exec/PreviewedNoticeError.java +17 -0
- data/embulk-core/src/main/java/org/embulk/exec/ResumeState.java +100 -0
- data/embulk-core/src/main/java/org/embulk/exec/SamplingParserPlugin.java +136 -0
- data/embulk-core/src/main/java/org/embulk/exec/SetCurrentThreadName.java +19 -0
- data/embulk-core/src/main/java/org/embulk/exec/SystemConfigModule.java +24 -0
- data/embulk-core/src/main/java/org/embulk/exec/TempFileAllocator.java +35 -0
- data/embulk-core/src/main/java/org/embulk/guice/Bootstrap.java +157 -0
- data/embulk-core/src/main/java/org/embulk/guice/CloseableInjector.java +22 -0
- data/embulk-core/src/main/java/org/embulk/guice/InjectorProxy.java +145 -0
- data/embulk-core/src/main/java/org/embulk/guice/LifeCycleInjector.java +26 -0
- data/embulk-core/src/main/java/org/embulk/guice/LifeCycleInjectorProxy.java +61 -0
- data/embulk-core/src/main/java/org/embulk/guice/LifeCycleManager.java +187 -0
- data/embulk-core/src/main/java/org/embulk/guice/LifeCycleMethods.java +89 -0
- data/embulk-core/src/main/java/org/embulk/guice/LifeCycleMethodsMap.java +38 -0
- data/embulk-core/src/main/java/org/embulk/guice/LifeCycleModule.java +97 -0
- data/embulk-core/src/main/java/org/embulk/jruby/JRubyPluginSource.java +72 -0
- data/embulk-core/src/main/java/org/embulk/jruby/JRubyScriptingModule.java +119 -0
- data/embulk-core/src/main/java/org/embulk/plugin/BuiltinPluginSourceModule.java +17 -0
- data/embulk-core/src/main/java/org/embulk/plugin/InjectedPluginSource.java +96 -0
- data/embulk-core/src/main/java/org/embulk/plugin/PluginClassLoader.java +168 -0
- data/embulk-core/src/main/java/org/embulk/plugin/PluginClassLoaderFactory.java +9 -0
- data/embulk-core/src/main/java/org/embulk/plugin/PluginClassLoaderModule.java +71 -0
- data/embulk-core/src/main/java/org/embulk/plugin/PluginManager.java +78 -0
- data/embulk-core/src/main/java/org/embulk/plugin/PluginSource.java +6 -0
- data/embulk-core/src/main/java/org/embulk/plugin/PluginSourceNotMatchException.java +19 -0
- data/embulk-core/src/main/java/org/embulk/plugin/PluginType.java +47 -0
- data/embulk-core/src/main/java/org/embulk/plugin/compat/InputPluginWrapper.java +102 -0
- data/embulk-core/src/main/java/org/embulk/plugin/compat/PluginWrappers.java +30 -0
- data/embulk-core/src/main/java/org/embulk/plugin/compat/TransactionalFileInputWrapper.java +96 -0
- data/embulk-core/src/main/java/org/embulk/plugin/compat/TransactionalFileOutputWrapper.java +102 -0
- data/embulk-core/src/main/java/org/embulk/plugin/compat/TransactionalPageOutputWrapper.java +95 -0
- data/embulk-core/src/main/java/org/embulk/spi/Buffer.java +148 -0
- data/embulk-core/src/main/java/org/embulk/spi/BufferAllocator.java +8 -0
- data/embulk-core/src/main/java/org/embulk/spi/Column.java +92 -0
- data/embulk-core/src/main/java/org/embulk/spi/ColumnConfig.java +112 -0
- data/embulk-core/src/main/java/org/embulk/spi/ColumnVisitor.java +14 -0
- data/embulk-core/src/main/java/org/embulk/spi/DecoderPlugin.java +16 -0
- data/embulk-core/src/main/java/org/embulk/spi/EncoderPlugin.java +16 -0
- data/embulk-core/src/main/java/org/embulk/spi/Exec.java +113 -0
- data/embulk-core/src/main/java/org/embulk/spi/ExecAction.java +6 -0
- data/embulk-core/src/main/java/org/embulk/spi/ExecSession.java +217 -0
- data/embulk-core/src/main/java/org/embulk/spi/ExecutorPlugin.java +19 -0
- data/embulk-core/src/main/java/org/embulk/spi/Extension.java +44 -0
- data/embulk-core/src/main/java/org/embulk/spi/FileInput.java +11 -0
- data/embulk-core/src/main/java/org/embulk/spi/FileInputPlugin.java +30 -0
- data/embulk-core/src/main/java/org/embulk/spi/FileInputRunner.java +162 -0
- data/embulk-core/src/main/java/org/embulk/spi/FileOutput.java +13 -0
- data/embulk-core/src/main/java/org/embulk/spi/FileOutputPlugin.java +28 -0
- data/embulk-core/src/main/java/org/embulk/spi/FileOutputRunner.java +202 -0
- data/embulk-core/src/main/java/org/embulk/spi/FilterPlugin.java +18 -0
- data/embulk-core/src/main/java/org/embulk/spi/FormatterPlugin.java +18 -0
- data/embulk-core/src/main/java/org/embulk/spi/GuessPlugin.java +9 -0
- data/embulk-core/src/main/java/org/embulk/spi/InputPlugin.java +33 -0
- data/embulk-core/src/main/java/org/embulk/spi/OutputPlugin.java +29 -0
- data/embulk-core/src/main/java/org/embulk/spi/Page.java +51 -0
- data/embulk-core/src/main/java/org/embulk/spi/PageBuilder.java +338 -0
- data/embulk-core/src/main/java/org/embulk/spi/PageFormat.java +47 -0
- data/embulk-core/src/main/java/org/embulk/spi/PageOutput.java +11 -0
- data/embulk-core/src/main/java/org/embulk/spi/PageReader.java +226 -0
- data/embulk-core/src/main/java/org/embulk/spi/ParserPlugin.java +17 -0
- data/embulk-core/src/main/java/org/embulk/spi/ProcessState.java +10 -0
- data/embulk-core/src/main/java/org/embulk/spi/ProcessTask.java +117 -0
- data/embulk-core/src/main/java/org/embulk/spi/Schema.java +134 -0
- data/embulk-core/src/main/java/org/embulk/spi/SchemaConfig.java +93 -0
- data/embulk-core/src/main/java/org/embulk/spi/SchemaConfigException.java +22 -0
- data/embulk-core/src/main/java/org/embulk/spi/TaskState.java +81 -0
- data/embulk-core/src/main/java/org/embulk/spi/TempFileException.java +19 -0
- data/embulk-core/src/main/java/org/embulk/spi/TempFileSpace.java +87 -0
- data/embulk-core/src/main/java/org/embulk/spi/Transactional.java +10 -0
- data/embulk-core/src/main/java/org/embulk/spi/TransactionalFileInput.java +17 -0
- data/embulk-core/src/main/java/org/embulk/spi/TransactionalFileOutput.java +19 -0
- data/embulk-core/src/main/java/org/embulk/spi/TransactionalPageOutput.java +17 -0
- data/embulk-core/src/main/java/org/embulk/spi/time/DateTimeZoneSerDe.java +55 -0
- data/embulk-core/src/main/java/org/embulk/spi/time/JRubyTimeParserHelper.java +8 -0
- data/embulk-core/src/main/java/org/embulk/spi/time/JRubyTimeParserHelperFactory.java +6 -0
- data/embulk-core/src/main/java/org/embulk/spi/time/Timestamp.java +159 -0
- data/embulk-core/src/main/java/org/embulk/spi/time/TimestampFormat.java +100 -0
- data/embulk-core/src/main/java/org/embulk/spi/time/TimestampFormatter.java +97 -0
- data/embulk-core/src/main/java/org/embulk/spi/time/TimestampParseException.java +10 -0
- data/embulk-core/src/main/java/org/embulk/spi/time/TimestampParser.java +104 -0
- data/embulk-core/src/main/java/org/embulk/spi/time/TimestampSerDe.java +49 -0
- data/embulk-core/src/main/java/org/embulk/spi/type/AbstractType.java +58 -0
- data/embulk-core/src/main/java/org/embulk/spi/type/BooleanType.java +12 -0
- data/embulk-core/src/main/java/org/embulk/spi/type/DoubleType.java +12 -0
- data/embulk-core/src/main/java/org/embulk/spi/type/LongType.java +12 -0
- data/embulk-core/src/main/java/org/embulk/spi/type/StringType.java +12 -0
- data/embulk-core/src/main/java/org/embulk/spi/type/TimestampType.java +41 -0
- data/embulk-core/src/main/java/org/embulk/spi/type/Type.java +15 -0
- data/embulk-core/src/main/java/org/embulk/spi/type/TypeDeserializer.java +44 -0
- data/embulk-core/src/main/java/org/embulk/spi/type/Types.java +14 -0
- data/embulk-core/src/main/java/org/embulk/spi/unit/ByteSize.java +156 -0
- data/embulk-core/src/main/java/org/embulk/spi/unit/LocalFile.java +106 -0
- data/embulk-core/src/main/java/org/embulk/spi/unit/LocalFileSerDe.java +113 -0
- data/embulk-core/src/main/java/org/embulk/spi/unit/ToString.java +54 -0
- data/embulk-core/src/main/java/org/embulk/spi/unit/ToStringMap.java +34 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/CharsetSerDe.java +55 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/Decoders.java +81 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/DynamicColumnNotFoundException.java +10 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/DynamicColumnSetter.java +18 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/DynamicColumnSetterFactory.java +94 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/DynamicPageBuilder.java +161 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/Encoders.java +81 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/Executors.java +95 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/FileInputInputStream.java +111 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/FileOutputOutputStream.java +119 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/Filters.java +100 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/InputStreamFileInput.java +190 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/InputStreamTransactionalFileInput.java +25 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/Inputs.java +65 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/LineDecoder.java +118 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/LineEncoder.java +123 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/ListFileInput.java +52 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/Newline.java +38 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/OutputStreamFileOutput.java +88 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/PagePrinter.java +102 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/Pages.java +139 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/ResumableInputStream.java +128 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/RetryExecutor.java +130 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/Timestamps.java +53 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/AbstractDynamicColumnSetter.java +79 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/BooleanColumnSetter.java +64 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/DefaultValueSetter.java +18 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/DoubleColumnSetter.java +61 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/LongColumnSetter.java +69 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/NullDefaultValueSetter.java +34 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/SkipColumnSetter.java +52 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/StringColumnSetter.java +56 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/TimestampColumnSetter.java +64 -0
- data/embulk-core/src/main/resources/embulk/logback-color.xml +72 -0
- data/embulk-core/src/main/resources/embulk/logback-console.xml +14 -0
- data/embulk-core/src/main/resources/embulk/logback-file.xml +25 -0
- data/embulk-core/src/main/resources/embulk/parent_first_packages.properties +70 -0
- data/embulk-core/src/main/resources/embulk/parent_first_resources.properties +28 -0
- data/embulk-core/src/test/java/org/embulk/EmbulkTestRuntime.java +114 -0
- data/embulk-core/src/test/java/org/embulk/GuiceBinder.java +72 -0
- data/embulk-core/src/test/java/org/embulk/RandomManager.java +53 -0
- data/embulk-core/src/test/java/org/embulk/TestPluginSourceModule.java +23 -0
- data/embulk-core/src/test/java/org/embulk/TestUtilityModule.java +17 -0
- data/embulk-core/src/test/java/org/embulk/config/TestConfigLoader.java +66 -0
- data/embulk-core/src/test/java/org/embulk/config/TestConfigSource.java +114 -0
- data/embulk-core/src/test/java/org/embulk/config/TestTaskSource.java +70 -0
- data/embulk-core/src/test/java/org/embulk/plugin/MockPluginSource.java +58 -0
- data/embulk-core/src/test/java/org/embulk/plugin/TestPluginType.java +18 -0
- data/embulk-core/src/test/java/org/embulk/spi/MockFileOutput.java +63 -0
- data/embulk-core/src/test/java/org/embulk/spi/MockFormatterPlugin.java +101 -0
- data/embulk-core/src/test/java/org/embulk/spi/MockParserPlugin.java +73 -0
- data/embulk-core/src/test/java/org/embulk/spi/PageTestUtils.java +57 -0
- data/embulk-core/src/test/java/org/embulk/spi/TestBuffer.java +24 -0
- data/embulk-core/src/test/java/org/embulk/spi/TestFileInputInputStream.java +89 -0
- data/embulk-core/src/test/java/org/embulk/spi/TestFileInputRunner.java +196 -0
- data/embulk-core/src/test/java/org/embulk/spi/TestFileOutputRunner.java +207 -0
- data/embulk-core/src/test/java/org/embulk/spi/TestInputStreamFileInput.java +188 -0
- data/embulk-core/src/test/java/org/embulk/spi/TestPageBuilderReader.java +319 -0
- data/embulk-core/src/test/java/org/embulk/spi/time/TestTimestamp.java +116 -0
- data/embulk-core/src/test/java/org/embulk/spi/time/TestTimestampFormatterParser.java +63 -0
- data/embulk-core/src/test/java/org/embulk/spi/time/TestTimestampFormatterParserDeprecated.java +67 -0
- data/embulk-core/src/test/java/org/embulk/spi/type/TestTypeSerDe.java +45 -0
- data/embulk-core/src/test/java/org/embulk/spi/unit/TestByteSize.java +79 -0
- data/embulk-core/src/test/java/org/embulk/spi/util/TestLineDecoder.java +132 -0
- data/embulk-core/src/test/java/org/embulk/spi/util/TestLineEncoder.java +123 -0
- data/embulk-docs/Makefile +178 -0
- data/embulk-docs/build.gradle +32 -0
- data/embulk-docs/make.bat +243 -0
- data/embulk-docs/push-gh-pages.sh +49 -0
- data/embulk-docs/src/_static/embulk-architecture.png +0 -0
- data/embulk-docs/src/_static/embulk-logo.png +0 -0
- data/embulk-docs/src/_static/embulk-logo.svg +133 -0
- data/embulk-docs/src/built-in.rst +440 -0
- data/embulk-docs/src/conf.py +260 -0
- data/embulk-docs/src/customization.rst +184 -0
- data/embulk-docs/src/index.rst +84 -0
- data/embulk-docs/src/recipe.rst +8 -0
- data/embulk-docs/src/recipe/scheduled-csv-load-to-elasticsearch-kibana4.rst +153 -0
- data/embulk-docs/src/release.rst +57 -0
- data/embulk-docs/src/release/release-0.1.0.rst +8 -0
- data/embulk-docs/src/release/release-0.2.0.rst +16 -0
- data/embulk-docs/src/release/release-0.2.1.rst +19 -0
- data/embulk-docs/src/release/release-0.3.0.rst +34 -0
- data/embulk-docs/src/release/release-0.3.1.rst +11 -0
- data/embulk-docs/src/release/release-0.3.2.rst +15 -0
- data/embulk-docs/src/release/release-0.4.0.rst +74 -0
- data/embulk-docs/src/release/release-0.4.1.rst +18 -0
- data/embulk-docs/src/release/release-0.4.10.rst +17 -0
- data/embulk-docs/src/release/release-0.4.2.rst +18 -0
- data/embulk-docs/src/release/release-0.4.3.rst +34 -0
- data/embulk-docs/src/release/release-0.4.4.rst +39 -0
- data/embulk-docs/src/release/release-0.4.5.rst +24 -0
- data/embulk-docs/src/release/release-0.4.6.rst +30 -0
- data/embulk-docs/src/release/release-0.4.7.rst +16 -0
- data/embulk-docs/src/release/release-0.4.8.rst +15 -0
- data/embulk-docs/src/release/release-0.4.9.rst +23 -0
- data/embulk-docs/src/release/release-0.5.0.rst +89 -0
- data/embulk-docs/src/release/release-0.5.1.rst +13 -0
- data/embulk-docs/src/release/release-0.5.2.rst +30 -0
- data/embulk-docs/src/release/release-0.5.3.rst +22 -0
- data/embulk-docs/src/release/release-0.5.4.rst +24 -0
- data/embulk-docs/src/release/release-0.5.5.rst +18 -0
- data/embulk-docs/src/release/release-0.6.0.rst +34 -0
- data/embulk-docs/src/release/release-0.6.1.rst +11 -0
- data/embulk-docs/src/release/release-0.6.10.rst +15 -0
- data/embulk-docs/src/release/release-0.6.11.rst +19 -0
- data/embulk-docs/src/release/release-0.6.12.rst +31 -0
- data/embulk-docs/src/release/release-0.6.13.rst +23 -0
- data/embulk-docs/src/release/release-0.6.14.rst +47 -0
- data/embulk-docs/src/release/release-0.6.15.rst +26 -0
- data/embulk-docs/src/release/release-0.6.16.rst +26 -0
- data/embulk-docs/src/release/release-0.6.17.rst +39 -0
- data/embulk-docs/src/release/release-0.6.18.rst +14 -0
- data/embulk-docs/src/release/release-0.6.19.rst +18 -0
- data/embulk-docs/src/release/release-0.6.2.rst +17 -0
- data/embulk-docs/src/release/release-0.6.20.rst +19 -0
- data/embulk-docs/src/release/release-0.6.21.rst +20 -0
- data/embulk-docs/src/release/release-0.6.22.rst +26 -0
- data/embulk-docs/src/release/release-0.6.23.rst +17 -0
- data/embulk-docs/src/release/release-0.6.24.rst +13 -0
- data/embulk-docs/src/release/release-0.6.25.rst +12 -0
- data/embulk-docs/src/release/release-0.6.3.rst +23 -0
- data/embulk-docs/src/release/release-0.6.4.rst +13 -0
- data/embulk-docs/src/release/release-0.6.5.rst +17 -0
- data/embulk-docs/src/release/release-0.6.6.rst +17 -0
- data/embulk-docs/src/release/release-0.6.7.rst +17 -0
- data/embulk-docs/src/release/release-0.6.8.rst +24 -0
- data/embulk-docs/src/release/release-0.6.9.rst +24 -0
- data/embulk-docs/src/release/release-0.7.0.rst +96 -0
- data/embulk-standards/build.gradle +5 -0
- data/embulk-standards/src/main/java/org/embulk/standards/CsvFormatterPlugin.java +284 -0
- data/embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java +379 -0
- data/embulk-standards/src/main/java/org/embulk/standards/CsvTokenizer.java +411 -0
- data/embulk-standards/src/main/java/org/embulk/standards/GzipFileDecoderPlugin.java +55 -0
- data/embulk-standards/src/main/java/org/embulk/standards/GzipFileEncoderPlugin.java +71 -0
- data/embulk-standards/src/main/java/org/embulk/standards/LocalFileInputPlugin.java +203 -0
- data/embulk-standards/src/main/java/org/embulk/standards/LocalFileOutputPlugin.java +148 -0
- data/embulk-standards/src/main/java/org/embulk/standards/NullOutputPlugin.java +59 -0
- data/embulk-standards/src/main/java/org/embulk/standards/RenameFilterPlugin.java +56 -0
- data/embulk-standards/src/main/java/org/embulk/standards/StandardPluginExtension.java +16 -0
- data/embulk-standards/src/main/java/org/embulk/standards/StandardPluginModule.java +53 -0
- data/embulk-standards/src/main/java/org/embulk/standards/StdoutOutputPlugin.java +85 -0
- data/embulk-standards/src/main/resources/META-INF/services/org.embulk.spi.Extension +1 -0
- data/embulk-standards/src/test/java/org/embulk/standards/TestCsvFormatterPlugin.java +312 -0
- data/embulk-standards/src/test/java/org/embulk/standards/TestCsvParserPlugin.java +75 -0
- data/embulk-standards/src/test/java/org/embulk/standards/TestCsvTokenizer.java +360 -0
- data/embulk-standards/src/test/java/org/embulk/standards/TestRenameFilterPlugin.java +88 -0
- data/embulk.gemspec +39 -0
- data/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +6 -0
- data/gradlew +164 -0
- data/gradlew.bat +90 -0
- data/lib/embulk.rb +72 -0
- data/lib/embulk/buffer.rb +22 -0
- data/lib/embulk/column.rb +70 -0
- data/lib/embulk/command/embulk_bundle.rb +56 -0
- data/lib/embulk/command/embulk_example.rb +32 -0
- data/lib/embulk/command/embulk_generate_bin.rb +62 -0
- data/lib/embulk/command/embulk_main.rb +2 -0
- data/lib/embulk/command/embulk_migrate_plugin.rb +170 -0
- data/lib/embulk/command/embulk_new_plugin.rb +124 -0
- data/lib/embulk/command/embulk_run.rb +470 -0
- data/lib/embulk/command/embulk_selfupdate.rb +84 -0
- data/lib/embulk/data/bundle/.bundle/config +3 -0
- data/lib/embulk/data/bundle/.ruby-version +1 -0
- data/lib/embulk/data/bundle/Gemfile +26 -0
- data/lib/embulk/data/bundle/embulk/filter/example.rb +42 -0
- data/lib/embulk/data/bundle/embulk/input/example.rb +54 -0
- data/lib/embulk/data/bundle/embulk/output/example.rb +58 -0
- data/lib/embulk/data/new/LICENSE.txt +21 -0
- data/lib/embulk/data/new/README.md.erb +111 -0
- data/lib/embulk/data/new/gitignore.erb +13 -0
- data/lib/embulk/data/new/java/build.gradle.erb +73 -0
- data/lib/embulk/data/new/java/decoder.java.erb +84 -0
- data/lib/embulk/data/new/java/encoder.java.erb +86 -0
- data/lib/embulk/data/new/java/file_input.java.erb +143 -0
- data/lib/embulk/data/new/java/file_output.java.erb +93 -0
- data/lib/embulk/data/new/java/filter.java.erb +56 -0
- data/lib/embulk/data/new/java/formatter.java.erb +54 -0
- data/lib/embulk/data/new/java/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/lib/embulk/data/new/java/gradle/wrapper/gradle-wrapper.properties +6 -0
- data/lib/embulk/data/new/java/gradlew +164 -0
- data/lib/embulk/data/new/java/gradlew.bat +90 -0
- data/lib/embulk/data/new/java/input.java.erb +87 -0
- data/lib/embulk/data/new/java/output.java.erb +77 -0
- data/lib/embulk/data/new/java/parser.java.erb +60 -0
- data/lib/embulk/data/new/java/plugin_loader.rb.erb +3 -0
- data/lib/embulk/data/new/java/test.java.erb +5 -0
- data/lib/embulk/data/new/ruby/.ruby-version +1 -0
- data/lib/embulk/data/new/ruby/Gemfile +2 -0
- data/lib/embulk/data/new/ruby/Rakefile +3 -0
- data/lib/embulk/data/new/ruby/decoder_guess.rb.erb +25 -0
- data/lib/embulk/data/new/ruby/filter.rb.erb +41 -0
- data/lib/embulk/data/new/ruby/formatter.rb.erb +49 -0
- data/lib/embulk/data/new/ruby/gemspec.erb +20 -0
- data/lib/embulk/data/new/ruby/input.rb.erb +59 -0
- data/lib/embulk/data/new/ruby/output.rb.erb +61 -0
- data/lib/embulk/data/new/ruby/parser.rb.erb +44 -0
- data/lib/embulk/data/new/ruby/parser_guess.rb.erb +65 -0
- data/lib/embulk/data/package_data.rb +50 -0
- data/lib/embulk/data_source.rb +220 -0
- data/lib/embulk/decoder_plugin.rb +27 -0
- data/lib/embulk/encoder_plugin.rb +27 -0
- data/lib/embulk/error.rb +8 -0
- data/lib/embulk/executor_plugin.rb +23 -0
- data/lib/embulk/file_input.rb +87 -0
- data/lib/embulk/file_input_plugin.rb +27 -0
- data/lib/embulk/file_output.rb +56 -0
- data/lib/embulk/file_output_plugin.rb +27 -0
- data/lib/embulk/filter_plugin.rb +105 -0
- data/lib/embulk/formatter_plugin.rb +105 -0
- data/lib/embulk/guess/charset.rb +44 -0
- data/lib/embulk/guess/csv.rb +327 -0
- data/lib/embulk/guess/gzip.rb +18 -0
- data/lib/embulk/guess/newline.rb +22 -0
- data/lib/embulk/guess/schema_guess.rb +118 -0
- data/lib/embulk/guess/time_format_guess.rb +394 -0
- data/lib/embulk/guess_plugin.rb +129 -0
- data/lib/embulk/input_plugin.rb +121 -0
- data/lib/embulk/java/bootstrap.rb +24 -0
- data/lib/embulk/java/imports.rb +69 -0
- data/lib/embulk/java/time_helper.rb +79 -0
- data/lib/embulk/java_plugin.rb +90 -0
- data/lib/embulk/logger.rb +154 -0
- data/lib/embulk/output_plugin.rb +150 -0
- data/lib/embulk/page.rb +30 -0
- data/lib/embulk/page_builder.rb +76 -0
- data/lib/embulk/parser_plugin.rb +78 -0
- data/lib/embulk/plugin.rb +239 -0
- data/lib/embulk/plugin_registry.rb +96 -0
- data/lib/embulk/runner.rb +184 -0
- data/lib/embulk/schema.rb +103 -0
- data/lib/embulk/version.rb +3 -0
- data/settings.gradle +6 -0
- data/test/guess/test_schema_guess.rb +11 -0
- data/test/guess/test_time_format_guess.rb +133 -0
- data/test/helper.rb +21 -0
- data/test/run-test.rb +14 -0
- metadata +566 -0
@@ -0,0 +1,49 @@
|
|
1
|
+
#!/bin/bash
|
2
|
+
|
3
|
+
function re() {
|
4
|
+
r "$@"
|
5
|
+
if [ $? -ne 0 ];then
|
6
|
+
exit $?
|
7
|
+
fi
|
8
|
+
}
|
9
|
+
|
10
|
+
function r() {
|
11
|
+
echo "$@"
|
12
|
+
"$@"
|
13
|
+
}
|
14
|
+
|
15
|
+
[ "$TRAVIS_PULL_REQUEST" != "false" ] && exit 0
|
16
|
+
[ "$TRAVIS_BRANCH" != "master" -a "$TRAVIS_BRANCH" != "$(git describe --tags --always HEAD)" ] && exit 0
|
17
|
+
|
18
|
+
revision="$(git rev-parse HEAD)"
|
19
|
+
remote="https://github.com/embulk/embulk.github.io.git"
|
20
|
+
re ./gradlew site
|
21
|
+
|
22
|
+
r git fetch --unshallow || echo "using complete repository."
|
23
|
+
|
24
|
+
re rm -rf gh_pages
|
25
|
+
re git clone . gh_pages
|
26
|
+
re cd gh_pages
|
27
|
+
|
28
|
+
re git remote add travis_push "$remote"
|
29
|
+
re git fetch travis_push
|
30
|
+
|
31
|
+
re git checkout -b gh-pages travis_push/master
|
32
|
+
re rm -rf docs
|
33
|
+
re cp -a ../embulk-docs/build/html docs
|
34
|
+
re git add --all docs
|
35
|
+
|
36
|
+
re git config user.name "$GIT_USER_NAME"
|
37
|
+
re git config user.email "$GIT_USER_EMAIL"
|
38
|
+
r git commit -m "Updated document $revision"
|
39
|
+
|
40
|
+
git show | grep -E '^[+-] ' | grep -Eqv 'Generated by|Generated on|Search.setIndex|meta name="date" content='
|
41
|
+
if [ $? -ne 0 ];then
|
42
|
+
echo "No document changes."
|
43
|
+
exit 0
|
44
|
+
fi
|
45
|
+
|
46
|
+
re git config credential.helper "store --file=$HOME/.git_credentials"
|
47
|
+
echo "https://$GITHUB_TOKEN:@github.com" > "$HOME/.git_credentials"
|
48
|
+
trap "rm -rf $HOME/.git_credentials" EXIT
|
49
|
+
re git push travis_push gh-pages:master
|
Binary file
|
Binary file
|
@@ -0,0 +1,133 @@
|
|
1
|
+
<?xml version="1.0" encoding="utf-8"?>
|
2
|
+
<!-- Generator: Adobe Illustrator 15.0.2, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
|
3
|
+
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
4
|
+
<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
|
5
|
+
width="466px" height="268px" viewBox="0 0 466 268" enable-background="new 0 0 466 268" xml:space="preserve">
|
6
|
+
<g>
|
7
|
+
<path fill="#EF4F39" d="M124.753,173.479c0,0,7.781,2.789,8.031,6.721c0.57,8.859,5.539,30.307-0.93,30.686
|
8
|
+
c-8.516,0.502-20.797-12.975-24.602-22.738c-5.82-14.951-4.18-19.992-4.18-19.992L124.753,173.479z"/>
|
9
|
+
<g>
|
10
|
+
<g>
|
11
|
+
<path fill="#1F1F1F" d="M38.015,225.368c1.477,4.021,3.523,7.008,6.141,8.953s5.891,2.918,9.828,2.918
|
12
|
+
c2.859,0,5.742-0.58,8.648-1.744c0.762-0.312,3.422-1.586,7.984-3.822c1.383-0.672,2.93-1.006,4.625-1.006
|
13
|
+
c2.328,0,4.297,1.051,5.906,3.152c1.34,1.699,2.012,3.398,2.012,5.096c0,3.803-2.707,7.246-8.113,10.33
|
14
|
+
c-6.488,3.713-13.844,5.57-22.07,5.57c-12.297,0-21.98-4.273-29.047-12.814c-5.094-6.17-7.645-13.592-7.645-22.268
|
15
|
+
c0-11.762,3.777-21.152,11.336-28.172c6.527-6.037,14.266-9.057,23.207-9.057c7.828,0,14.781,2.416,20.859,7.246
|
16
|
+
c5.859,4.695,9.75,11.066,11.672,19.115c0.672,2.863,1.008,5.725,1.008,8.586c0,5.277-2.93,7.916-8.789,7.916H38.015z
|
17
|
+
M64.511,211.616c-0.848-3.264-1.832-5.701-2.949-7.312c-2.414-3.531-5.812-5.299-10.195-5.299c-3.758,0-6.953,1.365-9.594,4.094
|
18
|
+
c-1.699,1.787-3.195,4.627-4.492,8.518H64.511z"/>
|
19
|
+
<path fill="#1F1F1F" d="M112.202,189.481c2.367-2.102,4.539-3.623,6.508-4.561c3.352-1.609,7.133-2.416,11.336-2.416
|
20
|
+
c5.23,0,9.812,1.117,13.75,3.354c1.742,0.984,3.598,2.797,5.566,5.434c3.355-2.727,6.215-4.693,8.586-5.902
|
21
|
+
c3.934-1.922,8.16-2.885,12.676-2.885c6.977,0,12.742,2.215,17.305,6.641c4.609,4.518,6.91,10.061,6.91,16.635v35.146
|
22
|
+
c0,3.848-0.449,6.666-1.34,8.451c-0.762,1.432-1.957,2.627-3.59,3.59c-1.633,0.961-3.344,1.443-5.129,1.443
|
23
|
+
c-1.793,0-3.5-0.469-5.133-1.41c-1.633-0.939-2.828-2.146-3.59-3.623c-0.895-1.785-1.34-4.604-1.34-8.451v-30.652
|
24
|
+
c0-2.951-0.895-5.342-2.684-7.176c-2.012-2.146-4.559-3.221-7.645-3.221c-3.133,0-5.75,1.051-7.852,3.154
|
25
|
+
c-1.875,1.832-2.816,4.246-2.816,7.242v30.652c0,3.805-0.449,6.621-1.34,8.451c-0.762,1.477-1.957,2.684-3.59,3.623
|
26
|
+
c-1.633,0.941-3.344,1.41-5.129,1.41c-1.789,0-3.5-0.469-5.133-1.41c-1.633-0.939-2.828-2.146-3.59-3.623
|
27
|
+
c-0.895-1.785-1.34-4.604-1.34-8.451v-28.641c0-4.066-0.961-7.152-2.887-9.254c-1.922-2.104-4.449-3.154-7.578-3.154
|
28
|
+
c-3.086,0-5.699,1.098-7.848,3.287c-1.789,1.879-2.68,4.248-2.68,7.109v30.652c0,3.805-0.453,6.621-1.344,8.451
|
29
|
+
c-0.762,1.477-1.957,2.684-3.59,3.623c-1.633,0.941-3.34,1.41-5.133,1.41c-1.785,0-3.496-0.469-5.129-1.41
|
30
|
+
c-1.633-0.939-2.828-2.146-3.59-3.623c-0.895-1.785-1.34-4.604-1.34-8.451v-46.414c0-3.398,0.848-6.127,2.547-8.186
|
31
|
+
c1.879-2.279,4.336-3.42,7.379-3.42c2.594,0,4.809,0.873,6.641,2.617C109.897,186.284,110.952,187.603,112.202,189.481z"/>
|
32
|
+
<path fill="#1F1F1F" d="M112.202,189.481c2.367-2.102,4.539-3.623,6.508-4.561c3.352-1.609,7.133-2.416,11.336-2.416
|
33
|
+
c5.23,0,9.812,1.117,13.75,3.354c1.742,0.984,3.598,2.797,5.566,5.434c3.355-2.727,6.215-4.693,8.586-5.902
|
34
|
+
c3.934-1.922,8.16-2.885,12.676-2.885c6.977,0,12.742,2.215,17.305,6.641c4.609,4.518,6.91,10.061,6.91,16.635v35.146
|
35
|
+
c0,3.848-0.449,6.666-1.34,8.451c-0.762,1.432-1.957,2.627-3.59,3.59c-1.633,0.961-3.344,1.443-5.129,1.443
|
36
|
+
c-1.793,0-3.5-0.469-5.133-1.41c-1.633-0.939-2.828-2.146-3.59-3.623c-0.895-1.785-1.34-4.604-1.34-8.451v-30.652
|
37
|
+
c0-2.951-0.895-5.342-2.684-7.176c-2.012-2.146-4.559-3.221-7.645-3.221c-3.133,0-5.75,1.051-7.852,3.154
|
38
|
+
c-1.875,1.832-2.816,4.246-2.816,7.242v30.652c0,3.805-0.449,6.621-1.34,8.451c-0.762,1.477-1.957,2.684-3.59,3.623
|
39
|
+
c-1.633,0.941-3.344,1.41-5.129,1.41c-1.789,0-3.5-0.469-5.133-1.41c-1.633-0.939-2.828-2.146-3.59-3.623
|
40
|
+
c-0.895-1.785-1.34-4.604-1.34-8.451v-28.641c0-4.066-0.961-7.152-2.887-9.254c-1.922-2.104-4.449-3.154-7.578-3.154
|
41
|
+
c-3.086,0-5.699,1.098-7.848,3.287c-1.789,1.879-2.68,4.248-2.68,7.109v30.652c0,3.805-0.453,6.621-1.344,8.451
|
42
|
+
c-0.762,1.477-1.957,2.684-3.59,3.623c-1.633,0.941-3.34,1.41-5.133,1.41c-1.785,0-3.496-0.469-5.129-1.41
|
43
|
+
c-1.633-0.939-2.828-2.146-3.59-3.623c-0.895-1.785-1.34-4.604-1.34-8.451v-46.414c0-3.398,0.848-6.127,2.547-8.186
|
44
|
+
c1.879-2.279,4.336-3.42,7.379-3.42c2.594,0,4.809,0.873,6.641,2.617C109.897,186.284,110.952,187.603,112.202,189.481z"/>
|
45
|
+
</g>
|
46
|
+
</g>
|
47
|
+
<g>
|
48
|
+
<path fill="#FFFFFF" d="M32.706,101.839c0,0,12.794,37.536,28.406,50.635c23.352,19.592,42.266,24.008,63.57,26.375
|
49
|
+
c27.992,3.109,68.055,0.375,76.672-3.381c19.914-8.666,36.523-27.725,41.93-43.279c5.414-15.557-82.844-32.125-82.844-32.125
|
50
|
+
l-30.766-13.865c0,0-19.617-10.143-26.719-6.76c-7.102,3.381-13.859,5.748-19.273,7.1c-5.406,1.354-13.523,0.338-22.656,2.029
|
51
|
+
C51.901,90.257,32.706,101.839,32.706,101.839z"/>
|
52
|
+
<path fill="#EF4F39" d="M32.331,101.245c0,0,0.391,10.795,3.898,14.678c4.727,5.23,3.203,11.189,13.461,25.664
|
53
|
+
c13.281,18.732,38.328,36.789,89.406,38.908c50.391,2.094,50.086-1.395,50.086-1.395s-6.891,1.152-44.25-0.252
|
54
|
+
c-23.625-0.889-56.344-5.354-75.016-20.842c-17.734-14.709-24.203-24.42-29.125-39.309c-2.094-6.34-5.797-6.422-5.992-15.996
|
55
|
+
C34.776,101.249,32.331,101.245,32.331,101.245z"/>
|
56
|
+
<g>
|
57
|
+
<path fill="#F16523" d="M195.331,176.63c0,0,5.219-1.023,11.352-4.504c8.711-4.943,14.773-11.02,15.852-11.918
|
58
|
+
c4.883-4.057,15.219-16.484,15.219-16.484c-0.414,0.613-5.562,9.764-12.305,15.977c-4.484,4.133-1.898,2.156-9.25,7.861
|
59
|
+
c-9.078,7.043-22,11.92-22,11.92"/>
|
60
|
+
<path fill="#F16523" d="M284.198,12.147c0,0-7.016,13.777-24.008,15.553c-11.57,1.211-13.188,11.16-13.188,11.16
|
61
|
+
s-4.055-10.652-14.453-12.174c-15.477-2.264-20.375-13.525-20.375-13.525c-4.391,16.23,0.68,24.346,3.383,28.404
|
62
|
+
c5.188,7.781,18.258,9.469,21.305,16.23c3.039,6.764,0.672,11.498-2.703,27.391c-3.383,15.893-19.281,12.848-32.125,11.156
|
63
|
+
c-7.117-0.934-18.672-8.105-41.508-16.402c0.242-4.438-2.695-9.434-1.016-16.23c6.086-24.6,24.094-34.361,12.938-32.715
|
64
|
+
c-8.805,1.301-50.031,32.121-56.523,36.102c-8.344-1.617-17.086-2.822-26.156-3.387c-31.359-1.947-39.305,14.033-48.945,20.627
|
65
|
+
c-17.148,11.734-8.07,17.84-8.07,17.84s28.812,1.553,40.109,3.975c9.469,2.027,32.125,8.453,33.812,11.496
|
66
|
+
c1.695,3.043,4.398,11.834,12.516,10.145c8.109-1.691,8.75-7.969,8.75-7.969c-0.344,9.131,6.805,21.156,14.242,31.639
|
67
|
+
s19.273,13.188,19.273,13.188c2.031,25.359-11.562,49.064-8.031,49.195c23.844,0.889,38.156-23.922,40.453-28.275
|
68
|
+
c4.82-9.131,3.297-19.656,3.297-19.656c42.266-8.113,49.461-34.107,53.203-45.637c6.344-19.529,3.055-59.438,3.055-59.438
|
69
|
+
s8.789-0.676,20.961-9.467C286.565,42.579,284.198,12.147,284.198,12.147z"/>
|
70
|
+
</g>
|
71
|
+
<path fill="#FFFFFF" d="M107.753,97.312c0,0-0.617-14.115,18.625-10.881s19.953,13.1,4.945,17.168
|
72
|
+
C116.323,107.669,108.065,103.565,107.753,97.312z"/>
|
73
|
+
<g>
|
74
|
+
<circle fill="#1F1F1F" cx="101.776" cy="92.2" r="4.312"/>
|
75
|
+
<path fill="#FFFFFF" d="M101.776,96.761c-2.516,0-4.562-2.046-4.562-4.561c0-2.516,2.047-4.562,4.562-4.562
|
76
|
+
s4.562,2.047,4.562,4.562C106.339,94.715,104.292,96.761,101.776,96.761z M101.776,88.138c-2.24,0-4.062,1.822-4.062,4.062
|
77
|
+
c0,2.239,1.822,4.061,4.062,4.061s4.062-1.822,4.062-4.061C105.839,89.96,104.017,88.138,101.776,88.138z"/>
|
78
|
+
<path fill="#FFFFFF" d="M104.394,90.595c0,0.512-0.414,0.928-0.93,0.928c-0.508,0-0.93-0.416-0.93-0.928
|
79
|
+
c0-0.516,0.422-0.932,0.93-0.932C103.979,89.663,104.394,90.079,104.394,90.595z"/>
|
80
|
+
</g>
|
81
|
+
<g>
|
82
|
+
<path fill="#EF4F39" d="M164.237,161.2c-1.523,0.254-4.141,0.254-9.383-1.184c-4.789-1.311-12.648-9.082-13.984-10.43
|
83
|
+
c0.438,0.631,0.875,1.254,1.312,1.875c7.438,10.482,19.273,13.188,19.273,13.188c1.969,24.617-10.773,47.672-8.297,49.121
|
84
|
+
c0.008-0.064,0.094-0.949,0.258-1.764c0.164-0.846,0.594-2.283,1.266-3.805c0.68-1.521,10.062-18.682,12.172-26.543
|
85
|
+
c2.117-7.861,1.188-21.727,1.188-21.727S165.761,160.948,164.237,161.2z"/>
|
86
|
+
<path fill="#EF4F39" d="M122.995,115.067c0,0,2.531-7.607,10.648-6.086s5.328,10.271,5.328,10.271s0.031-6.586-5.711-7.734
|
87
|
+
C128.19,110.503,122.995,115.067,122.995,115.067z"/>
|
88
|
+
<path fill="#EF4F39" d="M117.237,126.362c-5.328-0.635-5.711-8.242-9.891-11.287c-3.484-2.529-25.367-9.129-34.242-11.031
|
89
|
+
s-36.266-3.803-38.547-3.93s-2.914-3.043-3.297-3.553c-0.242-0.326,0.25-2.391,0.633-3.816c-3.898,6.225,0.859,9.432,0.859,9.432
|
90
|
+
s28.812,1.553,40.109,3.975c9.469,2.027,32.125,8.453,33.812,11.496c1.695,3.043,4.398,11.834,12.516,10.145
|
91
|
+
c8.109-1.691,8.75-7.969,8.75-7.969S122.144,126.946,117.237,126.362z"/>
|
92
|
+
<path fill="#EF4F39" d="M251.394,111.146c-1.352,7.1-6.594,22.994-16.398,32.799c-6.102,6.098-15.898,10.652-23.672,14.203
|
93
|
+
c-6.266,2.859-15.219,3.887-15.219,3.887l1.07,3.879c42.266-8.113,49.461-34.107,53.203-45.637
|
94
|
+
c2.305-7.115,3.344-16.934,3.719-26.541C253.448,98.396,252.308,106.351,251.394,111.146z"/>
|
95
|
+
</g>
|
96
|
+
</g>
|
97
|
+
<rect x="133.667" y="203.567" fill="#1F1F1F" width="20.055" height="21.766"/>
|
98
|
+
<g>
|
99
|
+
<g>
|
100
|
+
<path fill="#1F1F1F" d="M226.835,188.409c2.816-2.104,5.188-3.512,7.109-4.227c2.992-1.162,6.059-1.744,9.188-1.744
|
101
|
+
c8.32,0,15.586,3.379,21.801,10.127c6.348,6.889,9.527,15.496,9.527,25.826c0,11.402-3.555,20.658-10.668,27.77
|
102
|
+
c-5.723,5.768-12.543,8.654-20.457,8.654c-4.609,0-8.562-1.031-11.875-3.086c-1.516-0.984-3.148-2.619-4.895-4.896
|
103
|
+
c-2.059,5.051-5.23,7.578-9.527,7.578c-2.637,0-4.961-0.85-6.973-2.549c-2.238-1.967-3.355-4.697-3.355-8.182v-77.543
|
104
|
+
c0-3.799,0.449-6.594,1.344-8.383c0.758-1.477,1.953-2.684,3.59-3.623c1.629-0.939,3.34-1.408,5.129-1.408s3.5,0.469,5.133,1.408
|
105
|
+
s2.828,2.146,3.586,3.623c0.895,1.789,1.344,4.584,1.344,8.383V188.409z M240.519,201.22c-3.758,0-7.02,1.631-9.793,4.896
|
106
|
+
c-2.684,3.219-4.023,7.379-4.023,12.477c0,5.141,1.34,9.324,4.023,12.543c2.773,3.264,6.078,4.896,9.926,4.896
|
107
|
+
c4.16,0,7.488-1.746,9.996-5.23c2.457-3.49,3.688-7.535,3.688-12.143c0-4.695-1.23-8.742-3.688-12.143
|
108
|
+
C248.097,202.987,244.718,201.22,240.519,201.22z"/>
|
109
|
+
<path fill="#1F1F1F" d="M345.358,196.39v29.916c0,6.842-1.637,12.387-4.898,16.635c-2.773,3.578-6.352,6.396-10.734,8.451
|
110
|
+
c-4.918,2.324-10.148,3.488-15.695,3.488c-7.781,0-14.598-2.078-20.457-6.238c-4.559-3.221-7.645-7.064-9.258-11.537
|
111
|
+
c-1.074-3.037-1.605-6.641-1.605-10.799V196.39c0-3.801,0.445-6.617,1.336-8.451c0.762-1.477,1.949-2.684,3.559-3.623
|
112
|
+
c1.605-0.938,3.332-1.408,5.164-1.408c1.789,0,3.496,0.471,5.129,1.408c1.633,0.939,2.828,2.146,3.59,3.623
|
113
|
+
c0.895,1.789,1.34,4.607,1.34,8.451v27.434c0,2.818,0.449,4.945,1.344,6.373c2.281,3.668,5.57,5.502,9.859,5.502
|
114
|
+
c3.129,0,5.793-1.074,7.984-3.221c2.145-2.059,3.219-4.941,3.219-8.654V196.39c0-3.844,0.445-6.662,1.34-8.451
|
115
|
+
c0.762-1.477,1.957-2.684,3.59-3.623c1.633-0.938,3.344-1.408,5.133-1.408c1.785,0,3.5,0.471,5.129,1.408
|
116
|
+
c1.633,0.939,2.828,2.146,3.59,3.623C344.909,189.728,345.358,192.546,345.358,196.39z"/>
|
117
|
+
<path fill="#1F1F1F" d="M377.351,166.138v74.789c0,3.848-0.445,6.666-1.344,8.451c-0.758,1.477-1.953,2.684-3.586,3.623
|
118
|
+
c-1.633,0.941-3.344,1.41-5.133,1.41s-3.5-0.469-5.129-1.41c-1.633-0.939-2.832-2.146-3.59-3.623
|
119
|
+
c-0.895-1.785-1.344-4.604-1.344-8.451v-74.789c0-3.799,0.449-6.594,1.344-8.383c0.758-1.477,1.957-2.684,3.59-3.623
|
120
|
+
c1.629-0.939,3.34-1.408,5.129-1.408s3.5,0.469,5.133,1.408s2.828,2.146,3.586,3.623
|
121
|
+
C376.905,159.544,377.351,162.339,377.351,166.138z"/>
|
122
|
+
<path fill="#1F1F1F" d="M409.343,220.403v20.523c0,3.805-0.445,6.621-1.34,8.451c-0.762,1.477-1.957,2.684-3.586,3.623
|
123
|
+
c-1.633,0.941-3.344,1.41-5.133,1.41s-3.496-0.469-5.129-1.41c-1.633-0.939-2.828-2.146-3.594-3.623
|
124
|
+
c-0.891-1.785-1.34-4.604-1.34-8.451v-74.789c0-3.799,0.449-6.594,1.34-8.383c0.766-1.477,1.961-2.684,3.594-3.623
|
125
|
+
s3.34-1.408,5.129-1.408s3.5,0.469,5.133,1.408c1.629,0.939,2.824,2.146,3.586,3.623c0.895,1.789,1.34,4.584,1.34,8.383v40.582
|
126
|
+
l18.516-18.982c3.445-3.533,6.527-5.299,9.258-5.299c2.727,0,5.051,0.984,6.977,2.951c1.922,1.924,2.883,4.227,2.883,6.908
|
127
|
+
c0,1.969-0.738,3.936-2.215,5.904c-0.488,0.625-1.566,1.744-3.219,3.354l-12.609,12.072l17.641,21.936
|
128
|
+
c2.902,3.578,4.355,6.551,4.355,8.918c0,3.178-1.27,5.793-3.82,7.85c-1.836,1.432-3.957,2.146-6.375,2.146
|
129
|
+
c-2.41,0-4.672-0.961-6.77-2.885c-0.629-0.58-1.656-1.787-3.09-3.621L409.343,220.403z"/>
|
130
|
+
</g>
|
131
|
+
</g>
|
132
|
+
</g>
|
133
|
+
</svg>
|
@@ -0,0 +1,440 @@
|
|
1
|
+
Configuration
|
2
|
+
==================================
|
3
|
+
|
4
|
+
.. contents::
|
5
|
+
:local:
|
6
|
+
:depth: 2
|
7
|
+
|
8
|
+
Embulk configuration file format
|
9
|
+
------------------
|
10
|
+
|
11
|
+
Embulk uses a YAML file to define a bulk data loading. Here is an example of the file:
|
12
|
+
|
13
|
+
.. code-block:: yaml
|
14
|
+
|
15
|
+
in:
|
16
|
+
type: file
|
17
|
+
path_prefix: ./mydata/csv/
|
18
|
+
decoders:
|
19
|
+
- {type: gzip}
|
20
|
+
parser:
|
21
|
+
charset: UTF-8
|
22
|
+
newline: CRLF
|
23
|
+
type: csv
|
24
|
+
delimiter: ','
|
25
|
+
quote: '"'
|
26
|
+
escape: '"'
|
27
|
+
null_string: 'NULL'
|
28
|
+
skip_header_lines: 1
|
29
|
+
columns:
|
30
|
+
- {name: id, type: long}
|
31
|
+
- {name: account, type: long}
|
32
|
+
- {name: time, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
|
33
|
+
- {name: purchase, type: timestamp, format: '%Y%m%d'}
|
34
|
+
- {name: comment, type: string}
|
35
|
+
filters:
|
36
|
+
- type: speedometer
|
37
|
+
speed_limit: 250000
|
38
|
+
out:
|
39
|
+
type: stdout
|
40
|
+
|
41
|
+
A configuration file consists of following sections:
|
42
|
+
|
43
|
+
* **in:** Input plugin options. An input plugin is either record-based (`MySQL <https://github.com/embulk/embulk-input-jdbc>`_, `DynamoDB <https://github.com/lulichn/embulk-input-dynamodb>`_, etc) or file-based (`S3 <https://github.com/embulk/embulk-input-s3>`_, `HTTP <https://github.com/takumakanari/embulk-input-http>`_, etc).
|
44
|
+
|
45
|
+
* **parser:** If the input is file-based, parser plugin parses a file format (built-in csv, `json <https://github.com/takumakanari/embulk-parser-json>`_, etc).
|
46
|
+
|
47
|
+
* **decoder:** If the input is file-based, decoder plugin decodes compression or encryption (built-in gzip, `zip <https://github.com/hata/embulk-decoder-commons-compress>`_, `tar.gz <https://github.com/hata/embulk-decoder-commons-compress>`_, etc).
|
48
|
+
|
49
|
+
* **out:** Output plugin options. An output plugin is either record-based (`Oracle <https://github.com/embulk/embulk-output-jdbc>`_, `Elasticsearch <https://github.com/muga/embulk-output-elasticsearch>`_, etc) or file-based (`Google Cloud Storage <https://github.com/hakobera/embulk-output-gcs>`_, `Command <https://github.com/embulk/embulk-output-command>`_, etc)
|
50
|
+
|
51
|
+
* **formatter:** If the output is file-based, formatter plugin formats a file format (such as built-in csv, `JSON <https://github.com/takei-yuya/embulk-formatter-jsonl>`_)
|
52
|
+
|
53
|
+
* **encoder:** If the output is file-based, encoder plugin encodes compression or encryption (such as built-in gzip)
|
54
|
+
|
55
|
+
* **filters:** Filter plugins options (optional).
|
56
|
+
|
57
|
+
* **exec:** Executor plugin options. An executor plugin control parallel processing (such as built-in thread executor, `Hadoop MapReduce executor <https://github.com/embulk/embulk-executor-mapreduce>`_)
|
58
|
+
|
59
|
+
In many cases, what you need to write is **in:**, **out**: and **formatter** sections only because ``guess`` command guesses **parser** and **decoder** options for you. See also the `Quick Start <https://github.com/embulk/embulk#quick-start>`_.
|
60
|
+
|
61
|
+
|
62
|
+
Using variables
|
63
|
+
~~~~~~~~~~~~~~~~~~
|
64
|
+
|
65
|
+
You can embed environment variables in configuration file using `Liquid template engine <http://liquidmarkup.org/>`_ (This is experimental feature. Behavior might change or be removed in future releases).
|
66
|
+
|
67
|
+
To use template engine, configuration file name must end with ``.yml.liquid``.
|
68
|
+
|
69
|
+
Environment variables are set to ``env`` variable.
|
70
|
+
|
71
|
+
.. code-block:: yaml
|
72
|
+
|
73
|
+
in:
|
74
|
+
type: file
|
75
|
+
path_prefix: {{ env.path_prefix }}
|
76
|
+
decoders:
|
77
|
+
- {type: gzip}
|
78
|
+
parser:
|
79
|
+
...
|
80
|
+
out:
|
81
|
+
type: postgresql
|
82
|
+
host: {{ env.pg_host }}
|
83
|
+
port: {{ env.pg_port }}
|
84
|
+
user: {{ env.pg_user }}
|
85
|
+
password: "{{ env.pg_password }}"
|
86
|
+
database: embulk_load
|
87
|
+
mode: insert
|
88
|
+
table: {{ env.pg_table }}
|
89
|
+
|
90
|
+
|
91
|
+
Local file input plugin
|
92
|
+
------------------
|
93
|
+
|
94
|
+
The ``file`` input plugin reads files from local file system.
|
95
|
+
|
96
|
+
Options
|
97
|
+
~~~~~~~~~~~~~~~~~~
|
98
|
+
|
99
|
+
+----------------+----------+------------------------------------------------+-----------+
|
100
|
+
| name | type | description | required? |
|
101
|
+
+================+==========+================================================+===========+
|
102
|
+
| path\_prefix | string | Path prefix of input files | required |
|
103
|
+
+----------------+----------+------------------------------------------------+-----------+
|
104
|
+
| parsers | hash | Parsers configurations (see below) | required |
|
105
|
+
+----------------+----------+------------------------------------------------+-----------+
|
106
|
+
| decoders | array | Decoder configuration (see below) | |
|
107
|
+
+----------------+----------+------------------------------------------------+-----------+
|
108
|
+
| last\_path | string | Name of last read file in previous operation | |
|
109
|
+
+----------------+----------+------------------------------------------------+-----------+
|
110
|
+
|
111
|
+
The ``path_prefix`` option is required. If you have files as following, you may set ``path_prefix: /path/to/files/sample_``:
|
112
|
+
|
113
|
+
::
|
114
|
+
|
115
|
+
.
|
116
|
+
`-- path
|
117
|
+
`-- to
|
118
|
+
`-- files
|
119
|
+
|-- sample_01.csv -> read
|
120
|
+
|-- sample_02.csv -> read
|
121
|
+
|-- sample_03.csv -> read
|
122
|
+
|-- sample_04.csv -> read
|
123
|
+
|
124
|
+
The ``last_path`` option is used to skip files older than or same with the file in dictionary order.
|
125
|
+
For example, if you set ``last_path: /path/to/files/sample_02.csv``, Embulk reads following files:
|
126
|
+
|
127
|
+
::
|
128
|
+
|
129
|
+
.
|
130
|
+
`-- path
|
131
|
+
`-- to
|
132
|
+
`-- files
|
133
|
+
|-- sample_01.csv -> skip
|
134
|
+
|-- sample_02.csv -> skip
|
135
|
+
|-- sample_03.csv -> read
|
136
|
+
|-- sample_04.csv -> read
|
137
|
+
|
138
|
+
Example
|
139
|
+
~~~~~~~~~~~~~~~~~~
|
140
|
+
|
141
|
+
.. code-block:: yaml
|
142
|
+
|
143
|
+
in:
|
144
|
+
type: file
|
145
|
+
path_prefix: /path/to/files/sample_
|
146
|
+
last_path: /path/to/files/sample_02.csv
|
147
|
+
parser:
|
148
|
+
...
|
149
|
+
|
150
|
+
In most of cases, you'll use guess to configure the parsers and decoders. See also `Quick Start <https://github.com/embulk/embulk#quick-start>`_.
|
151
|
+
|
152
|
+
CSV parser plugin
|
153
|
+
------------------
|
154
|
+
|
155
|
+
The ``csv`` parser plugin parses CSV and TSV files.
|
156
|
+
|
157
|
+
Options
|
158
|
+
~~~~~~~~~~~~~~~~~~
|
159
|
+
|
160
|
+
+----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
|
161
|
+
| name | type | description | required? |
|
162
|
+
+============================+==========+================================================================================================================+========================+
|
163
|
+
| delimiter | string | Delimiter character such as ``,`` for CSV, ``"\t"`` for TSV, ``"|"`` or any single-byte character | ``,`` by default |
|
164
|
+
+----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
|
165
|
+
| quote | string | The character surrounding a quoted value. Setting ``null`` disables quoting. | ``\"`` by default |
|
166
|
+
+----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
|
167
|
+
| escape | string | Escape character to escape a special character. Setting ``null`` disables escaping. | ``\\`` by default |
|
168
|
+
+----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
|
169
|
+
| skip\_header\_lines | integer | Skip this number of lines first. Set 1 if the file has header line. | ``0`` by default |
|
170
|
+
+----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
|
171
|
+
| null\_string | string | If a value is this string, converts it to NULL. For example, set ``\N`` for CSV files created by mysqldump | |
|
172
|
+
+----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
|
173
|
+
| trim\_if\_not\_quoted | boolean | If true, remove spaces of a value if the value is not surrounded by the quote character | ``false`` by default |
|
174
|
+
+----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
|
175
|
+
| comment\_line\_marker | string | Skip a line if the line begins with this string | null by default |
|
176
|
+
+----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
|
177
|
+
| allow\_optional\_columns | boolean | If true, set null to insufficient columns. Otherwise, skip the row in case of insufficient number of columns | ``false`` by default |
|
178
|
+
+----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
|
179
|
+
| allow\_extra\_columns | boolean | If true, ignore too many columns. Otherwise, skip the row in case of too many columns | ``false`` by default |
|
180
|
+
+----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
|
181
|
+
| max\_quoted\_size\_limit | integer | Maximum number of bytes of a quoted value. If a value exceeds the limit, the row will be skipped | ``131072`` by default |
|
182
|
+
+----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
|
183
|
+
| default\_timezone | string | Time zone of timestamp columns if the value itself doesn't include time zone description (eg. Asia/Tokyo) | ``UTC`` by default |
|
184
|
+
+----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
|
185
|
+
| newline | enum | Newline character (CRLF, LF or CR) | ``CRLF`` by default |
|
186
|
+
+----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
|
187
|
+
| charset | enum | Character encoding (eg. ISO-8859-1, UTF-8) | ``UTF-8`` by default |
|
188
|
+
+----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
|
189
|
+
| columns | hash | Columns (see below) | required |
|
190
|
+
+----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
|
191
|
+
|
192
|
+
The ``columns`` option declares the list of columns. This CSV parser plugin ignores the header line.
|
193
|
+
|
194
|
+
+----------+-------------------------------------------------+
|
195
|
+
| name | description |
|
196
|
+
+==========+=================================================+
|
197
|
+
| name | Name of the column |
|
198
|
+
+----------+-------------------------------------------------+
|
199
|
+
| type | Type of the column (see below) |
|
200
|
+
+----------+-------------------------------------------------+
|
201
|
+
| format | Format of the timestamp if type is timestamp |
|
202
|
+
+----------+-------------------------------------------------+
|
203
|
+
|
204
|
+
List of types:
|
205
|
+
|
206
|
+
+-------------+----------------------------------------------+
|
207
|
+
| name | description |
|
208
|
+
+=============+==============================================+
|
209
|
+
| boolean | true or false |
|
210
|
+
+-------------+----------------------------------------------+
|
211
|
+
| long | 64-bit signed integers |
|
212
|
+
+-------------+----------------------------------------------+
|
213
|
+
| timestamp | Date and time with nano-seconds precision |
|
214
|
+
+-------------+----------------------------------------------+
|
215
|
+
| double | 64-bit floating point numbers |
|
216
|
+
+-------------+----------------------------------------------+
|
217
|
+
| string | Strings |
|
218
|
+
+-------------+----------------------------------------------+
|
219
|
+
|
220
|
+
You can use ``guess`` to automatically generate the column settings. See also `Quick Start <https://github.com/embulk/embulk#quick-start>`_.
|
221
|
+
|
222
|
+
Example
|
223
|
+
~~~~~~~~~~~~~~~~~~
|
224
|
+
|
225
|
+
.. code-block:: yaml
|
226
|
+
|
227
|
+
in:
|
228
|
+
...
|
229
|
+
parser:
|
230
|
+
type: csv
|
231
|
+
charset: UTF-8
|
232
|
+
newline: CRLF
|
233
|
+
delimiter: "\t"
|
234
|
+
quote: '"'
|
235
|
+
escape: '"'
|
236
|
+
null_string: 'NULL'
|
237
|
+
skip_header_lines: 1
|
238
|
+
comment_line_marker: '#'
|
239
|
+
columns:
|
240
|
+
- {name: id, type: long}
|
241
|
+
- {name: account, type: long}
|
242
|
+
- {name: time, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
|
243
|
+
- {name: purchase, type: timestamp, format: '%Y%m%d'}
|
244
|
+
- {name: comment, type: string}
|
245
|
+
|
246
|
+
Gzip decoder plugin
|
247
|
+
------------------
|
248
|
+
|
249
|
+
The ``gzip`` decoder plugin decompresses gzip files before input plugins read them.
|
250
|
+
|
251
|
+
Options
|
252
|
+
~~~~~~~~~~~~~~~~~~
|
253
|
+
|
254
|
+
This plugin doesn't have any options.
|
255
|
+
|
256
|
+
Example
|
257
|
+
~~~~~~~~~~~~~~~~~~
|
258
|
+
|
259
|
+
.. code-block:: yaml
|
260
|
+
|
261
|
+
in:
|
262
|
+
...
|
263
|
+
decoders:
|
264
|
+
- {type: gzip}
|
265
|
+
|
266
|
+
|
267
|
+
File output plugin
|
268
|
+
------------------
|
269
|
+
|
270
|
+
The ``file`` output plugin writes records to local file system.
|
271
|
+
|
272
|
+
Options
|
273
|
+
~~~~~~~~~~~~~~~~~~
|
274
|
+
|
275
|
+
+--------------------+----------+---------------------------------------------------+----------------------------+
|
276
|
+
| name | type | description | required? |
|
277
|
+
+====================+==========+===================================================+============================+
|
278
|
+
| path\_prefix | string | Path prefix of the output files | required |
|
279
|
+
+--------------------+----------+---------------------------------------------------+----------------------------+
|
280
|
+
| sequence\_format | string | Format of the sequence number of the output files | ``%03d.%02d.`` by default |
|
281
|
+
+--------------------+----------+---------------------------------------------------+----------------------------+
|
282
|
+
| file\_ext | string | Path suffix of the output files (e.g. ``"csv"``) | required |
|
283
|
+
+--------------------+----------+---------------------------------------------------+----------------------------+
|
284
|
+
|
285
|
+
For example, if you set ``path_prefix: /path/to/output/sample_``, ``sequence_format: "%03d.%02d."``, and ``file_ext: csv``, name of the output files will be as following:
|
286
|
+
|
287
|
+
::
|
288
|
+
|
289
|
+
.
|
290
|
+
`-- path
|
291
|
+
`-- to
|
292
|
+
`-- output
|
293
|
+
|-- sample_01.000.csv
|
294
|
+
|-- sample_02.000.csv
|
295
|
+
|-- sample_03.000.csv
|
296
|
+
|-- sample_04.000.csv
|
297
|
+
|
298
|
+
``sequence_format`` formats task index and sequence number in a task.
|
299
|
+
|
300
|
+
Example
|
301
|
+
~~~~~~~~~~~~~~~~~~
|
302
|
+
|
303
|
+
.. code-block:: yaml
|
304
|
+
|
305
|
+
out:
|
306
|
+
type: file
|
307
|
+
path_prefix: /path/to/output/sample_
|
308
|
+
file_ext: csv
|
309
|
+
formatter:
|
310
|
+
...
|
311
|
+
|
312
|
+
CSV formatter plugin
|
313
|
+
------------------
|
314
|
+
|
315
|
+
The ``csv`` formatter plugin formats records using CSV or TSV format.
|
316
|
+
|
317
|
+
Options
|
318
|
+
~~~~~~~~~~~~~~~~~~
|
319
|
+
|
320
|
+
+----------------------+---------+-------------------------------------------------------------------------------------------------------+------------------------+
|
321
|
+
| name | type | description | required? |
|
322
|
+
+======================+=========+=======================================================================================================+========================+
|
323
|
+
| delimiter | string | Delimiter character such as ``,`` for CSV, ``"\t"`` for TSV, ``"|"`` or any single-byte character | ``,`` by default |
|
324
|
+
+----------------------+---------+-------------------------------------------------------------------------------------------------------+------------------------+
|
325
|
+
| quote | string | The character surrounding a quoted value | ``\"`` by default |
|
326
|
+
+----------------------+---------+-------------------------------------------------------------------------------------------------------+------------------------+
|
327
|
+
| quote\_policy | enum | Policy for quote (ALL, MINIMAL, NONE) (see below) | ``MINIMAL`` by default |
|
328
|
+
+----------------------+---------+-------------------------------------------------------------------------------------------------------+------------------------+
|
329
|
+
| escape | string | Escape character to escape a quote character when quote\_policy is ALL or MINIMAL | ``\"`` by default |
|
330
|
+
+----------------------+---------+-------------------------------------------------------------------------------------------------------+------------------------+
|
331
|
+
| header\_line | boolean | If true, write the header line with column name at the first line | ``true`` by default |
|
332
|
+
+----------------------+---------+-------------------------------------------------------------------------------------------------------+------------------------+
|
333
|
+
| null_string | string | Expression of NULL values | empty by default |
|
334
|
+
+----------------------+---------+-------------------------------------------------------------------------------------------------------+------------------------+
|
335
|
+
| newline | enum | Newline character (CRLF, LF or CR) | ``CRLF`` by default |
|
336
|
+
+----------------------+---------+-------------------------------------------------------------------------------------------------------+------------------------+
|
337
|
+
| newline\_in\_field | enum | Newline character in each field (CRLF, LF, CR) | ``LF`` by default |
|
338
|
+
+----------------------+---------+-------------------------------------------------------------------------------------------------------+------------------------+
|
339
|
+
| charset | enum | Character encoding (eg. ISO-8859-1, UTF-8) | ``UTF-8`` by default |
|
340
|
+
+----------------------+---------+-------------------------------------------------------------------------------------------------------+------------------------+
|
341
|
+
| default\_timezone | string | Time zone of timestamp columns. This can be overwritten for each column using ``column_options`` | ``UTC`` by default |
|
342
|
+
+----------------------+---------+-------------------------------------------------------------------------------------------------------+------------------------+
|
343
|
+
| column\_options | hash | See bellow | optional |
|
344
|
+
+----------------------+---------+-------------------------------------------------------------------------------------------------------+------------------------+
|
345
|
+
|
346
|
+
The ``quote_policy`` option is used to determine field type to quote.
|
347
|
+
|
348
|
+
+------------+--------------------------------------------------------------------------------------------------------+
|
349
|
+
| name | description |
|
350
|
+
+============+========================================================================================================+
|
351
|
+
| ALL | Quote all fields |
|
352
|
+
+------------+--------------------------------------------------------------------------------------------------------+
|
353
|
+
| MINIMAL | Only quote those fields which contain delimiter, quote or any of the characters in lineterminator |
|
354
|
+
+------------+--------------------------------------------------------------------------------------------------------+
|
355
|
+
| NONE | Never quote fields. When the delimiter occurs in field, escape with escape char |
|
356
|
+
+------------+--------------------------------------------------------------------------------------------------------+
|
357
|
+
|
358
|
+
The ``column_options`` option is a map whose keys are name of columns, and values are configuration with following parameters:
|
359
|
+
|
360
|
+
+----------------------+---------+-------------------------------------------------------------------------------------------------------+-----------------------------------------+
|
361
|
+
| name | type | description | required? |
|
362
|
+
+======================+=========+=======================================================================================================+=========================================+
|
363
|
+
| timezone | string | Time zone if type of this column is timestamp. If not set, ``default\_timezone`` is used. | optional |
|
364
|
+
+----------------------+---------+-------------------------------------------------------------------------------------------------------+-----------------------------------------+
|
365
|
+
| format | string | Timestamp format if type of this column is timestamp. | ``%Y-%m-%d %H:%M:%S.%6N %z`` by default |
|
366
|
+
+----------------------+---------+-------------------------------------------------------------------------------------------------------+-----------------------------------------+
|
367
|
+
|
368
|
+
Example
|
369
|
+
~~~~~~~~~~~~~~~~~~
|
370
|
+
|
371
|
+
.. code-block:: yaml
|
372
|
+
|
373
|
+
out:
|
374
|
+
...
|
375
|
+
formatter:
|
376
|
+
type: csv
|
377
|
+
delimiter: '\t'
|
378
|
+
newline: CRLF
|
379
|
+
newline_in_field: LF
|
380
|
+
charset: UTF-8
|
381
|
+
quote_policy: MINIMAL
|
382
|
+
quote: '"'
|
383
|
+
escape: '\\'
|
384
|
+
null_string: '\\N'
|
385
|
+
default_timezone: 'UTC'
|
386
|
+
column_options:
|
387
|
+
mycol1: {format: '%Y-%m-%d %H:%M:%S'}
|
388
|
+
mycol2: {format: '%Y-%m-%d %H:%M:%S', timezone: 'America/Los_Angeles'}
|
389
|
+
|
390
|
+
Gzip encoder plugin
|
391
|
+
------------------
|
392
|
+
|
393
|
+
The ``gzip`` encoder plugin compresses output files using gzip.
|
394
|
+
|
395
|
+
Options
|
396
|
+
~~~~~~~~~~~~~~~~~~
|
397
|
+
|
398
|
+
+---------+----------+----------------------------------------------------------------------+--------------------+
|
399
|
+
| name | type | description | required? |
|
400
|
+
+=========+==========+======================================================================+====================+
|
401
|
+
| level | integer | Compression level. From 0 (no compression) to 9 (best compression). | ``6`` by default |
|
402
|
+
+---------+----------+----------------------------------------------------------------------+--------------------+
|
403
|
+
|
404
|
+
Example
|
405
|
+
~~~~~~~~~~~~~~~~~~
|
406
|
+
|
407
|
+
.. code-block:: yaml
|
408
|
+
|
409
|
+
out:
|
410
|
+
...
|
411
|
+
encoders:
|
412
|
+
- type: gzip
|
413
|
+
level: 1
|
414
|
+
|
415
|
+
Rename filter plugin
|
416
|
+
------------------
|
417
|
+
|
418
|
+
The ``rename`` filter plugin changes column names. This plugin has no impact on performance.
|
419
|
+
|
420
|
+
Options
|
421
|
+
~~~~~~~~~~~~~~~~~~
|
422
|
+
|
423
|
+
+---------+----------+----------------------------------------------------------------------+--------------------+
|
424
|
+
| name | type | description | required? |
|
425
|
+
+=========+==========+======================================================================+====================+
|
426
|
+
| columns | hash | A map whose keys are existing column names. values are new names. | ``{}`` by default |
|
427
|
+
+---------+----------+----------------------------------------------------------------------+--------------------+
|
428
|
+
|
429
|
+
Example
|
430
|
+
~~~~~~~~~~~~~~~~~~
|
431
|
+
|
432
|
+
.. code-block:: yaml
|
433
|
+
|
434
|
+
filters:
|
435
|
+
...
|
436
|
+
- type: rename
|
437
|
+
columns:
|
438
|
+
my_existing_column1: new_column1
|
439
|
+
my_existing_column2: new_column2
|
440
|
+
|