embulk 0.7.0-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +15 -0
- data/.ruby-version +1 -0
- data/.travis.yml +18 -0
- data/COPYING +14 -0
- data/Gemfile +2 -0
- data/Gemfile.lock +31 -0
- data/README.md +206 -0
- data/Rakefile +26 -0
- data/appveyor.yml +20 -0
- data/bin/embulk +106 -0
- data/build.gradle +338 -0
- data/embulk-cli/build.gradle +6 -0
- data/embulk-cli/src/main/java/org/embulk/cli/Main.java +22 -0
- data/embulk-cli/src/main/sh/selfrun.sh +158 -0
- data/embulk-cli/src/test/java/org/embulk/cli/DummyMain.java +23 -0
- data/embulk-cli/src/test/java/org/embulk/cli/SelfrunTest.java +281 -0
- data/embulk-core/build.gradle +59 -0
- data/embulk-core/src/main/java/org/embulk/EmbulkEmbed.java +315 -0
- data/embulk-core/src/main/java/org/embulk/EmbulkService.java +76 -0
- data/embulk-core/src/main/java/org/embulk/command/PreviewPrinter.java +84 -0
- data/embulk-core/src/main/java/org/embulk/command/TablePreviewPrinter.java +107 -0
- data/embulk-core/src/main/java/org/embulk/command/VerticalPreviewPrinter.java +47 -0
- data/embulk-core/src/main/java/org/embulk/config/CommitReport.java +33 -0
- data/embulk-core/src/main/java/org/embulk/config/Config.java +15 -0
- data/embulk-core/src/main/java/org/embulk/config/ConfigDefault.java +15 -0
- data/embulk-core/src/main/java/org/embulk/config/ConfigDiff.java +29 -0
- data/embulk-core/src/main/java/org/embulk/config/ConfigException.java +20 -0
- data/embulk-core/src/main/java/org/embulk/config/ConfigInject.java +14 -0
- data/embulk-core/src/main/java/org/embulk/config/ConfigLoader.java +141 -0
- data/embulk-core/src/main/java/org/embulk/config/ConfigSource.java +31 -0
- data/embulk-core/src/main/java/org/embulk/config/DataSource.java +39 -0
- data/embulk-core/src/main/java/org/embulk/config/DataSourceImpl.java +231 -0
- data/embulk-core/src/main/java/org/embulk/config/DataSourceSerDe.java +84 -0
- data/embulk-core/src/main/java/org/embulk/config/GenericTypeReference.java +20 -0
- data/embulk-core/src/main/java/org/embulk/config/ModelManager.java +123 -0
- data/embulk-core/src/main/java/org/embulk/config/Task.java +10 -0
- data/embulk-core/src/main/java/org/embulk/config/TaskInvocationHandler.java +180 -0
- data/embulk-core/src/main/java/org/embulk/config/TaskReport.java +29 -0
- data/embulk-core/src/main/java/org/embulk/config/TaskSerDe.java +345 -0
- data/embulk-core/src/main/java/org/embulk/config/TaskSource.java +31 -0
- data/embulk-core/src/main/java/org/embulk/config/TaskValidationException.java +38 -0
- data/embulk-core/src/main/java/org/embulk/config/TaskValidator.java +24 -0
- data/embulk-core/src/main/java/org/embulk/exec/BulkLoader.java +652 -0
- data/embulk-core/src/main/java/org/embulk/exec/ExecModule.java +52 -0
- data/embulk-core/src/main/java/org/embulk/exec/ExecutionInterruptedException.java +10 -0
- data/embulk-core/src/main/java/org/embulk/exec/ExecutionResult.java +26 -0
- data/embulk-core/src/main/java/org/embulk/exec/ExtensionServiceLoaderModule.java +43 -0
- data/embulk-core/src/main/java/org/embulk/exec/ForGuess.java +16 -0
- data/embulk-core/src/main/java/org/embulk/exec/ForSystemConfig.java +16 -0
- data/embulk-core/src/main/java/org/embulk/exec/GuessExecutor.java +373 -0
- data/embulk-core/src/main/java/org/embulk/exec/LocalExecutorPlugin.java +129 -0
- data/embulk-core/src/main/java/org/embulk/exec/LocalThreadExecutor.java +34 -0
- data/embulk-core/src/main/java/org/embulk/exec/LoggerProvider.java +60 -0
- data/embulk-core/src/main/java/org/embulk/exec/NoSampleException.java +10 -0
- data/embulk-core/src/main/java/org/embulk/exec/PartialExecutionException.java +18 -0
- data/embulk-core/src/main/java/org/embulk/exec/PooledBufferAllocator.java +77 -0
- data/embulk-core/src/main/java/org/embulk/exec/PreviewExecutor.java +183 -0
- data/embulk-core/src/main/java/org/embulk/exec/PreviewResult.java +27 -0
- data/embulk-core/src/main/java/org/embulk/exec/PreviewedNoticeError.java +17 -0
- data/embulk-core/src/main/java/org/embulk/exec/ResumeState.java +100 -0
- data/embulk-core/src/main/java/org/embulk/exec/SamplingParserPlugin.java +136 -0
- data/embulk-core/src/main/java/org/embulk/exec/SetCurrentThreadName.java +19 -0
- data/embulk-core/src/main/java/org/embulk/exec/SystemConfigModule.java +24 -0
- data/embulk-core/src/main/java/org/embulk/exec/TempFileAllocator.java +35 -0
- data/embulk-core/src/main/java/org/embulk/guice/Bootstrap.java +157 -0
- data/embulk-core/src/main/java/org/embulk/guice/CloseableInjector.java +22 -0
- data/embulk-core/src/main/java/org/embulk/guice/InjectorProxy.java +145 -0
- data/embulk-core/src/main/java/org/embulk/guice/LifeCycleInjector.java +26 -0
- data/embulk-core/src/main/java/org/embulk/guice/LifeCycleInjectorProxy.java +61 -0
- data/embulk-core/src/main/java/org/embulk/guice/LifeCycleManager.java +187 -0
- data/embulk-core/src/main/java/org/embulk/guice/LifeCycleMethods.java +89 -0
- data/embulk-core/src/main/java/org/embulk/guice/LifeCycleMethodsMap.java +38 -0
- data/embulk-core/src/main/java/org/embulk/guice/LifeCycleModule.java +97 -0
- data/embulk-core/src/main/java/org/embulk/jruby/JRubyPluginSource.java +72 -0
- data/embulk-core/src/main/java/org/embulk/jruby/JRubyScriptingModule.java +119 -0
- data/embulk-core/src/main/java/org/embulk/plugin/BuiltinPluginSourceModule.java +17 -0
- data/embulk-core/src/main/java/org/embulk/plugin/InjectedPluginSource.java +96 -0
- data/embulk-core/src/main/java/org/embulk/plugin/PluginClassLoader.java +168 -0
- data/embulk-core/src/main/java/org/embulk/plugin/PluginClassLoaderFactory.java +9 -0
- data/embulk-core/src/main/java/org/embulk/plugin/PluginClassLoaderModule.java +71 -0
- data/embulk-core/src/main/java/org/embulk/plugin/PluginManager.java +78 -0
- data/embulk-core/src/main/java/org/embulk/plugin/PluginSource.java +6 -0
- data/embulk-core/src/main/java/org/embulk/plugin/PluginSourceNotMatchException.java +19 -0
- data/embulk-core/src/main/java/org/embulk/plugin/PluginType.java +47 -0
- data/embulk-core/src/main/java/org/embulk/plugin/compat/InputPluginWrapper.java +102 -0
- data/embulk-core/src/main/java/org/embulk/plugin/compat/PluginWrappers.java +30 -0
- data/embulk-core/src/main/java/org/embulk/plugin/compat/TransactionalFileInputWrapper.java +96 -0
- data/embulk-core/src/main/java/org/embulk/plugin/compat/TransactionalFileOutputWrapper.java +102 -0
- data/embulk-core/src/main/java/org/embulk/plugin/compat/TransactionalPageOutputWrapper.java +95 -0
- data/embulk-core/src/main/java/org/embulk/spi/Buffer.java +148 -0
- data/embulk-core/src/main/java/org/embulk/spi/BufferAllocator.java +8 -0
- data/embulk-core/src/main/java/org/embulk/spi/Column.java +92 -0
- data/embulk-core/src/main/java/org/embulk/spi/ColumnConfig.java +112 -0
- data/embulk-core/src/main/java/org/embulk/spi/ColumnVisitor.java +14 -0
- data/embulk-core/src/main/java/org/embulk/spi/DecoderPlugin.java +16 -0
- data/embulk-core/src/main/java/org/embulk/spi/EncoderPlugin.java +16 -0
- data/embulk-core/src/main/java/org/embulk/spi/Exec.java +113 -0
- data/embulk-core/src/main/java/org/embulk/spi/ExecAction.java +6 -0
- data/embulk-core/src/main/java/org/embulk/spi/ExecSession.java +217 -0
- data/embulk-core/src/main/java/org/embulk/spi/ExecutorPlugin.java +19 -0
- data/embulk-core/src/main/java/org/embulk/spi/Extension.java +44 -0
- data/embulk-core/src/main/java/org/embulk/spi/FileInput.java +11 -0
- data/embulk-core/src/main/java/org/embulk/spi/FileInputPlugin.java +30 -0
- data/embulk-core/src/main/java/org/embulk/spi/FileInputRunner.java +162 -0
- data/embulk-core/src/main/java/org/embulk/spi/FileOutput.java +13 -0
- data/embulk-core/src/main/java/org/embulk/spi/FileOutputPlugin.java +28 -0
- data/embulk-core/src/main/java/org/embulk/spi/FileOutputRunner.java +202 -0
- data/embulk-core/src/main/java/org/embulk/spi/FilterPlugin.java +18 -0
- data/embulk-core/src/main/java/org/embulk/spi/FormatterPlugin.java +18 -0
- data/embulk-core/src/main/java/org/embulk/spi/GuessPlugin.java +9 -0
- data/embulk-core/src/main/java/org/embulk/spi/InputPlugin.java +33 -0
- data/embulk-core/src/main/java/org/embulk/spi/OutputPlugin.java +29 -0
- data/embulk-core/src/main/java/org/embulk/spi/Page.java +51 -0
- data/embulk-core/src/main/java/org/embulk/spi/PageBuilder.java +338 -0
- data/embulk-core/src/main/java/org/embulk/spi/PageFormat.java +47 -0
- data/embulk-core/src/main/java/org/embulk/spi/PageOutput.java +11 -0
- data/embulk-core/src/main/java/org/embulk/spi/PageReader.java +226 -0
- data/embulk-core/src/main/java/org/embulk/spi/ParserPlugin.java +17 -0
- data/embulk-core/src/main/java/org/embulk/spi/ProcessState.java +10 -0
- data/embulk-core/src/main/java/org/embulk/spi/ProcessTask.java +117 -0
- data/embulk-core/src/main/java/org/embulk/spi/Schema.java +134 -0
- data/embulk-core/src/main/java/org/embulk/spi/SchemaConfig.java +93 -0
- data/embulk-core/src/main/java/org/embulk/spi/SchemaConfigException.java +22 -0
- data/embulk-core/src/main/java/org/embulk/spi/TaskState.java +81 -0
- data/embulk-core/src/main/java/org/embulk/spi/TempFileException.java +19 -0
- data/embulk-core/src/main/java/org/embulk/spi/TempFileSpace.java +87 -0
- data/embulk-core/src/main/java/org/embulk/spi/Transactional.java +10 -0
- data/embulk-core/src/main/java/org/embulk/spi/TransactionalFileInput.java +17 -0
- data/embulk-core/src/main/java/org/embulk/spi/TransactionalFileOutput.java +19 -0
- data/embulk-core/src/main/java/org/embulk/spi/TransactionalPageOutput.java +17 -0
- data/embulk-core/src/main/java/org/embulk/spi/time/DateTimeZoneSerDe.java +55 -0
- data/embulk-core/src/main/java/org/embulk/spi/time/JRubyTimeParserHelper.java +8 -0
- data/embulk-core/src/main/java/org/embulk/spi/time/JRubyTimeParserHelperFactory.java +6 -0
- data/embulk-core/src/main/java/org/embulk/spi/time/Timestamp.java +159 -0
- data/embulk-core/src/main/java/org/embulk/spi/time/TimestampFormat.java +100 -0
- data/embulk-core/src/main/java/org/embulk/spi/time/TimestampFormatter.java +97 -0
- data/embulk-core/src/main/java/org/embulk/spi/time/TimestampParseException.java +10 -0
- data/embulk-core/src/main/java/org/embulk/spi/time/TimestampParser.java +104 -0
- data/embulk-core/src/main/java/org/embulk/spi/time/TimestampSerDe.java +49 -0
- data/embulk-core/src/main/java/org/embulk/spi/type/AbstractType.java +58 -0
- data/embulk-core/src/main/java/org/embulk/spi/type/BooleanType.java +12 -0
- data/embulk-core/src/main/java/org/embulk/spi/type/DoubleType.java +12 -0
- data/embulk-core/src/main/java/org/embulk/spi/type/LongType.java +12 -0
- data/embulk-core/src/main/java/org/embulk/spi/type/StringType.java +12 -0
- data/embulk-core/src/main/java/org/embulk/spi/type/TimestampType.java +41 -0
- data/embulk-core/src/main/java/org/embulk/spi/type/Type.java +15 -0
- data/embulk-core/src/main/java/org/embulk/spi/type/TypeDeserializer.java +44 -0
- data/embulk-core/src/main/java/org/embulk/spi/type/Types.java +14 -0
- data/embulk-core/src/main/java/org/embulk/spi/unit/ByteSize.java +156 -0
- data/embulk-core/src/main/java/org/embulk/spi/unit/LocalFile.java +106 -0
- data/embulk-core/src/main/java/org/embulk/spi/unit/LocalFileSerDe.java +113 -0
- data/embulk-core/src/main/java/org/embulk/spi/unit/ToString.java +54 -0
- data/embulk-core/src/main/java/org/embulk/spi/unit/ToStringMap.java +34 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/CharsetSerDe.java +55 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/Decoders.java +81 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/DynamicColumnNotFoundException.java +10 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/DynamicColumnSetter.java +18 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/DynamicColumnSetterFactory.java +94 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/DynamicPageBuilder.java +161 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/Encoders.java +81 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/Executors.java +95 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/FileInputInputStream.java +111 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/FileOutputOutputStream.java +119 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/Filters.java +100 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/InputStreamFileInput.java +190 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/InputStreamTransactionalFileInput.java +25 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/Inputs.java +65 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/LineDecoder.java +118 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/LineEncoder.java +123 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/ListFileInput.java +52 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/Newline.java +38 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/OutputStreamFileOutput.java +88 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/PagePrinter.java +102 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/Pages.java +139 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/ResumableInputStream.java +128 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/RetryExecutor.java +130 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/Timestamps.java +53 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/AbstractDynamicColumnSetter.java +79 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/BooleanColumnSetter.java +64 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/DefaultValueSetter.java +18 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/DoubleColumnSetter.java +61 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/LongColumnSetter.java +69 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/NullDefaultValueSetter.java +34 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/SkipColumnSetter.java +52 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/StringColumnSetter.java +56 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/TimestampColumnSetter.java +64 -0
- data/embulk-core/src/main/resources/embulk/logback-color.xml +72 -0
- data/embulk-core/src/main/resources/embulk/logback-console.xml +14 -0
- data/embulk-core/src/main/resources/embulk/logback-file.xml +25 -0
- data/embulk-core/src/main/resources/embulk/parent_first_packages.properties +70 -0
- data/embulk-core/src/main/resources/embulk/parent_first_resources.properties +28 -0
- data/embulk-core/src/test/java/org/embulk/EmbulkTestRuntime.java +114 -0
- data/embulk-core/src/test/java/org/embulk/GuiceBinder.java +72 -0
- data/embulk-core/src/test/java/org/embulk/RandomManager.java +53 -0
- data/embulk-core/src/test/java/org/embulk/TestPluginSourceModule.java +23 -0
- data/embulk-core/src/test/java/org/embulk/TestUtilityModule.java +17 -0
- data/embulk-core/src/test/java/org/embulk/config/TestConfigLoader.java +66 -0
- data/embulk-core/src/test/java/org/embulk/config/TestConfigSource.java +114 -0
- data/embulk-core/src/test/java/org/embulk/config/TestTaskSource.java +70 -0
- data/embulk-core/src/test/java/org/embulk/plugin/MockPluginSource.java +58 -0
- data/embulk-core/src/test/java/org/embulk/plugin/TestPluginType.java +18 -0
- data/embulk-core/src/test/java/org/embulk/spi/MockFileOutput.java +63 -0
- data/embulk-core/src/test/java/org/embulk/spi/MockFormatterPlugin.java +101 -0
- data/embulk-core/src/test/java/org/embulk/spi/MockParserPlugin.java +73 -0
- data/embulk-core/src/test/java/org/embulk/spi/PageTestUtils.java +57 -0
- data/embulk-core/src/test/java/org/embulk/spi/TestBuffer.java +24 -0
- data/embulk-core/src/test/java/org/embulk/spi/TestFileInputInputStream.java +89 -0
- data/embulk-core/src/test/java/org/embulk/spi/TestFileInputRunner.java +196 -0
- data/embulk-core/src/test/java/org/embulk/spi/TestFileOutputRunner.java +207 -0
- data/embulk-core/src/test/java/org/embulk/spi/TestInputStreamFileInput.java +188 -0
- data/embulk-core/src/test/java/org/embulk/spi/TestPageBuilderReader.java +319 -0
- data/embulk-core/src/test/java/org/embulk/spi/time/TestTimestamp.java +116 -0
- data/embulk-core/src/test/java/org/embulk/spi/time/TestTimestampFormatterParser.java +63 -0
- data/embulk-core/src/test/java/org/embulk/spi/time/TestTimestampFormatterParserDeprecated.java +67 -0
- data/embulk-core/src/test/java/org/embulk/spi/type/TestTypeSerDe.java +45 -0
- data/embulk-core/src/test/java/org/embulk/spi/unit/TestByteSize.java +79 -0
- data/embulk-core/src/test/java/org/embulk/spi/util/TestLineDecoder.java +132 -0
- data/embulk-core/src/test/java/org/embulk/spi/util/TestLineEncoder.java +123 -0
- data/embulk-docs/Makefile +178 -0
- data/embulk-docs/build.gradle +32 -0
- data/embulk-docs/make.bat +243 -0
- data/embulk-docs/push-gh-pages.sh +49 -0
- data/embulk-docs/src/_static/embulk-architecture.png +0 -0
- data/embulk-docs/src/_static/embulk-logo.png +0 -0
- data/embulk-docs/src/_static/embulk-logo.svg +133 -0
- data/embulk-docs/src/built-in.rst +440 -0
- data/embulk-docs/src/conf.py +260 -0
- data/embulk-docs/src/customization.rst +184 -0
- data/embulk-docs/src/index.rst +84 -0
- data/embulk-docs/src/recipe.rst +8 -0
- data/embulk-docs/src/recipe/scheduled-csv-load-to-elasticsearch-kibana4.rst +153 -0
- data/embulk-docs/src/release.rst +57 -0
- data/embulk-docs/src/release/release-0.1.0.rst +8 -0
- data/embulk-docs/src/release/release-0.2.0.rst +16 -0
- data/embulk-docs/src/release/release-0.2.1.rst +19 -0
- data/embulk-docs/src/release/release-0.3.0.rst +34 -0
- data/embulk-docs/src/release/release-0.3.1.rst +11 -0
- data/embulk-docs/src/release/release-0.3.2.rst +15 -0
- data/embulk-docs/src/release/release-0.4.0.rst +74 -0
- data/embulk-docs/src/release/release-0.4.1.rst +18 -0
- data/embulk-docs/src/release/release-0.4.10.rst +17 -0
- data/embulk-docs/src/release/release-0.4.2.rst +18 -0
- data/embulk-docs/src/release/release-0.4.3.rst +34 -0
- data/embulk-docs/src/release/release-0.4.4.rst +39 -0
- data/embulk-docs/src/release/release-0.4.5.rst +24 -0
- data/embulk-docs/src/release/release-0.4.6.rst +30 -0
- data/embulk-docs/src/release/release-0.4.7.rst +16 -0
- data/embulk-docs/src/release/release-0.4.8.rst +15 -0
- data/embulk-docs/src/release/release-0.4.9.rst +23 -0
- data/embulk-docs/src/release/release-0.5.0.rst +89 -0
- data/embulk-docs/src/release/release-0.5.1.rst +13 -0
- data/embulk-docs/src/release/release-0.5.2.rst +30 -0
- data/embulk-docs/src/release/release-0.5.3.rst +22 -0
- data/embulk-docs/src/release/release-0.5.4.rst +24 -0
- data/embulk-docs/src/release/release-0.5.5.rst +18 -0
- data/embulk-docs/src/release/release-0.6.0.rst +34 -0
- data/embulk-docs/src/release/release-0.6.1.rst +11 -0
- data/embulk-docs/src/release/release-0.6.10.rst +15 -0
- data/embulk-docs/src/release/release-0.6.11.rst +19 -0
- data/embulk-docs/src/release/release-0.6.12.rst +31 -0
- data/embulk-docs/src/release/release-0.6.13.rst +23 -0
- data/embulk-docs/src/release/release-0.6.14.rst +47 -0
- data/embulk-docs/src/release/release-0.6.15.rst +26 -0
- data/embulk-docs/src/release/release-0.6.16.rst +26 -0
- data/embulk-docs/src/release/release-0.6.17.rst +39 -0
- data/embulk-docs/src/release/release-0.6.18.rst +14 -0
- data/embulk-docs/src/release/release-0.6.19.rst +18 -0
- data/embulk-docs/src/release/release-0.6.2.rst +17 -0
- data/embulk-docs/src/release/release-0.6.20.rst +19 -0
- data/embulk-docs/src/release/release-0.6.21.rst +20 -0
- data/embulk-docs/src/release/release-0.6.22.rst +26 -0
- data/embulk-docs/src/release/release-0.6.23.rst +17 -0
- data/embulk-docs/src/release/release-0.6.24.rst +13 -0
- data/embulk-docs/src/release/release-0.6.25.rst +12 -0
- data/embulk-docs/src/release/release-0.6.3.rst +23 -0
- data/embulk-docs/src/release/release-0.6.4.rst +13 -0
- data/embulk-docs/src/release/release-0.6.5.rst +17 -0
- data/embulk-docs/src/release/release-0.6.6.rst +17 -0
- data/embulk-docs/src/release/release-0.6.7.rst +17 -0
- data/embulk-docs/src/release/release-0.6.8.rst +24 -0
- data/embulk-docs/src/release/release-0.6.9.rst +24 -0
- data/embulk-docs/src/release/release-0.7.0.rst +96 -0
- data/embulk-standards/build.gradle +5 -0
- data/embulk-standards/src/main/java/org/embulk/standards/CsvFormatterPlugin.java +284 -0
- data/embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java +379 -0
- data/embulk-standards/src/main/java/org/embulk/standards/CsvTokenizer.java +411 -0
- data/embulk-standards/src/main/java/org/embulk/standards/GzipFileDecoderPlugin.java +55 -0
- data/embulk-standards/src/main/java/org/embulk/standards/GzipFileEncoderPlugin.java +71 -0
- data/embulk-standards/src/main/java/org/embulk/standards/LocalFileInputPlugin.java +203 -0
- data/embulk-standards/src/main/java/org/embulk/standards/LocalFileOutputPlugin.java +148 -0
- data/embulk-standards/src/main/java/org/embulk/standards/NullOutputPlugin.java +59 -0
- data/embulk-standards/src/main/java/org/embulk/standards/RenameFilterPlugin.java +56 -0
- data/embulk-standards/src/main/java/org/embulk/standards/StandardPluginExtension.java +16 -0
- data/embulk-standards/src/main/java/org/embulk/standards/StandardPluginModule.java +53 -0
- data/embulk-standards/src/main/java/org/embulk/standards/StdoutOutputPlugin.java +85 -0
- data/embulk-standards/src/main/resources/META-INF/services/org.embulk.spi.Extension +1 -0
- data/embulk-standards/src/test/java/org/embulk/standards/TestCsvFormatterPlugin.java +312 -0
- data/embulk-standards/src/test/java/org/embulk/standards/TestCsvParserPlugin.java +75 -0
- data/embulk-standards/src/test/java/org/embulk/standards/TestCsvTokenizer.java +360 -0
- data/embulk-standards/src/test/java/org/embulk/standards/TestRenameFilterPlugin.java +88 -0
- data/embulk.gemspec +39 -0
- data/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +6 -0
- data/gradlew +164 -0
- data/gradlew.bat +90 -0
- data/lib/embulk.rb +72 -0
- data/lib/embulk/buffer.rb +22 -0
- data/lib/embulk/column.rb +70 -0
- data/lib/embulk/command/embulk_bundle.rb +56 -0
- data/lib/embulk/command/embulk_example.rb +32 -0
- data/lib/embulk/command/embulk_generate_bin.rb +62 -0
- data/lib/embulk/command/embulk_main.rb +2 -0
- data/lib/embulk/command/embulk_migrate_plugin.rb +170 -0
- data/lib/embulk/command/embulk_new_plugin.rb +124 -0
- data/lib/embulk/command/embulk_run.rb +470 -0
- data/lib/embulk/command/embulk_selfupdate.rb +84 -0
- data/lib/embulk/data/bundle/.bundle/config +3 -0
- data/lib/embulk/data/bundle/.ruby-version +1 -0
- data/lib/embulk/data/bundle/Gemfile +26 -0
- data/lib/embulk/data/bundle/embulk/filter/example.rb +42 -0
- data/lib/embulk/data/bundle/embulk/input/example.rb +54 -0
- data/lib/embulk/data/bundle/embulk/output/example.rb +58 -0
- data/lib/embulk/data/new/LICENSE.txt +21 -0
- data/lib/embulk/data/new/README.md.erb +111 -0
- data/lib/embulk/data/new/gitignore.erb +13 -0
- data/lib/embulk/data/new/java/build.gradle.erb +73 -0
- data/lib/embulk/data/new/java/decoder.java.erb +84 -0
- data/lib/embulk/data/new/java/encoder.java.erb +86 -0
- data/lib/embulk/data/new/java/file_input.java.erb +143 -0
- data/lib/embulk/data/new/java/file_output.java.erb +93 -0
- data/lib/embulk/data/new/java/filter.java.erb +56 -0
- data/lib/embulk/data/new/java/formatter.java.erb +54 -0
- data/lib/embulk/data/new/java/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/lib/embulk/data/new/java/gradle/wrapper/gradle-wrapper.properties +6 -0
- data/lib/embulk/data/new/java/gradlew +164 -0
- data/lib/embulk/data/new/java/gradlew.bat +90 -0
- data/lib/embulk/data/new/java/input.java.erb +87 -0
- data/lib/embulk/data/new/java/output.java.erb +77 -0
- data/lib/embulk/data/new/java/parser.java.erb +60 -0
- data/lib/embulk/data/new/java/plugin_loader.rb.erb +3 -0
- data/lib/embulk/data/new/java/test.java.erb +5 -0
- data/lib/embulk/data/new/ruby/.ruby-version +1 -0
- data/lib/embulk/data/new/ruby/Gemfile +2 -0
- data/lib/embulk/data/new/ruby/Rakefile +3 -0
- data/lib/embulk/data/new/ruby/decoder_guess.rb.erb +25 -0
- data/lib/embulk/data/new/ruby/filter.rb.erb +41 -0
- data/lib/embulk/data/new/ruby/formatter.rb.erb +49 -0
- data/lib/embulk/data/new/ruby/gemspec.erb +20 -0
- data/lib/embulk/data/new/ruby/input.rb.erb +59 -0
- data/lib/embulk/data/new/ruby/output.rb.erb +61 -0
- data/lib/embulk/data/new/ruby/parser.rb.erb +44 -0
- data/lib/embulk/data/new/ruby/parser_guess.rb.erb +65 -0
- data/lib/embulk/data/package_data.rb +50 -0
- data/lib/embulk/data_source.rb +220 -0
- data/lib/embulk/decoder_plugin.rb +27 -0
- data/lib/embulk/encoder_plugin.rb +27 -0
- data/lib/embulk/error.rb +8 -0
- data/lib/embulk/executor_plugin.rb +23 -0
- data/lib/embulk/file_input.rb +87 -0
- data/lib/embulk/file_input_plugin.rb +27 -0
- data/lib/embulk/file_output.rb +56 -0
- data/lib/embulk/file_output_plugin.rb +27 -0
- data/lib/embulk/filter_plugin.rb +105 -0
- data/lib/embulk/formatter_plugin.rb +105 -0
- data/lib/embulk/guess/charset.rb +44 -0
- data/lib/embulk/guess/csv.rb +327 -0
- data/lib/embulk/guess/gzip.rb +18 -0
- data/lib/embulk/guess/newline.rb +22 -0
- data/lib/embulk/guess/schema_guess.rb +118 -0
- data/lib/embulk/guess/time_format_guess.rb +394 -0
- data/lib/embulk/guess_plugin.rb +129 -0
- data/lib/embulk/input_plugin.rb +121 -0
- data/lib/embulk/java/bootstrap.rb +24 -0
- data/lib/embulk/java/imports.rb +69 -0
- data/lib/embulk/java/time_helper.rb +79 -0
- data/lib/embulk/java_plugin.rb +90 -0
- data/lib/embulk/logger.rb +154 -0
- data/lib/embulk/output_plugin.rb +150 -0
- data/lib/embulk/page.rb +30 -0
- data/lib/embulk/page_builder.rb +76 -0
- data/lib/embulk/parser_plugin.rb +78 -0
- data/lib/embulk/plugin.rb +239 -0
- data/lib/embulk/plugin_registry.rb +96 -0
- data/lib/embulk/runner.rb +184 -0
- data/lib/embulk/schema.rb +103 -0
- data/lib/embulk/version.rb +3 -0
- data/settings.gradle +6 -0
- data/test/guess/test_schema_guess.rb +11 -0
- data/test/guess/test_time_format_guess.rb +133 -0
- data/test/helper.rb +21 -0
- data/test/run-test.rb +14 -0
- metadata +566 -0
@@ -0,0 +1,260 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
#
|
3
|
+
# Embulk documentation build configuration file, created by
|
4
|
+
# sphinx-quickstart on Fri Feb 13 14:52:36 2015.
|
5
|
+
#
|
6
|
+
# This file is execfile()d with the current directory set to its
|
7
|
+
# containing dir.
|
8
|
+
#
|
9
|
+
# Note that not all possible configuration values are present in this
|
10
|
+
# autogenerated file.
|
11
|
+
#
|
12
|
+
# All configuration values have a default; values that are commented out
|
13
|
+
# serve to show the default.
|
14
|
+
|
15
|
+
import sys
|
16
|
+
import os
|
17
|
+
|
18
|
+
# If extensions (or modules to document with autodoc) are in another directory,
|
19
|
+
# add these directories to sys.path here. If the directory is relative to the
|
20
|
+
# documentation root, use os.path.abspath to make it absolute, like shown here.
|
21
|
+
#sys.path.insert(0, os.path.abspath('.'))
|
22
|
+
|
23
|
+
# -- General configuration ------------------------------------------------
|
24
|
+
|
25
|
+
# If your documentation needs a minimal Sphinx version, state it here.
|
26
|
+
#needs_sphinx = '1.0'
|
27
|
+
|
28
|
+
# Add any Sphinx extension module names here, as strings. They can be
|
29
|
+
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
|
30
|
+
# ones.
|
31
|
+
extensions = [
|
32
|
+
'sphinx.ext.autodoc',
|
33
|
+
]
|
34
|
+
|
35
|
+
# Add any paths that contain templates here, relative to this directory.
|
36
|
+
templates_path = ['_templates']
|
37
|
+
|
38
|
+
# The suffix of source filenames.
|
39
|
+
source_suffix = '.rst'
|
40
|
+
|
41
|
+
# The encoding of source files.
|
42
|
+
#source_encoding = 'utf-8-sig'
|
43
|
+
|
44
|
+
# The master toctree document.
|
45
|
+
master_doc = 'index'
|
46
|
+
|
47
|
+
# General information about the project.
|
48
|
+
project = u'Embulk'
|
49
|
+
copyright = u'2015, Embulk Project'
|
50
|
+
|
51
|
+
# The version info for the project you're documenting, acts as replacement for
|
52
|
+
# |version| and |release|, also used in various other places throughout the
|
53
|
+
# built documents.
|
54
|
+
#
|
55
|
+
# The short X.Y version.
|
56
|
+
version = '0.7'
|
57
|
+
# The full version, including alpha/beta/rc tags.
|
58
|
+
release = '0.7'
|
59
|
+
|
60
|
+
# The language for content autogenerated by Sphinx. Refer to documentation
|
61
|
+
# for a list of supported languages.
|
62
|
+
#language = None
|
63
|
+
|
64
|
+
# There are two options for replacing |today|: either, you set today to some
|
65
|
+
# non-false value, then it is used:
|
66
|
+
#today = ''
|
67
|
+
# Else, today_fmt is used as the format for a strftime call.
|
68
|
+
#today_fmt = '%B %d, %Y'
|
69
|
+
|
70
|
+
# List of patterns, relative to source directory, that match files and
|
71
|
+
# directories to ignore when looking for source files.
|
72
|
+
exclude_patterns = []
|
73
|
+
|
74
|
+
# The reST default role (used for this markup: `text`) to use for all
|
75
|
+
# documents.
|
76
|
+
#default_role = None
|
77
|
+
|
78
|
+
# If true, '()' will be appended to :func: etc. cross-reference text.
|
79
|
+
#add_function_parentheses = True
|
80
|
+
|
81
|
+
# If true, the current module name will be prepended to all description
|
82
|
+
# unit titles (such as .. function::).
|
83
|
+
#add_module_names = True
|
84
|
+
|
85
|
+
# If true, sectionauthor and moduleauthor directives will be shown in the
|
86
|
+
# output. They are ignored by default.
|
87
|
+
#show_authors = False
|
88
|
+
|
89
|
+
# The name of the Pygments (syntax highlighting) style to use.
|
90
|
+
pygments_style = 'sphinx'
|
91
|
+
|
92
|
+
# A list of ignored prefixes for module index sorting.
|
93
|
+
#modindex_common_prefix = []
|
94
|
+
|
95
|
+
# If true, keep warnings as "system message" paragraphs in the built documents.
|
96
|
+
#keep_warnings = False
|
97
|
+
|
98
|
+
|
99
|
+
# -- Options for HTML output ----------------------------------------------
|
100
|
+
|
101
|
+
# The theme to use for HTML and HTML Help pages. See the documentation for
|
102
|
+
# a list of builtin themes.
|
103
|
+
html_theme = 'haiku'
|
104
|
+
|
105
|
+
# Theme options are theme-specific and customize the look and feel of a theme
|
106
|
+
# further. For a list of options available for each theme, see the
|
107
|
+
# documentation.
|
108
|
+
html_theme_options = {}
|
109
|
+
|
110
|
+
# Add any paths that contain custom themes here, relative to this directory.
|
111
|
+
#html_theme_path = []
|
112
|
+
|
113
|
+
# The name for this set of Sphinx documents. If None, it defaults to
|
114
|
+
# "<project> v<release> documentation".
|
115
|
+
#html_title = None
|
116
|
+
|
117
|
+
# A shorter title for the navigation bar. Default is the same as html_title.
|
118
|
+
#html_short_title = None
|
119
|
+
|
120
|
+
# The name of an image file (relative to this directory) to place at the top
|
121
|
+
# of the sidebar.
|
122
|
+
#html_logo = None
|
123
|
+
|
124
|
+
# The name of an image file (within the static path) to use as favicon of the
|
125
|
+
# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
|
126
|
+
# pixels large.
|
127
|
+
#html_favicon = None
|
128
|
+
|
129
|
+
# Add any paths that contain custom static files (such as style sheets) here,
|
130
|
+
# relative to this directory. They are copied after the builtin static files,
|
131
|
+
# so a file named "default.css" will overwrite the builtin "default.css".
|
132
|
+
html_static_path = ['_static']
|
133
|
+
|
134
|
+
# Add any extra paths that contain custom files (such as robots.txt or
|
135
|
+
# .htaccess) here, relative to this directory. These files are copied
|
136
|
+
# directly to the root of the documentation.
|
137
|
+
#html_extra_path = []
|
138
|
+
|
139
|
+
# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
|
140
|
+
# using the given strftime format.
|
141
|
+
#html_last_updated_fmt = '%b %d, %Y'
|
142
|
+
|
143
|
+
# If true, SmartyPants will be used to convert quotes and dashes to
|
144
|
+
# typographically correct entities.
|
145
|
+
#html_use_smartypants = True
|
146
|
+
|
147
|
+
# Custom sidebar templates, maps document names to template names.
|
148
|
+
#html_sidebars = {}
|
149
|
+
|
150
|
+
# Additional templates that should be rendered to pages, maps page names to
|
151
|
+
# template names.
|
152
|
+
#html_additional_pages = {}
|
153
|
+
|
154
|
+
# If false, no module index is generated.
|
155
|
+
#html_domain_indices = True
|
156
|
+
|
157
|
+
# If false, no index is generated.
|
158
|
+
#html_use_index = True
|
159
|
+
|
160
|
+
# If true, the index is split into individual pages for each letter.
|
161
|
+
#html_split_index = False
|
162
|
+
|
163
|
+
# If true, links to the reST sources are added to the pages.
|
164
|
+
#html_show_sourcelink = True
|
165
|
+
|
166
|
+
# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
|
167
|
+
#html_show_sphinx = True
|
168
|
+
|
169
|
+
# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
|
170
|
+
#html_show_copyright = True
|
171
|
+
|
172
|
+
# If true, an OpenSearch description file will be output, and all pages will
|
173
|
+
# contain a <link> tag referring to it. The value of this option must be the
|
174
|
+
# base URL from which the finished HTML is served.
|
175
|
+
#html_use_opensearch = ''
|
176
|
+
|
177
|
+
# This is the file name suffix for HTML files (e.g. ".xhtml").
|
178
|
+
#html_file_suffix = None
|
179
|
+
|
180
|
+
# Output file base name for HTML help builder.
|
181
|
+
htmlhelp_basename = 'Embulkdoc'
|
182
|
+
|
183
|
+
|
184
|
+
# -- Options for LaTeX output ---------------------------------------------
|
185
|
+
|
186
|
+
latex_elements = {
|
187
|
+
# The paper size ('letterpaper' or 'a4paper').
|
188
|
+
#'papersize': 'letterpaper',
|
189
|
+
|
190
|
+
# The font size ('10pt', '11pt' or '12pt').
|
191
|
+
#'pointsize': '10pt',
|
192
|
+
|
193
|
+
# Additional stuff for the LaTeX preamble.
|
194
|
+
#'preamble': '',
|
195
|
+
}
|
196
|
+
|
197
|
+
# Grouping the document tree into LaTeX files. List of tuples
|
198
|
+
# (source start file, target name, title,
|
199
|
+
# author, documentclass [howto, manual, or own class]).
|
200
|
+
latex_documents = [
|
201
|
+
('index', 'Embulk.tex', u'Embulk Documentation',
|
202
|
+
u'Embulk Project', 'manual'),
|
203
|
+
]
|
204
|
+
|
205
|
+
# The name of an image file (relative to this directory) to place at the top of
|
206
|
+
# the title page.
|
207
|
+
#latex_logo = None
|
208
|
+
|
209
|
+
# For "manual" documents, if this is true, then toplevel headings are parts,
|
210
|
+
# not chapters.
|
211
|
+
#latex_use_parts = False
|
212
|
+
|
213
|
+
# If true, show page references after internal links.
|
214
|
+
#latex_show_pagerefs = False
|
215
|
+
|
216
|
+
# If true, show URL addresses after external links.
|
217
|
+
#latex_show_urls = False
|
218
|
+
|
219
|
+
# Documents to append as an appendix to all manuals.
|
220
|
+
#latex_appendices = []
|
221
|
+
|
222
|
+
# If false, no module index is generated.
|
223
|
+
#latex_domain_indices = True
|
224
|
+
|
225
|
+
|
226
|
+
# -- Options for manual page output ---------------------------------------
|
227
|
+
|
228
|
+
# One entry per manual page. List of tuples
|
229
|
+
# (source start file, name, description, authors, manual section).
|
230
|
+
man_pages = [
|
231
|
+
('index', 'embulk', u'Embulk Documentation',
|
232
|
+
[u'Embulk Project'], 1)
|
233
|
+
]
|
234
|
+
|
235
|
+
# If true, show URL addresses after external links.
|
236
|
+
#man_show_urls = False
|
237
|
+
|
238
|
+
|
239
|
+
# -- Options for Texinfo output -------------------------------------------
|
240
|
+
|
241
|
+
# Grouping the document tree into Texinfo files. List of tuples
|
242
|
+
# (source start file, target name, title, author,
|
243
|
+
# dir menu entry, description, category)
|
244
|
+
texinfo_documents = [
|
245
|
+
('index', 'Embulk', u'Embulk Documentation',
|
246
|
+
u'Embulk Project', 'Embulk', 'One line description of project.',
|
247
|
+
'Miscellaneous'),
|
248
|
+
]
|
249
|
+
|
250
|
+
# Documents to append as an appendix to all manuals.
|
251
|
+
#texinfo_appendices = []
|
252
|
+
|
253
|
+
# If false, no module index is generated.
|
254
|
+
#texinfo_domain_indices = True
|
255
|
+
|
256
|
+
# How to display URL addresses: 'footnote', 'no', or 'inline'.
|
257
|
+
#texinfo_show_urls = 'footnote'
|
258
|
+
|
259
|
+
# If true, do not generate a @detailmenu in the "Top" node's menu.
|
260
|
+
#texinfo_no_detailmenu = False
|
@@ -0,0 +1,184 @@
|
|
1
|
+
Customization
|
2
|
+
==================================
|
3
|
+
|
4
|
+
.. contents::
|
5
|
+
:local:
|
6
|
+
:depth: 2
|
7
|
+
|
8
|
+
|
9
|
+
Creating plugins
|
10
|
+
------------------
|
11
|
+
|
12
|
+
Creating a new plugin is 4 steps:
|
13
|
+
|
14
|
+
1. Create a new project using a template generator
|
15
|
+
2. Build the project (Java only)
|
16
|
+
3. Confirm it works
|
17
|
+
4. Modify the code as you need
|
18
|
+
|
19
|
+
This article describes how to create a plugin step by step.
|
20
|
+
|
21
|
+
Step 1: Creating a new project
|
22
|
+
~~~~~~~~~~~~~~~~~~
|
23
|
+
|
24
|
+
Embulk comes with a number of templates that generates a new project so that you can start development instantly. Because the generated project contains completely working code without additional code, you can focus on the necessary coding.
|
25
|
+
|
26
|
+
Usage of the generator is where ``<type>`` is one of the plugin templates:
|
27
|
+
|
28
|
+
::
|
29
|
+
|
30
|
+
$ embulk new <type> <name>
|
31
|
+
|
32
|
+
Here is the list of available templates. Please find the best option:
|
33
|
+
|
34
|
+
==================== =============================== =================
|
35
|
+
Type description example
|
36
|
+
==================== =============================== =================
|
37
|
+
**java-input** Java record input plugin ``mysql``
|
38
|
+
**java-output** Java record output plugin ``mysql``
|
39
|
+
**java-filter** Java record filter plugin ``add-hostname``
|
40
|
+
**java-file-input** Java file input plugin ``ftp``
|
41
|
+
**java-file-output** Java file output plugin ``ftp``
|
42
|
+
**java-parser** Java file parser plugin ``csv``
|
43
|
+
**java-formatter** Java file formatter plugin ``csv``
|
44
|
+
**java-decoder** Java file decoder plugin ``gzip``
|
45
|
+
**java-encoder** Java file encoder plugin ``gzip``
|
46
|
+
**ruby-input** Ruby record input plugin ``mysql``
|
47
|
+
**ruby-output** Ruby record output plugin ``mysql``
|
48
|
+
**ruby-filter** Ruby record filter plugin ``add-hostname``
|
49
|
+
**ruby-parser** Ruby file parser plugin ``csv``
|
50
|
+
**ruby-formatter** Ruby file formatter plugin ``csv```
|
51
|
+
==================== =============================== =================
|
52
|
+
|
53
|
+
For example, if you want to parse a new file format using Java, type this command:
|
54
|
+
|
55
|
+
::
|
56
|
+
|
57
|
+
$ embulk new java-parser myformat
|
58
|
+
|
59
|
+
This will create a Java-based parser plugin called ``myformat`` in ``embulk-parser-myformat`` directory.
|
60
|
+
|
61
|
+
Step 2: Building the project
|
62
|
+
~~~~~~~~~~~~~~~~~~
|
63
|
+
|
64
|
+
If the plugin is Java-based, you need to build the project. To build, type this command:
|
65
|
+
|
66
|
+
::
|
67
|
+
|
68
|
+
$ cd embulk-parser-myformat
|
69
|
+
$ ./gradlew package
|
70
|
+
|
71
|
+
Now, the plugin is ready to use.
|
72
|
+
|
73
|
+
Step 3: Confirm it works
|
74
|
+
~~~~~~~~~~~~~~~~~~
|
75
|
+
|
76
|
+
The next step is to actually use the plugin.
|
77
|
+
|
78
|
+
Let's suppose you have a configuration file named ``your-config.yml``. You can use the plugin using embulk with ``-L`` argument:
|
79
|
+
|
80
|
+
::
|
81
|
+
|
82
|
+
$ embulk run -L ./embulk-parser-myformat/ your-config.yml
|
83
|
+
|
84
|
+
Step 4: Modifying the code
|
85
|
+
~~~~~~~~~~~~~~~~~~
|
86
|
+
|
87
|
+
The final step is to modify code as you want!
|
88
|
+
|
89
|
+
The code is located at
|
90
|
+
|
91
|
+
* Java-based plugins
|
92
|
+
|
93
|
+
* src/org/embulk/*
|
94
|
+
|
95
|
+
* Ruby-based plugins
|
96
|
+
|
97
|
+
* lib/embulk/*
|
98
|
+
|
99
|
+
There are a lot of good code examples on Github. Search repositories by `embulk-<type> keyword <https://github.com/search?q=embulk-output>`_.
|
100
|
+
|
101
|
+
Releasing plugins
|
102
|
+
------------------
|
103
|
+
|
104
|
+
You can release publicly so that all people can use your awesome plugins.
|
105
|
+
|
106
|
+
Checking plugin description
|
107
|
+
~~~~~~~~~~~~~~~~~~
|
108
|
+
|
109
|
+
To prepare the plugin ready to release, you need to include some additional information. The plugin information is written in this file:
|
110
|
+
|
111
|
+
* Java-based plugins
|
112
|
+
|
113
|
+
* ``build.gradle`` file
|
114
|
+
|
115
|
+
* Ruby-based plugins
|
116
|
+
|
117
|
+
* ``embulk-<type>-<name>.gemspec`` file (``<type>`` is plugin type and ``<name>`` is plugin name)
|
118
|
+
|
119
|
+
You will find following section in the file.
|
120
|
+
|
121
|
+
.. code-block:: ruby
|
122
|
+
|
123
|
+
Gem::Specification.new do |spec|
|
124
|
+
# ...
|
125
|
+
|
126
|
+
spec.authors = ["Your Name"]
|
127
|
+
spec.summary = %[Myformat parser plugin for Embulk]
|
128
|
+
spec.description = %[Parses Myformat files read by other file input plugins.]
|
129
|
+
spec.email = ["you@example.org"]
|
130
|
+
spec.licenses = ["MIT"]
|
131
|
+
spec.homepage = "https://github.com/frsyuki/embulk-parser-myformat"
|
132
|
+
|
133
|
+
# ...
|
134
|
+
end
|
135
|
+
|
136
|
+
The items in above example are important. Please make sure that they are good.
|
137
|
+
|
138
|
+
Creating account on RubyGems.org
|
139
|
+
~~~~~~~~~~~~~~~~~~
|
140
|
+
|
141
|
+
Embulk uses `RubyGems.org <https://rubygems.org/>`_ as a package distribution service. Please create an account there to release plugins at `Sign Up <https://rubygems.org/sign_up>`_ page.
|
142
|
+
|
143
|
+
Don't forget the password! It will be necessary at the next step.
|
144
|
+
|
145
|
+
Releasing the plugin to RubyGems.org
|
146
|
+
~~~~~~~~~~~~~~~~~~
|
147
|
+
|
148
|
+
Now, you're ready to release the plugin. To release, type following command:
|
149
|
+
|
150
|
+
* Java-based plugins
|
151
|
+
|
152
|
+
* ``$ ./gradlew gemPush``
|
153
|
+
|
154
|
+
* Ruby-based plugins
|
155
|
+
|
156
|
+
* ``$ rake release``
|
157
|
+
|
158
|
+
If everything is good, you can find your plugin at https://rubygems.org/. Congratulations!
|
159
|
+
|
160
|
+
Installing your plugin
|
161
|
+
~~~~~~~~~~~~~~~~~~
|
162
|
+
|
163
|
+
Usage of plugin installer is:
|
164
|
+
|
165
|
+
::
|
166
|
+
|
167
|
+
$ embulk gem install embulk-<type>-<name>
|
168
|
+
|
169
|
+
``<type>`` is plugin type and ``<name>`` is plugin name.
|
170
|
+
|
171
|
+
If your plugin is ``embulk-parser-myformat``, then type this command:
|
172
|
+
|
173
|
+
::
|
174
|
+
|
175
|
+
$ embulk gem install embulk-parser-myformat
|
176
|
+
|
177
|
+
This command installs the plugin to ``~/.embulk`` directory.
|
178
|
+
|
179
|
+
To check the list of installed plugins and their versions, use this command:
|
180
|
+
|
181
|
+
::
|
182
|
+
|
183
|
+
$ embulk gem list
|
184
|
+
|
@@ -0,0 +1,84 @@
|
|
1
|
+
.. Embulk documentation master file, created by
|
2
|
+
sphinx-quickstart on Fri Feb 13 14:52:36 2015.
|
3
|
+
You can adapt this file completely to your liking, but it should at least
|
4
|
+
contain the root `toctree` directive.
|
5
|
+
|
6
|
+
Embulk
|
7
|
+
==================================
|
8
|
+
|
9
|
+
.. image:: _static/embulk-logo.png
|
10
|
+
:width: 512px
|
11
|
+
:target: https://github.com/embulk/embulk
|
12
|
+
|
13
|
+
What's Embulk?
|
14
|
+
------------------
|
15
|
+
|
16
|
+
Embulk is a open-source bulk data loader that helps data transfer between various databases, storages, file formats, and cloud services.
|
17
|
+
|
18
|
+
Embulk supports:
|
19
|
+
|
20
|
+
* Automatic guessing of input file formats
|
21
|
+
* Parallel & distributed execution to deal with big data sets
|
22
|
+
* Transaction control to guarantee All-or-Nothing
|
23
|
+
* Resuming
|
24
|
+
* Plugins released on RubyGems.org
|
25
|
+
|
26
|
+
You can define a bulk data loading using combination of input and output plugins:
|
27
|
+
|
28
|
+
.. image:: _static/embulk-architecture.png
|
29
|
+
:width: 640px
|
30
|
+
:target: https://github.com/embulk/embulk#quick-start
|
31
|
+
|
32
|
+
For example, `this tutorial <recipe/scheduled-csv-load-to-elasticsearch-kibana4.html>`_ describes how to use **file** input plugin with **csv** parser plugin and **gzip** decoder plugin to read CSV files, and **elasticsearch** output plugin to load the records to Elasticsearch.
|
33
|
+
|
34
|
+
Documents
|
35
|
+
------------------
|
36
|
+
|
37
|
+
* `Quick Start <https://github.com/embulk/embulk#quick-start>`_
|
38
|
+
|
39
|
+
* `Linux <https://github.com/embulk/embulk#linux--mac--bsd>`_
|
40
|
+
|
41
|
+
* `Mac OS X <https://github.com/embulk/embulk#linux--mac--bsd>`_
|
42
|
+
|
43
|
+
* `Windows <https://github.com/embulk/embulk#windows>`_
|
44
|
+
|
45
|
+
.. toctree::
|
46
|
+
:maxdepth: 2
|
47
|
+
|
48
|
+
recipe
|
49
|
+
|
50
|
+
* `List of Plugins by Category <http://www.embulk.org/plugins/>`_
|
51
|
+
|
52
|
+
* `Input plugins <http://www.embulk.org/plugins/#input>`_
|
53
|
+
|
54
|
+
* `Output plugins <http://www.embulk.org/plugins/#output>`_
|
55
|
+
|
56
|
+
* `File parser plugins <http://www.embulk.org/plugins/#file-parser>`_
|
57
|
+
|
58
|
+
* `File formatter plugins <http://www.embulk.org/plugins/#file-formatter>`_
|
59
|
+
|
60
|
+
* `Filter plugins <http://www.embulk.org/plugins/#filter>`_
|
61
|
+
|
62
|
+
* `Executor plugins <http://www.embulk.org/plugins/#executor>`_
|
63
|
+
|
64
|
+
.. toctree::
|
65
|
+
:maxdepth: 3
|
66
|
+
|
67
|
+
built-in
|
68
|
+
|
69
|
+
.. toctree::
|
70
|
+
:maxdepth: 3
|
71
|
+
|
72
|
+
customization
|
73
|
+
|
74
|
+
* `JavaDoc <javadoc/index.html>`_
|
75
|
+
|
76
|
+
* `RDoc <rdoc/_index.html>`_
|
77
|
+
|
78
|
+
* `Github <https://github.com/embulk/embulk>`_
|
79
|
+
|
80
|
+
.. toctree::
|
81
|
+
:maxdepth: 2
|
82
|
+
|
83
|
+
release
|
84
|
+
|