embulk 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +15 -0
- data/.gitignore +13 -0
- data/Gemfile +3 -0
- data/Gemfile.lock +33 -0
- data/README.md +117 -0
- data/Rakefile +58 -0
- data/bin/embulk +63 -0
- data/build.gradle +149 -0
- data/embulk-cli/build.gradle +6 -0
- data/embulk-cli/pom.xml +94 -0
- data/embulk-cli/src/main/java/org/embulk/cli/Main.java +15 -0
- data/embulk-core/build.gradle +6 -0
- data/embulk-core/pom.xml +143 -0
- data/embulk-core/src/main/java/org/embulk/EmbulkService.java +39 -0
- data/embulk-core/src/main/java/org/embulk/command/Runner.java +199 -0
- data/embulk-core/src/main/java/org/embulk/command/TablePrinter.java +119 -0
- data/embulk-core/src/main/java/org/embulk/config/CommitReport.java +26 -0
- data/embulk-core/src/main/java/org/embulk/config/Config.java +15 -0
- data/embulk-core/src/main/java/org/embulk/config/ConfigDefault.java +15 -0
- data/embulk-core/src/main/java/org/embulk/config/ConfigException.java +20 -0
- data/embulk-core/src/main/java/org/embulk/config/ConfigLoader.java +83 -0
- data/embulk-core/src/main/java/org/embulk/config/ConfigSource.java +28 -0
- data/embulk-core/src/main/java/org/embulk/config/DataSource.java +35 -0
- data/embulk-core/src/main/java/org/embulk/config/DataSourceImpl.java +208 -0
- data/embulk-core/src/main/java/org/embulk/config/DataSourceSerDe.java +80 -0
- data/embulk-core/src/main/java/org/embulk/config/GenericTypeReference.java +20 -0
- data/embulk-core/src/main/java/org/embulk/config/ModelManager.java +125 -0
- data/embulk-core/src/main/java/org/embulk/config/NextConfig.java +26 -0
- data/embulk-core/src/main/java/org/embulk/config/Task.java +10 -0
- data/embulk-core/src/main/java/org/embulk/config/TaskInvocationHandler.java +180 -0
- data/embulk-core/src/main/java/org/embulk/config/TaskSerDe.java +343 -0
- data/embulk-core/src/main/java/org/embulk/config/TaskSource.java +28 -0
- data/embulk-core/src/main/java/org/embulk/config/TaskValidationException.java +37 -0
- data/embulk-core/src/main/java/org/embulk/config/TaskValidator.java +24 -0
- data/embulk-core/src/main/java/org/embulk/exec/ExecModule.java +45 -0
- data/embulk-core/src/main/java/org/embulk/exec/ExecuteInterruptedException.java +10 -0
- data/embulk-core/src/main/java/org/embulk/exec/ExecuteResult.java +19 -0
- data/embulk-core/src/main/java/org/embulk/exec/ExtensionServiceLoaderModule.java +43 -0
- data/embulk-core/src/main/java/org/embulk/exec/ForSystemConfig.java +16 -0
- data/embulk-core/src/main/java/org/embulk/exec/GuessExecutor.java +307 -0
- data/embulk-core/src/main/java/org/embulk/exec/LocalExecutor.java +274 -0
- data/embulk-core/src/main/java/org/embulk/exec/LoggerProvider.java +30 -0
- data/embulk-core/src/main/java/org/embulk/exec/NoSampleException.java +10 -0
- data/embulk-core/src/main/java/org/embulk/exec/PooledBufferAllocator.java +58 -0
- data/embulk-core/src/main/java/org/embulk/exec/PreviewExecutor.java +138 -0
- data/embulk-core/src/main/java/org/embulk/exec/PreviewResult.java +27 -0
- data/embulk-core/src/main/java/org/embulk/exec/PreviewedNoticeError.java +17 -0
- data/embulk-core/src/main/java/org/embulk/exec/SamplingParserPlugin.java +116 -0
- data/embulk-core/src/main/java/org/embulk/exec/SystemConfigModule.java +24 -0
- data/embulk-core/src/main/java/org/embulk/jruby/JRubyPluginSource.java +69 -0
- data/embulk-core/src/main/java/org/embulk/jruby/JRubyScriptingModule.java +100 -0
- data/embulk-core/src/main/java/org/embulk/plugin/BuiltinPluginSourceModule.java +17 -0
- data/embulk-core/src/main/java/org/embulk/plugin/InjectedPluginSource.java +92 -0
- data/embulk-core/src/main/java/org/embulk/plugin/PluginManager.java +34 -0
- data/embulk-core/src/main/java/org/embulk/plugin/PluginSource.java +6 -0
- data/embulk-core/src/main/java/org/embulk/plugin/PluginSourceNotMatchException.java +19 -0
- data/embulk-core/src/main/java/org/embulk/plugin/PluginType.java +47 -0
- data/embulk-core/src/main/java/org/embulk/plugin/SetThreadContextClassLoader.java +19 -0
- data/embulk-core/src/main/java/org/embulk/spi/Buffer.java +113 -0
- data/embulk-core/src/main/java/org/embulk/spi/BufferAllocator.java +8 -0
- data/embulk-core/src/main/java/org/embulk/spi/Column.java +92 -0
- data/embulk-core/src/main/java/org/embulk/spi/ColumnConfig.java +79 -0
- data/embulk-core/src/main/java/org/embulk/spi/DecoderPlugin.java +16 -0
- data/embulk-core/src/main/java/org/embulk/spi/EncoderPlugin.java +16 -0
- data/embulk-core/src/main/java/org/embulk/spi/Exec.java +76 -0
- data/embulk-core/src/main/java/org/embulk/spi/ExecAction.java +6 -0
- data/embulk-core/src/main/java/org/embulk/spi/ExecSession.java +105 -0
- data/embulk-core/src/main/java/org/embulk/spi/Extension.java +42 -0
- data/embulk-core/src/main/java/org/embulk/spi/FileInput.java +11 -0
- data/embulk-core/src/main/java/org/embulk/spi/FileInputPlugin.java +19 -0
- data/embulk-core/src/main/java/org/embulk/spi/FileInputRunner.java +113 -0
- data/embulk-core/src/main/java/org/embulk/spi/FileOutput.java +13 -0
- data/embulk-core/src/main/java/org/embulk/spi/FileOutputPlugin.java +20 -0
- data/embulk-core/src/main/java/org/embulk/spi/FileOutputRunner.java +167 -0
- data/embulk-core/src/main/java/org/embulk/spi/FormatterPlugin.java +18 -0
- data/embulk-core/src/main/java/org/embulk/spi/GuessPlugin.java +9 -0
- data/embulk-core/src/main/java/org/embulk/spi/InputPlugin.java +20 -0
- data/embulk-core/src/main/java/org/embulk/spi/OutputPlugin.java +21 -0
- data/embulk-core/src/main/java/org/embulk/spi/Page.java +45 -0
- data/embulk-core/src/main/java/org/embulk/spi/PageBuilder.java +327 -0
- data/embulk-core/src/main/java/org/embulk/spi/PageFormat.java +47 -0
- data/embulk-core/src/main/java/org/embulk/spi/PageOutput.java +11 -0
- data/embulk-core/src/main/java/org/embulk/spi/PageReader.java +227 -0
- data/embulk-core/src/main/java/org/embulk/spi/ParserPlugin.java +17 -0
- data/embulk-core/src/main/java/org/embulk/spi/Schema.java +101 -0
- data/embulk-core/src/main/java/org/embulk/spi/SchemaConfig.java +52 -0
- data/embulk-core/src/main/java/org/embulk/spi/SchemaVisitor.java +14 -0
- data/embulk-core/src/main/java/org/embulk/spi/Transactional.java +10 -0
- data/embulk-core/src/main/java/org/embulk/spi/TransactionalFileInput.java +17 -0
- data/embulk-core/src/main/java/org/embulk/spi/TransactionalFileOutput.java +19 -0
- data/embulk-core/src/main/java/org/embulk/spi/TransactionalPageOutput.java +17 -0
- data/embulk-core/src/main/java/org/embulk/spi/time/DateTimeZoneSerDe.java +57 -0
- data/embulk-core/src/main/java/org/embulk/spi/time/JRubyTimeParserHelper.java +8 -0
- data/embulk-core/src/main/java/org/embulk/spi/time/JRubyTimeParserHelperFactory.java +6 -0
- data/embulk-core/src/main/java/org/embulk/spi/time/Timestamp.java +159 -0
- data/embulk-core/src/main/java/org/embulk/spi/time/TimestampFormat.java +98 -0
- data/embulk-core/src/main/java/org/embulk/spi/time/TimestampFormatter.java +55 -0
- data/embulk-core/src/main/java/org/embulk/spi/time/TimestampParseException.java +6 -0
- data/embulk-core/src/main/java/org/embulk/spi/time/TimestampParser.java +60 -0
- data/embulk-core/src/main/java/org/embulk/spi/time/TimestampSerDe.java +50 -0
- data/embulk-core/src/main/java/org/embulk/spi/type/AbstractType.java +55 -0
- data/embulk-core/src/main/java/org/embulk/spi/type/BooleanType.java +12 -0
- data/embulk-core/src/main/java/org/embulk/spi/type/DoubleType.java +12 -0
- data/embulk-core/src/main/java/org/embulk/spi/type/LongType.java +12 -0
- data/embulk-core/src/main/java/org/embulk/spi/type/StringType.java +12 -0
- data/embulk-core/src/main/java/org/embulk/spi/type/TimestampType.java +39 -0
- data/embulk-core/src/main/java/org/embulk/spi/type/Type.java +15 -0
- data/embulk-core/src/main/java/org/embulk/spi/type/TypeDeserializer.java +47 -0
- data/embulk-core/src/main/java/org/embulk/spi/type/Types.java +14 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/CharsetSerDe.java +55 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/Decoders.java +81 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/Encoders.java +81 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/FileInputInputStream.java +110 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/FileOutputOutputStream.java +94 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/InputStreamFileInput.java +111 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/Inputs.java +74 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/LineDecoder.java +118 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/LineEncoder.java +109 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/ListFileInput.java +52 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/Newline.java +38 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/PagePrinter.java +102 -0
- data/embulk-core/src/main/java/org/embulk/spi/util/Pages.java +139 -0
- data/embulk-core/src/test/java/org/embulk/EmbulkTestRuntime.java +110 -0
- data/embulk-core/src/test/java/org/embulk/GuiceBinder.java +72 -0
- data/embulk-core/src/test/java/org/embulk/RandomManager.java +53 -0
- data/embulk-core/src/test/java/org/embulk/TestPluginSourceModule.java +23 -0
- data/embulk-core/src/test/java/org/embulk/TestUtilityModule.java +17 -0
- data/embulk-core/src/test/java/org/embulk/config/TestConfigSource.java +114 -0
- data/embulk-core/src/test/java/org/embulk/config/TestTaskSource.java +70 -0
- data/embulk-core/src/test/java/org/embulk/plugin/MockPluginSource.java +57 -0
- data/embulk-core/src/test/java/org/embulk/plugin/TestPluginType.java +18 -0
- data/embulk-core/src/test/java/org/embulk/spi/MockFileOutput.java +63 -0
- data/embulk-core/src/test/java/org/embulk/spi/MockFormatterPlugin.java +101 -0
- data/embulk-core/src/test/java/org/embulk/spi/MockParserPlugin.java +73 -0
- data/embulk-core/src/test/java/org/embulk/spi/PageTestUtils.java +78 -0
- data/embulk-core/src/test/java/org/embulk/spi/TestFileInputInputStream.java +67 -0
- data/embulk-core/src/test/java/org/embulk/spi/TestFileInputRunner.java +180 -0
- data/embulk-core/src/test/java/org/embulk/spi/TestFileOutputRunner.java +192 -0
- data/embulk-core/src/test/java/org/embulk/spi/TestInputStreamFileInput.java +188 -0
- data/embulk-core/src/test/java/org/embulk/spi/TestPageBuilderReader.java +301 -0
- data/embulk-core/src/test/java/org/embulk/spi/time/TestTimestamp.java +116 -0
- data/embulk-core/src/test/java/org/embulk/spi/time/TestTimestampFormatterParser.java +52 -0
- data/embulk-core/src/test/java/org/embulk/spi/type/TestTypeSerDe.java +45 -0
- data/embulk-core/src/test/java/org/embulk/spi/util/TestLineDecoder.java +132 -0
- data/embulk-core/src/test/java/org/embulk/spi/util/TestLineEncoder.java +123 -0
- data/embulk-standards/build.gradle +6 -0
- data/embulk-standards/pom.xml +68 -0
- data/embulk-standards/src/main/java/org/embulk/standards/CsvFormatterPlugin.java +158 -0
- data/embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java +233 -0
- data/embulk-standards/src/main/java/org/embulk/standards/CsvTokenizer.java +355 -0
- data/embulk-standards/src/main/java/org/embulk/standards/GzipFileDecoderPlugin.java +55 -0
- data/embulk-standards/src/main/java/org/embulk/standards/GzipFileEncoderPlugin.java +39 -0
- data/embulk-standards/src/main/java/org/embulk/standards/LocalFileInputPlugin.java +138 -0
- data/embulk-standards/src/main/java/org/embulk/standards/LocalFileOutputPlugin.java +128 -0
- data/embulk-standards/src/main/java/org/embulk/standards/NullOutputPlugin.java +46 -0
- data/embulk-standards/src/main/java/org/embulk/standards/S3FileInputPlugin.java +238 -0
- data/embulk-standards/src/main/java/org/embulk/standards/StandardPluginExtension.java +16 -0
- data/embulk-standards/src/main/java/org/embulk/standards/StandardPluginModule.java +44 -0
- data/embulk-standards/src/main/java/org/embulk/standards/StdoutOutputPlugin.java +71 -0
- data/embulk-standards/src/main/resources/META-INF/services/org.embulk.spi.Extension +1 -0
- data/embulk-standards/src/test/java/org/embulk/standards/TestCsvParserPlugin.java +69 -0
- data/embulk-standards/src/test/java/org/embulk/standards/TestCsvTokenizer.java +291 -0
- data/embulk-standards/src/test/java/org/embulk/standards/TestS3FileInputPlugin.java +43 -0
- data/embulk.gemspec +27 -0
- data/examples/config.yml +34 -0
- data/examples/csv/sample.csv.gz +0 -0
- data/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +6 -0
- data/gradlew +164 -0
- data/gradlew.bat +90 -0
- data/lib/embulk.rb +16 -0
- data/lib/embulk/buffer.rb +17 -0
- data/lib/embulk/column.rb +47 -0
- data/lib/embulk/command/embulk.rb +39 -0
- data/lib/embulk/command/embulk_example.rb +32 -0
- data/lib/embulk/command/embulk_generate_bin.rb +62 -0
- data/lib/embulk/command/embulk_run.rb +243 -0
- data/lib/embulk/data/bundle/.bundle/config +3 -0
- data/lib/embulk/data/bundle/Gemfile +31 -0
- data/lib/embulk/data/bundle/Gemfile.lock +8 -0
- data/lib/embulk/data/bundle/embulk/input_example.rb +40 -0
- data/lib/embulk/data/bundle/embulk/output_example.rb +51 -0
- data/lib/embulk/data_source.rb +66 -0
- data/lib/embulk/error.rb +5 -0
- data/lib/embulk/guess_charset.rb +26 -0
- data/lib/embulk/guess_csv.rb +195 -0
- data/lib/embulk/guess_gzip.rb +18 -0
- data/lib/embulk/guess_newline.rb +20 -0
- data/lib/embulk/guess_plugin.rb +113 -0
- data/lib/embulk/input_plugin.rb +53 -0
- data/lib/embulk/java/bootstrap.rb +12 -0
- data/lib/embulk/java/imports.rb +26 -0
- data/lib/embulk/java/time_helper.rb +77 -0
- data/lib/embulk/output_plugin.rb +104 -0
- data/lib/embulk/page.rb +28 -0
- data/lib/embulk/page_builder.rb +22 -0
- data/lib/embulk/plugin.rb +152 -0
- data/lib/embulk/plugin_registry.rb +70 -0
- data/lib/embulk/schema.rb +85 -0
- data/lib/embulk/time_format_guess.rb +331 -0
- data/lib/embulk/version.rb +3 -0
- data/pom.xml +533 -0
- data/settings.gradle +5 -0
- metadata +370 -0
checksums.yaml
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
---
|
2
|
+
!binary "U0hBMQ==":
|
3
|
+
metadata.gz: !binary |-
|
4
|
+
MjM4MTAxYzU5YTZlYWZkMWM0NjYzYTZlMGRkMjgwN2I0Yjc0MDU5YQ==
|
5
|
+
data.tar.gz: !binary |-
|
6
|
+
MGJkZmQ2NDM0NWQ3NWIwNmY2MWUyYjY0OWM0MzA2MmIwM2RmMDYwOQ==
|
7
|
+
SHA512:
|
8
|
+
metadata.gz: !binary |-
|
9
|
+
MGY5Y2E2YjBiMjBhZTkxMzljNGU4MGNhZGE0MWM2N2YzZmM2YWM1MWRkZjUx
|
10
|
+
NTExZmEyN2Q5NDIyNThiNGNiYTE3ZDM0MzA3ZDMwNTE2NDI0N2E2ODM5ODE2
|
11
|
+
OTQxN2FjNWY1MzY3ODc5NGU5YTk3NmEwYjA4NDVjMTkwZTAxNjE=
|
12
|
+
data.tar.gz: !binary |-
|
13
|
+
NDZlMzkxOGVhN2VjY2U1ZGY4OTNlZWZlMzYxMWEyNDYyMDM3ZGMxNDlkOGY4
|
14
|
+
MWFhYzdhNzgyNTUxY2VkNGE4NWIzZDJkNmU1Nzc1ZTYyMjc0NTlkYzc3MGUx
|
15
|
+
ZWIwNTYzMzc3ZDA1NWQ0MzMwNmViZjZkNTdhZWQ0YjE1MDg0ZTA=
|
data/.gitignore
ADDED
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,33 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
embulk (0.1.0)
|
5
|
+
|
6
|
+
GEM
|
7
|
+
remote: https://rubygems.org/
|
8
|
+
specs:
|
9
|
+
diff-lcs (1.2.5)
|
10
|
+
json (1.8.1)
|
11
|
+
kramdown (1.5.0)
|
12
|
+
rake (10.4.2)
|
13
|
+
rspec (2.99.0)
|
14
|
+
rspec-core (~> 2.99.0)
|
15
|
+
rspec-expectations (~> 2.99.0)
|
16
|
+
rspec-mocks (~> 2.99.0)
|
17
|
+
rspec-core (2.99.2)
|
18
|
+
rspec-expectations (2.99.2)
|
19
|
+
diff-lcs (>= 1.1.3, < 2.0)
|
20
|
+
rspec-mocks (2.99.2)
|
21
|
+
yard (0.8.7.6)
|
22
|
+
|
23
|
+
PLATFORMS
|
24
|
+
ruby
|
25
|
+
|
26
|
+
DEPENDENCIES
|
27
|
+
bundler (>= 1.0)
|
28
|
+
embulk!
|
29
|
+
json (~> 1.7)
|
30
|
+
kramdown (~> 1.5.0)
|
31
|
+
rake (>= 0.10.0)
|
32
|
+
rspec (~> 2.11)
|
33
|
+
yard (~> 0.8.7)
|
data/README.md
ADDED
@@ -0,0 +1,117 @@
|
|
1
|
+
# Embulk: plugin-based parallel bulk data loader
|
2
|
+
|
3
|
+
## What's Embulk?
|
4
|
+
|
5
|
+
TODO
|
6
|
+
|
7
|
+
## Quick Start
|
8
|
+
|
9
|
+
The single-file package is the simplest way to try Embulk. You can download the latest embulk.jar from [releases]() and run it with java:
|
10
|
+
|
11
|
+
```
|
12
|
+
wget https://github.com/embulk/embulk/releases .... /latest
|
13
|
+
java -jar embulk.jar --help
|
14
|
+
```
|
15
|
+
|
16
|
+
Let's load a CSV file, for example. `embulk example` subcommand generates a csv file and config file for you.
|
17
|
+
|
18
|
+
```
|
19
|
+
java -jar embulk.jar example ./try1
|
20
|
+
java -jar embulk.jar guess ./try1/example.yml -o config.yml
|
21
|
+
java -jar embulk.jar preview config.yml
|
22
|
+
java -jar embulk.jar run config.yml
|
23
|
+
```
|
24
|
+
|
25
|
+
### Using plugins
|
26
|
+
|
27
|
+
You can use plugins to load data from/to various systems and file formats.
|
28
|
+
An example is [embulk-output-postgres-json]() plugin. It outputs data into PostgreSQL server using "json" column type.
|
29
|
+
|
30
|
+
```
|
31
|
+
java -jar embulk.jar gem install embulk-output-postgres-json
|
32
|
+
java -jar embulk.jar gem list
|
33
|
+
```
|
34
|
+
|
35
|
+
You can search plugins on RubyGems: [search for "embulk-"](https://rubygems.org/search?utf8=%E2%9C%93&query=embulk-).
|
36
|
+
|
37
|
+
### Using plugin bundle
|
38
|
+
|
39
|
+
`embulk bundle` subcommand creates (or updates if already exists) a *plugin bundle* directory.
|
40
|
+
You can use the bundle using `-b <bundle_dir>` option. `embulk bundle` also generates some example plugins to \<bundle_dir>/embulk/\*.rb directory.
|
41
|
+
|
42
|
+
See generated \<bundle_dir>/Gemfile file how to plugin bundles work.
|
43
|
+
|
44
|
+
```
|
45
|
+
java -jar embulk.jar bundle ./embulk_bundle
|
46
|
+
java -jar embulk.jar guess -b ./embulk_bundle ...
|
47
|
+
java -jar embulk.jar run -b ./embulk_bundle ...
|
48
|
+
```
|
49
|
+
|
50
|
+
### Releasing plugins to RubyGems
|
51
|
+
|
52
|
+
TODO: documents
|
53
|
+
|
54
|
+
```
|
55
|
+
embulk-plugin-xyz
|
56
|
+
```
|
57
|
+
|
58
|
+
## Embulk Development
|
59
|
+
|
60
|
+
### Build
|
61
|
+
|
62
|
+
```
|
63
|
+
rake # creates embulk-VERSION.jar
|
64
|
+
```
|
65
|
+
|
66
|
+
You can see JaCoCo's test coverage report at ${project}/target/site/jacoco/index.html
|
67
|
+
|
68
|
+
To build by Gradle, run:
|
69
|
+
```
|
70
|
+
./gradlew build
|
71
|
+
```
|
72
|
+
If you want to deploy artifacts on local maven repository like ~/.m2/repository/, run:
|
73
|
+
```
|
74
|
+
./gradlew install
|
75
|
+
```
|
76
|
+
If you want to compile the source code of embulk-core project only, run:
|
77
|
+
```
|
78
|
+
./gradlew :embulk-core:compileJava
|
79
|
+
```
|
80
|
+
The following command allows use to see the dependency tree of embulk-core project
|
81
|
+
```
|
82
|
+
./gradlew :embulk-core:dependencies
|
83
|
+
```
|
84
|
+
|
85
|
+
### Release
|
86
|
+
|
87
|
+
You need to add your bintray account information to ~/.gradle/gradle.properties
|
88
|
+
|
89
|
+
```
|
90
|
+
bintray_user=(bintray user name)
|
91
|
+
bintray_api_key=(bintray api key)
|
92
|
+
```
|
93
|
+
|
94
|
+
Increment version number written at following 3 files (TODO improve this):
|
95
|
+
|
96
|
+
* build.gradle
|
97
|
+
* pom.xml
|
98
|
+
* lib/embulk/version.rb
|
99
|
+
|
100
|
+
Then, build and upload using gradle:
|
101
|
+
|
102
|
+
```
|
103
|
+
./gradlew bintrayUpload
|
104
|
+
```
|
105
|
+
|
106
|
+
Finally, you need to manually upload the single-file jar package to bintray.
|
107
|
+
Run `rake` and upload embulk-VERSION.jar from "Upload Files" link at https://bintray.com/embulk/maven/embulk/VERSION/upload.
|
108
|
+
|
109
|
+
```
|
110
|
+
rake
|
111
|
+
# embulk-VERSION.jar is built
|
112
|
+
```
|
113
|
+
|
114
|
+
See also:
|
115
|
+
* [Bintray](https://bintray.com)
|
116
|
+
* [How to acquire bintray API Keys](https://bintray.com/docs/usermanual/interacting/interacting_apikeys.html)
|
117
|
+
|
data/Rakefile
ADDED
@@ -0,0 +1,58 @@
|
|
1
|
+
#!/usr/bin/env rake
|
2
|
+
|
3
|
+
require 'bundler/gem_tasks'
|
4
|
+
require 'rake/testtask'
|
5
|
+
require 'rake/clean'
|
6
|
+
require 'bundler'
|
7
|
+
require 'yard'
|
8
|
+
|
9
|
+
desc 'Generate YARD document'
|
10
|
+
YARD::Rake::YardocTask.new(:doc) do |t|
|
11
|
+
t.files = ['lib/**/*.rb']
|
12
|
+
t.options = %w[-M kramdown]
|
13
|
+
t.options << '--debug' << '--verbose' if $trace
|
14
|
+
end
|
15
|
+
|
16
|
+
desc "Clean java code and copy files"
|
17
|
+
task :clean do
|
18
|
+
sh "mvn clean"
|
19
|
+
rm_rf "classpath"
|
20
|
+
end
|
21
|
+
|
22
|
+
desc "Compile java code and copy jar files to classpath/"
|
23
|
+
task :compile do
|
24
|
+
sh "mvn package dependency:copy-dependencies"
|
25
|
+
rm_rf "classpath"
|
26
|
+
mkdir_p "classpath"
|
27
|
+
cp Dir["embulk-cli/target/dependency/*.jar"], "classpath"
|
28
|
+
targets = Dir["embulk-cli/target/embulk-cli-*.jar"]
|
29
|
+
targets.reject! {|target| target =~ /-sources.jar$/ || target =~ /-executable.jar$/ }
|
30
|
+
cp targets, "classpath"
|
31
|
+
end
|
32
|
+
|
33
|
+
desc "Create embulk-{version}.jar"
|
34
|
+
task :jar do
|
35
|
+
require_relative 'lib/embulk/version'
|
36
|
+
executable = Dir["embulk-cli/target/embulk-cli-*-executable.jar"].sort.last
|
37
|
+
executable_data = File.read(executable).force_encoding('ASCII-8BIT')
|
38
|
+
header = <<EOF
|
39
|
+
#!/bin/sh
|
40
|
+
exec java -jar "$0" "$@"
|
41
|
+
exit 127
|
42
|
+
EOF
|
43
|
+
data = header.force_encoding('ASCII-8BIT') + executable_data
|
44
|
+
path = "embulk-#{Embulk::VERSION}.jar"
|
45
|
+
rm_f path
|
46
|
+
File.open(path, 'wb', 0755) {|f| f.write data }
|
47
|
+
puts "Created #{path}"
|
48
|
+
end
|
49
|
+
|
50
|
+
desc "Run clean, compile, build and jar"
|
51
|
+
task :all do
|
52
|
+
Rake::Task["clean"].invoke
|
53
|
+
Rake::Task["compile"].invoke
|
54
|
+
Rake::Task["build"].invoke
|
55
|
+
Rake::Task["jar"].invoke
|
56
|
+
end
|
57
|
+
|
58
|
+
task :default => :all
|
data/bin/embulk
ADDED
@@ -0,0 +1,63 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
# TODO cygwin check
|
4
|
+
cygwin = false
|
5
|
+
|
6
|
+
java_args = []
|
7
|
+
while i = ARGV.find_index {|arg| arg =~ /^\-D/ }
|
8
|
+
if ARGV[i] == "-D"
|
9
|
+
prop_path = ARGV[i + 1]
|
10
|
+
unless prop_path
|
11
|
+
STDERR.puts "-D option requires an option"
|
12
|
+
exit 1
|
13
|
+
end
|
14
|
+
ARGV.slice!(i, 2)
|
15
|
+
else
|
16
|
+
java_args << ARGV[i]
|
17
|
+
ARGV.slice!(i)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
if prop_path
|
22
|
+
props = File.read(prop_path)
|
23
|
+
java_props = props.split("\n").reject {|prop| prop.strip.empty? }
|
24
|
+
java_args = java_props + java_args
|
25
|
+
end
|
26
|
+
|
27
|
+
java_cmd = ENV['JAVACMD']
|
28
|
+
unless java_cmd
|
29
|
+
if java_home = ENV['JAVA_HOME']
|
30
|
+
if cygwin
|
31
|
+
java_cmd = "#{`cygpath -u "#{java_home}"`.strip}/bin/java"
|
32
|
+
else
|
33
|
+
java_cmd = "#{java_home}/bin/java"
|
34
|
+
end
|
35
|
+
else
|
36
|
+
java_cmd = "java"
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
embulk_home = ENV['EMBULK_HOME']
|
41
|
+
unless embulk_home
|
42
|
+
embulk_home = File.dirname(File.dirname(__FILE__))
|
43
|
+
end
|
44
|
+
ENV['EMBULK_HOME'] = File.expand_path(embulk_home)
|
45
|
+
|
46
|
+
classpath_dir = File.join(embulk_home, 'classpath')
|
47
|
+
lib_dir = File.join(embulk_home, 'lib')
|
48
|
+
|
49
|
+
jruby_complete = Dir.entries(classpath_dir).find {|jar| jar =~ /jruby-complete-[\d\.]+\.jar/ }
|
50
|
+
unless jruby_complete
|
51
|
+
STDERR.puts "Could not find jruby-complete at $EMBULK_HOME/classpath directory."
|
52
|
+
STDERR.puts "Please confirm EMBULK_HOME is correctly set"
|
53
|
+
STDERR.puts "Current EMBULK_HOME = #{ENV['EMBULK_HOME'].to_s.dump}"
|
54
|
+
raise SystemExit.new(1)
|
55
|
+
end
|
56
|
+
|
57
|
+
# java ... -jar ruby-complete.jar $EMBULK_HOME/lib/embulk/command/embulk.rb "$@"
|
58
|
+
cmdline = [java_cmd]
|
59
|
+
cmdline.concat java_args
|
60
|
+
cmdline << '-jar' << File.join(classpath_dir, jruby_complete)
|
61
|
+
cmdline << File.join(lib_dir, 'embulk/command/embulk.rb')
|
62
|
+
cmdline.concat ARGV
|
63
|
+
exec *cmdline
|
data/build.gradle
ADDED
@@ -0,0 +1,149 @@
|
|
1
|
+
// TODO this build.gradle file just works but not well-written yet...
|
2
|
+
|
3
|
+
//plugins {
|
4
|
+
// id "com.jfrog.bintray" version "1.0"
|
5
|
+
//}
|
6
|
+
buildscript {
|
7
|
+
repositories {
|
8
|
+
jcenter()
|
9
|
+
}
|
10
|
+
dependencies {
|
11
|
+
classpath 'com.jfrog.bintray.gradle:gradle-bintray-plugin:1.0'
|
12
|
+
}
|
13
|
+
}
|
14
|
+
|
15
|
+
allprojects {
|
16
|
+
repositories {
|
17
|
+
mavenCentral()
|
18
|
+
jcenter()
|
19
|
+
}
|
20
|
+
|
21
|
+
apply plugin: 'idea'
|
22
|
+
apply plugin: 'maven-publish'
|
23
|
+
apply plugin: 'com.jfrog.bintray'
|
24
|
+
|
25
|
+
group = 'org.embulk'
|
26
|
+
version = '0.1.0'
|
27
|
+
|
28
|
+
// to upload artifacts to Bintray by gradle-bintray-plugin
|
29
|
+
// $ gradle bintrayUpload
|
30
|
+
bintray {
|
31
|
+
// write at your bintray user name and api key to ~/.gradle/gradle.properties file:
|
32
|
+
// bintray_user=frsyuki
|
33
|
+
// bintray_api_key=xxxxxxxxxxx
|
34
|
+
user = project.hasProperty('bintray_user') ? bintray_user : ''
|
35
|
+
key = project.hasProperty('bintray_api_key') ? bintray_api_key : ''
|
36
|
+
|
37
|
+
publications = ['mavenJava']
|
38
|
+
|
39
|
+
dryRun = false
|
40
|
+
publish = false // TODO publish = project.publish
|
41
|
+
|
42
|
+
pkg {
|
43
|
+
userOrg = 'embulk'
|
44
|
+
repo = 'maven'
|
45
|
+
name = 'embulk'
|
46
|
+
desc = 'Embulk, a plugin-based parallel bulk data loader'
|
47
|
+
websiteUrl = 'https://github.com/embulk/embulk'
|
48
|
+
issueTrackerUrl = 'https://github.com/embulk/embulk/issues'
|
49
|
+
vcsUrl = 'https://github.com/embulk/embulk.git'
|
50
|
+
licenses = ['Apache-2.0']
|
51
|
+
labels = ['embulk', 'ruby', 'java']
|
52
|
+
publicDownloadNumbers = true
|
53
|
+
|
54
|
+
version {
|
55
|
+
name = project.version
|
56
|
+
// TODO
|
57
|
+
//mavenCentralSync {
|
58
|
+
// sync = true
|
59
|
+
// user = 'userToken'
|
60
|
+
// password = 'paasword'
|
61
|
+
// close = '1'
|
62
|
+
//}
|
63
|
+
}
|
64
|
+
}
|
65
|
+
}
|
66
|
+
}
|
67
|
+
|
68
|
+
subprojects {
|
69
|
+
apply plugin: 'java'
|
70
|
+
|
71
|
+
// to install jar files to the local repo
|
72
|
+
// $ gradle install
|
73
|
+
apply plugin: 'maven'
|
74
|
+
|
75
|
+
compileJava.options.encoding = 'UTF-8' // source encoding
|
76
|
+
sourceCompatibility = 1.7
|
77
|
+
targetCompatibility = 1.7
|
78
|
+
|
79
|
+
dependencies {
|
80
|
+
compile 'commons-logging:commons-logging:1.2',
|
81
|
+
'com.google.guava:guava:17.0',
|
82
|
+
'com.google.inject:guice:3.0',
|
83
|
+
'com.google.inject.extensions:guice-multibindings:3.0',
|
84
|
+
'javax.inject:javax.inject:1',
|
85
|
+
'com.fasterxml.jackson.core:jackson-annotations:2.4.3',
|
86
|
+
'com.fasterxml.jackson.core:jackson-core:2.4.3',
|
87
|
+
'com.fasterxml.jackson.core:jackson-databind:2.4.3',
|
88
|
+
'com.fasterxml.jackson.datatype:jackson-datatype-guava:2.4.3',
|
89
|
+
'com.fasterxml.jackson.datatype:jackson-datatype-joda:2.4.3',
|
90
|
+
'com.fasterxml.jackson.module:jackson-module-guice:2.4.3',
|
91
|
+
'log4j:log4j:1.2.17',
|
92
|
+
'org.slf4j:slf4j-api:1.7.9',
|
93
|
+
'org.slf4j:slf4j-log4j12:1.7.9',
|
94
|
+
'org.jruby:jruby-complete:1.7.16.1',
|
95
|
+
'org.yaml:snakeyaml:1.14',
|
96
|
+
'javax.validation:validation-api:1.1.0.Final',
|
97
|
+
'org.apache.bval:bval-jsr303:0.5',
|
98
|
+
'io.airlift:slice:0.7',
|
99
|
+
'joda-time:joda-time:2.3',
|
100
|
+
'io.netty:netty-buffer:4.0.24.Final',
|
101
|
+
'com.ibm.icu:icu4j:53.1'
|
102
|
+
|
103
|
+
testCompile 'junit:junit:4.10',
|
104
|
+
'org.mockito:mockito-core:1.9.5'
|
105
|
+
}
|
106
|
+
|
107
|
+
// custom tasks for creating source/javadoc jars
|
108
|
+
task sourcesJar(type: Jar, dependsOn: classes) {
|
109
|
+
classifier = 'sources'
|
110
|
+
from sourceSets.main.allSource
|
111
|
+
}
|
112
|
+
|
113
|
+
task javadocJar(type: Jar, dependsOn: javadoc) {
|
114
|
+
classifier = 'javadoc'
|
115
|
+
from javadoc.destinationDir
|
116
|
+
}
|
117
|
+
|
118
|
+
// add javadoc/source jar tasks as artifacts
|
119
|
+
artifacts {
|
120
|
+
archives sourcesJar, javadocJar
|
121
|
+
}
|
122
|
+
|
123
|
+
// $ gradle copyDependencies
|
124
|
+
task copyDependencies(type: Copy) {
|
125
|
+
println ":copy depend to $buildDir/libs/dependencies"
|
126
|
+
new File("$buildDir/libs/dependencies").mkdirs()
|
127
|
+
|
128
|
+
into "$buildDir/libs/dependencies"
|
129
|
+
from configurations.runtime.filter {
|
130
|
+
// TODO avoid bytecode compatibility error
|
131
|
+
it.name != 'asm-3.1.jar' && it.name != 'cglib-2.2.1-v20090111.jar'
|
132
|
+
}
|
133
|
+
}
|
134
|
+
|
135
|
+
publishing {
|
136
|
+
publications {
|
137
|
+
mavenJava(MavenPublication) {
|
138
|
+
groupId project.group
|
139
|
+
artifactId project.name
|
140
|
+
version project.version
|
141
|
+
|
142
|
+
from components.java
|
143
|
+
|
144
|
+
artifact sourcesJar
|
145
|
+
artifact javadocJar
|
146
|
+
}
|
147
|
+
}
|
148
|
+
}
|
149
|
+
}
|