embulk 0.6.27 → 0.7.0

data/lib/embulk/page_builder.rb

@@ -7,7 +7,7 @@ module Embulk
   class PageBuilder
     def initialize(schema, java_page_output)
       # TODO get task as an argument
-      task = Java::Exec.newConfigSource.load_config(Java::DynamicPageBuilder::BuilderTask.java_class)
+      task = Java::SPI::Exec.newConfigSource.load_config(Java::DynamicPageBuilder::BuilderTask.java_class)
       @page_builder = Java::DynamicPageBuilder.new(task, Java::Injected::BufferAllocator, schema.to_java, java_page_output)
       @schema = schema
     end

data/lib/embulk/plugin.rb

@@ -15,7 +15,6 @@ module Embulk
   require 'embulk/guess_plugin'
   require 'embulk/executor_plugin'
   require 'embulk/java_plugin' if Embulk.java?
-  require 'embulk/exec' if Embulk.java?
 
   class PluginManager
     def initialize

data/lib/embulk/runner.rb

@@ -0,0 +1,184 @@
+module Embulk
+
+  # Embulk.setup initializes:
+  # Runner = EmbulkRunner.new
+
+  class EmbulkRunner
+    def initialize(embed)
+      @embed = embed  # org.embulk.EmbulkEmbed
+    end
+
+    def guess(config, options={})
+      configSource = read_config(config, options)
+      output_path = options[:next_config_output_path]
+
+      check_file_writable(output_path)
+
+      configDiff = @embed.guess(configSource)
+
+      guessedConfigSource = configSource.merge(configDiff)
+      yaml = write_config(output_path, guessedConfigSource)
+      STDERR.puts yaml
+      if output_path
+        puts "Created '#{output_path}' file."
+      else
+        puts "Use -o PATH option to write the guessed config file to a file."
+      end
+
+      nil
+    end
+
+    def preview(config, options={})
+      configSource = read_config(config, options)
+      format = options[:format]
+
+      previewResult = @embed.preview(configSource)
+
+      modelManager = @embed.getModelManager
+      printer =
+        case format || "table"
+        when "table"
+          org.embulk.command.TablePreviewPrinter.new(java.lang.System.out, modelManager, previewResult.getSchema)
+        when "vertical"
+          org.embulk.command.VerticalPreviewPrinter.new(java.lang.System.out, modelManager, previewResult.getSchema)
+        else
+          raise ArgumentError, "Unknown preview output format '#{format}'. Supported formats: table, vertical"
+        end
+
+      printer.printAllPages(previewResult.getPages)
+      printer.finish
+
+      nil
+    end
+
+    def run(config, options={})
+      configSource = read_config(config, options)
+      output_path = options[:next_config_output_path]
+      resume_state_path = options[:resume_state_path]
+
+      check_file_writable(output_path)
+      check_file_writable(resume_state_path)
+
+      if resume_state_path
+        begin
+          resumeConfig = read_yaml_config_file(resume_state_path)
+          resumeConfig = nil if resumeConfig.isEmpty
+        rescue
+          # TODO log?
+          resumeConfig = nil
+        end
+      end
+
+      if resumeConfig
+        resumableResult = @embed.resumeState(configSource, resumeConfig).resume
+      elsif resume_state_path
+        resumableResult = @embed.runResumable(configSource)
+      else
+        executionResult = @embed.run(configSource)
+      end
+
+      unless executionResult
+        unless resumableResult.isSuccessful
+          Embulk.logger.info "Writing resume state to '#{resume_state_path}'"
+          write_config(resume_state_path, resumableResult.getResumeState)
+          Embulk.logger.info "Resume state is written. Run the transaction again with -r option to resume or use \"cleanup\" subcommand to delete intermediate data."
+          raise resumableResult.getCause
+        end
+        executionResult = resumableResult.getSuccessfulResult
+      end
+
+      # delete resume file
+      if resume_state_path
+        File.delete(resume_state_path) rescue nil
+      end
+
+      configDiff = executionResult.getConfigDiff
+      Embulk.logger.info("Committed.")
+      Embulk.logger.info("Next config diff: #{configDiff.toString}")
+
+      write_config(output_path, configSource.merge(configDiff))
+    end
+
+    #def resume_state(config, options={})
+    #  configSource = read_config(config, options)
+    #  Resumed.new(self, DataSource.from_java(configSource), options)
+    #end
+
+    private
+
+    def read_config(config, options={})
+      case config
+      when String
+        case config
+        when /\.yml\.liquid$/
+          require 'liquid'
+          template_params = options[:template_params] || {}
+          @embed.newConfigLoader.fromYamlString run_liquid(File.read(config), template_params)
+        when /\.yml$/
+          @embed.newConfigLoader.fromYamlString File.read(config)
+        else
+          raise ConfigError, "Unsupported file extension. Supported file extensions are .yml and .yml.liquid: #{config}"
+        end
+
+      when Hash, DataSource
+        DataSource.new(config).to_java
+      end
+    end
+
+    def read_yaml_config_file(path)
+      @embed.newConfigLoader.fromYamlString File.read(config)
+    end
+
+    def run_liquid(source, params)
+      require 'liquid'
+      template = Liquid::Template.parse(source)
+
+      data = {
+        "env" => ENV.to_h,
+      }.merge(params)
+
+      template.render(data)
+    end
+
+    def check_file_writable(path)
+      if path
+        # Open file with append mode and do nothing.
+        # If file is not writable, it throws an exception.
+        File.open(path, "ab")
+      end
+    end
+
+    def write_config(path, modelObject)
+      yaml = dump_yaml(modelObject)
+      if path
+        File.write(path, yaml)
+      end
+      yaml
+    end
+
+    def dump_yaml(modelObject)
+      modelManager = @embed.getModelManager
+      obj = modelManager.readObject(java.lang.Object.java_class, modelManager.writeObject(modelObject))
+      return org.yaml.snakeyaml.Yaml.new.dump(obj)
+    end
+
+    #class Runnable
+    #  def initialize(runner, config, options)
+    #    @runner = runner
+    #    @config = config
+    #    @options = options
+    #  end
+    #
+    #  attr_reader :config
+    #
+    #  def preview(options={})
+    #    @runner.preview(@config, @options.merge(options))
+    #  end
+    #
+    #  def run(options={})
+    #    @runner.run(@config, @options.merge(options))
+    #  end
+    #end
+  end
+
+end

data/lib/embulk/version.rb

@@ -1,3 +1,3 @@
 module Embulk
-  VERSION = '0.6.27'
+  VERSION = '0.7.0'
 end

metadata

@@ -1,86 +1,102 @@
 --- !ruby/object:Gem::Specification
 name: embulk
 version: !ruby/object:Gem::Version
-  version: 0.6.27
+  version: 0.7.0
 platform: ruby
 authors:
 - Sadayuki Furuhashi
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-01 00:00:00.000000000 Z
+date: 2015-08-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: jruby-jars
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 9.0.0.0
+  type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - '='
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: 9.0.0.0
+- !ruby/object:Gem::Dependency
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.0'
-  prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: rake
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.0
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.10.0
-  prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: test-unit
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 3.0.9
+        version: 0.10.0
+- !ruby/object:Gem::Dependency
+  name: test-unit
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 3.0.9
-  prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: yard
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.8.7
+        version: 3.0.9
+- !ruby/object:Gem::Dependency
+  name: yard
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.8.7
-  prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: kramdown
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.5.0
+        version: 0.8.7
+- !ruby/object:Gem::Dependency
+  name: kramdown
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.5.0
-  prerelease: false
   type: :development
-description: Embulk is an open-source, plugin-based bulk data loader to scale and simplify data management across heterogeneous data stores. It can collect and ship any kinds of data in high throughput with transaction control.
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+description: Embulk is an open-source, plugin-based bulk data loader to scale and
+  simplify data management across heterogeneous data stores. It can collect and ship
+  any kinds of data in high throughput with transaction control.
 email:
 - frsyuki@gmail.com
 executables:
@@ -88,8 +104,9 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .travis.yml
+- ".gitignore"
+- ".ruby-version"
+- ".travis.yml"
 - COPYING
 - Gemfile
 - Gemfile.lock
@@ -104,12 +121,11 @@ files:
 - classpath/bval-jsr303-0.5.jar
 - classpath/commons-beanutils-core-1.8.3.jar
 - classpath/commons-lang3-3.1.jar
-- classpath/embulk-core-0.6.27.jar
-- classpath/embulk-standards-0.6.27.jar
+- classpath/embulk-core-0.7.0.jar
+- classpath/embulk-standards-0.7.0.jar
 - classpath/guava-18.0.jar
 - classpath/guice-4.0.jar
 - classpath/guice-multibindings-4.0.jar
-- classpath/icu4j-54.1.1.jar
 - classpath/jackson-annotations-2.5.3.jar
 - classpath/jackson-core-2.5.3.jar
 - classpath/jackson-databind-2.5.3.jar
@@ -119,7 +135,6 @@ files:
 - classpath/jansi-1.11.jar
 - classpath/javax.inject-1.jar
 - classpath/joda-time-2.8.1.jar
-- classpath/jruby-complete-1.7.21.jar
 - classpath/logback-classic-1.1.3.jar
 - classpath/logback-core-1.1.3.jar
 - classpath/netty-buffer-5.0.0.Alpha1.jar
@@ -136,9 +151,7 @@ files:
 - embulk-core/build.gradle
 - embulk-core/src/main/java/org/embulk/EmbulkEmbed.java
 - embulk-core/src/main/java/org/embulk/EmbulkService.java
-- embulk-core/src/main/java/org/embulk/command/LiquidTemplate.java
 - embulk-core/src/main/java/org/embulk/command/PreviewPrinter.java
-- embulk-core/src/main/java/org/embulk/command/Runner.java
 - embulk-core/src/main/java/org/embulk/command/TablePreviewPrinter.java
 - embulk-core/src/main/java/org/embulk/command/VerticalPreviewPrinter.java
 - embulk-core/src/main/java/org/embulk/config/CommitReport.java
@@ -156,6 +169,7 @@ files:
 - embulk-core/src/main/java/org/embulk/config/ModelManager.java
 - embulk-core/src/main/java/org/embulk/config/Task.java
 - embulk-core/src/main/java/org/embulk/config/TaskInvocationHandler.java
+- embulk-core/src/main/java/org/embulk/config/TaskReport.java
 - embulk-core/src/main/java/org/embulk/config/TaskSerDe.java
 - embulk-core/src/main/java/org/embulk/config/TaskSource.java
 - embulk-core/src/main/java/org/embulk/config/TaskValidationException.java
@@ -184,8 +198,9 @@ files:
 - embulk-core/src/main/java/org/embulk/exec/TempFileAllocator.java
 - embulk-core/src/main/java/org/embulk/guice/Bootstrap.java
 - embulk-core/src/main/java/org/embulk/guice/CloseableInjector.java
-- embulk-core/src/main/java/org/embulk/guice/CloseableInjectorProxy.java
 - embulk-core/src/main/java/org/embulk/guice/InjectorProxy.java
+- embulk-core/src/main/java/org/embulk/guice/LifeCycleInjector.java
+- embulk-core/src/main/java/org/embulk/guice/LifeCycleInjectorProxy.java
 - embulk-core/src/main/java/org/embulk/guice/LifeCycleManager.java
 - embulk-core/src/main/java/org/embulk/guice/LifeCycleMethods.java
 - embulk-core/src/main/java/org/embulk/guice/LifeCycleMethodsMap.java
@@ -201,6 +216,11 @@ files:
 - embulk-core/src/main/java/org/embulk/plugin/PluginSource.java
 - embulk-core/src/main/java/org/embulk/plugin/PluginSourceNotMatchException.java
 - embulk-core/src/main/java/org/embulk/plugin/PluginType.java
+- embulk-core/src/main/java/org/embulk/plugin/compat/InputPluginWrapper.java
+- embulk-core/src/main/java/org/embulk/plugin/compat/PluginWrappers.java
+- embulk-core/src/main/java/org/embulk/plugin/compat/TransactionalFileInputWrapper.java
+- embulk-core/src/main/java/org/embulk/plugin/compat/TransactionalFileOutputWrapper.java
+- embulk-core/src/main/java/org/embulk/plugin/compat/TransactionalPageOutputWrapper.java
 - embulk-core/src/main/java/org/embulk/spi/Buffer.java
 - embulk-core/src/main/java/org/embulk/spi/BufferAllocator.java
 - embulk-core/src/main/java/org/embulk/spi/Column.java
@@ -386,8 +406,6 @@ files:
 - embulk-docs/src/release/release-0.6.23.rst
 - embulk-docs/src/release/release-0.6.24.rst
 - embulk-docs/src/release/release-0.6.25.rst
-- embulk-docs/src/release/release-0.6.26.rst
-- embulk-docs/src/release/release-0.6.27.rst
 - embulk-docs/src/release/release-0.6.3.rst
 - embulk-docs/src/release/release-0.6.4.rst
 - embulk-docs/src/release/release-0.6.5.rst
@@ -395,6 +413,7 @@ files:
 - embulk-docs/src/release/release-0.6.7.rst
 - embulk-docs/src/release/release-0.6.8.rst
 - embulk-docs/src/release/release-0.6.9.rst
+- embulk-docs/src/release/release-0.7.0.rst
 - embulk-standards/build.gradle
 - embulk-standards/src/main/java/org/embulk/standards/CsvFormatterPlugin.java
 - embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java
@@ -421,15 +440,17 @@ files:
 - lib/embulk.rb
 - lib/embulk/buffer.rb
 - lib/embulk/column.rb
-- lib/embulk/command/embulk.rb
+- lib/embulk/command/embulk_bundle.rb
 - lib/embulk/command/embulk_example.rb
 - lib/embulk/command/embulk_generate_bin.rb
+- lib/embulk/command/embulk_main.rb
+- lib/embulk/command/embulk_migrate_plugin.rb
 - lib/embulk/command/embulk_new_plugin.rb
 - lib/embulk/command/embulk_run.rb
 - lib/embulk/command/embulk_selfupdate.rb
 - lib/embulk/data/bundle/.bundle/config
+- lib/embulk/data/bundle/.ruby-version
 - lib/embulk/data/bundle/Gemfile
-- lib/embulk/data/bundle/Gemfile.lock
 - lib/embulk/data/bundle/embulk/filter/example.rb
 - lib/embulk/data/bundle/embulk/input/example.rb
 - lib/embulk/data/bundle/embulk/output/example.rb
@@ -452,6 +473,7 @@ files:
 - lib/embulk/data/new/java/parser.java.erb
 - lib/embulk/data/new/java/plugin_loader.rb.erb
 - lib/embulk/data/new/java/test.java.erb
+- lib/embulk/data/new/ruby/.ruby-version
 - lib/embulk/data/new/ruby/Gemfile
 - lib/embulk/data/new/ruby/Rakefile
 - lib/embulk/data/new/ruby/decoder_guess.rb.erb
@@ -467,7 +489,6 @@ files:
 - lib/embulk/decoder_plugin.rb
 - lib/embulk/encoder_plugin.rb
 - lib/embulk/error.rb
-- lib/embulk/exec.rb
 - lib/embulk/executor_plugin.rb
 - lib/embulk/file_input.rb
 - lib/embulk/file_input_plugin.rb
@@ -475,7 +496,6 @@ files:
 - lib/embulk/file_output_plugin.rb
 - lib/embulk/filter_plugin.rb
 - lib/embulk/formatter_plugin.rb
-- lib/embulk/gems.rb
 - lib/embulk/guess/charset.rb
 - lib/embulk/guess/csv.rb
 - lib/embulk/guess/gzip.rb
@@ -486,7 +506,6 @@ files:
 - lib/embulk/input_plugin.rb
 - lib/embulk/java/bootstrap.rb
 - lib/embulk/java/imports.rb
-- lib/embulk/java/liquid_helper.rb
 - lib/embulk/java/time_helper.rb
 - lib/embulk/java_plugin.rb
 - lib/embulk/logger.rb
@@ -496,6 +515,7 @@ files:
 - lib/embulk/parser_plugin.rb
 - lib/embulk/plugin.rb
 - lib/embulk/plugin_registry.rb
+- lib/embulk/runner.rb
 - lib/embulk/schema.rb
 - lib/embulk/version.rb
 - settings.gradle
@@ -507,24 +527,24 @@ homepage: https://github.com/embulk/embulk
 licenses:
 - Apache 2.0
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.4.8
-signing_key:
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
 specification_version: 4
 summary: Embulk, a plugin-based parallel bulk data loader
 test_files:
@@ -532,3 +552,4 @@ test_files:
 - test/guess/test_time_format_guess.rb
 - test/helper.rb
 - test/run-test.rb
+has_rdoc: false