embulk 0.6.9 → 0.6.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 19a42567d7e841a769ba329935d942926a062f66
-  data.tar.gz: 0bac8733cc4f5028e62865c08193520fce7f9d79
+  metadata.gz: 5f9bbe21037e6186e386b39176e66b8336ce8d20
+  data.tar.gz: 29fef3403d44ff6dc571d3e3c050efc23f77b770
 SHA512:
-  metadata.gz: b74b82f6d740d513cce0eadc365824a78737a2cd40bcac32082ff51f03259cd720140eb7463530b3ffe8d8a7d8601a4d91de889a891760888c4b8ddfe9b4bede
-  data.tar.gz: 5a77c5eba35a06074289aab700f1351ced771f1f181cf5831d4ccf747692444063be0c32133a8d5bbf5d30fe6e636db398f591898135b351e81aeba7b90bb3db
+  metadata.gz: 1481af06ce9c9aa2497ee25ee2c7b4f3be2578acf33e2be01ba06a0727cdc20523af16a4e7bae64658e286b2060e118e6eedd97cec085186843063f5246a6001
+  data.tar.gz: a3ab3e1c0e0f8477127dd30c5484210ae09845410ba0a747be942868061a002ad35e27e4deed4b63c6d49bca6477103728bb35378d12ae8363e40b24d8bea94b

data/build.gradle

@@ -11,7 +11,7 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards")]
 
 allprojects {
     group = 'org.embulk'
-    version = '0.6.9'
+    version = '0.6.10'
 
     ext {
         jrubyVersion = '1.7.19'

data/embulk-core/src/main/java/org/embulk/spi/ProcessTask.java

@@ -1,6 +1,9 @@
 package org.embulk.spi;
 
 import java.util.List;
+import com.google.common.base.Function;
+import com.google.common.collect.Lists;
+import com.google.common.collect.ImmutableList;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonIgnore;
@@ -8,6 +11,7 @@ import org.embulk.plugin.PluginType;
 import org.embulk.config.TaskSource;
 import org.embulk.spi.Schema;
 import org.embulk.spi.util.Executors;
+import org.embulk.spi.type.TimestampType;
 
 public class ProcessTask
 {
@@ -21,17 +25,16 @@ public class ProcessTask
     private final Schema executorSchema;
     private TaskSource executorTaskSource;
 
-    @JsonCreator
     public ProcessTask(
-            @JsonProperty("inputType") PluginType inputPluginType,
-            @JsonProperty("outputType") PluginType outputPluginType,
-            @JsonProperty("filterTypes") List<PluginType> filterPluginTypes,
-            @JsonProperty("inputTask") TaskSource inputTaskSource,
-            @JsonProperty("outputTask") TaskSource outputTaskSource,
-            @JsonProperty("filterTasks") List<TaskSource> filterTaskSources,
-            @JsonProperty("schemas") List<Schema> schemas,
-            @JsonProperty("executorSchema") Schema executorSchema,
-            @JsonProperty("executorTask") TaskSource executorTaskSource)
+            PluginType inputPluginType,
+            PluginType outputPluginType,
+            List<PluginType> filterPluginTypes,
+            TaskSource inputTaskSource,
+            TaskSource outputTaskSource,
+            List<TaskSource> filterTaskSources,
+            List<Schema> schemas,
+            Schema executorSchema,
+            TaskSource executorTaskSource)
     {
         this.inputPluginType = inputPluginType;
         this.outputPluginType = outputPluginType;
@@ -44,6 +47,36 @@ public class ProcessTask
         this.executorTaskSource = executorTaskSource;
     }
 
+    // TODO Because TimestampType doesn't store timestamp_format, serializing and deserializing
+    // Schema loses timestamp_format information. Here uses SchemaConfig instead to preseve it.
+
+    @JsonCreator
+    ProcessTask(
+            @JsonProperty("inputType") PluginType inputPluginType,
+            @JsonProperty("outputType") PluginType outputPluginType,
+            @JsonProperty("filterTypes") List<PluginType> filterPluginTypes,
+            @JsonProperty("inputTask") TaskSource inputTaskSource,
+            @JsonProperty("outputTask") TaskSource outputTaskSource,
+            @JsonProperty("filterTasks") List<TaskSource> filterTaskSources,
+            @JsonProperty("schemas") List<SchemaConfig> schemas,
+            @JsonProperty("executorSchema") SchemaConfig executorSchema,
+            @JsonProperty("executorTask") TaskSource executorTaskSource)
+    {
+        this(inputPluginType, outputPluginType, filterPluginTypes,
+                inputTaskSource, outputTaskSource, filterTaskSources,
+                ImmutableList.copyOf(Lists.transform(schemas,
+                        new Function<SchemaConfig, Schema>()
+                        {
+                            public Schema apply(SchemaConfig s)
+                            {
+                                return s.toSchema();
+                            }
+                        }
+                    )),
+                executorSchema.toSchema(),
+                executorTaskSource);
+    }
+
     @JsonProperty("inputType")
     public PluginType getInputPluginType()
     {
@@ -80,18 +113,54 @@ public class ProcessTask
         return filterTaskSources;
     }
 
-    @JsonProperty("schemas")
+    @JsonIgnore
     public List<Schema> getFilterSchemas()
     {
         return schemas;
     }
 
-    @JsonProperty("executorSchema")
+    @JsonProperty("schemas")
+    public List<SchemaConfig> getFilterSchemaConfigs()
+    {
+        return Lists.transform(schemas,
+                new Function<Schema, SchemaConfig>()
+                {
+                    public SchemaConfig apply(Schema schema)
+                    {
+                        return schemaToSchemaConfig(schema);
+                    }
+                });
+    }
+
+    @JsonIgnore
     public Schema getExecutorSchema()
     {
         return executorSchema;
     }
 
+    @JsonProperty("executorSchema")
+    SchemaConfig getExecutorSchemaConfig()
+    {
+        return schemaToSchemaConfig(executorSchema);
+    }
+
+    private static SchemaConfig schemaToSchemaConfig(Schema s)
+    {
+        return new SchemaConfig(Lists.transform(s.getColumns(),
+                    new Function<Column, ColumnConfig>()
+                    {
+                        public ColumnConfig apply(Column c)
+                        {
+                            if (c.getType() instanceof TimestampType) {
+                                return new ColumnConfig(c.getName(), c.getType(), ((TimestampType) c.getType()).getFormat());
+                            } else {
+                                return new ColumnConfig(c.getName(), c.getType(), null);
+                            }
+                        }
+                    }
+                ));
+    }
+
     @JsonIgnore
     public Schema getInputSchema()
     {

data/embulk-docs/src/release.rst

@@ -4,6 +4,7 @@ Release Notes
 .. toctree::
     :maxdepth: 1
 
+    release/release-0.6.10
     release/release-0.6.9
     release/release-0.6.8
     release/release-0.6.7

data/embulk-docs/src/release/release-0.6.10.rst

@@ -0,0 +1,15 @@
+Release 0.6.10
+==================================
+
+General Changes
+------------------
+
+* PluginRegistry shows loaded plugin name and version to stdout
+* Fixed a problem where ``embulk gem install`` command can't install gems which include a native extension
+* Added ``embulk irb`` subcommand
+* Fixed ``spi.ProcessTask`` serialization to preserve ``timestamp_format`` of a column. This fixes compatibility with embulk-executor-mapreduce.
+
+
+Release Date
+------------------
+2015-05-21

data/lib/embulk/command/embulk_run.rb

@@ -14,6 +14,10 @@ module Embulk
       Gem.clear_paths  # force rubygems to reload GEM_HOME
     end
 
+    # Gem.path is called when GemRunner installs a gem with native extension.
+    # Running extconf.rb fails without this hack.
+    fix_gem_ruby_path
+
     # to make sure org.embulk.jruby.JRubyScriptingModule can require 'embulk/java/bootstrap'
     $LOAD_PATH << Embulk.home('lib')
 
@@ -192,6 +196,11 @@ examples:
       exec(*argv)
       exit 127
 
+    when :irb
+      require 'irb'
+      IRB.start
+      exit 0
+
     else
       usage "Unknown subcommand #{subcmd.dump}."
     end
@@ -339,7 +348,7 @@ examples:
 
   def self.home(dir)
     jar, resource = __FILE__.split("!")
-    if resource
+    if resource && jar =~ /^file:/
       home = resource.split("/")[0..-3].join("/")
       "#{jar}!#{home}/#{dir}"
     else
@@ -361,6 +370,23 @@ examples:
 
   private
 
+  def self.fix_gem_ruby_path
+    ruby_path = File.join(RbConfig::CONFIG["bindir"], RbConfig::CONFIG["ruby_install_name"])
+    jar, resource = ruby_path.split("!")
+
+    if resource && jar =~ /^file:/
+      # java
+      manifest = File.read("#{jar}!/META-INF/MANIFEST.MF") rescue ""
+      m = /Main-Class: ([^\r\n]+)/.match(manifest)
+      if m && m[1] != "org.jruby.Main"
+        # Main-Class is not jruby
+        Gem.define_singleton_method(:ruby) do
+          "java -cp #{jar_path(jar)} org.jruby.Main"
+        end
+      end
+    end
+  end
+
   def self.setup_gem_paths(path)
     # install bundler gem here & use bundler installed here
     ENV['GEM_HOME'] = File.expand_path File.join(path, Gem.ruby_engine, RbConfig::CONFIG['ruby_version'])

data/lib/embulk/plugin_registry.rb

@@ -6,6 +6,7 @@ module Embulk
     def initialize(category, search_prefix)
       @category = category
       @search_prefix = search_prefix
+      @loaded_gems = {}
       @map = {}
     end
 
@@ -34,7 +35,7 @@ module Embulk
     def search(type)
       name = "#{@search_prefix}#{type}"
       begin
-        require name
+        require_and_show name
         return true
       rescue LoadError => e
         # catch LoadError but don't catch ClassNotFoundException
@@ -50,7 +51,7 @@ module Embulk
 
       paths = load_path_files.compact.sort  # sort to prefer newer version
       paths.each do |path|
-        require path
+        require_and_show path
         return true
       end
 
@@ -64,7 +65,7 @@ module Embulk
         specs = specs.sort_by {|spec| spec.version }
         if spec = specs.last
           spec.require_paths.each do |lib|
-            require "#{spec.full_gem_path}/#{lib}/#{name}"
+            require_and_show "#{spec.full_gem_path}/#{lib}/#{name}"
           end
           return true
         end
@@ -72,5 +73,20 @@ module Embulk
 
       return false
     end
+
+    def require_and_show(name)
+      require name
+      show_loaded_gems
+    end
+
+    def show_loaded_gems
+      # TODO use logger
+      Gem.loaded_specs.each do |name,spec|
+        if !@loaded_gems[name] && name =~ /^embulk/
+          puts "#{Time.now.strftime("%Y-%m-%d %H:%M:%S.%3N %z")}: Loaded plugin #{name} (#{spec.version})"
+          @loaded_gems[name] = true
+        end
+      end
+    end
   end
 end

data/lib/embulk/version.rb

@@ -1,3 +1,3 @@
 module Embulk
-  VERSION = '0.6.9'
+  VERSION = '0.6.10'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk
 version: !ruby/object:Gem::Version
-  version: 0.6.9
+  version: 0.6.10
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-05-15 00:00:00.000000000 Z
+date: 2015-05-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -293,6 +293,7 @@ files:
 - embulk-docs/src/release/release-0.5.5.rst
 - embulk-docs/src/release/release-0.6.0.rst
 - embulk-docs/src/release/release-0.6.1.rst
+- embulk-docs/src/release/release-0.6.10.rst
 - embulk-docs/src/release/release-0.6.2.rst
 - embulk-docs/src/release/release-0.6.3.rst
 - embulk-docs/src/release/release-0.6.4.rst
@@ -408,8 +409,8 @@ files:
 - classpath/bval-jsr303-0.5.jar
 - classpath/commons-beanutils-core-1.8.3.jar
 - classpath/commons-lang3-3.1.jar
-- classpath/embulk-core-0.6.9.jar
-- classpath/embulk-standards-0.6.9.jar
+- classpath/embulk-core-0.6.10.jar
+- classpath/embulk-standards-0.6.10.jar
 - classpath/guava-18.0.jar
 - classpath/guice-4.0.jar
 - classpath/guice-multibindings-4.0.jar