embulk 0.6.21 → 0.6.22

data/embulk-core/src/main/java/org/embulk/guice/LifeCycleMethods.java

@@ -0,0 +1,89 @@
+/*
+ * Copyright 2010 Proofpoint, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/*
+ * Copyright 2015 Sadayuki Furuhashi
+ */
+package org.embulk.guice;
+
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Multimap;
+
+import javax.annotation.PostConstruct;
+import javax.annotation.PreDestroy;
+import java.lang.annotation.Annotation;
+import java.lang.reflect.Method;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+
+class LifeCycleMethods
+{
+    private final Multimap<Class<? extends Annotation>, Method> methodMap = ArrayListMultimap.create();
+
+    LifeCycleMethods(Class<?> clazz)
+    {
+        addLifeCycleMethods(clazz, new HashSet<String>(), new HashSet<String>());
+    }
+
+    boolean hasFor(Class<? extends Annotation> annotation)
+    {
+        Collection<Method> methods = methodMap.get(annotation);
+        return (methods != null) && (methods.size() > 0);
+    }
+
+    Collection<Method> methodsFor(Class<? extends Annotation> annotation)
+    {
+        Collection<Method> methods = methodMap.get(annotation);
+        return (methods != null) ? methods : Lists.<Method>newArrayList();
+    }
+
+    private void addLifeCycleMethods(Class<?> clazz, Set<String> usedConstructNames, Set<String> usedDestroyNames)
+    {
+        if (clazz == null) {
+            return;
+        }
+
+        for (Method method : clazz.getDeclaredMethods()) {
+            if (method.isSynthetic() || method.isBridge()) {
+                continue;
+            }
+
+            processMethod(method, PostConstruct.class, usedConstructNames);
+            processMethod(method, PreDestroy.class, usedDestroyNames);
+        }
+
+        addLifeCycleMethods(clazz.getSuperclass(), usedConstructNames, usedDestroyNames);
+        for (Class<?> face : clazz.getInterfaces()) {
+            addLifeCycleMethods(face, usedConstructNames, usedDestroyNames);
+        }
+    }
+
+    private void processMethod(Method method, Class<? extends Annotation> annotationClass, Set<String> usedSet)
+    {
+        if (method.isAnnotationPresent(annotationClass)) {
+            if (!usedSet.contains(method.getName())) {
+                if (method.getParameterTypes().length != 0) {
+                    throw new UnsupportedOperationException(String.format("@PostConstruct/@PreDestroy methods cannot have arguments: %s", method.getDeclaringClass().getName() + "." + method.getName() + "(...)"));
+                }
+
+                method.setAccessible(true);
+                usedSet.add(method.getName());
+                methodMap.put(annotationClass, method);
+            }
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/guice/LifeCycleMethodsMap.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2010 Proofpoint, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/*
+ * Copyright 2015 Sadayuki Furuhashi
+ */
+package org.embulk.guice;
+
+import com.google.common.collect.Maps;
+
+import java.util.Map;
+
+class LifeCycleMethodsMap
+{
+    private final Map<Class<?>, LifeCycleMethods> map = Maps.newHashMap();
+
+    synchronized LifeCycleMethods get(Class<?> clazz)
+    {
+        LifeCycleMethods methods = map.get(clazz);
+        if (methods == null) {
+            methods = new LifeCycleMethods(clazz);
+            map.put(clazz, methods);
+        }
+        return methods;
+    }
+}

data/embulk-core/src/main/java/org/embulk/guice/LifeCycleModule.java

@@ -0,0 +1,97 @@
+/*
+ * Copyright 2010 Proofpoint, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/*
+ * Copyright 2015 Sadayuki Furuhashi
+ */
+package org.embulk.guice;
+
+import com.google.common.collect.Lists;
+import com.google.inject.Binder;
+import com.google.inject.Module;
+import com.google.inject.Provides;
+import com.google.inject.Singleton;
+import com.google.inject.TypeLiteral;
+import com.google.inject.spi.InjectionListener;
+import com.google.inject.spi.TypeEncounter;
+import com.google.inject.spi.TypeListener;
+
+import javax.annotation.PostConstruct;
+import javax.annotation.PreDestroy;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static com.google.inject.matcher.Matchers.any;
+
+/**
+ * Guice module for binding the LifeCycle manager
+ */
+public class LifeCycleModule implements Module
+{
+    private final List<Object> injectedInstances = Lists.newArrayList();
+    private final LifeCycleMethodsMap lifeCycleMethodsMap = new LifeCycleMethodsMap();
+    private final AtomicReference<LifeCycleManager> lifeCycleManagerRef = new AtomicReference<LifeCycleManager>(null);
+
+    @Override
+    public void configure(Binder binder)
+    {
+        binder.disableCircularProxies();
+
+        binder.bindListener(any(), new TypeListener()
+        {
+            @Override
+            public <T> void hear(TypeLiteral<T> type, TypeEncounter<T> encounter)
+            {
+                encounter.register(new InjectionListener<T>()
+                {
+                    @Override
+                    public void afterInjection(T obj)
+                    {
+                        if (isLifeCycleClass(obj.getClass())) {
+                            LifeCycleManager lifeCycleManager = lifeCycleManagerRef.get();
+                            if (lifeCycleManager != null) {
+                                try {
+                                    lifeCycleManager.addInstance(obj);
+                                }
+                                catch (Exception e) {
+                                    throw new Error(e);
+                                }
+                            }
+                            else {
+                                injectedInstances.add(obj);
+                            }
+                        }
+                    }
+                });
+            }
+        });
+    }
+
+    @Provides
+    @Singleton
+    public LifeCycleManager getServerManager()
+            throws Exception
+    {
+        LifeCycleManager lifeCycleManager = new LifeCycleManager(injectedInstances, lifeCycleMethodsMap);
+        lifeCycleManagerRef.set(lifeCycleManager);
+        return lifeCycleManager;
+    }
+
+    private boolean isLifeCycleClass(Class<?> clazz)
+    {
+        LifeCycleMethods methods = lifeCycleMethodsMap.get(clazz);
+        return methods.hasFor(PostConstruct.class) || methods.hasFor(PreDestroy.class);
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/TempFileSpace.java

@@ -3,6 +3,11 @@ package org.embulk.spi;
 import java.util.List;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.nio.file.FileVisitResult;
 import java.io.File;
 import java.io.IOException;
 import com.google.common.base.Preconditions;
@@ -43,14 +48,43 @@ public class TempFileSpace
 
     public void cleanup()
     {
-        File[] files = dir.listFiles();
-        if (files != null) {
-            for (File e : files) {
-                e.delete();
-                // TODO delete directory recursively
-            }
+        try {
+            deleteFilesIfExistsRecursively(dir);
+        }
+        catch (IOException ex) {
+            // ignore IOException
         }
-        dir.delete();
         dirCreated = false;
     }
+
+    private void deleteFilesIfExistsRecursively(File dir)
+            throws IOException
+    {
+        Files.walkFileTree(dir.toPath(), new SimpleFileVisitor<Path>()
+        {
+            @Override
+            public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
+            {
+                try {
+                    Files.deleteIfExists(file);
+                }
+                catch (IOException ex) {
+                    // ignore IOException
+                }
+                return FileVisitResult.CONTINUE;
+            }
+
+            @Override
+            public FileVisitResult postVisitDirectory(Path dir, IOException exc)
+            {
+                try {
+                    Files.deleteIfExists(dir);
+                }
+                catch (IOException ex) {
+                    // ignore IOException
+                }
+                return FileVisitResult.CONTINUE;
+            }
+        });
+    }
 }

data/embulk-docs/build.gradle

@@ -23,8 +23,9 @@ task javadoc_html(type: Copy, dependsOn: [':embulk-core:javadoc']) {
 
 task rdoc_html(type: JRubyExec) {
     workingDir '..'
-    jrubyArgs '-ryard', '-eYARD::CLI::Yardoc.run(*ARGV)', '--', '-o', 'embulk-docs/build/html/rdoc'
-    script 'lib'
+    jrubyArgs '-ryard', '-eYARD::CLI::Yardoc.run(*ARGV)'
+    script './lib/embulk/version.rb'  // dummy
+    scriptArgs 'lib', '-o', 'embulk-docs/build/html/rdoc'
 }
 
 task site(type: Copy, dependsOn: ['sphinx_html', 'rdoc_html', 'javadoc_html']) {

data/embulk-docs/src/built-in.rst

@@ -59,6 +59,35 @@ A configuration file consists of following sections:
 In many cases, what you need to write is **in:**, **out**: and **formatter** sections only because ``guess`` command guesses **parser** and **decoder** options for you. See also the `Quick Start <https://github.com/embulk/embulk#quick-start>`_.
 
 
+Using variables
+~~~~~~~~~~~~~~~~~~
+
+You can embed environment variables in configuration file using `Liquid template engine <http://liquidmarkup.org/>`_ (This is experimental feature. Behavior might change or be removed in future releases).
+
+To use template engine, configuration file name must end with ``.yml.liquid``.
+
+Environment variables are set to ``env`` variable.
+
+.. code-block:: yaml
+
+    in:
+      type: file
+      path_prefix: {{ env.path_prefix }}
+      decoders:
+      - {type: gzip}
+      parser:
+        ...
+    out:
+      type: postgresql
+      host: {{ env.pg_host }}
+      port: {{ env.pg_port }}
+      user: {{ env.pg_user }}
+      password: "{{ env.pg_password }}"
+      database: embulk_load
+      mode: insert
+      table: {{ env.pg_table }}
+
+
 Local file input plugin
 ------------------
 
@@ -248,7 +277,7 @@ Options
 +====================+==========+===================================================+============================+
 | path\_prefix       | string   | Path prefix of the output files                   | required                   |
 +--------------------+----------+---------------------------------------------------+----------------------------+
-| sequence\_format   | string   | Format of the sequence number of the output files | ``%03d.%02d.`` by default |
+| sequence\_format   | string   | Format of the sequence number of the output files | ``%03d.%02d.`` by default  |
 +--------------------+----------+---------------------------------------------------+----------------------------+
 | file\_ext          | string   | Path suffix of the output files (e.g. ``"csv"``)  | required                   |
 +--------------------+----------+---------------------------------------------------+----------------------------+

data/embulk-docs/src/index.rst

@@ -62,7 +62,7 @@ Documents
   * `Executor plugins <http://www.embulk.org/plugins/#executor>`_
 
 .. toctree::
-   :maxdepth: 2
+   :maxdepth: 3
 
    built-in
 

data/embulk-docs/src/release.rst

@@ -4,6 +4,7 @@ Release Notes
 .. toctree::
     :maxdepth: 1
 
+    release/release-0.6.22
     release/release-0.6.21
     release/release-0.6.20
     release/release-0.6.19

data/embulk-docs/src/release/release-0.6.22.rst

@@ -0,0 +1,26 @@
+Release 0.6.22
+==================================
+
+General Changes
+------------------
+
+* Added experimental support for Liquid template engine of configuration files. If configuration file name ends with ``.yml.liquid``, embulk runs embeds variables using Liquid template engine.
+
+Ruby Plugin API
+------------------
+
+* Added ``Embulk.require_classpath`` method to initialize plugin test environment (@cosmo0920++)
+* ``Embulk::ConfigError`` extends ``org.embulk.config.ConfigException``.
+* ``Embulk::DataSource`` raises ``Embulk::ConfigError`` instead of StandardError.
+
+Java Plugin API
+------------------
+
+* Added ``org.embulk.EmbulkEmbed`` to embed Embulk in an applications as a library.
+* Added ``ConfigLoader.newConfigSource`` to create an empty ConfigSource.
+* ``TempFileSpace.cleanup`` deletes files and directories recursively (@cosmo0920++)
+
+
+Release Date
+------------------
+2015-08-11

data/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
-#Wed Feb 04 13:46:12 PST 2015
+#Tue Aug 11 00:26:20 PDT 2015
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-2.2.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-2.6-bin.zip

data/lib/embulk/command/embulk_run.rb

@@ -206,6 +206,8 @@ examples:
       args = 0..0
 
     when :gem
+      require 'embulk/gems'
+      Embulk.add_embedded_gem_path
       require 'rubygems/gem_runner'
       Gem::GemRunner.new.run argv
       exit 0
@@ -357,11 +359,7 @@ examples:
         java.lang.Class.forName('org.embulk.command.Runner')
       rescue java.lang.ClassNotFoundException
         # load classpath
-        classpath_dir = Embulk.home('classpath')
-        jars = Dir.entries(classpath_dir).select {|f| f =~ /\.jar$/ }.sort
-        jars.each do |jar|
-          require File.join(classpath_dir, jar)
-        end
+        Embulk.require_classpath
       end
 
       setup_plugin_paths(plugin_paths)
@@ -390,6 +388,14 @@ examples:
     end
   end
 
+  def self.require_classpath
+    classpath_dir = Embulk.home("classpath")
+    jars = Dir.entries(classpath_dir).select{|f| f =~ /\.jar$/ }.sort
+    jars.each do |jar|
+      require File.join(classpath_dir, jar)
+    end
+  end
+
   def self.default_gem_home
     if RUBY_PLATFORM =~ /java/i
       user_home = java.lang.System.properties["user.home"]

data/lib/embulk/data_source.rb

@@ -142,31 +142,51 @@ module Embulk
         value =
           case type
           when :integer
-            Integer(v)
+            begin
+              Integer(v)
+            rescue => e
+              raise ConfigError, e
+            end
           when :float
-            Float(v)
+            begin
+              Float(v)
+            rescue => e
+              raise ConfigError, e
+            end
           when :string
-            String(v).dup
+            begin
+              String(v).dup
+            rescue => e
+              raise ConfigError, e
+            end
           when :bool
-            !!v  # TODO validation
+            begin
+              !!v  # TODO validation
+            rescue => e
+              raise ConfigError, e
+            end
           when :hash
-            raise ArgumentError, "Invalid value for :hash" unless v.is_a?(Hash)
+            raise ConfigError, "Invalid value for :hash" unless v.is_a?(Hash)
             DataSource.new.merge!(v)
           when :array
-            raise ArgumentError, "Invalid value for :array" unless v.is_a?(Array)
+            raise ConfigError, "Invalid value for :array" unless v.is_a?(Array)
             v.dup
           else
             unless type.respond_to?(:load)
               raise ArgumentError, "Unknown type #{type.to_s.dump}"
             end
-            type.load(v)
+            begin
+              type.load(v)
+            rescue => e
+              raise ConfigError, e
+            end
           end
 
       elsif options.has_key?(:default)
         value = options[:default]
 
       else
-        raise "Required field #{key.to_s.dump} is not set"
+        raise ConfigError, "Required field #{key.to_s.dump} is not set"
       end
 
       return value