embulk 0.3.2 → 0.4.0

data/lib/embulk/data/new/java/gradlew.bat

@@ -0,0 +1,90 @@
+@if "%DEBUG%" == "" @echo off
+@rem ##########################################################################
+@rem
+@rem  Gradle startup script for Windows
+@rem
+@rem ##########################################################################
+
+@rem Set local scope for the variables with windows NT shell
+if "%OS%"=="Windows_NT" setlocal
+
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS=
+
+set DIRNAME=%~dp0
+if "%DIRNAME%" == "" set DIRNAME=.
+set APP_BASE_NAME=%~n0
+set APP_HOME=%DIRNAME%
+
+@rem Find java.exe
+if defined JAVA_HOME goto findJavaFromJavaHome
+
+set JAVA_EXE=java.exe
+%JAVA_EXE% -version >NUL 2>&1
+if "%ERRORLEVEL%" == "0" goto init
+
+echo.
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:findJavaFromJavaHome
+set JAVA_HOME=%JAVA_HOME:"=%
+set JAVA_EXE=%JAVA_HOME%/bin/java.exe
+
+if exist "%JAVA_EXE%" goto init
+
+echo.
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:init
+@rem Get command-line arguments, handling Windowz variants
+
+if not "%OS%" == "Windows_NT" goto win9xME_args
+if "%@eval[2+2]" == "4" goto 4NT_args
+
+:win9xME_args
+@rem Slurp the command line arguments.
+set CMD_LINE_ARGS=
+set _SKIP=2
+
+:win9xME_args_slurp
+if "x%~1" == "x" goto execute
+
+set CMD_LINE_ARGS=%*
+goto execute
+
+:4NT_args
+@rem Get arguments from the 4NT Shell from JP Software
+set CMD_LINE_ARGS=%$
+
+:execute
+@rem Setup the command line
+
+set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+
+@rem Execute Gradle
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
+
+:end
+@rem End local scope for the variables with windows NT shell
+if "%ERRORLEVEL%"=="0" goto mainEnd
+
+:fail
+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
+rem the _cmd.exe /c_ return code!
+if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
+exit /b 1
+
+:mainEnd
+if "%OS%"=="Windows_NT" endlocal
+
+:omega

data/lib/embulk/data/new/java/input.java.erb

@@ -0,0 +1,69 @@
+package org.embulk.<%= embulk_category %>;
+
+import java.util.List;
+import org.embulk.config.CommitReport;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.Exec;
+import org.embulk.spi.InputPlugin;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.Schema;
+import org.embulk.spi.SchemaConfig;
+
+public class <%= java_class_name %>
+        implements InputPlugin
+{
+    public interface PluginTask
+            extends Task
+    {
+        @Config("property1")
+        public String getProperty1();
+
+        @Config("property2")
+        @ConfigDefault("0")
+        public int getProperty2();
+
+        // TODO get schema from config or data source
+        @Config("columns")
+        public SchemaConfig getColumns();
+    }
+
+    public ConfigDiff transaction(ConfigSource config,
+            InputPlugin.Control control)
+    {
+        PluginTask task = config.loadConfig(PluginTask.class);
+
+        Schema schema = task.getColumns().toSchema();
+        int processorCount = 1;  // number of run() method calls
+
+        return resume(task.dump(), schema, processorCount, control);
+    }
+
+    public ConfigDiff resume(TaskSource taskSource,
+            Schema schema, int processorCount,
+            InputPlugin.Control control)
+    {
+        control.run(taskSource, schema, processorCount);
+        return Exec.newConfigDiff();
+    }
+
+    public void cleanup(TaskSource taskSource,
+            Schema schema, int processorCount,
+            List<CommitReport> successCommitReports)
+    {
+    }
+
+    public CommitReport run(TaskSource taskSource,
+            Schema schema, int processorIndex,
+            PageOutput output)
+    {
+        PluginTask task = taskSource.loadTask(PluginTask.class);
+
+        // TODO
+        throw new UnsupportedOperationException("The 'run' method needs to be implemented");
+    }
+}

data/lib/embulk/data/new/java/output.java.erb

@@ -0,0 +1,65 @@
+package org.embulk.<%= embulk_category %>;
+
+import java.util.List;
+import org.embulk.config.CommitReport;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.Exec;
+import org.embulk.spi.OutputPlugin;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.Schema;
+import org.embulk.spi.TransactionalPageOutput;
+
+public class <%= java_class_name %>
+        implements OutputPlugin
+{
+    public interface PluginTask
+            extends Task
+    {
+        @Config("property1")
+        public String getProperty1();
+
+        @Config("property2")
+        @ConfigDefault("0")
+        public int getProperty2();
+    }
+
+    public ConfigDiff transaction(ConfigSource config,
+            Schema schema, int processorCount,
+            OutputPlugin.Control control)
+    {
+        PluginTask task = config.loadConfig(PluginTask.class);
+
+        // retryable (idempotent) output:
+        // return resume(task.dump(), schema, processorCount, control);
+
+        // non-retryable (non-idempotent) output:
+        control.run(task.dump());
+        return Exec.newConfigDiff();
+    }
+
+    public ConfigDiff resume(TaskSource taskSource,
+            Schema schema, int processorCount,
+            OutputPlugin.Control control)
+    {
+        throw new UnsupportedOperationException("<%= name %> output plugin does not support resuming");
+    }
+
+    public void cleanup(TaskSource taskSource,
+            Schema schema, int processorCount,
+            List<CommitReport> successCommitReports)
+    {
+    }
+
+    public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int processorIndex)
+    {
+        PluginTask task = taskSource.loadTask(PluginTask.class);
+
+        // TODO
+        throw new UnsupportedOperationException("The 'open' method needs to be implemented");
+    }
+}

data/lib/embulk/data/new/java/parser.java.erb

@@ -0,0 +1,51 @@
+package org.embulk.<%= embulk_category %>;
+
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.ParserPlugin;
+import org.embulk.spi.FileInput;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.Schema;
+import org.embulk.spi.SchemaConfig;
+
+public class <%= java_class_name %>
+        implements ParserPlugin
+{
+    public interface PluginTask
+            extends Task
+    {
+        @Config("property1")
+        public String getProperty1();
+
+        @Config("property2")
+        @ConfigDefault("0")
+        public int getProperty2();
+
+        // TODO get schema from config or data source
+        @Config("columns")
+        public SchemaConfig getColumns();
+    }
+
+    @Override
+    public void transaction(ConfigSource config, ParserPlugin.Control control)
+    {
+        PluginTask task = config.loadConfig(PluginTask.class);
+
+        Schema schema = task.getColumns().toSchema();
+
+        control.run(task.dump(), schema);
+    }
+
+    @Override
+    public void run(TaskSource taskSource, Schema schema,
+            FileInput input, PageOutput output)
+    {
+        PluginTask task = taskSource.loadTask(PluginTask.class);
+
+        // TODO
+    }
+}

data/lib/embulk/data/new/java/plugin_loader.rb.erb

@@ -0,0 +1,3 @@
+Embulk::JavaPlugin.register_<%= embulk_category %>(
+  <%= name.to_sym.inspect %>, <%= "org.embulk.#{embulk_category}.#{java_class_name}".dump %>,
+  File.expand_path('../../../../classpath', __FILE__))

data/lib/embulk/data/new/java/test.java.erb

@@ -0,0 +1,5 @@
+package org.embulk.<%= embulk_category %>;
+
+public class Test<%= java_class_name %>
+{
+}

data/lib/embulk/data/new/ruby/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org/'
+gemspec

data/lib/embulk/data/new/ruby/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/lib/embulk/data/new/ruby/filter.rb.erb

@@ -0,0 +1,39 @@
+module Embulk
+  module Filter
+
+    class <%= ruby_class_name %> < FilterPlugin
+      Plugin.register_filter(<%= name %>, self)
+
+      def self.transaction(config, in_schema, &control)
+        # configuration code:
+        task = {
+          "property1" => config.param("property1", :string)
+          "property2" => config.param("property2", :integer, default: 0)
+        }
+
+        yield(task, out_columns)
+      end
+
+      def init
+        # initialization code:
+        @property1 = task["property1"]
+        @property2 = task["property2"]
+      end
+
+      def close
+      end
+
+      def add(page)
+        # filtering code:
+        page.each do |record|
+          @page_builder.add(record)
+        end
+      end
+
+      def finish
+        @page_builder.finish
+      end
+    end
+
+  end
+end

data/lib/embulk/data/new/ruby/gemspec.erb

@@ -0,0 +1,19 @@
+
+Gem::Specification.new do |spec|
+  spec.name          = "<%= project_name %>"
+  spec.version       = "0.1.0"
+  spec.authors       = [<%= author.dump %>]
+  spec.summary       = "<%= display_name %> <%= display_category %> plugin for Embulk"
+  spec.description   = "<%= display_name %> <%= display_category %> plugin is an Embulk plugin <%= description %>"
+  spec.email         = [<%= email.dump %>]
+  spec.licenses      = ["MIT"]
+  # TODO: spec.homepage      = "https://github.com/<%= email[/([^@]*)/] %>/<%= project_name %>"
+
+  spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
+  spec.test_files    = spec.files.grep(%r{^(test|spec)/})
+  spec.require_paths = ["lib"]
+
+  #spec.add_dependency 'YOUR_GEM_DEPENDENCY', ['~> YOUR_GEM_DEPENDENCY_VERSION']
+  spec.add_development_dependency 'bundler', ['~> 1.0']
+  spec.add_development_dependency 'rake', ['>= 10.0']
+end

data/lib/embulk/data/new/ruby/input.rb.erb

@@ -0,0 +1,47 @@
+module Embulk
+  module Input
+
+    class <%= ruby_class_name %> < InputPlugin
+      Plugin.register_input(<%= name %>, self)
+
+      def self.transaction(config, &control)
+        # configuration code:
+        task = {
+          "property1" => config.param("property1", :string)
+          "property2" => config.param("property2", :integer, default: 0)
+        }
+
+        columns = [
+          Column.new(0, "example", :string),
+          Column.new(1, "column", :integer),
+          Column.new(2, "name", :double),
+        ]
+
+        resume(task, columns, 1, &control)
+      end
+
+      def self.resume(task, columns, count, &control)
+        commit_reports = yield(task, columns, count)
+
+        next_config_diff = {}
+        return next_config_diff
+      end
+
+      def init
+        # initialization code:
+        @property1 = task["property1"]
+        @property2 = task["property2"]
+      end
+
+      def run
+        @page_builder.add(["example-value", 1, 0.1])
+        @page_builder.add(["example-value", 2, 0.2])
+        @page_builder.finish
+
+        commit_report = {}
+        return commit_report
+      end
+    end
+
+  end
+end

data/lib/embulk/data/new/ruby/output.rb.erb

@@ -0,0 +1,59 @@
+module Embulk
+  module Output
+
+    class <%= ruby_class_name %> < OutputPlugin
+      Plugin.register_output(<%= name.dump %>, self)
+
+      def self.transaction(config, schema, count, &control)
+        # configuration code:
+        task = {
+          "property1" => config.param("property1", :string)
+          "property2" => config.param("property2", :integer, default: 0)
+        }
+
+        # resumable output:
+        # resume(task, schema, count, &control)
+
+        # non-resumable output:
+        commit_reports = yield(task)
+        next_config_diff = {}
+        return next_config_diff
+      end
+
+      #def self.resume(task, schema, count, &control)
+      #  commit_reports = yield(task)
+      #
+      #  next_config_diff = {}
+      #  return next_config_diff
+      #end
+
+      def init
+        # initialization code:
+        @property1 = task["property1"]
+        @property2 = task["property2"]
+      end
+
+      def close
+      end
+
+      def add(page)
+        # output code:
+        page.each do |record|
+          #hash = Hash[schema.names.zip(record)]
+        end
+      end
+
+      def finish
+      end
+
+      def abort
+      end
+
+      def commit
+        commit_report = {}
+        return commit_report
+      end
+    end
+
+  end
+end