embulk 0.1.0

data/embulk-cli/build.gradle

@@ -0,0 +1,6 @@
+dependencies {
+    compile project(':embulk-core')
+    compile project(':embulk-standards')
+
+    testCompile project(':embulk-core').sourceSets.test.output
+}

data/embulk-cli/pom.xml

@@ -0,0 +1,94 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>org.embulk</groupId>
+        <artifactId>embulk-parent</artifactId>
+        <version>0.1.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>embulk-cli</artifactId>
+    <name>embulk-cli</name>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.jruby</groupId>
+            <artifactId>jruby-complete</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.embulk</groupId>
+            <artifactId>embulk-core</artifactId>
+        </dependency>
+
+        <!-- TODO move embulk-standards to another repository -->
+        <!-- TODO once embulk-standards is moved to another repository, replace this dependency with gem from rubygems-proxy -->
+        <!--     and change Rakefile to include embulk-core instead of embulk-cli and gemspec depends on embulk-standards -->
+        <dependency>
+            <groupId>org.embulk</groupId>
+            <artifactId>embulk-standards</artifactId>
+        </dependency>
+    </dependencies>
+
+    <repositories>
+        <!-- See https://github.com/jruby/jruby/wiki/Jruby-Scripting-container-using-Gems-with-a-Maven-Project -->
+        <repository>
+            <id>rubygems-proxy</id>
+            <name>RubyGems Proxy</name>
+            <url>http://rubygems-proxy.torquebox.org/releases</url>
+            <layout>default</layout>
+            <releases>
+                <enabled>true</enabled>
+            </releases>
+            <snapshots>
+                <enabled>false</enabled>
+                <updatePolicy>never</updatePolicy>
+            </snapshots>
+        </repository>
+    </repositories>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>de.saumya.mojo</groupId>
+                <artifactId>gem-maven-plugin</artifactId>
+                <version>1.0.0</version>
+                <configuration>
+                    <includeRubygemsInResources>true</includeRubygemsInResources>
+                </configuration>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>initialize</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+                            <shadedArtifactAttached>true</shadedArtifactAttached>
+                            <shadedClassifierName>executable</shadedClassifierName>
+                            <transformers>
+                                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                                    <manifestEntries>
+                                        <Main-Class>org.embulk.cli.Main</Main-Class>
+                                    </manifestEntries>
+                                </transformer>
+                            </transformers>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>

data/embulk-cli/src/main/java/org/embulk/cli/Main.java

@@ -0,0 +1,15 @@
+package org.embulk.cli;
+
+public class Main
+{
+    public static void main(String[] args)
+    {
+        // TODO set GEM_HOME to point the internal gem repository created by gem-maven-plugin?
+
+        // $ java -jar jruby-complete.jar classpath:embulk/command/embulk.rb "$@"
+        String[] jrubyArgs = new String[args.length + 1];
+        jrubyArgs[0] = "classpath:embulk/command/embulk.rb";
+        System.arraycopy(args, 0, jrubyArgs, 1, args.length);
+        org.jruby.Main.main(jrubyArgs);
+    }
+}

data/embulk-core/build.gradle

@@ -0,0 +1,6 @@
+sourceSets {
+    main.resources {
+        srcDirs 'src/main/resources'
+        srcDirs '../lib'
+    }
+}

data/embulk-core/pom.xml

@@ -0,0 +1,143 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>org.embulk</groupId>
+        <artifactId>embulk-parent</artifactId>
+        <version>0.1.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>embulk-core</artifactId>
+    <name>embulk-core</name>
+
+    <dependencies>
+        <dependency>
+            <groupId>commons-logging</groupId>
+            <artifactId>commons-logging</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.google.inject</groupId>
+            <artifactId>guice</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.google.inject.extensions</groupId>
+            <artifactId>guice-multibindings</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>javax.inject</groupId>
+            <artifactId>javax.inject</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-annotations</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-core</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.fasterxml.jackson.datatype</groupId>
+            <artifactId>jackson-datatype-guava</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.fasterxml.jackson.module</groupId>
+            <artifactId>jackson-module-guice</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.fasterxml.jackson.datatype</groupId>
+            <artifactId>jackson-datatype-joda</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-log4j12</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>log4j</groupId>
+            <artifactId>log4j</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>joda-time</groupId>
+            <artifactId>joda-time</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.yaml</groupId>
+            <artifactId>snakeyaml</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>javax.validation</groupId>
+            <artifactId>validation-api</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.bval</groupId>
+            <artifactId>bval-jsr303</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>io.airlift</groupId>
+            <artifactId>slice</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>io.netty</groupId>
+            <artifactId>netty-buffer</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.jruby</groupId>
+            <artifactId>jruby-complete</artifactId>
+        </dependency>
+
+        <!-- for guess_charset plugin -->
+        <dependency>
+            <groupId>com.ibm.icu</groupId>
+            <artifactId>icu4j</artifactId>
+            <scope>runtime</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <resources>
+            <resource>
+                <directory>src/main/resources</directory>
+            </resource>
+            <resource>
+                <directory>../lib</directory>
+            </resource>
+        </resources>
+    </build>
+</project>

data/embulk-core/src/main/java/org/embulk/EmbulkService.java

@@ -0,0 +1,39 @@
+package org.embulk;
+
+import com.google.common.collect.ImmutableList;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.Module;
+import org.embulk.config.ConfigSource;
+import org.embulk.exec.SystemConfigModule;
+import org.embulk.exec.ExecModule;
+import org.embulk.exec.ExtensionServiceLoaderModule;
+import org.embulk.plugin.BuiltinPluginSourceModule;
+import org.embulk.jruby.JRubyScriptingModule;
+
+public class EmbulkService
+{
+    protected final Injector injector;
+
+    public EmbulkService(ConfigSource systemConfig)
+    {
+        ImmutableList.Builder<Module> modules = ImmutableList.builder();
+        modules.add(new SystemConfigModule(systemConfig));
+        modules.add(new ExecModule());
+        modules.add(new ExtensionServiceLoaderModule(systemConfig));
+        modules.add(new BuiltinPluginSourceModule());
+        modules.add(new JRubyScriptingModule(systemConfig));
+        modules.addAll(getAdditionalModules());
+        injector = Guice.createInjector(modules.build());
+    }
+
+    protected Iterable<? extends Module> getAdditionalModules()
+    {
+        return ImmutableList.of();
+    }
+
+    public Injector getInjector()
+    {
+        return injector;
+    }
+}

data/embulk-core/src/main/java/org/embulk/command/Runner.java

@@ -0,0 +1,199 @@
+package org.embulk.command;
+
+import java.util.List;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.Locale;
+import java.io.File;
+import java.io.IOException;
+import java.io.FileOutputStream;
+import java.io.BufferedWriter;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.text.NumberFormat;
+import org.yaml.snakeyaml.Yaml;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.inject.Injector;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.DataSource;
+import org.embulk.config.ConfigLoader;
+import org.embulk.config.NextConfig;
+import org.embulk.config.ModelManager;
+import org.embulk.config.ConfigException;
+import org.embulk.exec.LocalExecutor;
+import org.embulk.exec.ExecuteResult;
+import org.embulk.exec.GuessExecutor;
+import org.embulk.exec.PreviewExecutor;
+import org.embulk.exec.PreviewResult;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.ExecSession;
+import org.embulk.spi.util.Pages;
+import org.embulk.EmbulkService;
+
+public class Runner
+{
+    private static class Options
+    {
+        private String nextConfigOutputPath;
+        public String getNextConfigOutputPath() { return nextConfigOutputPath; }
+    }
+
+    private final Options options;
+    private final ConfigSource systemConfig;
+    private final EmbulkService service;
+    private final Injector injector;
+
+    public Runner(String optionJson)
+    {
+        ModelManager bootstrapModelManager = new ModelManager(null, new ObjectMapper());
+        this.options = bootstrapModelManager.readObject(Options.class, optionJson);
+        this.systemConfig = new ConfigLoader(bootstrapModelManager).fromPropertiesYamlLiteral(System.getProperties(), "embulk.");
+        this.service = new EmbulkService(systemConfig);
+        this.injector = service.getInjector();
+    }
+
+    public void main(String command, String[] args)
+    {
+        switch (command) {
+        case "run":
+            run(args[0]);
+            break;
+        case "guess":
+            guess(args[0]);
+            break;
+        case "preview":
+            preview(args[0]);
+            break;
+        default:
+            throw new RuntimeException("Unsupported command: "+command);
+        }
+    }
+
+    public void run(String configPath)
+    {
+        ConfigSource config = loadYamlConfig(configPath);
+        checkNextConfigOutputPath(options.getNextConfigOutputPath());
+
+        ExecSession exec = newExecSession(config);
+        LocalExecutor local = injector.getInstance(LocalExecutor.class);
+        ExecuteResult result = local.run(exec, config);
+        NextConfig nextConfig = result.getNextConfig();
+
+        exec.getLogger(Runner.class).info("next config: {}", nextConfig.toString());
+        writeNextConfig(options.getNextConfigOutputPath(), config, nextConfig);
+    }
+
+    public void guess(String partialConfigPath)
+    {
+        ConfigSource config = loadYamlConfig(partialConfigPath);
+        checkNextConfigOutputPath(options.getNextConfigOutputPath());
+
+        ExecSession exec = newExecSession(config);
+        GuessExecutor guess = injector.getInstance(GuessExecutor.class);
+        NextConfig nextConfig = guess.guess(exec, config);
+
+        String yml = writeNextConfig(options.getNextConfigOutputPath(), config, nextConfig);
+        System.err.println(yml);
+    }
+
+    private void checkNextConfigOutputPath(String path)
+    {
+        if (path != null) {
+            try (FileOutputStream in = new FileOutputStream(path, true)) {
+                // open with append mode and do nothing. just check availability of the path to not cause exceptiosn later
+            } catch (IOException ex) {
+                throw new RuntimeException(ex);
+            }
+        }
+    }
+
+    private String writeNextConfig(String path, ConfigSource originalConfig, NextConfig nextConfigDiff)
+    {
+        String yml = dumpConfigInYaml(originalConfig.merge(nextConfigDiff));
+        if (path != null) {
+            if (path.equals("-")) {
+                System.out.print(yml);
+            } else {
+                try (Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(path), "UTF-8"))) {
+                    writer.write(yml);
+                } catch (IOException ex) {
+                    throw new RuntimeException(ex);
+                }
+            }
+        }
+        return yml;
+    }
+
+    public void preview(String partialConfigPath)
+    {
+        ConfigSource config = loadYamlConfig(partialConfigPath);
+        ExecSession exec = newExecSession(config);
+        PreviewExecutor preview = injector.getInstance(PreviewExecutor.class);
+        PreviewResult result = preview.preview(exec, config);
+        List<Object[]> records = Pages.toObjects(result.getSchema(), result.getPages());
+        final ModelManager model = injector.getInstance(ModelManager.class);
+
+        String[] header = new String[result.getSchema().getColumnCount()];
+        for (int i=0; i < header.length; i++) {
+            header[i] = result.getSchema().getColumnName(i) + ":" + result.getSchema().getColumnType(i);
+        }
+
+        TablePrinter printer = new TablePrinter(System.out, header) {
+            private NumberFormat numberFormat = NumberFormat.getNumberInstance(Locale.ENGLISH);
+
+            protected String valueToString(Object obj)
+            {
+                if (obj instanceof String) {
+                    return (String) obj;
+                } else if (obj instanceof Number) {
+                    if (obj instanceof Integer) {
+                        return numberFormat.format(((Integer) obj).longValue());
+                    }
+                    if (obj instanceof Long) {
+                        return numberFormat.format(((Long) obj).longValue());
+                    }
+                    return obj.toString();
+                } else if (obj instanceof Timestamp) {
+                    return obj.toString();
+                } else {
+                    return model.writeObject(obj);
+                }
+            }
+        };
+
+        try {
+            for (Object[] record : records) {
+                printer.add(record);
+            }
+            printer.finish();
+        } catch (IOException ex) {
+            throw new RuntimeException(ex);
+        }
+    }
+
+    private ConfigSource loadYamlConfig(String yamlPath)
+    {
+        try {
+            return injector.getInstance(ConfigLoader.class).fromYamlFile(new File(yamlPath));
+
+        } catch (IOException ex) {
+            throw new ConfigException(ex);
+        }
+    }
+
+    private String dumpConfigInYaml(DataSource config)
+    {
+        ModelManager model = injector.getInstance(ModelManager.class);
+        Map<String, Object> map = model.readObject(MapType.class, model.writeObject(config));
+        return new Yaml().dump(map);
+    }
+
+    private ExecSession newExecSession(ConfigSource config)
+    {
+        return new ExecSession(injector, config.getNestedOrSetEmpty("exec"));
+    }
+
+    private static class MapType extends HashMap<String, Object> {
+        public MapType() { }
+    };
+}