embulk 0.4.6 → 0.4.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b02e8762879a557c331e6fba758a1be4bd2d0575
-  data.tar.gz: 1d75577656dbf789d30674e5e689abbb2048f28e
+  metadata.gz: 2f4db59dbfae683eff9877c8794fab684f562224
+  data.tar.gz: 7969e968bae40633fb8c3d70a9ed2f80a0ee4a37
 SHA512:
-  metadata.gz: b3e674c5d0e1a9fa3a09b516339cba7a7f587ff70521287e11c64caa0ec8af1d4a229b88bb168e3d10135c1e574e34c40c264587ab4c3c169e194323ef2040be
-  data.tar.gz: d51979854aa840ff03cbbf0ee118a93aae2066f06485503f3451d31486f222e56203ac11e819e2fa42238a9914b3023a276d0e6a2dbe5010367da837fc504938
+  metadata.gz: 592576af399e8f184b1b28dfa32c298fb7ff236f9d956a37e5587e073ee64af0280e481a4a568e114b04391e0cea6977b364f4d9ac03f0a1a194a12b3a7cd19e
+  data.tar.gz: f2238b2e4c0bb06c1ad33754b86f9944845565c75ea0f36fb801fd2c1e5b1a87255a5ddbad763dc2f72b36724692cd11cd862cb05ee64827655a7ca797df408c

data/README.md

@@ -25,10 +25,10 @@ The single-file package is the simplest way to try Embulk. You can download the
 
 ### Linux & Mac & BSD
 
-Following 4 commans install embulk to your home directory:
+Following 4 commands install embulk to your home directory:
 
 ```
-curl --create-dirs -o ~/.embulk/bin/embulk -L https://bintray.com/artifact/download/embulk/maven/embulk-0.4.6.jar
+curl --create-dirs -o ~/.embulk/bin/embulk -L https://bintray.com/artifact/download/embulk/maven/embulk-0.4.7.jar
 chmod +x ~/.embulk/bin/embulk
 echo 'export PATH="$HOME/.embulk/bin:$PATH"' >> ~/.bashrc
 source ~/.bashrc

data/build.gradle

@@ -12,7 +12,7 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards")]
 
 allprojects {
     group = 'org.embulk'
-    version = '0.4.6'
+    version = '0.4.7'
 
     apply plugin: 'java'
     apply plugin: 'maven-publish'

data/embulk-core/src/main/java/org/embulk/command/PreviewPrinter.java

@@ -0,0 +1,84 @@
+package org.embulk.command;
+
+import java.util.List;
+import java.util.Locale;
+import java.io.PrintStream;
+import java.io.Closeable;
+import java.io.IOException;
+import java.text.NumberFormat;
+import org.embulk.config.ModelManager;
+import org.embulk.spi.Schema;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.Page;
+import org.embulk.spi.util.Pages;
+
+public abstract class PreviewPrinter
+        implements Closeable
+{
+    protected final PrintStream out;
+    protected final ModelManager modelManager;
+    protected final Schema schema;
+    private final String[] stringValues;
+
+    private final NumberFormat numberFormat = NumberFormat.getNumberInstance(Locale.ENGLISH);
+
+    public PreviewPrinter(PrintStream out, ModelManager modelManager, Schema schema)
+    {
+        this.out = out;
+        this.modelManager = modelManager;
+        this.schema = schema;
+        this.stringValues = new String[schema.getColumnCount()];
+    }
+
+    public void printAllPages(List<Page> pages) throws IOException
+    {
+        List<Object[]> records = Pages.toObjects(schema, pages);
+        for (Object[] record : records) {
+            printRecord(record);
+        }
+    }
+
+    public void printRecord(Object... values) throws IOException
+    {
+        int min = Math.min(schema.getColumnCount(), values.length);
+        for (int i=0; i < min; i++) {
+            stringValues[i] = valueToString(values[i]);
+        }
+        for (int i=min; i < schema.getColumnCount(); i++) {
+            stringValues[i] = valueToString(null);
+        }
+        printRecord(stringValues);
+    }
+
+    protected abstract void printRecord(String[] values) throws IOException;
+
+    protected String valueToString(Object obj)
+    {
+        if (obj == null) {
+            return "";
+        } else if (obj instanceof String) {
+            return (String) obj;
+        } else if (obj instanceof Number) {
+            if (obj instanceof Integer) {
+                return numberFormat.format(((Integer) obj).longValue());
+            }
+            if (obj instanceof Long) {
+                return numberFormat.format(((Long) obj).longValue());
+            }
+            return obj.toString();
+        } else if (obj instanceof Timestamp) {
+            return obj.toString();
+        } else {
+            return modelManager.writeObject(obj);
+        }
+    }
+
+    public void finish() throws IOException
+    { }
+
+    @Override
+    public void close() throws IOException
+    {
+        out.close();
+    }
+}

data/embulk-core/src/main/java/org/embulk/command/Runner.java

@@ -3,16 +3,15 @@ package org.embulk.command;
 import java.util.List;
 import java.util.Map;
 import java.util.HashMap;
-import java.util.Locale;
 import java.io.File;
 import java.io.IOException;
 import java.io.FileOutputStream;
 import java.io.BufferedWriter;
 import java.io.OutputStreamWriter;
 import java.io.Writer;
-import java.text.NumberFormat;
 import org.yaml.snakeyaml.Yaml;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Function;
 import com.google.inject.Injector;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.DataSource;
@@ -29,7 +28,6 @@ import org.embulk.exec.ResumeState;
 import org.embulk.exec.PartialExecutionException;
 import org.embulk.spi.time.Timestamp;
 import org.embulk.spi.ExecSession;
-import org.embulk.spi.util.Pages;
 import org.embulk.EmbulkService;
 
 public class Runner
@@ -44,6 +42,9 @@ public class Runner
 
         private String logLevel;
         public String getLogLevel() { return logLevel; }
+
+        private String previewOutputFormat;
+        public String getPreviewOutputFormat() { return previewOutputFormat; };
     }
 
     private final Options options;
@@ -218,41 +219,27 @@ public class Runner
         ExecSession exec = newExecSession(config);
         PreviewExecutor preview = injector.getInstance(PreviewExecutor.class);
         PreviewResult result = preview.preview(exec, config);
-        List<Object[]> records = Pages.toObjects(result.getSchema(), result.getPages());
-        final ModelManager model = injector.getInstance(ModelManager.class);
-
-        String[] header = new String[result.getSchema().getColumnCount()];
-        for (int i=0; i < header.length; i++) {
-            header[i] = result.getSchema().getColumnName(i) + ":" + result.getSchema().getColumnType(i);
-        }
+        ModelManager modelManager = injector.getInstance(ModelManager.class);
 
-        TablePrinter printer = new TablePrinter(System.out, header) {
-            private NumberFormat numberFormat = NumberFormat.getNumberInstance(Locale.ENGLISH);
+        PreviewPrinter printer;
 
-            protected String valueToString(Object obj)
-            {
-                if (obj instanceof String) {
-                    return (String) obj;
-                } else if (obj instanceof Number) {
-                    if (obj instanceof Integer) {
-                        return numberFormat.format(((Integer) obj).longValue());
-                    }
-                    if (obj instanceof Long) {
-                        return numberFormat.format(((Long) obj).longValue());
-                    }
-                    return obj.toString();
-                } else if (obj instanceof Timestamp) {
-                    return obj.toString();
-                } else {
-                    return model.writeObject(obj);
-                }
-            }
-        };
+        String format = options.getPreviewOutputFormat();
+        if (format == null) {
+            format = "table";
+        }
+        switch (format) {
+        case "table":
+            printer = new TablePreviewPrinter(System.out, modelManager, result.getSchema());
+            break;
+        case "vertical":
+            printer = new VerticalPreviewPrinter(System.out, modelManager, result.getSchema());
+            break;
+        default:
+            throw new IllegalArgumentException(String.format("Unknown preview output format '%s'. Supported formats: table, vertical", format));
+        }
 
         try {
-            for (Object[] record : records) {
-                printer.add(record);
-            }
+            printer.printAllPages(result.getPages());
             printer.finish();
         } catch (IOException ex) {
             throw new RuntimeException(ex);

data/embulk-core/src/main/java/org/embulk/command/TablePreviewPrinter.java

@@ -0,0 +1,108 @@
+package org.embulk.command;
+
+import java.util.List;
+import java.util.Arrays;
+import java.util.ArrayList;
+import java.io.PrintStream;
+import java.io.IOException;
+import com.google.common.base.Function;
+import org.embulk.config.ModelManager;
+import org.embulk.spi.Schema;
+
+public class TablePreviewPrinter
+        extends PreviewPrinter
+{
+    private static final int MAX_SAMPLE_SIZE = 32000;
+
+    private List<String[]> samples;
+    private int sampleSize;
+    private String format = null;
+    private String border = null;
+
+    public TablePreviewPrinter(PrintStream out, ModelManager modelManager, Schema schema)
+    {
+        super(out, modelManager, schema);
+        this.samples = new ArrayList<String[]>();
+        String[] header = new String[schema.getColumnCount()];
+        for (int i=0; i < header.length; i++) {
+            header[i] = schema.getColumnName(i) + ":" + schema.getColumnType(i);
+        }
+        samples.add(header);
+    }
+
+    @Override
+    protected void printRecord(String[] values) throws IOException
+    {
+        if (samples == null) {
+            // header is already written
+            out.format(format, Arrays.copyOf(values, values.length, Object[].class));
+        } else {
+            // estimating size of columns
+            samples.add(Arrays.copyOf(values, values.length));
+            for (String v : values) {
+                sampleSize += v.length();
+            }
+            if (sampleSize > MAX_SAMPLE_SIZE) {
+                // enough number of rows to estimate size of the columns.
+                flushSamples();
+            }
+        }
+    }
+
+    private void flushSamples()
+    {
+        StringBuilder borderBuilder = new StringBuilder();
+
+        StringBuilder sb = new StringBuilder();
+        sb.append("| ");
+        borderBuilder.append("+-");
+        for (int i=0; i < schema.getColumnCount(); i++) {
+            if (i != 0) {
+                sb.append(" | ");
+                borderBuilder.append("-+-");
+            }
+            int colLen = maxLengthInColumn(i);
+            sb.append("%"+colLen+"s");
+            for (int b=0; b < colLen; b++) {
+                borderBuilder.append("-");
+            }
+        }
+        sb.append(" |");
+        borderBuilder.append("-+");
+        sb.append("\n");
+
+        this.format = sb.toString();
+        this.border = borderBuilder.toString();
+
+        out.println(border);
+
+        for (int i=0; i < samples.size(); i++) {
+            String[] values = samples.get(i);
+            out.format(format, Arrays.copyOf(values, values.length, Object[].class));
+            if (i == 0) {
+                // i == 0 is header. write border after the header
+                out.println(border);
+            }
+        }
+
+        this.samples = null;
+    }
+
+    private int maxLengthInColumn(int i)
+    {
+        int max = 0;
+        for (String[] values : samples) {
+            max = Math.max(max, values[i].length());
+        }
+        return max;
+    }
+
+    @Override
+    public void finish() throws IOException
+    {
+        if (samples != null) {
+            flushSamples();
+        }
+        out.println(border);
+    }
+}

data/embulk-core/src/main/java/org/embulk/command/VerticalPreviewPrinter.java

@@ -0,0 +1,47 @@
+package org.embulk.command;
+
+import java.io.PrintStream;
+import java.io.IOException;
+import org.embulk.config.ModelManager;
+import org.embulk.spi.Schema;
+
+public class VerticalPreviewPrinter
+        extends PreviewPrinter
+{
+    private final String format;
+    private int count = 0;
+
+    public VerticalPreviewPrinter(PrintStream out, ModelManager modelManager, Schema schema)
+    {
+        super(out, modelManager, schema);
+        this.format = "%" + maxColumnNameLength(schema) + "s (%" + maxColumnTypeNameLength(schema)+ "s) : %s\n";
+    }
+
+    private static int maxColumnNameLength(Schema schema)
+    {
+        int max = 0;
+        for (int i=0; i < schema.getColumnCount(); i++) {
+            max = Math.max(max, schema.getColumnName(i).length());
+        }
+        return max;
+    }
+
+    private static int maxColumnTypeNameLength(Schema schema)
+    {
+        int max = 0;
+        for (int i=0; i < schema.getColumnCount(); i++) {
+            max = Math.max(max, schema.getColumnType(i).toString().length());
+        }
+        return max;
+    }
+
+    @Override
+    protected void printRecord(String[] values) throws IOException
+    {
+        count++;
+        out.format("*************************** %d ***************************\n", count);
+        for (int i=0; i < schema.getColumnCount(); i++) {
+            out.format(format, schema.getColumnName(i), schema.getColumnType(i), values[i]);
+        }
+    }
+}

data/embulk-docs/src/release.rst

@@ -17,4 +17,5 @@ Release Notes
     release/release-0.4.4
     release/release-0.4.5
     release/release-0.4.6
+    release/release-0.4.7
 

data/embulk-docs/src/release/release-0.4.7.rst

@@ -0,0 +1,16 @@
+Release 0.4.7
+==================================
+
+CLI
+------------------
+
+* Added ``-G, --vertical`` option to ``preview`` subcommand. It shows records in vertical style like MySQL's ``\G`` or PostgreSQL's ``\x`` (@yaggytter++).
+
+General Changes
+------------------
+
+* Fixed a problem on Windows where classloader raises (@hito4_t++)
+
+Release Date
+------------------
+2015-02-23

data/lib/embulk/command/embulk_run.rb

@@ -102,6 +102,9 @@ module Embulk
       op.on('-C', '--classpath PATH', "Add java classpath separated by #{classpath_separator} (CLASSPATH)") do |classpath|
         classpaths.concat classpath.split(classpath_separator)
       end
+      op.on('-G', '--vertical', "Use vertical output format", TrueClass) do |b|
+        options[:previewOutputFormat] = "vertical"
+      end
       args = 1..1
 
     when :guess

data/lib/embulk/java_plugin.rb

@@ -5,7 +5,7 @@ module Embulk
   class JavaPlugin
     def self.classloader(dir)
       jars = Dir["#{dir}/**/*.jar"]
-      urls = jars.map {|jar| java.net.URL.new "file://#{File.expand_path(jar)}" }
+      urls = jars.map {|jar| java.io.File.new(File.expand_path(jar)).toURI.toURL }
       classloader = Java::PluginClassLoader.new(JRuby.runtime, urls)
     end
 

data/lib/embulk/version.rb

@@ -1,3 +1,3 @@
 module Embulk
-  VERSION = '0.4.6'
+  VERSION = '0.4.7'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk
 version: !ruby/object:Gem::Version
-  version: 0.4.6
+  version: 0.4.7
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-02-21 00:00:00.000000000 Z
+date: 2015-02-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -102,8 +102,10 @@ files:
 - embulk-cli/src/main/sh/selfrun.sh
 - embulk-core/build.gradle
 - embulk-core/src/main/java/org/embulk/EmbulkService.java
+- embulk-core/src/main/java/org/embulk/command/PreviewPrinter.java
 - embulk-core/src/main/java/org/embulk/command/Runner.java
-- embulk-core/src/main/java/org/embulk/command/TablePrinter.java
+- embulk-core/src/main/java/org/embulk/command/TablePreviewPrinter.java
+- embulk-core/src/main/java/org/embulk/command/VerticalPreviewPrinter.java
 - embulk-core/src/main/java/org/embulk/config/CommitReport.java
 - embulk-core/src/main/java/org/embulk/config/Config.java
 - embulk-core/src/main/java/org/embulk/config/ConfigDefault.java
@@ -260,6 +262,7 @@ files:
 - embulk-docs/src/release/release-0.4.4.rst
 - embulk-docs/src/release/release-0.4.5.rst
 - embulk-docs/src/release/release-0.4.6.rst
+- embulk-docs/src/release/release-0.4.7.rst
 - embulk-standards/build.gradle
 - embulk-standards/src/main/java/org/embulk/standards/CsvFormatterPlugin.java
 - embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java
@@ -360,10 +363,8 @@ files:
 - classpath/bval-jsr303-0.5.jar
 - classpath/commons-beanutils-core-1.8.3.jar
 - classpath/commons-lang3-3.1.jar
-- classpath/embulk-core-0.4.5.jar
-- classpath/embulk-core-0.4.6.jar
-- classpath/embulk-standards-0.4.5.jar
-- classpath/embulk-standards-0.4.6.jar
+- classpath/embulk-core-0.4.7.jar
+- classpath/embulk-standards-0.4.7.jar
 - classpath/guava-18.0.jar
 - classpath/guice-3.0.jar
 - classpath/guice-multibindings-3.0.jar

data/embulk-core/src/main/java/org/embulk/command/TablePrinter.java

@@ -1,119 +0,0 @@
-package org.embulk.command;
-
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Locale;
-import java.io.PrintStream;
-import java.io.Flushable;
-import java.io.IOException;
-
-class TablePrinter
-{
-    private static final int SAMPLES = 10;
-
-    private PrintStream out;
-
-    private String[] header;
-
-    private List<Object[]> samples;
-    private String format;
-    private String border;
-
-    public TablePrinter(PrintStream out, String... header)
-    {
-        this.out = out;
-        this.header = header;
-        this.samples = new ArrayList<Object[]>(SAMPLES);
-        samples.add(header);
-    }
-
-    public void add(Object... values) throws IOException
-    {
-        int min = header.length < values.length ? header.length : values.length;
-        Object[] cols = new Object[header.length];
-        for(int i=0; i < min; i++) {
-            if(values[i] == null) {
-                cols[i] = "";
-            } else {
-                cols[i] = valueToString(values[i]);
-            }
-        }
-        for(int i=min; i < header.length; i++) {
-            cols[i] = "";
-        }
-
-        if(samples == null) {
-            out.format(format, cols);
-            return;
-        }
-
-        samples.add(cols);
-        if(samples.size() < SAMPLES) {
-            return;
-        }
-    }
-
-    protected String valueToString(Object obj)
-    {
-        return obj.toString();
-    }
-
-    private void flushSamples()
-    {
-        StringBuilder borderBuilder = new StringBuilder();
-
-        StringBuilder sb = new StringBuilder();
-        sb.append("| ");
-        borderBuilder.append("+-");
-        for(int i=0; i < header.length; i++) {
-            if(i != 0) {
-                sb.append(" | ");
-                borderBuilder.append("-+-");
-            }
-            int colLen = maxLengthInColumn(i);
-            sb.append("%"+colLen+"s");
-            for(int b=0; b < colLen; b++) {
-                borderBuilder.append("-");
-            }
-        }
-        sb.append(" |");
-        borderBuilder.append("-+");
-        sb.append("\n");
-
-        this.format = sb.toString();
-        this.border = borderBuilder.toString();
-
-        out.println(border);
-
-        for(int i=0; i < samples.size(); i++) {
-            out.format(format, samples.get(i));
-            if(i == 0) {
-                out.println(border);
-            }
-        }
-
-        this.samples = null;
-    }
-
-    private int maxLengthInColumn(int i)
-    {
-        int max = 0;
-        for(Object[] cols : samples) {
-            String s = (String) cols[i];
-            int len = (s == null) ? 0 : s.length();
-            if(max < len) {
-                max = len;
-            }
-        }
-        return max;
-    }
-
-    public void finish() throws IOException
-    {
-        if(!samples.isEmpty()) {
-            flushSamples();
-        }
-        out.println(border);
-    }
-}
-