embulk 0.8.18-java → 0.8.19-java

data/embulk-docs/src/release.rst

@@ -4,6 +4,7 @@ Release Notes
 .. toctree::
     :maxdepth: 1
 
+    release/release-0.8.19
     release/release-0.8.18
     release/release-0.8.17
     release/release-0.8.16

data/embulk-docs/src/release/release-0.4.0.rst

@@ -2,7 +2,7 @@ Release 0.4.0
 ==================================
 
 Plugin Template Generator
-------------------
+-------------------------
 
 A new CLI subcommand ``new`` generate a plugin template.
 

data/embulk-docs/src/release/release-0.5.0.rst

@@ -2,7 +2,7 @@ Release 0.5.0
 ==================================
 
 New Guess Plugin Architecture
-------------------
+-----------------------------
 
 Embulk v0.5.0 supports dynamically loadable guess plugins.
 

data/embulk-docs/src/release/release-0.6.0.rst

@@ -2,7 +2,7 @@ Release 0.6.0
 ==================================
 
 Executor Plugin Mechanism
-------------------
+-------------------------
 
 Now executor of Embulk is fully extensible using plugins. Executor plugins get input, filter and output plugins from the Embulk framework and runs them using multiple threads, processes, or servers. While input, filter and output plugins are response for data processing, executor plugins are responsible for scheduling the processing tasks and managing parallelism for performance.
 

data/embulk-docs/src/release/release-0.6.20.rst

@@ -2,7 +2,7 @@ Release 0.6.20
 ==================================
 
 Command line interface
-------------------
+----------------------
 
 * Added ``-X key=value`` argument to set system config. This argument is intended to overwrite performance parameters such as number of threads (``max_threads``) or page buffer size (``page_size``).
 

data/embulk-docs/src/release/release-0.8.19.rst

@@ -0,0 +1,43 @@
+Release 0.8.19
+==================================
+
+General Changes
+------------------
+
+* Fixed LocalFileInputPlugin to follow link if path prefix is symlink. (@hiroyuki-sato) [#585]
+
+* Fixed CsvParserPlugin to refresh the number of skipped header lines for each file. [#567]
+
+* Fixed embulk/guess/csv.rb to finish immidiately if CSV parser could not parse sample_lines. [#556]
+
+* Fixed PageBuilder to avoid saving unexpected string values saving to stringReferences. [#598]
+
+* Fixed PreviewExecutor to avoid to call SamplingPageOutput#finish twice. [#571]
+
+* Fixed TempFileSpace to change temporary filename format for Windows safe. (@hiroyuki-sato) [#589]
+
+* Added 'preview_sample_buffer_bytes' option for 'preview' command to make sampling buffer bytes configurable. [#572]
+
+* Added 'guess_sample_buffer_bytes' option for 'guess' command to make sampling buffer bytes configurable. [#594]
+
+* Updated snakeyaml from 1.14 to 1.18. (@hiroyuki-sato) [#575]
+
+* Updated liquid from 3.0.6 to 4.0.0. (@hiroyuki-sato)  [#587]
+
+* Minor fix:
+
+  * Removed lib/embulk/command/embulk_generate_bin.rb that is not called anywhere. [#578]
+
+  * Deprecated lib/embulk/version.rb 'Embulk::VERSION', and used the jar manifest (META-INF/MANIFEST.MF) to provide the Embulk version at runtime [#596, #597]
+
+  * Rewrote the "example" subcommand in Java. [#558]
+
+  * Rewrote the "selfupdate" subcommand in Java. [#563]
+
+  * Rewrote the "migrate" subcommand in Java. [#568]
+
+  * Rewrote the "new" subcommand in Java. [#569]
+
+Release Date
+------------------
+2017-05-12

data/embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java

@@ -236,12 +236,12 @@ public class CsvParserPlugin
         final boolean allowOptionalColumns = task.getAllowOptionalColumns();
         final boolean allowExtraColumns = task.getAllowExtraColumns();
         final boolean stopOnInvalidRecord = task.getStopOnInvalidRecord();
-        int skipHeaderLines = task.getSkipHeaderLines();
+        final int skipHeaderLines = task.getSkipHeaderLines();
 
         try (final PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), schema, output)) {
             while (tokenizer.nextFile()) {
                 // skip the header lines for each file
-                for (; skipHeaderLines > 0; skipHeaderLines--) {
+                for (int skipHeaderLineNumber = skipHeaderLines; skipHeaderLineNumber > 0; skipHeaderLineNumber--) {
                     if (!tokenizer.skipHeaderLine()) {
                         break;
                     }

data/embulk-standards/src/main/java/org/embulk/standards/LocalFileInputPlugin.java

@@ -30,6 +30,10 @@ import org.embulk.spi.TransactionalFileInput;
 import org.embulk.spi.util.InputStreamTransactionalFileInput;
 import org.slf4j.Logger;
 
+import java.nio.file.FileVisitOption;
+import java.util.EnumSet;
+import java.util.Set;
+
 public class LocalFileInputPlugin
         implements FileInputPlugin
 {
@@ -43,6 +47,10 @@ public class LocalFileInputPlugin
         @ConfigDefault("null")
         Optional<String> getLastPath();
 
+        @Config("follow_symlinks")
+        @ConfigDefault("false")
+        boolean getFollowSymlinks();
+
         List<String> getFiles();
         void setFiles(List<String> files);
 
@@ -120,7 +128,17 @@ public class LocalFileInputPlugin
         final String lastPath = task.getLastPath().orNull();
         try {
             log.info("Listing local files at directory '{}' filtering filename by prefix '{}'", directory.equals(CURRENT_DIR) ? "." : directory.toString(), fileNamePrefix);
-            Files.walkFileTree(directory, new SimpleFileVisitor<Path>() {
+
+            int maxDepth = Integer.MAX_VALUE;
+            Set<FileVisitOption> opts;
+            if (task.getFollowSymlinks()) {
+                opts = EnumSet.of(FileVisitOption.FOLLOW_LINKS);
+            } else {
+                opts = EnumSet.noneOf(FileVisitOption.class);
+                log.info("\"follow_symlinks\" is set false. Note that symbolic links to directories are skipped.");
+            }
+
+            Files.walkFileTree(directory, opts, maxDepth, new SimpleFileVisitor<Path>() {
                 @Override
                 public FileVisitResult preVisitDirectory(Path path, BasicFileAttributes attrs)
                 {
@@ -148,6 +166,17 @@ public class LocalFileInputPlugin
                 @Override
                 public FileVisitResult visitFile(Path path, BasicFileAttributes attrs)
                 {
+                    try {
+                        // Avoid directories from listing.
+                        // Directories are normally unvisited with |FileVisitor#visitFile|, but symbolic links to
+                        // directories are visited like files unless |FOLLOW_LINKS| is set in |Files#walkFileTree|.
+                        // Symbolic links to directories are explicitly skipped here by checking with |Path#toReadlPath|.
+                        if (Files.isDirectory(path.toRealPath())) {
+                            return FileVisitResult.CONTINUE;
+                        }
+                    } catch (IOException ex){
+                        throw new RuntimeException("Can't resolve symbolic link", ex);
+                    }
                     if (lastPath != null && path.toString().compareTo(lastPath) <= 0) {
                         return FileVisitResult.CONTINUE;
                     } else {

data/embulk-standards/src/test/java/org/embulk/standards/guess/TestCsvGuessPlugin.java

@@ -219,6 +219,16 @@ public class TestCsvGuessPlugin
                 "test_backslash_escape_guessed.yml");
     }
 
+    @Test
+    public void skipSuggestIfEmptySampleRecords()
+            throws Exception
+    {
+        // This test checks that the CSV guess doesn't suggest anything by invalid formatted CSV file.
+        assertGuessByResource(embulk,
+                "test_skip_suggest_if_empty_sample_records_seed.yml", "test_skip_suggest_if_empty_sample_records.csv",
+                "test_skip_suggest_if_empty_sample_records_guessed.yml");
+    }
+
     static void assertGuessByResource(TestingEmbulk embulk, String seedYamlResourceName, String sourceCsvResourceName,
             String resultResourceName)
             throws IOException

data/embulk-standards/src/test/java/org/embulk/standards/preview/TestFilePreview.java

@@ -0,0 +1,73 @@
+package org.embulk.standards.preview;
+
+import org.embulk.config.ConfigSource;
+import org.embulk.exec.PreviewResult;
+import org.embulk.test.TestingEmbulk;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.file.Path;
+
+import static org.embulk.test.EmbulkTests.copyResource;
+import static org.embulk.test.EmbulkTests.readFile;
+import static org.embulk.test.EmbulkTests.readResource;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+
+public class TestFilePreview
+{
+    private static final String RESOURCE_NAME_PREFIX = "org/embulk/standards/preview/file/test/";
+
+    @Rule
+    public TestingEmbulk embulk = TestingEmbulk.builder()
+            .build();
+
+    @Test
+    public void testSimple()
+            throws Exception
+    {
+        assertPreviewedRecords(embulk, "test_simple_load.yml", "test_simple.csv", "test_simple_previewed.csv");
+    }
+
+    @Test
+    public void changePreviewSampleBufferBytes()
+            throws Exception
+    {
+        assertPreviewedRecords(embulk, "test_sample_buffer_bytes_load.yml", "test_sample_buffer_bytes_exec.yml",
+                "test_sample_buffer_bytes.csv", "test_sample_buffer_bytes_previewed.csv");
+    }
+
+    private static void assertPreviewedRecords(TestingEmbulk embulk,
+            String loadYamlResourceName, String sourceCsvResourceName, String resultCsvResourceName)
+            throws IOException
+    {
+        assertPreviewedRecords(embulk, loadYamlResourceName, null, sourceCsvResourceName, resultCsvResourceName);
+    }
+
+    private static void assertPreviewedRecords(TestingEmbulk embulk,
+            String loadYamlResourceName, String execYamlResourceName, String sourceCsvResourceName, String resultCsvResourceName)
+            throws IOException
+    {
+        Path inputPath = embulk.createTempFile("csv");
+        Path outputPath = embulk.createTempFile("csv");
+
+        // in: config
+        copyResource(RESOURCE_NAME_PREFIX + sourceCsvResourceName, inputPath);
+        ConfigSource load = embulk.loadYamlResource(RESOURCE_NAME_PREFIX + loadYamlResourceName)
+                .set("path_prefix", inputPath.toAbsolutePath().toString());
+
+        // exec: config
+        final TestingEmbulk.InputBuilder builder = embulk.inputBuilder();
+        if (execYamlResourceName != null) {
+            final ConfigSource execConfig = embulk.loadYamlResource(RESOURCE_NAME_PREFIX + execYamlResourceName);
+            builder.exec(execConfig);
+        }
+
+        // execute preview
+        final PreviewResult result = builder.in(load).outputPath(outputPath).preview();
+
+        assertThat(readFile(outputPath), is(readResource(RESOURCE_NAME_PREFIX + resultCsvResourceName)));
+    }
+}
+

data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_skip_suggest_if_empty_sample_records.csv

@@ -0,0 +1,5 @@
+id,comment
+1,"{"account":"embulk"}"
+2,"{"account":"embulk jruby"}"
+3,"{"account":"Embulk ""csv"" parser plugin"}"
+4,"{"account":null}"

data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_skip_suggest_if_empty_sample_records_guessed.yml

@@ -0,0 +1,2 @@
+charset: UTF-8
+newline: LF

data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_skip_suggest_if_empty_sample_records_seed.yml

@@ -0,0 +1 @@
+{}

data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes.csv

@@ -0,0 +1,5 @@
+id,account,time,purchase,comment
+1,32864,2015-01-27 19:23:49,20150127,embulk
+2,14824,2015-01-27 19:01:23,20150127,embulk jruby
+3,27559,2015-01-28 02:20:02,20150128,"Embulk ""csv"" parser plugin"
+4,11270,2015-01-29 11:54:36,20150129,NULL

data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes_exec.yml

@@ -0,0 +1 @@
+preview_sample_buffer_bytes: 96

data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes_load.yml

@@ -0,0 +1,19 @@
+type: file
+parser:
+  charset: UTF-8
+  newline: LF
+  type: csv
+  delimiter: ','
+  quote: '"'
+  escape: '"'
+  null_string: "NULL"
+  trim_if_not_quoted: false
+  skip_header_lines: 1
+  allow_extra_columns: false
+  allow_optional_columns: false
+  columns:
+  - {name: id, type: long}
+  - {name: account, type: long}
+  - {name: time, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
+  - {name: purchase, type: timestamp, format: '%Y%m%d'}
+  - {name: comment, type: string}

data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes_previewed.csv

@@ -0,0 +1 @@
+1,32864,2015-01-27 19:23:49.000000 +0000,2015-01-27 00:00:00.000000 +0000,embulk

data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_simple.csv

@@ -0,0 +1,5 @@
+id,account,time,purchase,comment
+1,32864,2015-01-27 19:23:49,20150127,embulk
+2,14824,2015-01-27 19:01:23,20150127,embulk jruby
+3,27559,2015-01-28 02:20:02,20150128,"Embulk ""csv"" parser plugin"
+4,11270,2015-01-29 11:54:36,20150129,NULL

data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_simple_load.yml

@@ -0,0 +1,19 @@
+type: file
+parser:
+  charset: UTF-8
+  newline: LF
+  type: csv
+  delimiter: ','
+  quote: '"'
+  escape: '"'
+  null_string: "NULL"
+  trim_if_not_quoted: false
+  skip_header_lines: 1
+  allow_extra_columns: false
+  allow_optional_columns: false
+  columns:
+  - {name: id, type: long}
+  - {name: account, type: long}
+  - {name: time, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
+  - {name: purchase, type: timestamp, format: '%Y%m%d'}
+  - {name: comment, type: string}

data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_simple_previewed.csv

@@ -0,0 +1,4 @@
+1,32864,2015-01-27 19:23:49.000000 +0000,2015-01-27 00:00:00.000000 +0000,embulk
+2,14824,2015-01-27 19:01:23.000000 +0000,2015-01-27 00:00:00.000000 +0000,embulk jruby
+3,27559,2015-01-28 02:20:02.000000 +0000,2015-01-28 00:00:00.000000 +0000,"Embulk ""csv"" parser plugin"
+4,11270,2015-01-29 11:54:36.000000 +0000,2015-01-29 00:00:00.000000 +0000,

data/embulk-test/src/main/java/org/embulk/test/PreviewResultInputPlugin.java

@@ -0,0 +1,65 @@
+package org.embulk.test;
+
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
+import org.embulk.config.TaskReport;
+import org.embulk.config.TaskSource;
+import org.embulk.exec.PreviewResult;
+import org.embulk.spi.Exec;
+import org.embulk.spi.InputPlugin;
+import org.embulk.spi.Page;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.Schema;
+
+import java.util.List;
+
+import static com.google.common.base.Preconditions.checkState;
+
+/**
+ * This plugin is used for TestingEmbulk.InputBuilder.preview().
+ */
+public final class PreviewResultInputPlugin
+        implements InputPlugin
+{
+    private static PreviewResult previewResult;
+
+    public static void setPreviewResult(PreviewResult result)
+    {
+        previewResult = result;
+    }
+
+    @Override
+    public ConfigDiff transaction(ConfigSource config, Control control)
+    {
+        checkState(previewResult != null, "PreviewResult object must be set");
+        return resume(config.loadConfig(Task.class).dump(), previewResult.getSchema(), 1, control);
+    }
+
+    @Override
+    public ConfigDiff resume(TaskSource taskSource, Schema schema, int taskCount, Control control)
+    {
+        control.run(taskSource, schema, taskCount);
+        return Exec.newConfigDiff();
+    }
+
+    @Override
+    public void cleanup(TaskSource taskSource, Schema schema, int taskCount, List<TaskReport> successTaskReports)
+    {
+    }
+
+    @Override
+    public TaskReport run(TaskSource taskSource, Schema schema, int taskIndex, PageOutput output)
+    {
+        for (Page page : previewResult.getPages()) {
+            output.add(page);
+        }
+        return Exec.newTaskReport();
+    }
+
+    @Override
+    public ConfigDiff guess(ConfigSource config)
+    {
+        return Exec.newConfigDiff();
+    }
+}

data/embulk-test/src/main/java/org/embulk/test/TestingBulkLoader.java

@@ -9,6 +9,7 @@ import com.google.inject.Injector;
 import com.google.inject.Module;
 import com.google.inject.util.Modules;
 import java.util.List;
+
 import org.embulk.config.ConfigSource;
 import org.embulk.config.TaskReport;
 import org.embulk.exec.BulkLoader;
@@ -17,9 +18,12 @@ import org.embulk.exec.ForSystemConfig;
 import org.embulk.exec.ResumeState;
 import org.embulk.spi.Exec;
 import org.embulk.spi.ExecSession;
+import org.embulk.spi.InputPlugin;
 import org.embulk.spi.Schema;
 import org.slf4j.Logger;
 
+import static org.embulk.plugin.InjectedPluginSource.registerPluginTo;
+
 class TestingBulkLoader
         extends BulkLoader
 {
@@ -33,6 +37,7 @@ class TestingBulkLoader
                     public void configure(Binder binder)
                     {
                         binder.bind(BulkLoader.class).to(TestingBulkLoader.class);
+                        registerPluginTo(binder, InputPlugin.class, "preview_result", PreviewResultInputPlugin.class);
                     }
                 };
                 return ImmutableList.of(Modules.override(modules).with(ImmutableList.of(override)));

data/embulk-test/src/main/java/org/embulk/test/TestingEmbulk.java

@@ -25,7 +25,11 @@ import org.embulk.config.ConfigLoader;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.ModelManager;
 import org.embulk.config.TaskReport;
+import org.embulk.exec.PreviewResult;
+import org.embulk.plugin.PluginClassLoader;
 import org.embulk.spi.ColumnConfig;
+import org.embulk.spi.FileOutputRunner;
+import org.embulk.spi.Page;
 import org.embulk.spi.Schema;
 import org.embulk.spi.SchemaConfig;
 import org.embulk.spi.TempFileException;
@@ -241,6 +245,47 @@ public class TestingEmbulk
             return embed.guess(config).getNested("in");
         }
 
+        /**
+         * This method returns PreviewResult.
+         *
+         * @return PreviewResult returns the result by PreviewExecutor
+         * @throws IOException
+         */
+        public PreviewResult preview()
+                throws IOException
+        {
+            checkState(inConfig != null, "inputPath must be set");
+            checkState(outputPath != null, "outputPath must be set");
+
+            // Execute preview to get PreviewResult
+            ConfigSource previewConfig = newConfig()
+                    .set("exec", execConfig.set("min_output_tasks", 1)) // exec: config
+                    .set("in", inConfig)
+                    .set("filters", filtersConfig);
+            PreviewResult result = embed.preview(previewConfig);
+            PreviewResultInputPlugin.setPreviewResult(result);
+
+            String fileName = outputPath.getFileName().toString();
+            checkArgument(fileName.endsWith(".csv"), "outputPath must end with .csv");
+            Path dir = outputPath.getParent().resolve(fileName.substring(0, fileName.length() - 4));
+            Files.createDirectories(dir);
+
+            // Execute run to write PreviewResult's Page objects to output files
+            ConfigSource runConfig = newConfig()
+                    .set("in", newConfig().set("type", "preview_result")) // in: config
+                    .set("out", newConfig() // out: config
+                            .set("type", "file")
+                            .set("path_prefix", dir.resolve("fragments_").toString())
+                            .set("file_ext", "csv")
+                            .set("formatter", newConfig()
+                                    .set("type", "csv")
+                                    .set("header_line", false)
+                                    .set("newline", "LF")));
+            embed.run(runConfig);
+
+            return buildPreviewResultWithOutput(result, dir, outputPath);
+        }
+
         public RunResult run()
                 throws IOException
         {
@@ -514,8 +559,22 @@ public class TestingEmbulk
         }
     }
 
+    private PreviewResult buildPreviewResultWithOutput(PreviewResult result, Path outputDir, Path outputPath)
+            throws IOException
+    {
+        copyToPath(outputDir, outputPath);
+        return result;
+    }
+
     private RunResult buildRunResultWithOutput(RunResult result, Path outputDir, Path outputPath)
             throws IOException
+    {
+        copyToPath(outputDir, outputPath);
+        return result;
+    }
+
+    private void copyToPath(Path outputDir, Path outputPath)
+            throws IOException
     {
         try (OutputStream out = Files.newOutputStream(outputPath)) {
             List<Path> fragments = new ArrayList<Path>();
@@ -531,8 +590,6 @@ public class TestingEmbulk
                 }
             }
         }
-
-        return result;
     }
 
     public InputBuilder inputBuilder()