embulk-input-filesplit 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 16ce42eb0c98f4e0f5092b4ddf29b5d77954e106
+  data.tar.gz: 961132ebfbadeb9650f480e27946cf4de4b3763a
+SHA512:
+  metadata.gz: fdc723070b206a2d2bc1c9de1445f610acc10227c74e6fc13966521391fe430eca9d9d267b6e9dfabbe03f513fc810a5cb869ed88108cf9223d9402e123f406b
+  data.tar.gz: 220a17b5af68e4f6fe8c80f1d3f1a98cb3e3fc7ee6466ff85911fbb8dd4c145d2ce60cc362b6db15467d15458eef9411595eada0272ef799b41e8e2c2cf6ab04

data/README.md

@@ -0,0 +1,40 @@
+# Splitting input file plugin for Embulk
+
+This Embulk plugin splits and inputs a text file.
+By splitting a file, input tasks will be executed in multithreads and the performance will be improved.   
+
+Lines of the text file should be separated by CR or LF or CRLF. 
+The plugin searches line separators and splits a file properly.
+
+## Overview
+
+* **Plugin type**: input
+
+## Configuration
+
+- **path**: the path of a text file (string, required)
+- **header_line**: whether the first line is a header or not (boolean, default: false)
+- **tasks**: number of tasks (integer, default: number of available processors * 2)
+
+### Example
+
+```yaml
+in:
+  type: filesplit
+  path: '/data/address.csv'
+  header_line: true
+  tasks: 4
+  parser:
+    charset: UTF-8
+    newline: CRLF
+    type: csv
+    header_line: true
+    delimiter: ','
+    ...
+```
+
+### Build
+
+```
+$ ./gradle gem
+```

data/build.gradle

@@ -0,0 +1,64 @@
+plugins {
+    id 'com.jfrog.bintray' version '1.1'
+    id 'com.github.jruby-gradle.base' version '0.1.5'
+    id 'java'
+}
+import com.github.jrubygradle.JRubyExec
+
+apply plugin: 'java'
+apply plugin: 'com.github.jruby-gradle.base'
+
+[compileJava, compileTestJava]*.options*.encoding = 'UTF-8'
+
+project.version = '0.1.1'
+
+repositories {
+    mavenCentral()
+    jcenter()
+}
+
+configurations {
+    provided
+}
+
+dependencies {
+    compile     'org.embulk:embulk-core:0.5.0'
+    provided    'org.embulk:embulk-core:0.5.0'
+    testCompile 'org.embulk:embulk-standards:0.5.0'
+    testCompile 'junit:junit:4.+'
+}
+
+task classpath(type: Copy, dependsOn: ["jar"]) {
+    doFirst { file('classpath').deleteDir() }
+    from (configurations.runtime - configurations.provided + files(jar.archivePath))
+    into 'classpath'
+}
+clean { delete 'classpath' }
+
+task gem(type: JRubyExec, dependsOn: ['build', 'gemspec', 'classpath']) {
+    jrubyArgs '-rrubygems/gem_runner', "-eGem::GemRunner.new.run(ARGV)", 'build'
+    script "build/gemspec"
+    doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
+}
+
+task gemspec << {
+    file('build').mkdirs();
+    file('build/gemspec').write($/
+Gem::Specification.new do |spec|
+  spec.name          = "${project.name}"
+  spec.version       = "${project.version}"
+  spec.authors       = ["Hitoshi Tanaka"]
+  spec.homepage      = "https://github.com/hito4t/embulk-input-filesplit"
+  spec.summary       = "Embulk plugin for splitting input file"
+  spec.licenses      = ["Apache 2.0"]
+  spec.files         = `git ls-files`.split("\n").grep(%r"^(?!\.)").grep(%r"^(?!depends/)") + Dir["classpath/*.jar"]
+  spec.test_files    = spec.files.grep(%r"^(test|spec)/")
+  spec.require_paths = ["lib"]
+end
+/$)
+}
+
+
+task gempush << {
+    "gem push pkg/embulk-input-filesplit-${project.version}.gem".execute().waitFor()
+}

data/lib/embulk/input/filesplit.rb

@@ -0,0 +1,3 @@
+Embulk::JavaPlugin.register_input(
+  :filesplit, "org.embulk.input.filesplit.LocalFileSplitInputPlugin",
+  File.expand_path('../../../../classpath', __FILE__))

data/src/main/java/org/embulk/input/filesplit/LocalFileSplitInputPlugin.java

@@ -0,0 +1,187 @@
+package org.embulk.input.filesplit;
+
+import java.io.BufferedInputStream;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.SequenceInputStream;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.embulk.config.CommitReport;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigInject;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.BufferAllocator;
+import org.embulk.spi.Exec;
+import org.embulk.spi.FileInputPlugin;
+import org.embulk.spi.TransactionalFileInput;
+import org.embulk.spi.util.InputStreamFileInput;
+
+import com.google.common.base.Optional;
+
+
+public class LocalFileSplitInputPlugin
+        implements FileInputPlugin
+{
+	public interface PluginTask
+            extends Task
+    {
+        @Config("path")
+        public String getPath();
+        
+        @Config("tasks")
+        @ConfigDefault("null")
+        public Optional<Integer> getTasks();
+        
+        @Config("header_line")
+        @ConfigDefault("false")
+        public boolean getHeaderLine();
+
+        public List<PartialFile> getFiles();
+        public void setFiles(List<PartialFile> files);
+
+        @ConfigInject
+        public BufferAllocator getBufferAllocator();
+    }
+
+    @Override
+    public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control)
+    {
+        PluginTask task = config.loadConfig(PluginTask.class);
+
+        int tasks;
+        if (task.getTasks().isPresent()) {
+        	tasks = task.getTasks().get();
+        	if (tasks <= 0) {
+        		throw new IllegalArgumentException(String.format("'tasks' is %d but must be greater than 0", tasks));
+        	}
+        } else {
+        	tasks = Runtime.getRuntime().availableProcessors() * 2;
+        }
+
+        long size = new File(task.getPath()).length();
+        List<PartialFile> files = new ArrayList<PartialFile>();
+        for (int i = 0; i < tasks; i++) {
+        	long start = size * i / tasks;
+        	long end = size * (i + 1) / tasks;
+        	if (start < end) {
+        		files.add(new PartialFile(task.getPath(), start, end));
+        	}
+        }
+        
+        task.setFiles(files);
+
+        return resume(task.dump(), task.getFiles().size(), control);
+    }
+
+    @Override
+    public ConfigDiff resume(TaskSource taskSource,
+            int taskCount,
+            FileInputPlugin.Control control)
+    {
+        control.run(taskSource, taskCount);
+
+        return Exec.newConfigDiff();
+    }
+
+    @Override
+    public void cleanup(TaskSource taskSource,
+            int taskCount,
+            List<CommitReport> successCommitReports)
+    { }
+
+    @Override
+    public TransactionalFileInput open(TaskSource taskSource, int taskIndex)
+    {
+        PluginTask task = taskSource.loadTask(PluginTask.class);
+        return new LocalFileSplitInput(task, taskIndex);
+    }
+
+    public static class LocalFileSplitInput
+            extends InputStreamFileInput
+            implements TransactionalFileInput
+    {
+        public static class FileSplitProvider
+                implements InputStreamFileInput.Provider
+        {
+            private final PartialFile file;
+            private final boolean hasHeader;
+            private boolean opened = false;
+
+            public FileSplitProvider(PartialFile file, boolean hasHeader)
+            {
+                this.file = file;
+                this.hasHeader = hasHeader;
+            }
+
+            @Override
+            public InputStream openNext() throws IOException
+            {
+                if (opened) {
+                    return null;
+                }
+                opened = true;
+                
+                InputStream in = new PartialFileInputStream(new FileInputStream(file.getPath()), file.getStart(), file.getEnd());
+                if (file.getStart() > 0 && hasHeader) {
+                	in = new SequenceInputStream(openHeader(file.getPath()), in);
+                }
+                return in;
+            }
+
+            @Override
+            public void close() { }
+            
+            private InputStream openHeader(String path) throws IOException
+            {
+            	ByteArrayOutputStream header = new ByteArrayOutputStream();
+            	try (BufferedInputStream in = new BufferedInputStream(new FileInputStream(path))) {
+            		while (true) {
+            			int c = in.read();
+            			if (c < 0) {
+            				break;
+            			}
+            			
+        				header.write(c);
+        				
+            			if (c == '\n') {
+            				break;
+            			}
+            			
+            			if (c == '\r') {
+            				int c2 = in.read();
+            				if (c2 == '\n') {
+            					header.write(c2);
+            				}
+            				break;
+            			}
+            		}
+            	}
+            	header.close();
+            	return new ByteArrayInputStream(header.toByteArray());
+            }
+        }
+
+        public LocalFileSplitInput(PluginTask task, int taskIndex)
+        {
+            super(task.getBufferAllocator(), new FileSplitProvider(task.getFiles().get(taskIndex), task.getHeaderLine()));
+        }
+
+        @Override
+        public void abort() { }
+
+        @Override
+        public CommitReport commit()
+        {
+            return Exec.newCommitReport();
+        }
+    }
+}

data/src/main/java/org/embulk/input/filesplit/PartialFile.java

@@ -0,0 +1,53 @@
+/*
+ * $Id: typical.epf 2627 2010-03-18 01:40:13Z tiba $
+ */
+package org.embulk.input.filesplit;
+
+public class PartialFile 
+{
+	private String path;
+	private long start;
+	private long end;
+
+	
+	public PartialFile(String path, long start, long end) 
+	{
+		this.path = path;
+		this.start = start;
+		this.end = end;
+	}
+
+	public PartialFile() {
+	}
+	
+	public String getPath() 
+	{
+		return path;
+	}
+
+	public void setPath(String path)
+	{
+		this.path = path;
+	}
+
+
+	public long getStart() 
+	{
+		return start;
+	}
+
+	public void setStart(long start)
+	{
+		this.start = start;
+	}
+
+	public long getEnd() 
+	{
+		return end;
+	}
+
+	public void setEnd(long end)
+	{
+		this.end = end;
+	}
+}

data/src/main/java/org/embulk/input/filesplit/PartialFileInputStream.java

@@ -0,0 +1,154 @@
+package org.embulk.input.filesplit;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.PushbackInputStream;
+
+
+public class PartialFileInputStream extends InputStream
+{
+	private final PushbackInputStream original;
+	private long start;
+	private long end;
+	private long current;
+	private boolean eof;
+	
+	public PartialFileInputStream(InputStream original, long start, long end)
+	{
+		this.original = new PushbackInputStream(new BufferedInputStream(original));
+		this.start = start;
+		this.end = end;
+		current = -1;
+	}
+	
+	@Override
+	public int read(byte[] b) throws IOException
+	{
+		return read(b, 0, b.length);
+	}
+	
+	@Override
+	public int read(byte[] b, int off, int len) throws IOException 
+	{
+		initializeIfNeeded();
+
+		if (eof) {
+			return -1;
+		}
+		
+		int read = original.read(b, off, len);
+		if (read < 0) {
+			eof = true;
+			return -1;
+		}
+		
+		current += read;
+		if (current >= end) {
+			for (int i = Math.max((int)(end - 1 - current + read), 0); i < read; i++) {
+				if (b[off + i] == '\n') {
+					eof = true;
+					return i + 1;
+				}
+				
+				if (b[off + i] == '\r') {
+					int next = (i < read ? b[off + i + 1] : prefetch());
+					if (next != '\n') {
+						eof = true;
+						return i + 1;
+					}
+				}
+			}
+		}
+		
+		return read;
+	}
+
+	@Override
+	public int read() throws IOException 
+	{
+		initializeIfNeeded();
+
+		if (eof) {
+			return -1;
+		}
+		
+		int read = original.read();
+		current++;
+		
+		if (read < 0) {
+			eof = true;
+			return -1;
+		}
+		
+		if (current >= end) {
+			if (read == '\n' || read == '\r' && prefetch() != '\n') {
+				eof = true;
+			}
+		}
+		
+		return read;
+	}
+	
+	@Override
+	public long skip(long n) throws IOException
+	{
+		throw new IOException("Skip not supported.");
+		/*
+		long skip = original.skip(n);
+		current += skip;
+		return skip;
+		*/
+	}
+	
+	@Override
+	public int available() throws IOException 
+	{
+		return 0;
+	}
+	
+	@Override
+	public void close() throws IOException 
+	{
+		original.close();
+	}
+	
+	private void initializeIfNeeded() throws IOException 
+	{
+		if (current >= start) {
+			return;
+			
+		}
+		if (start == 0) {
+			current = 0;
+		} else {
+			current = original.skip(--start);
+			if (current != start) {
+				throw new IOException("Cannot skip.");
+			}
+			
+			int c;
+			while ((c = original.read()) >= 0) {
+				start++;
+				current++;
+				
+				if (c == '\n' || c == '\r' && prefetch() != '\n') {
+					break;
+				}
+			}
+		}
+		
+		if (start >= end) {
+			eof = true;
+		}
+	}
+	
+	private int prefetch() throws IOException 
+	{
+		int c = original.read();
+		if (c >= 0) {
+			original.unread(c);
+		}
+		return c;
+	}
+}