embulk 0.3.2 → 0.4.0

data/ChangeLog

@@ -1,46 +0,0 @@
-
-2015-02-04 version 0.3.2:
-
-* Fixed a problem where ruby input plugins can't use timestamp type (reported
-  by @shun0102)
-* Embulk::Page includes Enumerable to include map, each_with_index, and other
-   a lot of convenient methods (@niku++)
-* Fixed TimestampType::DEFAULT_FORMAT to use ':' as the separator of times
-
-
-2015-02-04 version 0.3.1:
-
-* Added -C, --classpath option to add java classpath
-
-
-2015-02-03 version 0.3.0:
-
-* Added resume functionality. InputPlugin and OutputPlugin needs to implement
-  resume and cleanup methods.
-* cli: embulk-run supports -r, --resume-state PATH option.
-* Added FilterInputPlugin Java API.
-* Added FilterInputPlugin JRuby API.
-* Configuration file accepts filters: array entry.
-* Added gradle-versions-plugin to build.gradle (@seratch++)
-* Fixed broken dependencies at build.gradle (@thagikura++)
-
-
-2015-01-29 version 0.2.1:
-
-* Fixed LineEncoder#finish to flush all remaining buffer (reported by @aibou)
-* Fixed NextConfig to be merged to in: or out: rather than the top-level
-  (reported by enukane) [#41]
-* ./bin/embulk shows warns to run `rake` if ./classpath doesn't exist
-* Embulk::PageBuilder#add accepts nil
-
-
-2015-01-26 version 0.2.0:
-
-* Changed JRuby InputPlugin API to use #run instead of .run
-* Fixed 'example' subcommand to quote file path for Windows environment
-
-
-2015-01-26 version 0.1.0:
-
-* The first release
-

data/embulk-cli/pom.xml

@@ -1,94 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-
-    <parent>
-        <groupId>org.embulk</groupId>
-        <artifactId>embulk-parent</artifactId>
-        <version>0.3.2-SNAPSHOT</version>
-    </parent>
-
-    <artifactId>embulk-cli</artifactId>
-    <name>embulk-cli</name>
-
-    <dependencies>
-        <dependency>
-            <groupId>org.jruby</groupId>
-            <artifactId>jruby-complete</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>org.embulk</groupId>
-            <artifactId>embulk-core</artifactId>
-        </dependency>
-
-        <!-- TODO move embulk-standards to another repository -->
-        <!-- TODO once embulk-standards is moved to another repository, replace this dependency with gem from rubygems-proxy -->
-        <!--     and change Rakefile to include embulk-core instead of embulk-cli and gemspec depends on embulk-standards -->
-        <dependency>
-            <groupId>org.embulk</groupId>
-            <artifactId>embulk-standards</artifactId>
-        </dependency>
-    </dependencies>
-
-    <repositories>
-        <!-- See https://github.com/jruby/jruby/wiki/Jruby-Scripting-container-using-Gems-with-a-Maven-Project -->
-        <repository>
-            <id>rubygems-proxy</id>
-            <name>RubyGems Proxy</name>
-            <url>http://rubygems-proxy.torquebox.org/releases</url>
-            <layout>default</layout>
-            <releases>
-                <enabled>true</enabled>
-            </releases>
-            <snapshots>
-                <enabled>false</enabled>
-                <updatePolicy>never</updatePolicy>
-            </snapshots>
-        </repository>
-    </repositories>
-
-    <build>
-        <plugins>
-            <plugin>
-                <groupId>de.saumya.mojo</groupId>
-                <artifactId>gem-maven-plugin</artifactId>
-                <version>1.0.0</version>
-                <configuration>
-                    <includeRubygemsInResources>true</includeRubygemsInResources>
-                </configuration>
-                <executions>
-                    <execution>
-                        <goals>
-                            <goal>initialize</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
-
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-shade-plugin</artifactId>
-                <executions>
-                    <execution>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>shade</goal>
-                        </goals>
-                        <configuration>
-                            <shadedArtifactAttached>true</shadedArtifactAttached>
-                            <shadedClassifierName>executable</shadedClassifierName>
-                            <transformers>
-                                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
-                                    <manifestEntries>
-                                        <Main-Class>org.embulk.cli.Main</Main-Class>
-                                    </manifestEntries>
-                                </transformer>
-                            </transformers>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-        </plugins>
-    </build>
-</project>

data/embulk-core/pom.xml

@@ -1,148 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-
-    <parent>
-        <groupId>org.embulk</groupId>
-        <artifactId>embulk-parent</artifactId>
-        <version>0.3.2-SNAPSHOT</version>
-    </parent>
-
-    <artifactId>embulk-core</artifactId>
-    <name>embulk-core</name>
-
-    <dependencies>
-        <dependency>
-            <groupId>commons-logging</groupId>
-            <artifactId>commons-logging</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>com.google.guava</groupId>
-            <artifactId>guava</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>com.google.inject</groupId>
-            <artifactId>guice</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>com.google.inject.extensions</groupId>
-            <artifactId>guice-multibindings</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>javax.inject</groupId>
-            <artifactId>javax.inject</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-databind</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-annotations</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-core</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>com.fasterxml.jackson.datatype</groupId>
-            <artifactId>jackson-datatype-guava</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>com.fasterxml.jackson.module</groupId>
-            <artifactId>jackson-module-guice</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>com.fasterxml.jackson.datatype</groupId>
-            <artifactId>jackson-datatype-joda</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-api</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-log4j12</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>log4j</groupId>
-            <artifactId>log4j</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>joda-time</groupId>
-            <artifactId>joda-time</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>org.yaml</groupId>
-            <artifactId>snakeyaml</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>javax.validation</groupId>
-            <artifactId>validation-api</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>org.apache.bval</groupId>
-            <artifactId>bval-jsr303</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>io.airlift</groupId>
-            <artifactId>slice</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>io.netty</groupId>
-            <artifactId>netty-buffer</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>org.jruby</groupId>
-            <artifactId>jruby-complete</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>com.google.code.findbugs</groupId>
-            <artifactId>annotations</artifactId>
-        </dependency>
-
-        <!-- for guess_charset plugin -->
-        <dependency>
-            <groupId>com.ibm.icu</groupId>
-            <artifactId>icu4j</artifactId>
-            <scope>runtime</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-        </dependency>
-    </dependencies>
-
-    <build>
-        <resources>
-            <resource>
-                <directory>src/main/resources</directory>
-            </resource>
-            <resource>
-                <directory>../lib</directory>
-            </resource>
-        </resources>
-    </build>
-</project>

data/embulk-core/src/main/java/org/embulk/config/NextConfig.java

@@ -1,26 +0,0 @@
-package org.embulk.config;
-
-public interface NextConfig
-        extends DataSource
-{
-    @Override
-    public NextConfig getNested(String attrName);
-
-    @Override
-    public NextConfig getNestedOrSetEmpty(String attrName);
-
-    @Override
-    public NextConfig set(String attrName, Object v);
-
-    @Override
-    public NextConfig setNested(String attrName, DataSource v);
-
-    @Override
-    public NextConfig setAll(DataSource other);
-
-    @Override
-    public NextConfig deepCopy();
-
-    @Override
-    public NextConfig merge(DataSource other);
-}

data/embulk-standards/pom.xml

@@ -1,68 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-
-    <parent>
-        <groupId>org.embulk</groupId>
-        <artifactId>embulk-parent</artifactId>
-        <version>0.3.2-SNAPSHOT</version>
-    </parent>
-
-    <artifactId>embulk-standards</artifactId>
-    <name>embulk-standards</name>
-
-    <dependencies>
-        <dependency>
-            <groupId>org.embulk</groupId>
-            <artifactId>embulk-core</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>org.embulk</groupId>
-            <artifactId>embulk-core</artifactId>
-            <type>test-jar</type>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>com.google.guava</groupId>
-            <artifactId>guava</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>com.google.inject</groupId>
-            <artifactId>guice</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>javax.validation</groupId>
-            <artifactId>validation-api</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-databind</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-api</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>com.amazonaws</groupId>
-            <artifactId>aws-java-sdk</artifactId>
-            <version>1.5.2</version>
-        </dependency>
-
-        <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>org.mockito</groupId>
-            <artifactId>mockito-core</artifactId>
-        </dependency>
-    </dependencies>
-</project>

data/embulk-standards/src/main/java/org/embulk/standards/S3FileInputPlugin.java

@@ -1,250 +0,0 @@
-package org.embulk.standards;
-
-import java.util.List;
-import java.io.IOException;
-import java.io.InputStream;
-
-import com.amazonaws.AmazonClientException;
-import com.amazonaws.AmazonServiceException;
-import com.google.common.collect.ImmutableList;
-import com.google.common.base.Optional;
-import com.fasterxml.jackson.annotation.JacksonInject;
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.BasicAWSCredentials;
-import com.amazonaws.services.s3.AmazonS3Client;
-import com.amazonaws.services.s3.model.ListObjectsRequest;
-import com.amazonaws.services.s3.model.S3ObjectSummary;
-import com.amazonaws.services.s3.model.ObjectListing;
-import com.amazonaws.services.s3.model.GetObjectRequest;
-import com.amazonaws.services.s3.model.S3Object;
-import com.amazonaws.ClientConfiguration;
-import org.embulk.config.Config;
-import org.embulk.config.Task;
-import org.embulk.config.TaskSource;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.NextConfig;
-import org.embulk.config.CommitReport;
-import org.embulk.spi.BufferAllocator;
-import org.embulk.spi.Exec;
-import org.embulk.spi.FileInputPlugin;
-import org.embulk.spi.TransactionalFileInput;
-import org.embulk.spi.util.InputStreamFileInput;
-import org.slf4j.Logger;
-
-import static org.embulk.spi.util.Inputs.formatPath;
-
-public class S3FileInputPlugin
-        implements FileInputPlugin
-{
-    public interface PluginTask
-            extends Task
-    {
-        @Config("bucket")
-        public String getBucket();
-
-        @Config("paths")
-        public List<String> getPathPrefixes();
-
-        @Config("endpoint")
-        public Optional<String> getEndpoint();
-
-        // TODO timeout, ssl, etc
-
-        @Config("access_key_id")
-        public String getAccessKeyId();
-
-        @Config("secret_access_key")
-        public String getSecretAccessKey();
-
-        // TODO support more options such as STS
-
-        public List<String> getFiles();
-        public void setFiles(List<String> files);
-
-        @JacksonInject
-        public BufferAllocator getBufferAllocator();
-    }
-
-    private final Logger log = Exec.getLogger(getClass());
-
-    @Override
-    public NextConfig transaction(ConfigSource config, FileInputPlugin.Control control)
-    {
-        PluginTask task = config.loadConfig(PluginTask.class);
-
-        // list files recursively
-        task.setFiles(listFiles(task));
-
-        // number of processors is same with number of files
-        int processorCount = task.getFiles().size();
-        return resume(task.dump(), processorCount, control);
-    }
-
-    @Override
-    public NextConfig resume(TaskSource taskSource,
-            int processorCount,
-            FileInputPlugin.Control control)
-    {
-        control.run(taskSource, processorCount);
-        return Exec.newNextConfig();
-    }
-
-    @Override
-    public void cleanup(TaskSource taskSource,
-            int processorCount,
-            List<CommitReport> successCommitReports)
-    { }
-
-    public static AWSCredentialsProvider getCredentialsProvider(PluginTask task)
-    {
-        final AWSCredentials cred = new BasicAWSCredentials(
-                task.getAccessKeyId(), task.getSecretAccessKey());
-        return new AWSCredentialsProvider() {
-            public AWSCredentials getCredentials()
-            {
-                return cred;
-            }
-
-            public void refresh()
-            {
-            }
-        };
-    }
-
-    private static AmazonS3Client newS3Client(PluginTask task)
-    {
-        AWSCredentialsProvider credentials = getCredentialsProvider(task);
-        AmazonS3Client client = newS3Client(credentials, task.getEndpoint());
-        return client;
-    }
-
-    private static AmazonS3Client newS3Client(AWSCredentialsProvider credentials,
-            Optional<String> endpoint)
-    {
-        // TODO get config from AmazonS3Task
-        ClientConfiguration clientConfig = new ClientConfiguration();
-        //clientConfig.setProtocol(Protocol.HTTP);
-        clientConfig.setMaxConnections(50); // SDK default: 50
-        clientConfig.setMaxErrorRetry(3); // SDK default: 3
-        clientConfig.setSocketTimeout(8*60*1000); // SDK default: 50*1000
-
-        AmazonS3Client client = new AmazonS3Client(credentials, clientConfig);
-
-        if (endpoint.isPresent()) {
-            client.setEndpoint(endpoint.get());
-        }
-
-        return client;
-    }
-
-    public List<String> listFiles(PluginTask task)
-    {
-        AmazonS3Client client = newS3Client(task);
-        String bucketName = task.getBucket();
-
-        ImmutableList.Builder<String> builder = ImmutableList.builder();
-        for (String prefix : task.getPathPrefixes()) {
-            String formatted = formatPath(prefix);
-            try {
-                log.info("Listing S3 files with prefix '{}'", formatted);
-                builder.addAll(listS3FilesByPrefix(client, bucketName, formatted));
-            } catch (RuntimeException e) {
-                throw new RuntimeException(String.format("Failed get a list of S3 files at '%s'", formatted), e);
-            }
-        }
-
-        return builder.build();
-    }
-
-    /**
-     * Lists S3 filenames filtered by prefix.
-     *
-     * The resulting list does not include the file that's size == 0.
-     */
-    public static List<String> listS3FilesByPrefix(AmazonS3Client client, String bucketName, String prefix)
-    {
-        // TODO implement retrying
-        ImmutableList.Builder<String> builder = ImmutableList.builder();
-
-        String lastKey = null;
-        do {
-            ListObjectsRequest req = new ListObjectsRequest(bucketName, prefix, lastKey, null, 1024);
-            ObjectListing ol = client.listObjects(req);
-            for(S3ObjectSummary s : ol.getObjectSummaries()) {
-                if (s.getSize() > 0) {
-                    builder.add(s.getKey());
-                }
-            }
-            lastKey = ol.getNextMarker();
-        } while(lastKey != null);
-
-        return builder.build();
-    }
-
-    @Override
-    public TransactionalFileInput open(TaskSource taskSource, int processorIndex)
-    {
-        PluginTask task = taskSource.loadTask(PluginTask.class);
-        return new S3FileInput(task, processorIndex);
-    }
-
-    public static class S3FileInput
-            extends InputStreamFileInput
-            implements TransactionalFileInput
-    {
-        // TODO create single-file InputStreamFileInput utility
-        private static class SingleFileProvider
-                implements InputStreamFileInput.Provider
-        {
-            private AmazonS3Client client;
-            private final String bucket;
-            private final String key;
-            private boolean opened = false;
-
-            public SingleFileProvider(PluginTask task, int processorIndex)
-            {
-                this.client = newS3Client(task);
-                this.bucket = task.getBucket();
-                this.key = task.getFiles().get(processorIndex);
-            }
-
-            @Override
-            public InputStream openNext() throws IOException
-            {
-                if (opened) {
-                    return null;
-                }
-                opened = true;
-                GetObjectRequest request = new GetObjectRequest(bucket, key);
-                //if (pos > 0) {
-                //    request.setRange(pos, contentLength);
-                //}
-                S3Object obj = client.getObject(request);
-                //if (pos <= 0) {
-                //    // first call
-                //    contentLength = obj.getObjectMetadata().getContentLength();
-                //}
-                return obj.getObjectContent();
-            }
-
-            @Override
-            public void close() { }
-        }
-
-        public S3FileInput(PluginTask task, int processorIndex)
-        {
-            super(task.getBufferAllocator(), new SingleFileProvider(task, processorIndex));
-        }
-
-        public void abort() { }
-
-        public CommitReport commit()
-        {
-            return Exec.newCommitReport();
-        }
-
-        @Override
-        public void close() { }
-    }
-}