embulk 0.2.0 → 0.2.1

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NDBmNzE4N2NiODA5MmM5NzBlMjFjMTg5ZDNkMjIxM2VmYzUzM2I5NA==
+    NGUwOTc0ZDE1MWZlZjJhYjdhNmJmMjQwZjliOWU3MmEyYmM5ZTczNQ==
   data.tar.gz: !binary |-
-    ZmRmZDUwNTU0ZmE1NjNhZDc5NzQyMzEwMGM2ODBmNjM1ZTY5ZmY3NA==
+    ZjA1YTE5NDlhZGViMTU1NjVmOTBhZDVlZDY5NGZjODI0NGU5OGViZA==
 SHA512:
   metadata.gz: !binary |-
-    Njk2ZDU4MDNkODFmZTJiNmEyNmNhNWIwOWYzZjViYWI0YmYxNmMwMjBhNDQ5
-    ODMwY2YxOGI0ZWFiNjExNTdjMGE5ZmVlMDA2YWUwYTY2YWUxZDRjYTQ0OTY5
-    NzI0MGVkODQwYWI1YjUzNzdhNmNkMmVhMGRmZjRlZWIxZmQxYzU=
+    NDY3NzQ1NTkxNTk5MzAzMGQ2ZmIzYjM0YjMyMTczOGM1YjhmYzFkYTg0YTY3
+    ZDBlNzdiYWIwZmVkMWU5YzA3NTEyYzA2ZGI3YjQyMTQ5ZDI2MWI3ZWEwZTM0
+    YjQ3MTllMTZkYzdlYTM2YjNlZDVjNGIwYjEwNDVhNjlmN2IxYTk=
   data.tar.gz: !binary |-
-    N2Q3NTEwN2NlMDg5OGQ1NmRlNmVkMzcyNzRiMjBhODY3MDExODA1ZDI5YTYy
-    ZmNjNTM3YjU5NGE4ZmM4NGFlNWE4ZmEwZWI2YTIzNjM3YmVmODE5MDc2OTQ1
-    NWMxNzM1OGUzZTExNTY0ZmM0MTBmYWRmNDJhZTc4MjhlNTZkNmQ=
+    MGZjMzM1NmVhNzdhZDhjODg3ZWZiNGRmOWQwMTU5MzUwZmEwYTBkMDY1MTgz
+    NDBhOTAwM2Y3NDNjM2VlZTE1YjRkZjA4MWNiZjZjN2QzOTBjYTliMzJlYTgw
+    OGY2ZGZmMDJmMTI4ZWU1YjNmMTMxNTc5NDdjN2NiODkxYzQ4MmI=

data/ChangeLog

@@ -1,4 +1,13 @@
 
+2015-01-29 version 0.2.1:
+
+* Fixed LineEncoder#finish to flush all remaining buffer (reported by @aibou)
+* Fixed NextConfig to be merged to in: or out: rather than the top-level
+  (reported by enukane) [#41]
+* ./bin/embulk shows warns to run `rake` if ./classpath doesn't exist
+* Embulk::PageBuilder#add accepts nil
+
+
 2015-01-26 version 0.2.0:
 
 * Changed JRuby InputPlugin API to use #run instead of .run

data/README.md

@@ -4,14 +4,24 @@ A plugin-based parallel bulk data loader that makes painful data integration wor
 
 ## What's Embulk?
 
-TODO
+Embulk is a plugin-based parallel bulk data loader that helps **data transfer** between various **storages**, **databases**, **NoSQL** and **cloud services**.
+
+You can install input and output plugins to integrate many other file formats and storages.
+
+You also can release plugins to share your efforts of data cleaning, error handling, transaction control, and retrying. 
+Packaging effrots into plugins **brings OSS-style development to the data scripts** which **was tend to be one-time adhoc scripts**.
+
+[Embuk, an open-source plugin-based parallel bulk data loader](http://www.slideshare.net/frsyuki/embuk-making-data-integration-works-relaxed) at Slideshare
+
+[![Embulk](https://gist.githubusercontent.com/frsyuki/f322a77ee2766a508ba9/raw/e8539b6b4fda1b3357e8c79d3966aa8148dbdbd3/embulk-overview.png)](http://www.slideshare.net/frsyuki/embuk-making-data-integration-works-relaxed/12)
+
 
 ## Quick Start
 
 The single-file package is the simplest way to try Embulk. You can download the latest embulk-VERSION.jar from [the releases page](https://bintray.com/embulk/maven/embulk/view#files) and run it with java:
 
 ```
-wget https://bintray.com/artifact/download/embulk/maven/embulk-0.2.0.jar -O embulk.jar
+wget https://bintray.com/artifact/download/embulk/maven/embulk-0.2.1.jar -O embulk.jar
 java -jar embulk.jar --help
 ```
 
@@ -27,14 +37,14 @@ java -jar embulk.jar run     config.yml
 ### Using plugins
 
 You can use plugins to load data from/to various systems and file formats.
-An example is [embulk-output-postgres-json]() plugin. It outputs data into PostgreSQL server using "json" column type.
+An example is [embulk-output-postgres-json](https://github.com/frsyuki/embulk-plugin-postgres-json) plugin. It outputs data into PostgreSQL server using "json" column type.
 
 ```
 java -jar embulk.jar gem install embulk-output-postgres-json
 java -jar embulk.jar gem list
 ```
 
-You can search plugins on RubyGems: [search for "embulk-"](https://rubygems.org/search?utf8=%E2%9C%93&query=embulk-).
+You can search plugins on RubyGems: [search for "embulk-plugin"](https://rubygems.org/search?utf8=%E2%9C%93&query=embulk-plugin).
 
 ### Using plugin bundle
 

data/bin/embulk

@@ -37,20 +37,16 @@ unless java_cmd
   end
 end
 
-embulk_home = ENV['EMBULK_HOME']
-unless embulk_home
-  embulk_home = File.dirname(File.dirname(__FILE__))
-end
-ENV['EMBULK_HOME'] = File.expand_path(embulk_home)
-
+embulk_home = File.dirname(File.dirname(__FILE__))
 classpath_dir = File.join(embulk_home, 'classpath')
 lib_dir = File.join(embulk_home, 'lib')
 
-jruby_complete = Dir.entries(classpath_dir).find {|jar| jar =~ /jruby-complete-[\d\.]+\.jar/ }
+jruby_complete = Dir.entries(classpath_dir).find {|jar| jar =~ /jruby-complete-[\d\.]+\.jar/ } rescue nil
 unless jruby_complete
-  STDERR.puts "Could not find jruby-complete at $EMBULK_HOME/classpath directory."
-  STDERR.puts "Please confirm EMBULK_HOME is correctly set"
-  STDERR.puts "Current EMBULK_HOME = #{ENV['EMBULK_HOME'].to_s.dump}"
+  STDERR.puts "Could not find jruby-complete at #{embulk_home}/classpath directory."
+  if embulk_home == '.'
+    STDERR.puts "Did you run \`rake\`? You need to build java code and create ./classpath directory first."
+  end
   raise SystemExit.new(1)
 end
 

data/build.gradle

@@ -23,7 +23,7 @@ allprojects {
     apply plugin: 'com.jfrog.bintray'
 
     group = 'org.embulk'
-    version = '0.2.0'
+    version = '0.2.1'
 
     // to upload artifacts to Bintray by gradle-bintray-plugin
     // $ gradle bintrayUpload
@@ -92,6 +92,7 @@ subprojects {
                 'org.slf4j:slf4j-api:1.7.9',
                 'org.slf4j:slf4j-log4j12:1.7.9',
                 'org.jruby:jruby-complete:1.7.16.1',
+                'com.google.code.findbugs:annotations:3.0.0',
                 'org.yaml:snakeyaml:1.14',
                 'javax.validation:validation-api:1.1.0.Final',
                 'org.apache.bval:bval-jsr303:0.5',

data/embulk-cli/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>org.embulk</groupId>
         <artifactId>embulk-parent</artifactId>
-        <version>0.2.0-SNAPSHOT</version>
+        <version>0.2.1-SNAPSHOT</version>
     </parent>
 
     <artifactId>embulk-cli</artifactId>

data/embulk-core/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>org.embulk</groupId>
         <artifactId>embulk-parent</artifactId>
-        <version>0.2.0-SNAPSHOT</version>
+        <version>0.2.1-SNAPSHOT</version>
     </parent>
 
     <artifactId>embulk-core</artifactId>
@@ -117,6 +117,11 @@
             <artifactId>jruby-complete</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>com.google.code.findbugs</groupId>
+            <artifactId>annotations</artifactId>
+        </dependency>
+
         <!-- for guess_charset plugin -->
         <dependency>
             <groupId>com.ibm.icu</groupId>

data/embulk-core/src/main/java/org/embulk/EmbulkService.java

@@ -23,11 +23,11 @@ public class EmbulkService
         modules.add(new ExtensionServiceLoaderModule(systemConfig));
         modules.add(new BuiltinPluginSourceModule());
         modules.add(new JRubyScriptingModule(systemConfig));
-        modules.addAll(getAdditionalModules());
+        modules.addAll(getAdditionalModules(systemConfig));
         injector = Guice.createInjector(modules.build());
     }
 
-    protected Iterable<? extends Module> getAdditionalModules()
+    protected Iterable<? extends Module> getAdditionalModules(ConfigSource systemConfig)
     {
         return ImmutableList.of();
     }

data/embulk-core/src/main/java/org/embulk/exec/LocalExecutor.java

@@ -109,7 +109,9 @@ public class LocalExecutor
             if (outputNextConfig == null) {
                 outputNextConfig = Exec.newNextConfig();
             }
-            NextConfig nextConfig = inputNextConfig.deepCopy().merge(outputNextConfig);
+            NextConfig nextConfig = Exec.newNextConfig();
+            nextConfig.getNestedOrSetEmpty("in").merge(inputNextConfig);
+            nextConfig.getNestedOrSetEmpty("out").merge(outputNextConfig);
             return new ExecuteResult(nextConfig);
         }
     }

data/embulk-core/src/main/java/org/embulk/spi/Buffer.java

@@ -2,6 +2,8 @@ package org.embulk.spi;
 
 import java.util.Arrays;
 
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+
 public class Buffer
 {
     public static final Buffer EMPTY = Buffer.allocate(0);
@@ -48,6 +50,7 @@ public class Buffer
         return new Buffer(src, offset, size).limit(size);
     }
 
+    @SuppressFBWarnings(value = "EI_EXPOSE_REP")
     public byte[] array()
     {
         return array;

data/embulk-core/src/main/java/org/embulk/spi/Extension.java

@@ -13,6 +13,7 @@ import org.embulk.config.ConfigSource;
  *
  * An example extention to add a custom PluginSource will be as following:
  *
+ * <code>
  * class MyPluginSourceExtension
  *         implements Extension, Module
  * {
@@ -22,19 +23,20 @@ import org.embulk.config.ConfigSource;
  *         // ...
  *     }
  *
- *     @Override
+ *     {@literal @}Override
  *     public void configure(Binder binder)
  *     {
- *         Multibinder<PluginSource> multibinder = Multibinder.newSetBinder(binder, PluginSource.class);
+ *         Multibinder&lt;PluginSource&gt; multibinder = Multibinder.newSetBinder(binder, PluginSource.class);
  *         multibinder.addBinding().to(MyPluginSource.class);
  *     }
  *
- *     @Override
- *     public List<Module> getModules()
+ *     {@literal @}Override
+ *     public List&lt;Module&gt; getModules()
  *     {
- *         return ImmutableList.<Module>of(this);
+ *         return ImmutableList.&lt;Module&gt;of(this);
  *     }
  * }
+ * </code>
  */
 public interface Extension
 {

data/embulk-core/src/main/java/org/embulk/spi/type/AbstractType.java

@@ -1,5 +1,7 @@
 package org.embulk.spi.type;
 
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+
 public abstract class AbstractType
         implements Type
 {
@@ -32,6 +34,7 @@ public abstract class AbstractType
         return fixedStorageSize;
     }
 
+    @SuppressFBWarnings(value = "EQ_UNUSUAL")
     @Override
     public boolean equals(Object o)
     {

data/embulk-core/src/main/java/org/embulk/spi/util/LineEncoder.java

@@ -92,8 +92,13 @@ public class LineEncoder
 
     public void finish()
     {
-        close();   // flush all remaining buffer in writer
-        outputStream.finish();
+        try {
+            writer.flush();   // flush all remaining buffer in writer because FileOutputOutputStream.close() doesn't flush buffer
+            outputStream.finish();
+            writer.close();
+        } catch (IOException ex) {
+            throw new RuntimeException(ex);
+        }
     }
 
     @Override

data/embulk-standards/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>org.embulk</groupId>
         <artifactId>embulk-parent</artifactId>
-        <version>0.2.0-SNAPSHOT</version>
+        <version>0.2.1-SNAPSHOT</version>
     </parent>
 
     <artifactId>embulk-standards</artifactId>

data/embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java

@@ -205,11 +205,6 @@ public class CsvParserPlugin
     private static String nextColumn(Schema schema, CsvTokenizer tokenizer, String nullStringOrNull)
     {
         String v = tokenizer.nextColumn();
-        if (v == null) {
-            throw new RuntimeException(String.format("Expected %d columns but line %d has fewer number of columns",
-                        schema.getColumnCount(), tokenizer.getCurrentLineNumber()));
-        }
-
         if (!v.isEmpty()) {
             if (v.equals(nullStringOrNull)) {
                 return null;

data/embulk-standards/src/main/java/org/embulk/standards/CsvTokenizer.java

@@ -5,7 +5,6 @@ import java.util.List;
 import java.util.ArrayList;
 import java.util.Deque;
 import java.util.ArrayDeque;
-import java.util.Iterator;
 import org.embulk.spi.util.LineDecoder;
 
 public class CsvTokenizer
@@ -71,7 +70,7 @@ public class CsvTokenizer
             quotedValueLines.clear();
         }
         recordState = RecordState.END;
-        return line;
+        return skippedLine;
     }
 
     public boolean nextFile()

data/lib/embulk/schema.rb

@@ -33,21 +33,25 @@ module Embulk
       record_writer_script = "lambda do |builder,record|\n"
       record_writer_script << "java_timestamp_class = ::Embulk::Java::Timestamp\n"
       each do |column|
-        column_script =
+        idx = column.index
+        column_script = "if record[#{idx}].nil?\n" <<
+          "builder.setNull(#{idx})\n" <<
+          "else\n" <<
           case column.type
           when :boolean
-            "builder.setBoolean(#{column.index}, record[#{column.index}])"
+            "builder.setBoolean(#{idx}, record[#{idx}])"
           when :long
-            "builder.setLong(#{column.index}, record[#{column.index}])"
+            "builder.setLong(#{idx}, record[#{idx}])"
           when :double
-            "builder.setDouble(#{column.index}, record[#{column.index}])"
+            "builder.setDouble(#{idx}, record[#{idx}])"
           when :string
-            "builder.setString(#{column.index}, record[#{column.index}])"
+            "builder.setString(#{idx}, record[#{idx}])"
           when :timestamp
-            "builder.setTimestamp(#{column.index}, java_timestamp_class.fromRubyTime(record[#{column.index}]))"
+            "builder.setTimestamp(#{idx}, java_timestamp_class.fromRubyTime(record[#{idx}]))"
           else
             raise "Unknown type #{column.type.inspect}"
-          end
+          end <<
+          "end\n"
         record_writer_script << column_script << "\n"
       end
       record_writer_script << "builder.addRecord\n"

data/lib/embulk/version.rb

@@ -1,3 +1,3 @@
 module Embulk
-  VERSION = "0.2.0"
+  VERSION = "0.2.1"
 end

data/pom.xml

@@ -4,7 +4,7 @@
 
     <groupId>org.embulk</groupId>
     <artifactId>embulk-parent</artifactId>
-    <version>0.2.0-SNAPSHOT</version>
+    <version>0.2.1-SNAPSHOT</version>
     <packaging>pom</packaging>
 
     <name>Embulk</name>
@@ -222,6 +222,14 @@
                 <version>1.9.5</version>
                 <scope>test</scope>
             </dependency>
+
+            <dependency>
+                <groupId>com.google.code.findbugs</groupId>
+                <artifactId>annotations</artifactId>
+                <version>3.0.0</version>
+                <scope>compile</scope>
+            </dependency>
+
         </dependencies>
     </dependencyManagement>
 
@@ -410,7 +418,7 @@
                 <plugin>
                     <groupId>org.codehaus.mojo</groupId>
                     <artifactId>findbugs-maven-plugin</artifactId>
-                    <version>2.5.2</version>
+                    <version>3.0.0</version>
                     <configuration>
                         <skip>${project.check.skip-findbugs}</skip>
                         <jvmArgs>-Xmx${project.build.jvmsize}</jvmArgs>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-27 00:00:00.000000000 Z
+date: 2015-01-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -113,6 +113,7 @@ files:
 - Rakefile
 - bin/embulk
 - build.gradle
+- classpath/annotations-3.0.0.jar
 - classpath/aopalliance-1.0.jar
 - classpath/aws-java-sdk-1.5.2.jar
 - classpath/bval-core-0.5.jar
@@ -121,9 +122,9 @@ files:
 - classpath/commons-codec-1.3.jar
 - classpath/commons-lang3-3.1.jar
 - classpath/commons-logging-1.2.jar
-- classpath/embulk-cli-0.2.0-SNAPSHOT.jar
-- classpath/embulk-core-0.2.0-SNAPSHOT.jar
-- classpath/embulk-standards-0.2.0-SNAPSHOT.jar
+- classpath/embulk-cli-0.2.1-SNAPSHOT.jar
+- classpath/embulk-core-0.2.1-SNAPSHOT.jar
+- classpath/embulk-standards-0.2.1-SNAPSHOT.jar
 - classpath/guava-17.0.jar
 - classpath/guice-3.0.jar
 - classpath/guice-multibindings-3.0.jar
@@ -305,8 +306,6 @@ files:
 - embulk-standards/src/test/java/org/embulk/standards/TestCsvTokenizer.java
 - embulk-standards/src/test/java/org/embulk/standards/TestS3FileInputPlugin.java
 - embulk.gemspec
-- examples/config.yml
-- examples/csv/sample.csv.gz
 - gradle/wrapper/gradle-wrapper.jar
 - gradle/wrapper/gradle-wrapper.properties
 - gradlew

data/examples/config.yml

@@ -1,34 +0,0 @@
-#exec:
-#  transaction_time: 2015-01-17 00:00:00 UTC
-#  transaction_timezone: UTC
-in:
-  type: file
-  paths:
-    - examples/csv/
-  #parser:
-  #  type: csv
-  #  newline: LF
-  #  columns:
-  #    - {name: date_code, type: string}
-  #    - {name: customer_code, type: long}
-  #    - {name: product_code, type: string}
-  #    - {name: employee_code, type: string}
-  #  file_decoders:
-  #    - type: gzip
-out:
-  type: file
-  directory: tmp/
-  file_name: output
-  file_ext: csv
-  #encoders:
-  #  - type: gzip
-  #    level: 9
-  formatter:
-    type: csv
-    timezone: Etc/UTC
-#in:
-#  type: file
-#  paths:
-#    - benchmark/
-#out:
-#  type: "null"