embulk-guess-csv_verify 0.10.29-java → 0.10.30-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 560fef03cacdda19b9365be2a7352d5a00c47b415fb89cd1f7ae6664d0603091
-  data.tar.gz: 79ab9de8f15c812cd3093207329674370d37563bdc05360f2652072a15afe12c
+  metadata.gz: c980881f2e59cd43b69eebe56691108f9a9ff2436bd9d2446b2b22330bf8f88c
+  data.tar.gz: da1cf6e385c0b77d8b3d1aca9076e5ab5f2a44e81ceb357207837bcfd255de0f
 SHA512:
-  metadata.gz: ced3fb4071fa98eb20d4411272184217e9fc28e2c19e29192830d9fbcc158b67cdb87cfa7191c3fdf78794017009feef25982f60ae34bbae7cee08dc9257eafb
-  data.tar.gz: 357413c30817f29b0d2bdd1e22f093387374d4b1affff93e4ff88463f3468dfcfba4d1115da4f0f8daf48eb7f3a511ac012b0173af16a867fb1f2c63d9272bb6
+  metadata.gz: a3c223339d066fd2a4241232cdd6362e0836d051eb7a209353bbb4879c800fcc561ede18005b4c257e062119271e000f36cf3972afbce164f37fb7b6e51fad86
+  data.tar.gz: 61a444758472772ff192f932dabe6ed9fea417c413ec03203782931a0efe0d44f270686cb546870ec7bdc6a732edd89df27d97605e23a2d81838c08eb4e95511

data/lib/embulk/guess/csv_verify.rb

@@ -25,13 +25,13 @@ module Embulk
       CONFIG_MAPPER_FACTORY_CLASS = CLASSLOADER.loadClass("org.embulk.util.config.ConfigMapperFactory").ruby_class
       TYPE_MODULE_CLASS = CLASSLOADER.loadClass("org.embulk.util.config.modules.TypeModule").ruby_class
       CONFIG_MAPPER_FACTORY = CONFIG_MAPPER_FACTORY_CLASS.builder.addDefaultModules.addModule(TYPE_MODULE_CLASS.new).build
-      PLUGIN_TASK_CLASS = CLASSLOADER.loadClass("org.embulk.parser.csv.CsvParserPlugin$PluginTask")
+      LEGACY_PLUGIN_TASK_CLASS = CLASSLOADER.loadClass("org.embulk.standards.CsvParserPlugin$PluginTask")
       LIST_FILE_INPUT_CLASS = CLASSLOADER.loadClass("org.embulk.util.file.ListFileInput").ruby_class
       LINE_DECODER_CLASS = CLASSLOADER.loadClass("org.embulk.util.text.LineDecoder").ruby_class
       CSV_GUESS_PLUGIN_CLASS = CLASSLOADER.loadClass("org.embulk.guess.csv.CsvGuessPlugin").ruby_class
-      CSV_TOKENIZER_CLASS = CLASSLOADER.loadClass("org.embulk.parser.csv.CsvTokenizer").ruby_class
-      TOO_FEW_COLUMNS_EXCEPTION_CLASS = CLASSLOADER.loadClass("org.embulk.parser.csv.CsvTokenizer$TooFewColumnsException").ruby_class
-      INVALID_VALUE_EXCEPTION_CLASS = CLASSLOADER.loadClass("org.embulk.parser.csv.CsvTokenizer$InvalidValueException").ruby_class
+      LEGACY_CSV_TOKENIZER_CLASS = CLASSLOADER.loadClass("org.embulk.standards.CsvTokenizer").ruby_class
+      LEGACY_TOO_FEW_COLUMNS_EXCEPTION_CLASS = CLASSLOADER.loadClass("org.embulk.parser.csv.CsvTokenizer$TooFewColumnsException").ruby_class
+      LEGACY_INVALID_VALUE_EXCEPTION_CLASS = CLASSLOADER.loadClass("org.embulk.parser.csv.CsvTokenizer$InvalidValueException").ruby_class
 
       DELIMITER_CANDIDATES = [
         ",", "\t", "|", ";"
@@ -71,7 +71,9 @@ module Embulk
           end
           guessed_ruby_converted = config_to_java(guessed_ruby)
           if !guessed_java.equals(guessed_ruby_converted)
-            raise_and_log_guess_diff(guessed_ruby, guessed_java)
+            log_guess_diff(guessed_ruby, guessed_java, "decoders")
+            log_guess_diff(guessed_ruby, guessed_java, "parser")
+            raise "embulk-guess-csv has difference between Java/Ruby."
           end
         rescue Exception => e
           # Any error from the Java-based guess plugin should pass-through just with logging.
@@ -237,34 +239,23 @@ module Embulk
 
       private
 
-      def raise_and_log_guess_diff(guessed_ruby_entire, guessed_java_entire)
-        guessed_ruby = guessed_ruby_entire["parser"] || {}
-        guessed_java = guessed_java_entire.getNestedOrGetEmpty("parser")
-
-        require 'set'
-        keys = Set.new(guessed_ruby.keys) + Set.new(guessed_java.getAttributeNames)
+      def log_guess_diff(guessed_ruby_entire, guessed_java_entire, key)
+        guessed_ruby = guessed_ruby_entire[key] || {}
+        guessed_java = guessed_java_entire.getNestedOrGetEmpty(key)
 
         begin
           require 'json'
         rescue LoadError
-          Embulk.logger.warn "The 'json' gem is not installed. No details compared."
-          guessed_java_hash = nil
+          raise "The 'json' gem is not installed. No details compared."
         else
           guessed_java_hash = JSON.parse(guessed_java.toJson)
         end
 
-        diffs = []
-        keys.each do |key|
-          if !guessed_ruby.has_key?(key)
-            diffs << "Only embulk-guess-csv (Java) has: \"#{key}\""
-          elsif !guessed_java.has(key.to_java)
-            diffs << "Only embulk-guess-csv (Ruby) has: \"#{key}\""
-          elsif guessed_java_hash && guessed_ruby[key] != guessed_java_hash[key]
-            diffs << "embulk-guess-csv has difference between Java/Ruby: \"#{key}\""
-          end
+        if guessed_java_hash && guessed_ruby != guessed_java_hash
+          Embulk.logger.error "[Embulk CSV guess verify] '#{key}' has difference."
+          Embulk.logger.error "[Embulk CSV guess verify] Java => #{guessed_java_hash.to_json}"
+          Embulk.logger.error "[Embulk CSV guess verify] Ruby => #{guessed_ruby.to_json}"
         end
-
-        raise "embulk-guess-csv has difference between Java/Ruby: #{diffs.inspect}"
       end
 
       def config_to_java(config_ruby)
@@ -289,12 +280,11 @@ module Embulk
       def split_lines(parser_config, skip_empty_lines, sample_lines, delim, extra_config)
         null_string = parser_config["null_string"]
         config = parser_config.merge(extra_config).merge({"charset" => "UTF-8", "columns" => []})
-        parser_task = CONFIG_MAPPER_FACTORY.createConfigMapper.map(config_to_java(parser_config), PLUGIN_TASK_CLASS)
+        parser_task = config.load_config(LEGACY_PLUGIN_TASK_CLASS)
         data = sample_lines.map {|line| line.force_encoding('UTF-8') }.join(parser_task.getNewline.getString.encode('UTF-8'))
         sample = Buffer.from_ruby_string(data)
-        decoder = LINE_DECODER_CLASS.of(
-          LIST_FILE_INPUT_CLASS.new([[sample.to_java]]), parser_task.getCharset, parser_task.getLineDelimiterRecognized.orElse(nil))
-        tokenizer = CSV_TOKENIZER_CLASS.new(decoder, parser_task)
+        decoder = Java::LineDecoder.new(Java::ListFileInput.new([[sample.to_java]]), parser_task)
+        tokenizer = LEGACY_CSV_TOKENIZER_CLASS.new(decoder, parser_task)
         rows = []
         while tokenizer.nextFile
           while tokenizer.nextRecord(skip_empty_lines)
@@ -308,12 +298,12 @@ module Embulk
                     column = nil
                   end
                   columns << column
-                rescue TOO_FEW_COLUMNS_EXCEPTION_CLASS
+                rescue LEGACY_TOO_FEW_COLUMNS_EXCEPTION_CLASS
                   rows << columns
                   break
                 end
               end
-            rescue INVALID_VALUE_EXCEPTION_CLASS
+            rescue LEGACY_INVALID_VALUE_EXCEPTION_CLASS
               # TODO warning
               tokenizer.skipCurrentLine
             end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-guess-csv_verify
 version: !ruby/object:Gem::Version
-  version: 0.10.29
+  version: 0.10.30
 platform: java
 authors:
 - Sadayuki Furuhashi
@@ -10,19 +10,29 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-04-07 00:00:00.000000000 Z
+date: 2021-04-15 00:00:00.000000000 Z
 dependencies: []
 description: Verification-purpose Embulk CSV guess plugin to compare the old Ruby-based
-  one and the new Java-based one (not for your production use)
+  one and the new Java-based one (not for your production use; note that 'decoders'
+  and 'parser' sections in your configuration can be logged even if they contain confidential
+  information)
 email:
 - dmikurube@treasure-data.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- classpath/embulk-guess-csv-0.10.29.jar
-- classpath/embulk-guess-csv_verify-0.10.29.jar
-- classpath/embulk-parser-csv-0.10.29.jar
+- classpath/aopalliance-1.0.jar
+- classpath/bval-core-0.5.jar
+- classpath/bval-jsr303-0.5.jar
+- classpath/commons-beanutils-core-1.8.3.jar
+- classpath/commons-lang3-3.4.jar
+- classpath/embulk-api-0.10.30.jar
+- classpath/embulk-core-0.10.30.jar
+- classpath/embulk-guess-csv-0.10.30.jar
+- classpath/embulk-guess-csv_verify-0.10.30.jar
+- classpath/embulk-parser-csv-0.10.30.jar
+- classpath/embulk-spi-0.10.30.jar
 - classpath/embulk-util-config-0.2.1.jar
 - classpath/embulk-util-file-0.1.3.jar
 - classpath/embulk-util-guess-0.1.1.jar
@@ -30,11 +40,19 @@ files:
 - classpath/embulk-util-rubytime-0.3.2.jar
 - classpath/embulk-util-text-0.1.0.jar
 - classpath/embulk-util-timestamp-0.2.1.jar
+- classpath/guava-18.0.jar
+- classpath/guice-4.0.jar
+- classpath/guice-multibindings-4.0.jar
 - classpath/icu4j-54.1.1.jar
 - classpath/jackson-annotations-2.6.7.jar
 - classpath/jackson-core-2.6.7.jar
 - classpath/jackson-databind-2.6.7.jar
+- classpath/jackson-datatype-guava-2.6.7.jar
 - classpath/jackson-datatype-jdk8-2.6.7.jar
+- classpath/jackson-module-guice-2.6.7.jar
+- classpath/javax.inject-1.jar
+- classpath/msgpack-core-0.8.11.jar
+- classpath/slf4j-api-1.7.30.jar
 - classpath/validation-api-1.1.0.Final.jar
 - lib/embulk/guess/csv_verify.rb
 homepage: https://github.com/embulk/embulk