embulk-parser-xml2 0.1.0

data/src/test/java/org/embulk/parser/xml2/TestXml2ParserPlugin.java

@@ -0,0 +1,240 @@
+/**
+ * The MIT License (MIT)
+ *
+ * Copyright (C) 2016 Yahoo Japan Corporation. All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+package org.embulk.parser.xml2;
+
+import static org.junit.Assert.*;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskReport;
+import org.embulk.config.TaskSource;
+import org.embulk.parser.xml2.Xml2ParserPlugin.PluginTask;
+import org.embulk.spi.Column;
+import org.embulk.spi.ColumnVisitor;
+import org.embulk.spi.Exec;
+import org.embulk.spi.Page;
+import org.embulk.spi.PageReader;
+import org.embulk.spi.ParserPlugin;
+import org.embulk.spi.Schema;
+import org.embulk.spi.TransactionalPageOutput;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.util.InputStreamFileInput;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+
+public class TestXml2ParserPlugin {
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+    private Xml2ParserPlugin plugin;
+
+    private static String PATH_PREFIX;
+
+    @BeforeClass
+    public static void initializeConstant() {
+        PATH_PREFIX = Xml2ParserPlugin.class.getClassLoader().getResource("sample_01.xml").getPath();
+    }
+
+    @Before
+    public void createResource() {
+        plugin = new Xml2ParserPlugin();
+    }
+
+    @Test
+    public void testTransaction() {
+        ConfigSource config = config();
+        plugin.transaction(config, new ParserPlugin.Control() {
+            @Override
+            public void run(TaskSource taskSource, Schema schema) {
+            }
+        });
+    }
+
+    @Test
+    public void testFile() throws FileNotFoundException {
+        
+        ConfigSource config = config();
+        final Schema schema = config.loadConfig(Xml2ParserPlugin.PluginTask.class).getSchema().toSchema();
+        PluginTask task = config.loadConfig(PluginTask.class);
+        plugin.transaction(config, new ParserPlugin.Control() {
+            @Override
+            public void run(TaskSource taskSource, Schema schema) {
+            }
+        });
+        
+        // the list contains result.
+        final List<Map<String,Object>> resultList = new LinkedList<Map<String,Object>>();
+        
+        plugin.run(task.dump(), schema,
+                new InputStreamFileInput(Exec.getBufferAllocator(), new FileInputStream(new File(PATH_PREFIX))),
+                new TransactionalPageOutput() {
+
+                    private final PageReader reader = new PageReader(schema);
+
+                    @Override
+                    public void add(Page page) {
+                        reader.setPage(page);
+                        
+                        while (reader.nextRecord()) {
+                            final Map<String, Object> record = new HashMap<String, Object>();
+                            for (Column column : schema.getColumns()) {
+                                column.visit(new ColumnVisitor() {
+                                    @Override
+                                    public void timestampColumn(Column column) {
+                                        if (reader.isNull(column)) {
+                                            record.put(column.getName(), null);
+                                            return;
+                                        }
+                                        record.put(column.getName(), reader.getTimestamp(column));
+                                    }
+
+                                    @Override
+                                    public void stringColumn(Column column) {
+                                        if (reader.isNull(column)) {
+                                            record.put(column.getName(), null);
+                                            return;
+                                        }
+                                        record.put(column.getName(), reader.getString(column));
+                                    }
+
+                                    @Override
+                                    public void longColumn(Column column) {
+                                        if (reader.isNull(column)) {
+                                            record.put(column.getName(), null);
+                                            return;
+                                        }
+                                        record.put(column.getName(), reader.getLong(column));
+                                    }
+
+                                    @Override
+                                    public void doubleColumn(Column column) {
+                                        if (reader.isNull(column)) {
+                                            record.put(column.getName(), null);
+                                            return;
+                                        }
+                                        record.put(column.getName(), reader.getDouble(column));
+                                    }
+
+                                    @Override
+                                    public void booleanColumn(Column column) {
+                                        if (reader.isNull(column)) {
+                                            record.put(column.getName(), null);
+                                            return;
+                                        }
+                                        record.put(column.getName(), reader.getBoolean(column));
+                                    }
+
+                                    @Override
+                                    public void jsonColumn(Column column) {
+                                        if (reader.isNull(column)) {
+                                            record.put(column.getName(), null);
+                                            return;
+                                        }
+                                        record.put(column.getName(), reader.getString(column));
+                                    }
+                                });
+                            }
+                            resultList.add(record);
+                        }
+                    }
+
+                    @Override
+                    public void finish() {
+                    }
+
+                    @Override
+                    public void close() {
+                    }
+
+                    @Override
+                    public void abort() {
+                    }
+
+                    @Override
+                    public TaskReport commit() {
+                        return Exec.newTaskReport();
+                    }
+                });
+        
+        //assert...
+        for (Map<String,Object> r : resultList) {
+            System.out.println(r);
+        }
+        
+        assertEquals(2,resultList.size());
+        
+        Map<String, Object> record0 = resultList.get(0);
+        assertEquals("Wikipedia:アップロードログ 2004年4月",record0.get("title"));
+        assertEquals(1L,record0.get("id"));
+        assertEquals("なんか書く",record0.get("revision/text"));
+        assertEquals(1083336360L * 1000L, ((Timestamp)record0.get("revision/timestamp")).toEpochMilli());
+        
+        Map<String, Object> record1 = resultList.get(1);
+        assertEquals("アンパサンド",record1.get("title"));
+        assertEquals(5L,record1.get("id"));
+        assertEquals("アンパサンドとは\n「…と…」を意味する記号である。",record1.get("revision/text"));
+        assertEquals(1449883580L * 1000L, ((Timestamp)record1.get("revision/timestamp")).toEpochMilli());   
+    }
+
+    private ConfigSource config() {
+        return Exec.newConfigSource().set("in", inputConfig()).set("root", "mediawiki/page")
+                .set("schema", schemaConfig()).set("out", outputConfig());
+    }
+
+    private ImmutableMap<String, Object> inputConfig() {
+        ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
+        builder.put("type", "file");
+        builder.put("path_prefix", PATH_PREFIX);
+        builder.put("last_path", "");
+        return builder.build();
+    }
+
+    private ImmutableMap<String, Object> outputConfig() {
+        ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
+        builder.put("type", "stdout");
+        return builder.build();
+    }
+
+    private ImmutableList<Object> schemaConfig() {
+        ImmutableList.Builder<Object> builder = new ImmutableList.Builder<>();
+        builder.add(ImmutableMap.of("name", "id", "type", "long"));
+        builder.add(ImmutableMap.of("name", "title", "type", "string"));
+        builder.add(ImmutableMap.of("name", "revision/timestamp", "type", "timestamp", "format", "%Y-%m-%dT%H:%M:%SZ", "timezone", "UTC"));
+        builder.add(ImmutableMap.of("name", "revision/text", "type", "string"));
+        return builder.build();
+    }
+}

data/src/test/resources/sample_01.xml

@@ -0,0 +1,80 @@
+<mediawiki xmlns="http://www.mediawiki.org/xml/export-0.10/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.mediawiki.org/xml/export-0.10/ http://www.mediawiki.org/xml/export-0.10.xsd" version="0.10" xml:lang="ja">
+  <siteinfo>
+    <sitename>Wikipedia</sitename>
+    <dbname>jawiki</dbname>
+    <base>https://ja.wikipedia.org/wiki/%E3%83%A1%E3%82%A4%E3%83%B3%E3%83%9A%E3%83%BC%E3%82%B8</base>
+    <generator>MediaWiki 1.27.0-wmf.10</generator>
+    <case>first-letter</case>
+    <namespaces>
+      <namespace key="-2" case="first-letter">メディア</namespace>
+      <namespace key="-1" case="first-letter">特別</namespace>
+      <namespace key="0" case="first-letter" />
+      <namespace key="1" case="first-letter">ノート</namespace>
+      <namespace key="2" case="first-letter">利用者</namespace>
+      <namespace key="3" case="first-letter">利用者‐会話</namespace>
+      <namespace key="4" case="first-letter">Wikipedia</namespace>
+      <namespace key="5" case="first-letter">Wikipedia‐ノート</namespace>
+      <namespace key="6" case="first-letter">ファイル</namespace>
+      <namespace key="7" case="first-letter">ファイル‐ノート</namespace>
+      <namespace key="8" case="first-letter">MediaWiki</namespace>
+      <namespace key="9" case="first-letter">MediaWiki‐ノート</namespace>
+      <namespace key="10" case="first-letter">Template</namespace>
+      <namespace key="11" case="first-letter">Template‐ノート</namespace>
+      <namespace key="12" case="first-letter">Help</namespace>
+      <namespace key="13" case="first-letter">Help‐ノート</namespace>
+      <namespace key="14" case="first-letter">Category</namespace>
+      <namespace key="15" case="first-letter">Category‐ノート</namespace>
+      <namespace key="100" case="first-letter">Portal</namespace>
+      <namespace key="101" case="first-letter">Portal‐ノート</namespace>
+      <namespace key="102" case="first-letter">プロジェクト</namespace>
+      <namespace key="103" case="first-letter">プロジェクト‐ノート</namespace>
+      <namespace key="828" case="first-letter">モジュール</namespace>
+      <namespace key="829" case="first-letter">モジュール‐ノート</namespace>
+      <namespace key="2300" case="first-letter">Gadget</namespace>
+      <namespace key="2301" case="first-letter">Gadget talk</namespace>
+      <namespace key="2302" case="case-sensitive">Gadget definition</namespace>
+      <namespace key="2303" case="case-sensitive">Gadget definition talk</namespace>
+      <namespace key="2600" case="first-letter">Topic</namespace>
+    </namespaces>
+  </siteinfo>
+  <page>
+    <title>Wikipedia:アップロードログ 2004年4月</title>
+    <ns>4</ns>
+    <id>1</id>
+    <restrictions>sysop</restrictions>
+    <revision>
+      <id>2168855</id>
+      <parentid>299151</parentid>
+      <timestamp>2004-04-30T14:46:00Z</timestamp>
+      <contributor>
+        <username>Oxhop</username>
+        <id>2551</id>
+      </contributor>
+      <minor />
+      <comment>&quot;LocationMacedonia.png&quot;をアップロードしました。: マケドニアの位置 - 英語版より</comment>
+      <model>wikitext</model>
+      <format>text/x-wiki</format>
+      <text xml:space="preserve">なんか書く</text>
+      <sha1>gbhvqlwj7ga4v9ghhy0n88iqmlo19vz</sha1>
+    </revision>
+  </page>
+  <page>
+    <title>アンパサンド</title>
+    <ns>0</ns>
+    <id>5</id>
+    <revision>
+      <id>57857555</id>
+      <parentid>56511929</parentid>
+      <timestamp>2015-12-12T01:26:20Z</timestamp>
+      <contributor>
+        <username>＆＆＆&amp;&amp;&amp;COXOXO</username>
+        <id>1043200</id>
+      </contributor>
+      <model>wikitext</model>
+      <format>text/x-wiki</format>
+      <text xml:space="preserve">アンパサンドとは
+「…と…」を意味する記号である。</text>
+      <sha1>esme5un4ixvpdpy5ri44s948usq3uhd</sha1>
+    </revision>
+  </page>
+</mediawiki>

metadata

@@ -0,0 +1,88 @@
+--- !ruby/object:Gem::Specification
+name: embulk-parser-xml2
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- yoshi0309
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2016-08-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: bundler
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  name: rake
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: Parses Xml2 files read by other file input plugins.
+email:
+- takumyos@yahoo-corp.jp
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- LICENSE.txt
+- README.md
+- build.gradle
+- config/checkstyle/checkstyle.xml
+- config/checkstyle/default.xml
+- gradle/wrapper/gradle-wrapper.jar
+- gradle/wrapper/gradle-wrapper.properties
+- gradlew
+- gradlew.bat
+- lib/embulk/guess/xml2.rb
+- lib/embulk/parser/xml2.rb
+- src/main/java/org/embulk/parser/xml2/Xml2ParserPlugin.java
+- src/test/java/org/embulk/parser/xml2/TestXml2ParserPlugin.java
+- src/test/resources/sample_01.xml
+- classpath/embulk-parser-xml2-0.1.0.jar
+homepage:
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.1.9
+signing_key:
+specification_version: 4
+summary: Xml2 parser plugin for Embulk
+test_files: []