embulk-output-utf8parquet 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 217f5b1b8a369ad9d2327b1c59aff5d9c0333eb6
+  data.tar.gz: 862c3aec856f11163ba30d008e98afe469aea538
+SHA512:
+  metadata.gz: 6bb6be1333e7869d5b8d711f5a410288b7fa9df9cc18d32674b33908736ae9daf99d1d4314a5a7fa2ddc32f189214256be6c757ab71f1091c2629a9aaae5c3e9
+  data.tar.gz: f10c1e3f01d42f5826bf8edcca966a05ab7c9abfc5e7e8a3ccb7df866ea9aa28f78e2c960a0e61e09fc39451ef8491b6603b8e7d8757603ba142264eaabfb2bd

data/.idea/gradle.xml

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="GradleSettings">
+    <option name="linkedExternalProjectsSettings">
+      <GradleProjectSettings>
+        <option name="distributionType" value="DEFAULT_WRAPPED" />
+        <option name="externalProjectPath" value="$PROJECT_DIR$" />
+      </GradleProjectSettings>
+    </option>
+  </component>
+</project>

data/.idea/misc.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" languageLevel="JDK_1_9" project-jdk-name="9.0" project-jdk-type="JavaSDK">
+    <output url="file://$PROJECT_DIR$/classes" />
+  </component>
+</project>

data/.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/embulk-output-utf8parquet.iml" filepath="$PROJECT_DIR$/embulk-output-utf8parquet.iml" />
+    </modules>
+  </component>
+</project>

data/.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>

data/README.md

@@ -0,0 +1,49 @@
+# Parquet output plugin for Embulk
+
+## Overview
+
+* **Plugin type**: output
+* **Load all or nothing**: no
+* **Resume supported**: no
+* **Cleanup supported**: no
+
+## Configuration
+
+- **path_prefix**: A prefix of output path. This is hadoop Path URI, and you can also include `scheme` and `authority` within this parameter. (string, required)
+- **file_ext**: An extension of output path. (string, default: .parquet)
+- **sequence_format**: (string, default: .%03d)
+- **block_size**: A block size of parquet file. (int, default: 134217728(128M))
+- **page_size**: A page size of parquet file. (int, default: 1048576(1M))
+- **compression_codec**: A compression codec. available: UNCOMPRESSED, SNAPPY, GZIP (string, default: UNCOMPRESSED)
+- **default_timezone**: Time zone of timestamp columns. This can be overwritten for each column using column_options
+- **default_timestamp_format**: Format of timestamp columns. This can be overwritten for each column using column_options
+- **column_options**: Specify timezone and timestamp format for each column. Format of this option is the same as the official csv formatter. See [document](
+http://www.embulk.org/docs/built-in.html#csv-formatter-plugin).
+- **extra_configurations**: Add extra entries to Configuration which will be passed to ParquetWriter
+- **overwrite**: Overwrite if output files already exist. (default: fail if files exist)
+- **addUTF8**: If true, string columns are stored with OriginalType.UTF8 (boolean, default false)
+
+## Example
+
+```yaml
+out:
+  type: parquet
+  path_prefix: file:///data/output
+```
+
+### How to write parquet files into S3
+
+```yaml
+out:
+  type: parquet
+  path_prefix: s3a://bucket/keys
+  extra_configurations:
+    fs.s3a.access.key: 'your_access_key'
+    fs.s3a.secret.key: 'your_secret_access_key'
+```
+
+## Build
+
+```
+$ ./gradlew gem
+```

data/embulk-output-utf8parquet.iml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module external.linked.project.id="embulk-output-utf8parquet" external.linked.project.path="$MODULE_DIR$" external.root.project.path="$MODULE_DIR$" external.system.id="GRADLE" type="JAVA_MODULE" version="4">
+  <component name="NewModuleRootManager" inherit-compiler-output="true">
+    <exclude-output />
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

metadata

@@ -0,0 +1,164 @@
+--- !ruby/object:Gem::Specification
+name: embulk-output-utf8parquet
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- OKUNO Akihiro
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2018-05-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: bundler
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  name: rake
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: Parquet output plugin is an Embulk plugin that loads records to Parquet read by any input plugins. Search the input plugins by "embulk-input" keyword.
+email:
+- alexopoulos7@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .idea/gradle.xml
+- .idea/misc.xml
+- .idea/modules.xml
+- .idea/vcs.xml
+- README.md
+- embulk-output-utf8parquet.iml
+- classpath/jaxb-impl-2.2.3-1.jar
+- classpath/hadoop-aws-2.7.1.jar
+- classpath/hadoop-auth-2.7.1.jar
+- classpath/activation-1.1.jar
+- classpath/commons-configuration-1.6.jar
+- classpath/commons-beanutils-1.7.0.jar
+- classpath/hadoop-yarn-client-2.7.1.jar
+- classpath/xz-1.0.jar
+- classpath/commons-httpclient-3.1.jar
+- classpath/stax-api-1.0-2.jar
+- classpath/apacheds-i18n-2.0.0-M15.jar
+- classpath/joda-time-2.9.9.jar
+- classpath/httpclient-4.2.5.jar
+- classpath/jline-0.9.94.jar
+- classpath/jaxb-api-2.2.2.jar
+- classpath/hadoop-annotations-2.7.1.jar
+- classpath/hadoop-mapreduce-client-jobclient-2.7.1.jar
+- classpath/hadoop-hdfs-2.7.1.jar
+- classpath/jackson-jaxrs-1.9.13.jar
+- classpath/xercesImpl-2.9.1.jar
+- classpath/commons-logging-1.1.3.jar
+- classpath/hadoop-yarn-server-common-2.7.1.jar
+- classpath/curator-recipes-2.7.1.jar
+- classpath/hadoop-yarn-server-nodemanager-2.7.1.jar
+- classpath/jersey-json-1.9.jar
+- classpath/avro-1.7.4.jar
+- classpath/log4j-1.2.17.jar
+- classpath/commons-cli-1.2.jar
+- classpath/parquet-column-1.8.1.jar
+- classpath/xml-apis-1.3.04.jar
+- classpath/commons-digester-1.8.jar
+- classpath/servlet-api-2.5.jar
+- classpath/parquet-format-2.3.0-incubating.jar
+- classpath/protobuf-java-2.5.0.jar
+- classpath/hadoop-mapreduce-client-common-2.7.1.jar
+- classpath/xmlenc-0.52.jar
+- classpath/jackson-xc-1.9.13.jar
+- classpath/jetty-util-6.1.26.jar
+- classpath/hadoop-mapreduce-client-shuffle-2.7.1.jar
+- classpath/commons-compress-1.4.1.jar
+- classpath/hadoop-yarn-common-2.7.1.jar
+- classpath/commons-io-2.4.jar
+- classpath/hadoop-mapreduce-client-core-2.7.1.jar
+- classpath/jackson-core-asl-1.9.13.jar
+- classpath/jersey-core-1.9.jar
+- classpath/jsp-api-2.1.jar
+- classpath/commons-codec-1.6.jar
+- classpath/snappy-java-1.1.1.6.jar
+- classpath/jetty-6.1.26.jar
+- classpath/hadoop-yarn-api-2.7.1.jar
+- classpath/jersey-server-1.9.jar
+- classpath/java-xmlbuilder-0.4.jar
+- classpath/netty-3.7.0.Final.jar
+- classpath/hadoop-common-2.7.1.jar
+- classpath/jersey-client-1.9.jar
+- classpath/jersey-guice-1.9.jar
+- classpath/paranamer-2.3.jar
+- classpath/zookeeper-3.4.6.jar
+- classpath/parquet-encoding-1.8.1.jar
+- classpath/jettison-1.1.jar
+- classpath/api-asn1-api-1.0.0-M20.jar
+- classpath/apacheds-kerberos-codec-2.0.0-M15.jar
+- classpath/parquet-hadoop-1.8.1.jar
+- classpath/commons-collections-3.2.1.jar
+- classpath/asm-3.1.jar
+- classpath/parquet-common-1.8.1.jar
+- classpath/hadoop-client-2.7.1.jar
+- classpath/api-util-1.0.0-M20.jar
+- classpath/embulk-output-utf8parquet-1.0.0.jar
+- classpath/curator-framework-2.7.1.jar
+- classpath/commons-net-3.1.jar
+- classpath/gson-2.2.4.jar
+- classpath/jets3t-0.9.0.jar
+- classpath/commons-lang-2.6.jar
+- classpath/parquet-jackson-1.8.1.jar
+- classpath/jsch-0.1.42.jar
+- classpath/leveldbjni-all-1.8.jar
+- classpath/httpcore-4.2.4.jar
+- classpath/hadoop-mapreduce-client-app-2.7.1.jar
+- classpath/jackson-mapper-asl-1.9.13.jar
+- classpath/commons-math3-3.1.1.jar
+- classpath/netty-all-4.0.23.Final.jar
+- classpath/aws-java-sdk-1.7.4.jar
+- classpath/htrace-core-3.1.0-incubating.jar
+- classpath/jsr305-3.0.0.jar
+- classpath/curator-client-2.7.1.jar
+homepage: https://github.com/alexopoulos7/embulk-output-utf8parquet
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.1.9
+signing_key:
+specification_version: 4
+summary: Parquet output plugin for Embulk with UTF8 support
+test_files: []