embulk 0.7.0-java

data/embulk-docs/push-gh-pages.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+function re() {
+    r "$@"
+    if [ $? -ne 0 ];then
+        exit $?
+    fi
+}
+
+function r() {
+    echo "$@"
+    "$@"
+}
+
+[ "$TRAVIS_PULL_REQUEST" != "false" ] && exit 0
+[ "$TRAVIS_BRANCH" != "master" -a "$TRAVIS_BRANCH" != "$(git describe --tags --always HEAD)" ] && exit 0
+
+revision="$(git rev-parse HEAD)"
+remote="https://github.com/embulk/embulk.github.io.git"
+re ./gradlew site
+
+r git fetch --unshallow || echo "using complete repository."
+
+re rm -rf gh_pages
+re git clone . gh_pages
+re cd gh_pages
+
+re git remote add travis_push "$remote"
+re git fetch travis_push
+
+re git checkout -b gh-pages travis_push/master
+re rm -rf docs
+re cp -a ../embulk-docs/build/html docs
+re git add --all docs
+
+re git config user.name "$GIT_USER_NAME"
+re git config user.email "$GIT_USER_EMAIL"
+r git commit -m "Updated document $revision"
+
+git show | grep -E '^[+-] ' | grep -Eqv 'Generated by|Generated on|Search.setIndex|meta name="date" content='
+if [ $? -ne 0 ];then
+    echo "No document changes."
+    exit 0
+fi
+
+re git config credential.helper "store --file=$HOME/.git_credentials"
+echo "https://$GITHUB_TOKEN:@github.com" > "$HOME/.git_credentials"
+trap "rm -rf $HOME/.git_credentials" EXIT
+re git push travis_push gh-pages:master

data/embulk-docs/src/_static/embulk-logo.svg

@@ -0,0 +1,133 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Generator: Adobe Illustrator 15.0.2, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+	 width="466px" height="268px" viewBox="0 0 466 268" enable-background="new 0 0 466 268" xml:space="preserve">
+<g>
+	<path fill="#EF4F39" d="M124.753,173.479c0,0,7.781,2.789,8.031,6.721c0.57,8.859,5.539,30.307-0.93,30.686
+		c-8.516,0.502-20.797-12.975-24.602-22.738c-5.82-14.951-4.18-19.992-4.18-19.992L124.753,173.479z"/>
+	<g>
+		<g>
+			<path fill="#1F1F1F" d="M38.015,225.368c1.477,4.021,3.523,7.008,6.141,8.953s5.891,2.918,9.828,2.918
+				c2.859,0,5.742-0.58,8.648-1.744c0.762-0.312,3.422-1.586,7.984-3.822c1.383-0.672,2.93-1.006,4.625-1.006
+				c2.328,0,4.297,1.051,5.906,3.152c1.34,1.699,2.012,3.398,2.012,5.096c0,3.803-2.707,7.246-8.113,10.33
+				c-6.488,3.713-13.844,5.57-22.07,5.57c-12.297,0-21.98-4.273-29.047-12.814c-5.094-6.17-7.645-13.592-7.645-22.268
+				c0-11.762,3.777-21.152,11.336-28.172c6.527-6.037,14.266-9.057,23.207-9.057c7.828,0,14.781,2.416,20.859,7.246
+				c5.859,4.695,9.75,11.066,11.672,19.115c0.672,2.863,1.008,5.725,1.008,8.586c0,5.277-2.93,7.916-8.789,7.916H38.015z
+				 M64.511,211.616c-0.848-3.264-1.832-5.701-2.949-7.312c-2.414-3.531-5.812-5.299-10.195-5.299c-3.758,0-6.953,1.365-9.594,4.094
+				c-1.699,1.787-3.195,4.627-4.492,8.518H64.511z"/>
+			<path fill="#1F1F1F" d="M112.202,189.481c2.367-2.102,4.539-3.623,6.508-4.561c3.352-1.609,7.133-2.416,11.336-2.416
+				c5.23,0,9.812,1.117,13.75,3.354c1.742,0.984,3.598,2.797,5.566,5.434c3.355-2.727,6.215-4.693,8.586-5.902
+				c3.934-1.922,8.16-2.885,12.676-2.885c6.977,0,12.742,2.215,17.305,6.641c4.609,4.518,6.91,10.061,6.91,16.635v35.146
+				c0,3.848-0.449,6.666-1.34,8.451c-0.762,1.432-1.957,2.627-3.59,3.59c-1.633,0.961-3.344,1.443-5.129,1.443
+				c-1.793,0-3.5-0.469-5.133-1.41c-1.633-0.939-2.828-2.146-3.59-3.623c-0.895-1.785-1.34-4.604-1.34-8.451v-30.652
+				c0-2.951-0.895-5.342-2.684-7.176c-2.012-2.146-4.559-3.221-7.645-3.221c-3.133,0-5.75,1.051-7.852,3.154
+				c-1.875,1.832-2.816,4.246-2.816,7.242v30.652c0,3.805-0.449,6.621-1.34,8.451c-0.762,1.477-1.957,2.684-3.59,3.623
+				c-1.633,0.941-3.344,1.41-5.129,1.41c-1.789,0-3.5-0.469-5.133-1.41c-1.633-0.939-2.828-2.146-3.59-3.623
+				c-0.895-1.785-1.34-4.604-1.34-8.451v-28.641c0-4.066-0.961-7.152-2.887-9.254c-1.922-2.104-4.449-3.154-7.578-3.154
+				c-3.086,0-5.699,1.098-7.848,3.287c-1.789,1.879-2.68,4.248-2.68,7.109v30.652c0,3.805-0.453,6.621-1.344,8.451
+				c-0.762,1.477-1.957,2.684-3.59,3.623c-1.633,0.941-3.34,1.41-5.133,1.41c-1.785,0-3.496-0.469-5.129-1.41
+				c-1.633-0.939-2.828-2.146-3.59-3.623c-0.895-1.785-1.34-4.604-1.34-8.451v-46.414c0-3.398,0.848-6.127,2.547-8.186
+				c1.879-2.279,4.336-3.42,7.379-3.42c2.594,0,4.809,0.873,6.641,2.617C109.897,186.284,110.952,187.603,112.202,189.481z"/>
+			<path fill="#1F1F1F" d="M112.202,189.481c2.367-2.102,4.539-3.623,6.508-4.561c3.352-1.609,7.133-2.416,11.336-2.416
+				c5.23,0,9.812,1.117,13.75,3.354c1.742,0.984,3.598,2.797,5.566,5.434c3.355-2.727,6.215-4.693,8.586-5.902
+				c3.934-1.922,8.16-2.885,12.676-2.885c6.977,0,12.742,2.215,17.305,6.641c4.609,4.518,6.91,10.061,6.91,16.635v35.146
+				c0,3.848-0.449,6.666-1.34,8.451c-0.762,1.432-1.957,2.627-3.59,3.59c-1.633,0.961-3.344,1.443-5.129,1.443
+				c-1.793,0-3.5-0.469-5.133-1.41c-1.633-0.939-2.828-2.146-3.59-3.623c-0.895-1.785-1.34-4.604-1.34-8.451v-30.652
+				c0-2.951-0.895-5.342-2.684-7.176c-2.012-2.146-4.559-3.221-7.645-3.221c-3.133,0-5.75,1.051-7.852,3.154
+				c-1.875,1.832-2.816,4.246-2.816,7.242v30.652c0,3.805-0.449,6.621-1.34,8.451c-0.762,1.477-1.957,2.684-3.59,3.623
+				c-1.633,0.941-3.344,1.41-5.129,1.41c-1.789,0-3.5-0.469-5.133-1.41c-1.633-0.939-2.828-2.146-3.59-3.623
+				c-0.895-1.785-1.34-4.604-1.34-8.451v-28.641c0-4.066-0.961-7.152-2.887-9.254c-1.922-2.104-4.449-3.154-7.578-3.154
+				c-3.086,0-5.699,1.098-7.848,3.287c-1.789,1.879-2.68,4.248-2.68,7.109v30.652c0,3.805-0.453,6.621-1.344,8.451
+				c-0.762,1.477-1.957,2.684-3.59,3.623c-1.633,0.941-3.34,1.41-5.133,1.41c-1.785,0-3.496-0.469-5.129-1.41
+				c-1.633-0.939-2.828-2.146-3.59-3.623c-0.895-1.785-1.34-4.604-1.34-8.451v-46.414c0-3.398,0.848-6.127,2.547-8.186
+				c1.879-2.279,4.336-3.42,7.379-3.42c2.594,0,4.809,0.873,6.641,2.617C109.897,186.284,110.952,187.603,112.202,189.481z"/>
+		</g>
+	</g>
+	<g>
+		<path fill="#FFFFFF" d="M32.706,101.839c0,0,12.794,37.536,28.406,50.635c23.352,19.592,42.266,24.008,63.57,26.375
+			c27.992,3.109,68.055,0.375,76.672-3.381c19.914-8.666,36.523-27.725,41.93-43.279c5.414-15.557-82.844-32.125-82.844-32.125
+			l-30.766-13.865c0,0-19.617-10.143-26.719-6.76c-7.102,3.381-13.859,5.748-19.273,7.1c-5.406,1.354-13.523,0.338-22.656,2.029
+			C51.901,90.257,32.706,101.839,32.706,101.839z"/>
+		<path fill="#EF4F39" d="M32.331,101.245c0,0,0.391,10.795,3.898,14.678c4.727,5.23,3.203,11.189,13.461,25.664
+			c13.281,18.732,38.328,36.789,89.406,38.908c50.391,2.094,50.086-1.395,50.086-1.395s-6.891,1.152-44.25-0.252
+			c-23.625-0.889-56.344-5.354-75.016-20.842c-17.734-14.709-24.203-24.42-29.125-39.309c-2.094-6.34-5.797-6.422-5.992-15.996
+			C34.776,101.249,32.331,101.245,32.331,101.245z"/>
+		<g>
+			<path fill="#F16523" d="M195.331,176.63c0,0,5.219-1.023,11.352-4.504c8.711-4.943,14.773-11.02,15.852-11.918
+				c4.883-4.057,15.219-16.484,15.219-16.484c-0.414,0.613-5.562,9.764-12.305,15.977c-4.484,4.133-1.898,2.156-9.25,7.861
+				c-9.078,7.043-22,11.92-22,11.92"/>
+			<path fill="#F16523" d="M284.198,12.147c0,0-7.016,13.777-24.008,15.553c-11.57,1.211-13.188,11.16-13.188,11.16
+				s-4.055-10.652-14.453-12.174c-15.477-2.264-20.375-13.525-20.375-13.525c-4.391,16.23,0.68,24.346,3.383,28.404
+				c5.188,7.781,18.258,9.469,21.305,16.23c3.039,6.764,0.672,11.498-2.703,27.391c-3.383,15.893-19.281,12.848-32.125,11.156
+				c-7.117-0.934-18.672-8.105-41.508-16.402c0.242-4.438-2.695-9.434-1.016-16.23c6.086-24.6,24.094-34.361,12.938-32.715
+				c-8.805,1.301-50.031,32.121-56.523,36.102c-8.344-1.617-17.086-2.822-26.156-3.387c-31.359-1.947-39.305,14.033-48.945,20.627
+				c-17.148,11.734-8.07,17.84-8.07,17.84s28.812,1.553,40.109,3.975c9.469,2.027,32.125,8.453,33.812,11.496
+				c1.695,3.043,4.398,11.834,12.516,10.145c8.109-1.691,8.75-7.969,8.75-7.969c-0.344,9.131,6.805,21.156,14.242,31.639
+				s19.273,13.188,19.273,13.188c2.031,25.359-11.562,49.064-8.031,49.195c23.844,0.889,38.156-23.922,40.453-28.275
+				c4.82-9.131,3.297-19.656,3.297-19.656c42.266-8.113,49.461-34.107,53.203-45.637c6.344-19.529,3.055-59.438,3.055-59.438
+				s8.789-0.676,20.961-9.467C286.565,42.579,284.198,12.147,284.198,12.147z"/>
+		</g>
+		<path fill="#FFFFFF" d="M107.753,97.312c0,0-0.617-14.115,18.625-10.881s19.953,13.1,4.945,17.168
+			C116.323,107.669,108.065,103.565,107.753,97.312z"/>
+		<g>
+			<circle fill="#1F1F1F" cx="101.776" cy="92.2" r="4.312"/>
+			<path fill="#FFFFFF" d="M101.776,96.761c-2.516,0-4.562-2.046-4.562-4.561c0-2.516,2.047-4.562,4.562-4.562
+				s4.562,2.047,4.562,4.562C106.339,94.715,104.292,96.761,101.776,96.761z M101.776,88.138c-2.24,0-4.062,1.822-4.062,4.062
+				c0,2.239,1.822,4.061,4.062,4.061s4.062-1.822,4.062-4.061C105.839,89.96,104.017,88.138,101.776,88.138z"/>
+			<path fill="#FFFFFF" d="M104.394,90.595c0,0.512-0.414,0.928-0.93,0.928c-0.508,0-0.93-0.416-0.93-0.928
+				c0-0.516,0.422-0.932,0.93-0.932C103.979,89.663,104.394,90.079,104.394,90.595z"/>
+		</g>
+		<g>
+			<path fill="#EF4F39" d="M164.237,161.2c-1.523,0.254-4.141,0.254-9.383-1.184c-4.789-1.311-12.648-9.082-13.984-10.43
+				c0.438,0.631,0.875,1.254,1.312,1.875c7.438,10.482,19.273,13.188,19.273,13.188c1.969,24.617-10.773,47.672-8.297,49.121
+				c0.008-0.064,0.094-0.949,0.258-1.764c0.164-0.846,0.594-2.283,1.266-3.805c0.68-1.521,10.062-18.682,12.172-26.543
+				c2.117-7.861,1.188-21.727,1.188-21.727S165.761,160.948,164.237,161.2z"/>
+			<path fill="#EF4F39" d="M122.995,115.067c0,0,2.531-7.607,10.648-6.086s5.328,10.271,5.328,10.271s0.031-6.586-5.711-7.734
+				C128.19,110.503,122.995,115.067,122.995,115.067z"/>
+			<path fill="#EF4F39" d="M117.237,126.362c-5.328-0.635-5.711-8.242-9.891-11.287c-3.484-2.529-25.367-9.129-34.242-11.031
+				s-36.266-3.803-38.547-3.93s-2.914-3.043-3.297-3.553c-0.242-0.326,0.25-2.391,0.633-3.816c-3.898,6.225,0.859,9.432,0.859,9.432
+				s28.812,1.553,40.109,3.975c9.469,2.027,32.125,8.453,33.812,11.496c1.695,3.043,4.398,11.834,12.516,10.145
+				c8.109-1.691,8.75-7.969,8.75-7.969S122.144,126.946,117.237,126.362z"/>
+			<path fill="#EF4F39" d="M251.394,111.146c-1.352,7.1-6.594,22.994-16.398,32.799c-6.102,6.098-15.898,10.652-23.672,14.203
+				c-6.266,2.859-15.219,3.887-15.219,3.887l1.07,3.879c42.266-8.113,49.461-34.107,53.203-45.637
+				c2.305-7.115,3.344-16.934,3.719-26.541C253.448,98.396,252.308,106.351,251.394,111.146z"/>
+		</g>
+	</g>
+	<rect x="133.667" y="203.567" fill="#1F1F1F" width="20.055" height="21.766"/>
+	<g>
+		<g>
+			<path fill="#1F1F1F" d="M226.835,188.409c2.816-2.104,5.188-3.512,7.109-4.227c2.992-1.162,6.059-1.744,9.188-1.744
+				c8.32,0,15.586,3.379,21.801,10.127c6.348,6.889,9.527,15.496,9.527,25.826c0,11.402-3.555,20.658-10.668,27.77
+				c-5.723,5.768-12.543,8.654-20.457,8.654c-4.609,0-8.562-1.031-11.875-3.086c-1.516-0.984-3.148-2.619-4.895-4.896
+				c-2.059,5.051-5.23,7.578-9.527,7.578c-2.637,0-4.961-0.85-6.973-2.549c-2.238-1.967-3.355-4.697-3.355-8.182v-77.543
+				c0-3.799,0.449-6.594,1.344-8.383c0.758-1.477,1.953-2.684,3.59-3.623c1.629-0.939,3.34-1.408,5.129-1.408s3.5,0.469,5.133,1.408
+				s2.828,2.146,3.586,3.623c0.895,1.789,1.344,4.584,1.344,8.383V188.409z M240.519,201.22c-3.758,0-7.02,1.631-9.793,4.896
+				c-2.684,3.219-4.023,7.379-4.023,12.477c0,5.141,1.34,9.324,4.023,12.543c2.773,3.264,6.078,4.896,9.926,4.896
+				c4.16,0,7.488-1.746,9.996-5.23c2.457-3.49,3.688-7.535,3.688-12.143c0-4.695-1.23-8.742-3.688-12.143
+				C248.097,202.987,244.718,201.22,240.519,201.22z"/>
+			<path fill="#1F1F1F" d="M345.358,196.39v29.916c0,6.842-1.637,12.387-4.898,16.635c-2.773,3.578-6.352,6.396-10.734,8.451
+				c-4.918,2.324-10.148,3.488-15.695,3.488c-7.781,0-14.598-2.078-20.457-6.238c-4.559-3.221-7.645-7.064-9.258-11.537
+				c-1.074-3.037-1.605-6.641-1.605-10.799V196.39c0-3.801,0.445-6.617,1.336-8.451c0.762-1.477,1.949-2.684,3.559-3.623
+				c1.605-0.938,3.332-1.408,5.164-1.408c1.789,0,3.496,0.471,5.129,1.408c1.633,0.939,2.828,2.146,3.59,3.623
+				c0.895,1.789,1.34,4.607,1.34,8.451v27.434c0,2.818,0.449,4.945,1.344,6.373c2.281,3.668,5.57,5.502,9.859,5.502
+				c3.129,0,5.793-1.074,7.984-3.221c2.145-2.059,3.219-4.941,3.219-8.654V196.39c0-3.844,0.445-6.662,1.34-8.451
+				c0.762-1.477,1.957-2.684,3.59-3.623c1.633-0.938,3.344-1.408,5.133-1.408c1.785,0,3.5,0.471,5.129,1.408
+				c1.633,0.939,2.828,2.146,3.59,3.623C344.909,189.728,345.358,192.546,345.358,196.39z"/>
+			<path fill="#1F1F1F" d="M377.351,166.138v74.789c0,3.848-0.445,6.666-1.344,8.451c-0.758,1.477-1.953,2.684-3.586,3.623
+				c-1.633,0.941-3.344,1.41-5.133,1.41s-3.5-0.469-5.129-1.41c-1.633-0.939-2.832-2.146-3.59-3.623
+				c-0.895-1.785-1.344-4.604-1.344-8.451v-74.789c0-3.799,0.449-6.594,1.344-8.383c0.758-1.477,1.957-2.684,3.59-3.623
+				c1.629-0.939,3.34-1.408,5.129-1.408s3.5,0.469,5.133,1.408s2.828,2.146,3.586,3.623
+				C376.905,159.544,377.351,162.339,377.351,166.138z"/>
+			<path fill="#1F1F1F" d="M409.343,220.403v20.523c0,3.805-0.445,6.621-1.34,8.451c-0.762,1.477-1.957,2.684-3.586,3.623
+				c-1.633,0.941-3.344,1.41-5.133,1.41s-3.496-0.469-5.129-1.41c-1.633-0.939-2.828-2.146-3.594-3.623
+				c-0.891-1.785-1.34-4.604-1.34-8.451v-74.789c0-3.799,0.449-6.594,1.34-8.383c0.766-1.477,1.961-2.684,3.594-3.623
+				s3.34-1.408,5.129-1.408s3.5,0.469,5.133,1.408c1.629,0.939,2.824,2.146,3.586,3.623c0.895,1.789,1.34,4.584,1.34,8.383v40.582
+				l18.516-18.982c3.445-3.533,6.527-5.299,9.258-5.299c2.727,0,5.051,0.984,6.977,2.951c1.922,1.924,2.883,4.227,2.883,6.908
+				c0,1.969-0.738,3.936-2.215,5.904c-0.488,0.625-1.566,1.744-3.219,3.354l-12.609,12.072l17.641,21.936
+				c2.902,3.578,4.355,6.551,4.355,8.918c0,3.178-1.27,5.793-3.82,7.85c-1.836,1.432-3.957,2.146-6.375,2.146
+				c-2.41,0-4.672-0.961-6.77-2.885c-0.629-0.58-1.656-1.787-3.09-3.621L409.343,220.403z"/>
+		</g>
+	</g>
+</g>
+</svg>

data/embulk-docs/src/built-in.rst

@@ -0,0 +1,440 @@
+Configuration
+==================================
+
+.. contents::
+   :local:
+   :depth: 2
+
+Embulk configuration file format
+------------------
+
+Embulk uses a YAML file to define a bulk data loading. Here is an example of the file:
+
+.. code-block:: yaml
+
+    in:
+      type: file
+      path_prefix: ./mydata/csv/
+      decoders:
+      - {type: gzip}
+      parser:
+        charset: UTF-8
+        newline: CRLF
+        type: csv
+        delimiter: ','
+        quote: '"'
+        escape: '"'
+        null_string: 'NULL'
+        skip_header_lines: 1
+        columns:
+        - {name: id, type: long}
+        - {name: account, type: long}
+        - {name: time, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
+        - {name: purchase, type: timestamp, format: '%Y%m%d'}
+        - {name: comment, type: string}
+    filters:
+      - type: speedometer
+        speed_limit: 250000
+    out:
+      type: stdout
+
+A configuration file consists of following sections:
+
+* **in:** Input plugin options. An input plugin is either record-based (`MySQL <https://github.com/embulk/embulk-input-jdbc>`_, `DynamoDB <https://github.com/lulichn/embulk-input-dynamodb>`_, etc) or file-based (`S3 <https://github.com/embulk/embulk-input-s3>`_, `HTTP <https://github.com/takumakanari/embulk-input-http>`_, etc).
+
+  * **parser:** If the input is file-based, parser plugin parses a file format (built-in csv, `json <https://github.com/takumakanari/embulk-parser-json>`_, etc).
+
+  * **decoder:** If the input is file-based, decoder plugin decodes compression or encryption (built-in gzip, `zip <https://github.com/hata/embulk-decoder-commons-compress>`_, `tar.gz <https://github.com/hata/embulk-decoder-commons-compress>`_, etc).
+
+* **out:** Output plugin options. An output plugin is either record-based (`Oracle <https://github.com/embulk/embulk-output-jdbc>`_, `Elasticsearch <https://github.com/muga/embulk-output-elasticsearch>`_, etc) or file-based (`Google Cloud Storage <https://github.com/hakobera/embulk-output-gcs>`_, `Command <https://github.com/embulk/embulk-output-command>`_, etc)
+
+  * **formatter:** If the output is file-based, formatter plugin formats a file format (such as built-in csv, `JSON <https://github.com/takei-yuya/embulk-formatter-jsonl>`_)
+
+  * **encoder:** If the output is file-based, encoder plugin encodes compression or encryption (such as built-in gzip)
+
+* **filters:** Filter plugins options (optional).
+
+* **exec:** Executor plugin options. An executor plugin control parallel processing (such as built-in thread executor, `Hadoop MapReduce executor <https://github.com/embulk/embulk-executor-mapreduce>`_)
+
+In many cases, what you need to write is **in:**, **out**: and **formatter** sections only because ``guess`` command guesses **parser** and **decoder** options for you. See also the `Quick Start <https://github.com/embulk/embulk#quick-start>`_.
+
+
+Using variables
+~~~~~~~~~~~~~~~~~~
+
+You can embed environment variables in configuration file using `Liquid template engine <http://liquidmarkup.org/>`_ (This is experimental feature. Behavior might change or be removed in future releases).
+
+To use template engine, configuration file name must end with ``.yml.liquid``.
+
+Environment variables are set to ``env`` variable.
+
+.. code-block:: yaml
+
+    in:
+      type: file
+      path_prefix: {{ env.path_prefix }}
+      decoders:
+      - {type: gzip}
+      parser:
+        ...
+    out:
+      type: postgresql
+      host: {{ env.pg_host }}
+      port: {{ env.pg_port }}
+      user: {{ env.pg_user }}
+      password: "{{ env.pg_password }}"
+      database: embulk_load
+      mode: insert
+      table: {{ env.pg_table }}
+
+
+Local file input plugin
+------------------
+
+The ``file`` input plugin reads files from local file system.
+
+Options
+~~~~~~~~~~~~~~~~~~
+
++----------------+----------+------------------------------------------------+-----------+
+| name           | type     | description                                    | required? |
++================+==========+================================================+===========+
+| path\_prefix   | string   | Path prefix of input files                     | required  |
++----------------+----------+------------------------------------------------+-----------+
+| parsers        | hash     | Parsers configurations (see below)             | required  |
++----------------+----------+------------------------------------------------+-----------+
+| decoders       | array    | Decoder configuration (see below)              |           |
++----------------+----------+------------------------------------------------+-----------+
+| last\_path     | string   | Name of last read file in previous operation   |           |
++----------------+----------+------------------------------------------------+-----------+
+
+The ``path_prefix`` option is required. If you have files as following, you may set ``path_prefix: /path/to/files/sample_``:
+
+::
+
+    .
+    `-- path
+        `-- to
+            `-- files
+                |-- sample_01.csv   -> read
+                |-- sample_02.csv   -> read
+                |-- sample_03.csv   -> read
+                |-- sample_04.csv   -> read
+
+The ``last_path`` option is used to skip files older than or same with the file in dictionary order.
+For example, if you set ``last_path: /path/to/files/sample_02.csv``, Embulk reads following files:
+
+::
+
+    .
+    `-- path
+        `-- to
+            `-- files
+                |-- sample_01.csv   -> skip
+                |-- sample_02.csv   -> skip
+                |-- sample_03.csv   -> read
+                |-- sample_04.csv   -> read
+
+Example
+~~~~~~~~~~~~~~~~~~
+
+.. code-block:: yaml
+
+    in:
+      type: file
+      path_prefix: /path/to/files/sample_
+      last_path: /path/to/files/sample_02.csv
+      parser:
+        ...
+
+In most of cases, you'll use guess to configure the parsers and decoders. See also `Quick Start <https://github.com/embulk/embulk#quick-start>`_.
+
+CSV parser plugin
+------------------
+
+The ``csv`` parser plugin parses CSV and TSV files.
+
+Options
+~~~~~~~~~~~~~~~~~~
+
++----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
+| name                       | type     | description                                                                                                    |              required? |
++============================+==========+================================================================================================================+========================+
+| delimiter                  | string   | Delimiter character such as ``,`` for CSV, ``"\t"`` for TSV, ``"|"`` or any single-byte character              | ``,`` by default       |
++----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
+| quote                      | string   | The character surrounding a quoted value. Setting ``null`` disables quoting.                                   | ``\"`` by default      |
++----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
+| escape                     | string   | Escape character to escape a special character. Setting ``null`` disables escaping.                            | ``\\`` by default      |
++----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
+| skip\_header\_lines        | integer  | Skip this number of lines first. Set 1 if the file has header line.                                            | ``0`` by default       |
++----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
+| null\_string               | string   | If a value is this string, converts it to NULL. For example, set ``\N`` for CSV files created by mysqldump     |                        |
++----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
+| trim\_if\_not\_quoted      | boolean  | If true, remove spaces of a value if the value is not surrounded by the quote character                        | ``false`` by default   |
++----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
+| comment\_line\_marker      | string   | Skip a line if the line begins with this string                                                                | null by default        |
++----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
+| allow\_optional\_columns   | boolean  | If true, set null to insufficient columns. Otherwise, skip the row in case of insufficient number of columns   | ``false`` by default   |
++----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
+| allow\_extra\_columns      | boolean  | If true, ignore too many columns. Otherwise, skip the row in case of too many columns                          | ``false`` by default   |
++----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
+| max\_quoted\_size\_limit   | integer  | Maximum number of bytes of a quoted value. If a value exceeds the limit, the row will be skipped               | ``131072`` by default  |
++----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
+| default\_timezone          | string   | Time zone of timestamp columns if the value itself doesn't include time zone description (eg. Asia/Tokyo)      | ``UTC`` by default     |
++----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
+| newline                    | enum     | Newline character (CRLF, LF or CR)                                                                             | ``CRLF`` by default    |
++----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
+| charset                    | enum     | Character encoding (eg. ISO-8859-1, UTF-8)                                                                     | ``UTF-8`` by default   |
++----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
+| columns                    | hash     | Columns (see below)                                                                                            | required               |
++----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
+
+The ``columns`` option declares the list of columns. This CSV parser plugin ignores the header line.
+
++----------+-------------------------------------------------+
+| name     | description                                     |
++==========+=================================================+
+| name     | Name of the column                              |
++----------+-------------------------------------------------+
+| type     | Type of the column (see below)                  |
++----------+-------------------------------------------------+
+| format   | Format of the timestamp if type is timestamp    |
++----------+-------------------------------------------------+
+
+List of types:
+
++-------------+----------------------------------------------+
+| name        | description                                  |
++=============+==============================================+
+| boolean     | true or false                                |
++-------------+----------------------------------------------+
+| long        | 64-bit signed integers                       |
++-------------+----------------------------------------------+
+| timestamp   | Date and time with nano-seconds precision    |
++-------------+----------------------------------------------+
+| double      | 64-bit floating point numbers                |
++-------------+----------------------------------------------+
+| string      | Strings                                      |
++-------------+----------------------------------------------+
+
+You can use ``guess`` to automatically generate the column settings. See also `Quick Start <https://github.com/embulk/embulk#quick-start>`_.
+
+Example
+~~~~~~~~~~~~~~~~~~
+
+.. code-block:: yaml
+
+    in:
+      ...
+      parser:
+        type: csv
+        charset: UTF-8
+        newline: CRLF
+        delimiter: "\t"
+        quote: '"'
+        escape: '"'
+        null_string: 'NULL'
+        skip_header_lines: 1
+        comment_line_marker: '#'
+        columns:
+        - {name: id, type: long}
+        - {name: account, type: long}
+        - {name: time, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
+        - {name: purchase, type: timestamp, format: '%Y%m%d'}
+        - {name: comment, type: string}
+
+Gzip decoder plugin
+------------------
+
+The ``gzip`` decoder plugin decompresses gzip files before input plugins read them.
+
+Options
+~~~~~~~~~~~~~~~~~~
+
+This plugin doesn't have any options.
+
+Example
+~~~~~~~~~~~~~~~~~~
+
+.. code-block:: yaml
+
+    in:
+      ...
+      decoders:
+      - {type: gzip}
+
+
+File output plugin
+------------------
+
+The ``file`` output plugin writes records to local file system.
+
+Options
+~~~~~~~~~~~~~~~~~~
+
++--------------------+----------+---------------------------------------------------+----------------------------+
+| name               | type     | description                                       | required?                  |
++====================+==========+===================================================+============================+
+| path\_prefix       | string   | Path prefix of the output files                   | required                   |
++--------------------+----------+---------------------------------------------------+----------------------------+
+| sequence\_format   | string   | Format of the sequence number of the output files | ``%03d.%02d.`` by default  |
++--------------------+----------+---------------------------------------------------+----------------------------+
+| file\_ext          | string   | Path suffix of the output files (e.g. ``"csv"``)  | required                   |
++--------------------+----------+---------------------------------------------------+----------------------------+
+
+For example, if you set ``path_prefix: /path/to/output/sample_``, ``sequence_format: "%03d.%02d."``, and ``file_ext: csv``, name of the output files will be as following:
+
+::
+
+    .
+    `-- path
+        `-- to
+            `-- output
+                |-- sample_01.000.csv
+                |-- sample_02.000.csv
+                |-- sample_03.000.csv
+                |-- sample_04.000.csv
+
+``sequence_format`` formats task index and sequence number in a task.
+
+Example
+~~~~~~~~~~~~~~~~~~
+
+.. code-block:: yaml
+
+    out:
+      type: file
+      path_prefix: /path/to/output/sample_
+      file_ext: csv
+      formatter:
+        ...
+
+CSV formatter plugin
+------------------
+
+The ``csv`` formatter plugin formats records using CSV or TSV format.
+
+Options
+~~~~~~~~~~~~~~~~~~
+
++----------------------+---------+-------------------------------------------------------------------------------------------------------+------------------------+
+| name                 | type    | description                                                                                           | required?              |
++======================+=========+=======================================================================================================+========================+
+| delimiter            | string  | Delimiter character such as ``,`` for CSV, ``"\t"`` for TSV, ``"|"`` or any single-byte character     | ``,`` by default       |
++----------------------+---------+-------------------------------------------------------------------------------------------------------+------------------------+
+| quote                | string  | The character surrounding a quoted value                                                              | ``\"`` by default      |
++----------------------+---------+-------------------------------------------------------------------------------------------------------+------------------------+
+| quote\_policy        | enum    | Policy for quote (ALL, MINIMAL, NONE) (see below)                                                     | ``MINIMAL`` by default |
++----------------------+---------+-------------------------------------------------------------------------------------------------------+------------------------+
+| escape               | string  | Escape character to escape a quote character when quote\_policy is ALL or MINIMAL                     | ``\"`` by default      |
++----------------------+---------+-------------------------------------------------------------------------------------------------------+------------------------+
+| header\_line         | boolean | If true, write the header line with column name at the first line                                     | ``true`` by default    |
++----------------------+---------+-------------------------------------------------------------------------------------------------------+------------------------+
+| null_string          | string  | Expression of NULL values                                                                             | empty by default       |
++----------------------+---------+-------------------------------------------------------------------------------------------------------+------------------------+
+| newline              | enum    | Newline character (CRLF, LF or CR)                                                                    | ``CRLF`` by default    |
++----------------------+---------+-------------------------------------------------------------------------------------------------------+------------------------+
+| newline\_in\_field   | enum    | Newline character in each field (CRLF, LF, CR)                                                        | ``LF`` by default      |
++----------------------+---------+-------------------------------------------------------------------------------------------------------+------------------------+
+| charset              | enum    | Character encoding (eg. ISO-8859-1, UTF-8)                                                            | ``UTF-8`` by default   |
++----------------------+---------+-------------------------------------------------------------------------------------------------------+------------------------+
+| default\_timezone    | string  | Time zone of timestamp columns. This can be overwritten for each column using ``column_options``      | ``UTC`` by default     |
++----------------------+---------+-------------------------------------------------------------------------------------------------------+------------------------+
+| column\_options      | hash    | See bellow                                                                                            | optional               |
++----------------------+---------+-------------------------------------------------------------------------------------------------------+------------------------+
+
+The ``quote_policy`` option is used to determine field type to quote.
+
++------------+--------------------------------------------------------------------------------------------------------+
+| name       | description                                                                                            |
++============+========================================================================================================+
+| ALL        | Quote all fields                                                                                       |
++------------+--------------------------------------------------------------------------------------------------------+
+| MINIMAL    | Only quote those fields which contain delimiter, quote or any of the characters in lineterminator      |
++------------+--------------------------------------------------------------------------------------------------------+
+| NONE       | Never quote fields. When the delimiter occurs in field, escape with escape char                        |
++------------+--------------------------------------------------------------------------------------------------------+
+
+The ``column_options`` option is a map whose keys are name of columns, and values are configuration with following parameters:
+
++----------------------+---------+-------------------------------------------------------------------------------------------------------+-----------------------------------------+
+| name                 | type    | description                                                                                           | required?                               |
++======================+=========+=======================================================================================================+=========================================+
+| timezone             | string  | Time zone if type of this column is timestamp. If not set, ``default\_timezone`` is used.             | optional                                |
++----------------------+---------+-------------------------------------------------------------------------------------------------------+-----------------------------------------+
+| format               | string  | Timestamp format if type of this column is timestamp.                                                 | ``%Y-%m-%d %H:%M:%S.%6N %z`` by default |
++----------------------+---------+-------------------------------------------------------------------------------------------------------+-----------------------------------------+
+
+Example
+~~~~~~~~~~~~~~~~~~
+
+.. code-block:: yaml
+
+    out:
+      ...
+      formatter:
+        type: csv
+        delimiter: '\t'
+        newline: CRLF
+        newline_in_field: LF
+        charset: UTF-8
+        quote_policy: MINIMAL
+        quote: '"'
+        escape: '\\'
+        null_string: '\\N'
+        default_timezone: 'UTC'
+        column_options:
+          mycol1: {format: '%Y-%m-%d %H:%M:%S'}
+          mycol2: {format: '%Y-%m-%d %H:%M:%S', timezone: 'America/Los_Angeles'}
+
+Gzip encoder plugin
+------------------
+
+The ``gzip`` encoder plugin compresses output files using gzip.
+
+Options
+~~~~~~~~~~~~~~~~~~
+
++---------+----------+----------------------------------------------------------------------+--------------------+
+| name    | type     | description                                                          | required?          |
++=========+==========+======================================================================+====================+
+| level   | integer  | Compression level. From 0 (no compression) to 9 (best compression).  | ``6`` by default   |
++---------+----------+----------------------------------------------------------------------+--------------------+
+
+Example
+~~~~~~~~~~~~~~~~~~
+
+.. code-block:: yaml
+
+    out:
+      ...
+      encoders:
+      - type: gzip
+        level: 1
+
+Rename filter plugin
+------------------
+
+The ``rename`` filter plugin changes column names. This plugin has no impact on performance.
+
+Options
+~~~~~~~~~~~~~~~~~~
+
++---------+----------+----------------------------------------------------------------------+--------------------+
+| name    | type     | description                                                          | required?          |
++=========+==========+======================================================================+====================+
+| columns | hash     | A map whose keys are existing column names. values are new names.    | ``{}`` by default  |
++---------+----------+----------------------------------------------------------------------+--------------------+
+
+Example
+~~~~~~~~~~~~~~~~~~
+
+.. code-block:: yaml
+
+    filters:
+      ...
+      - type: rename
+	columns:
+	  my_existing_column1: new_column1
+	  my_existing_column2: new_column2
+