embulk 0.7.0-java

data/embulk-docs/src/conf.py

@@ -0,0 +1,260 @@
+# -*- coding: utf-8 -*-
+#
+# Embulk documentation build configuration file, created by
+# sphinx-quickstart on Fri Feb 13 14:52:36 2015.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+import sys
+import os
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#sys.path.insert(0, os.path.abspath('.'))
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinx.ext.autodoc',
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix of source filenames.
+source_suffix = '.rst'
+
+# The encoding of source files.
+#source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'Embulk'
+copyright = u'2015, Embulk Project'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = '0.7'
+# The full version, including alpha/beta/rc tags.
+release = '0.7'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#today = ''
+# Else, today_fmt is used as the format for a strftime call.
+#today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = []
+
+# The reST default role (used for this markup: `text`) to use for all
+# documents.
+#default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# A list of ignored prefixes for module index sorting.
+#modindex_common_prefix = []
+
+# If true, keep warnings as "system message" paragraphs in the built documents.
+#keep_warnings = False
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+html_theme = 'haiku'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+#html_theme_path = []
+
+# The name for this set of Sphinx documents.  If None, it defaults to
+# "<project> v<release> documentation".
+#html_title = None
+
+# A shorter title for the navigation bar.  Default is the same as html_title.
+#html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+#html_logo = None
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+#html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Add any extra paths that contain custom files (such as robots.txt or
+# .htaccess) here, relative to this directory. These files are copied
+# directly to the root of the documentation.
+#html_extra_path = []
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+#html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+#html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#html_additional_pages = {}
+
+# If false, no module index is generated.
+#html_domain_indices = True
+
+# If false, no index is generated.
+#html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+#html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+#html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+#html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+#html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it.  The value of this option must be the
+# base URL from which the finished HTML is served.
+#html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+#html_file_suffix = None
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'Embulkdoc'
+
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+# The paper size ('letterpaper' or 'a4paper').
+#'papersize': 'letterpaper',
+
+# The font size ('10pt', '11pt' or '12pt').
+#'pointsize': '10pt',
+
+# Additional stuff for the LaTeX preamble.
+#'preamble': '',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+  ('index', 'Embulk.tex', u'Embulk Documentation',
+   u'Embulk Project', 'manual'),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+#latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+#latex_use_parts = False
+
+# If true, show page references after internal links.
+#latex_show_pagerefs = False
+
+# If true, show URL addresses after external links.
+#latex_show_urls = False
+
+# Documents to append as an appendix to all manuals.
+#latex_appendices = []
+
+# If false, no module index is generated.
+#latex_domain_indices = True
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    ('index', 'embulk', u'Embulk Documentation',
+     [u'Embulk Project'], 1)
+]
+
+# If true, show URL addresses after external links.
+#man_show_urls = False
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+  ('index', 'Embulk', u'Embulk Documentation',
+   u'Embulk Project', 'Embulk', 'One line description of project.',
+   'Miscellaneous'),
+]
+
+# Documents to append as an appendix to all manuals.
+#texinfo_appendices = []
+
+# If false, no module index is generated.
+#texinfo_domain_indices = True
+
+# How to display URL addresses: 'footnote', 'no', or 'inline'.
+#texinfo_show_urls = 'footnote'
+
+# If true, do not generate a @detailmenu in the "Top" node's menu.
+#texinfo_no_detailmenu = False

data/embulk-docs/src/customization.rst

@@ -0,0 +1,184 @@
+Customization
+==================================
+
+.. contents::
+   :local:
+   :depth: 2
+
+
+Creating plugins
+------------------
+
+Creating a new plugin is 4 steps:
+
+1. Create a new project using a template generator
+2. Build the project (Java only)
+3. Confirm it works
+4. Modify the code as you need
+
+This article describes how to create a plugin step by step.
+
+Step 1: Creating a new project
+~~~~~~~~~~~~~~~~~~
+
+Embulk comes with a number of templates that generates a new project so that you can start development instantly. Because the generated project contains completely working code without additional code, you can focus on the necessary coding.
+
+Usage of the generator is where ``<type>`` is one of the plugin templates:
+
+::
+
+    $ embulk new <type> <name>
+
+Here is the list of available templates. Please find the best option:
+
+====================  ===============================  =================
+Type                  description                      example
+====================  ===============================  =================
+**java-input**        Java record input plugin         ``mysql``
+**java-output**       Java record output plugin        ``mysql``
+**java-filter**       Java record filter plugin        ``add-hostname``
+**java-file-input**   Java file input plugin           ``ftp``
+**java-file-output**  Java file output plugin          ``ftp``
+**java-parser**       Java file parser plugin          ``csv``
+**java-formatter**    Java file formatter plugin       ``csv``
+**java-decoder**      Java file decoder plugin         ``gzip``
+**java-encoder**      Java file encoder plugin         ``gzip``
+**ruby-input**        Ruby record input plugin         ``mysql``
+**ruby-output**       Ruby record output plugin        ``mysql``
+**ruby-filter**       Ruby record filter plugin        ``add-hostname``
+**ruby-parser**       Ruby file parser plugin          ``csv``
+**ruby-formatter**    Ruby file formatter plugin       ``csv```
+====================  ===============================  =================
+
+For example, if you want to parse a new file format using Java, type this command:
+
+::
+
+    $ embulk new java-parser myformat
+
+This will create a Java-based parser plugin called ``myformat`` in ``embulk-parser-myformat`` directory.
+
+Step 2: Building the project
+~~~~~~~~~~~~~~~~~~
+
+If the plugin is Java-based, you need to build the project. To build, type this command:
+
+::
+
+    $ cd embulk-parser-myformat
+    $ ./gradlew package
+
+Now, the plugin is ready to use.
+
+Step 3: Confirm it works
+~~~~~~~~~~~~~~~~~~
+
+The next step is to actually use the plugin.
+
+Let's suppose you have a configuration file named ``your-config.yml``. You can use the plugin using embulk with ``-L`` argument:
+
+::
+
+    $ embulk run -L ./embulk-parser-myformat/ your-config.yml
+
+Step 4: Modifying the code
+~~~~~~~~~~~~~~~~~~
+
+The final step is to modify code as you want!
+
+The code is located at
+
+* Java-based plugins
+
+  * src/org/embulk/*
+
+* Ruby-based plugins
+
+  * lib/embulk/*
+
+There are a lot of good code examples on Github. Search repositories by `embulk-<type> keyword <https://github.com/search?q=embulk-output>`_.
+
+Releasing plugins
+------------------
+
+You can release publicly so that all people can use your awesome plugins.
+
+Checking plugin description
+~~~~~~~~~~~~~~~~~~
+
+To prepare the plugin ready to release, you need to include some additional information. The plugin information is written in this file:
+
+* Java-based plugins
+
+  * ``build.gradle`` file
+
+* Ruby-based plugins
+
+  * ``embulk-<type>-<name>.gemspec`` file (``<type>`` is plugin type and ``<name>`` is plugin name)
+
+You will find following section in the file.
+
+.. code-block:: ruby
+
+    Gem::Specification.new do |spec|
+        # ...
+
+        spec.authors       = ["Your Name"]
+        spec.summary       = %[Myformat parser plugin for Embulk]
+        spec.description   = %[Parses Myformat files read by other file input plugins.]
+        spec.email         = ["you@example.org"]
+        spec.licenses      = ["MIT"]
+        spec.homepage      = "https://github.com/frsyuki/embulk-parser-myformat"
+
+        # ...
+    end
+
+The items in above example are important. Please make sure that they are good.
+
+Creating account on RubyGems.org
+~~~~~~~~~~~~~~~~~~
+
+Embulk uses `RubyGems.org <https://rubygems.org/>`_ as a package distribution service. Please create an account there to release plugins at `Sign Up <https://rubygems.org/sign_up>`_ page.
+
+Don't forget the password! It will be necessary at the next step.
+
+Releasing the plugin to RubyGems.org
+~~~~~~~~~~~~~~~~~~
+
+Now, you're ready to release the plugin. To release, type following command:
+
+* Java-based plugins
+
+  * ``$ ./gradlew gemPush``
+
+* Ruby-based plugins
+
+  * ``$ rake release``
+
+If everything is good, you can find your plugin at https://rubygems.org/. Congratulations!
+
+Installing your plugin
+~~~~~~~~~~~~~~~~~~
+
+Usage of plugin installer is:
+
+::
+
+    $ embulk gem install embulk-<type>-<name>
+
+``<type>`` is plugin type and ``<name>`` is plugin name.
+
+If your plugin is ``embulk-parser-myformat``, then type this command:
+
+::
+
+    $ embulk gem install embulk-parser-myformat
+
+This command installs the plugin to ``~/.embulk`` directory.
+
+To check the list of installed plugins and their versions, use this command:
+
+::
+
+    $ embulk gem list
+

data/embulk-docs/src/index.rst

@@ -0,0 +1,84 @@
+.. Embulk documentation master file, created by
+   sphinx-quickstart on Fri Feb 13 14:52:36 2015.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Embulk
+==================================
+
+.. image:: _static/embulk-logo.png
+   :width: 512px
+   :target: https://github.com/embulk/embulk
+
+What's Embulk?
+------------------
+
+Embulk is a open-source bulk data loader that helps data transfer between various databases, storages, file formats, and cloud services.
+
+Embulk supports:
+
+* Automatic guessing of input file formats
+* Parallel & distributed execution to deal with big data sets
+* Transaction control to guarantee All-or-Nothing
+* Resuming
+* Plugins released on RubyGems.org
+
+You can define a bulk data loading using combination of input and output plugins:
+
+.. image:: _static/embulk-architecture.png
+   :width: 640px
+   :target: https://github.com/embulk/embulk#quick-start
+
+For example, `this tutorial <recipe/scheduled-csv-load-to-elasticsearch-kibana4.html>`_ describes how to use **file** input plugin with **csv** parser plugin and **gzip** decoder plugin to read CSV files, and **elasticsearch** output plugin to load the records to Elasticsearch.
+
+Documents
+------------------
+
+* `Quick Start <https://github.com/embulk/embulk#quick-start>`_
+
+  * `Linux <https://github.com/embulk/embulk#linux--mac--bsd>`_
+
+  * `Mac OS X <https://github.com/embulk/embulk#linux--mac--bsd>`_
+
+  * `Windows <https://github.com/embulk/embulk#windows>`_
+
+.. toctree::
+   :maxdepth: 2
+
+   recipe
+
+* `List of Plugins by Category <http://www.embulk.org/plugins/>`_
+
+  * `Input plugins <http://www.embulk.org/plugins/#input>`_
+
+  * `Output plugins <http://www.embulk.org/plugins/#output>`_
+
+  * `File parser plugins <http://www.embulk.org/plugins/#file-parser>`_
+
+  * `File formatter plugins <http://www.embulk.org/plugins/#file-formatter>`_
+
+  * `Filter plugins <http://www.embulk.org/plugins/#filter>`_
+
+  * `Executor plugins <http://www.embulk.org/plugins/#executor>`_
+
+.. toctree::
+   :maxdepth: 3
+
+   built-in
+
+.. toctree::
+   :maxdepth: 3
+
+   customization
+
+* `JavaDoc <javadoc/index.html>`_
+
+* `RDoc <rdoc/_index.html>`_
+
+* `Github <https://github.com/embulk/embulk>`_
+
+.. toctree::
+   :maxdepth: 2
+
+   release
+