RubyGems - rroonga - Versions diffs - 4.0.6-x86-mingw32 → 4.0.7-x86-mingw32 - Mend

rroonga 4.0.6-x86-mingw32 → 4.0.7-x86-mingw32

Files changed (598) hide show

data/vendor/local/share/doc/groonga/en/html/reference/commands/table_tokenize.html ADDED Viewed

@@ -0,0 +1,276 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+  <head>
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+    <title>7.3.33. table_tokenize &mdash; Groonga v4.0.8 documentation</title>
+    <link rel="stylesheet" href="../../_static/groonga.css" type="text/css" />
+    <link rel="stylesheet" href="../../_static/pygments.css" type="text/css" />
+    <script type="text/javascript">
+      var DOCUMENTATION_OPTIONS = {
+        URL_ROOT:    '../../',
+        VERSION:     '4.0.8',
+        COLLAPSE_INDEX: false,
+        FILE_SUFFIX: '.html',
+        HAS_SOURCE:  true
+      };
+    </script>
+    <script type="text/javascript" src="../../_static/jquery.js"></script>
+    <script type="text/javascript" src="../../_static/underscore.js"></script>
+    <script type="text/javascript" src="../../_static/doctools.js"></script>
+    <link rel="shortcut icon" href="../../_static/favicon.ico"/>
+    <link rel="top" title="Groonga v4.0.8 documentation" href="../../index.html" />
+    <link rel="up" title="7.3. Command" href="../command.html" />
+    <link rel="next" title="7.3.34. tokenize" href="tokenize.html" />
+    <link rel="prev" title="7.3.32. table_remove" href="table_remove.html" />
+  </head>
+  <body role="document">
+<div class="header">
+  <h1 class="title">
+    <a id="top-link" href="../../index.html">
+      <span class="project">groonga</span>
+      <span class="separator">-</span>
+      <span class="description">An open-source fulltext search engine and column store.</span>
+    </a>
+  </h1>
+  <div class="other-language-links">
+    <ul>
+      <li><a href="../../../../ja/html/reference/commands/table_tokenize.html"><img src="../../_static/jp.png" alt="日本語">日本語版はこちら</a></li>
+    </ul>
+  </div>
+</div>
+    <div class="related" role="navigation" aria-label="related navigation">
+      <h3>Navigation</h3>
+      <ul>
+        <li class="right" style="margin-right: 10px">
+          <a href="../../genindex.html" title="General Index"
+             accesskey="I">index</a></li>
+        <li class="right" >
+          <a href="tokenize.html" title="7.3.34. tokenize"
+             accesskey="N">next</a> |</li>
+        <li class="right" >
+          <a href="table_remove.html" title="7.3.32. table_remove"
+             accesskey="P">previous</a> |</li>
+        <li><a href="../../index.html">Groonga v4.0.8 documentation</a> &raquo;</li>
+          <li><a href="../../reference.html" >7. Reference manual</a> &raquo;</li>
+          <li><a href="../command.html" accesskey="U">7.3. Command</a> &raquo;</li>
+      </ul>
+    </div>
+    <div class="document">
+      <div class="documentwrapper">
+        <div class="bodywrapper">
+          <div class="body" role="main">
+  <div class="section" id="table-tokenize">
+<h1>7.3.33. <code class="docutils literal"><span class="pre">table_tokenize</span></code><a class="headerlink" href="#table-tokenize" title="Permalink to this headline">¶</a></h1>
+<div class="section" id="summary">
+<h2>7.3.33.1. Summary<a class="headerlink" href="#summary" title="Permalink to this headline">¶</a></h2>
+<p><code class="docutils literal"><span class="pre">table_tokenize</span></code> command tokenizes text by the specified table's tokenizer.</p>
+</div>
+<div class="section" id="syntax">
+<h2>7.3.33.2. Syntax<a class="headerlink" href="#syntax" title="Permalink to this headline">¶</a></h2>
+<p><code class="docutils literal"><span class="pre">table_tokenize</span></code> command has required parameters and optional parameters.
+<code class="docutils literal"><span class="pre">table</span></code> and <code class="docutils literal"><span class="pre">string</span></code> are required parameters. Others are
+optional:</p>
+<div class="highlight-none"><div class="highlight"><pre>table_tokenize table
+               string
+               [flags=NONE]
+               [mode=GET]
+</pre></div>
+</div>
+</div>
+<div class="section" id="usage">
+<h2>7.3.33.3. Usage<a class="headerlink" href="#usage" title="Permalink to this headline">¶</a></h2>
+<p>Here is a simple example.</p>
+<p>Execution example:</p>
+<div class="highlight-none"><div class="highlight"><pre>register token_filters/stop_word
+# [[0,0.0,0.0],true]
+table_create Terms TABLE_PAT_KEY ShortText \
+  --default_tokenizer TokenBigram \
+  --normalizer NormalizerAuto \
+  --token_filters TokenFilterStopWord
+# [[0,0.0,0.0],true]
+column_create Terms is_stop_word COLUMN_SCALAR Bool
+# [[0,0.0,0.0],true]
+load --table Terms
+[
+{&quot;_key&quot;: &quot;and&quot;, &quot;is_stop_word&quot;: true}
+]
+# [[0,0.0,0.0],1]
+table_tokenize Terms &quot;Hello and Good-bye&quot; --mode GET
+# [
+#  [
+#    0,
+#    0.0,
+#    0.0
+#  ],
+#  [
+#    {
+#      &quot;value&quot;: &quot;hello&quot;,
+#      &quot;position&quot;: 0
+#    },
+#    {
+#      &quot;value&quot;: &quot;good&quot;,
+#      &quot;position&quot;: 2
+#    },
+#    {
+#      &quot;value&quot;: &quot;-&quot;,
+#      &quot;position&quot;: 3
+#    },
+#    {
+#      &quot;value&quot;: &quot;bye&quot;,
+#      &quot;position&quot;: 4
+#    }
+#  ]
+# ]
+</pre></div>
+</div>
+<p><code class="docutils literal"><span class="pre">Terms</span></code> table is set <code class="docutils literal"><span class="pre">TokenBigram</span></code> tokenizer, <code class="docutils literal"><span class="pre">NormalizerAuto</span></code> normalizer,
+<code class="docutils literal"><span class="pre">TokenFilterStopWord</span></code> token filter. It returns tokens that is
+generated by tokenizeing <code class="docutils literal"><span class="pre">&quot;Hello</span> <span class="pre">and</span> <span class="pre">Good-bye&quot;</span></code> with <code class="docutils literal"><span class="pre">TokenBigram</span></code> tokenizer.
+It is normalized by <code class="docutils literal"><span class="pre">NormalizerAuto</span></code> normalizer.
+<code class="docutils literal"><span class="pre">and</span></code> token is removed with <code class="docutils literal"><span class="pre">TokenFilterStopWord</span></code> token filter.</p>
+</div>
+<div class="section" id="parameters">
+<h2>7.3.33.4. Parameters<a class="headerlink" href="#parameters" title="Permalink to this headline">¶</a></h2>
+<p>This section describes all parameters. Parameters are categorized.</p>
+<div class="section" id="required-parameters">
+<h3>7.3.33.4.1. Required parameters<a class="headerlink" href="#required-parameters" title="Permalink to this headline">¶</a></h3>
+<p>There are required parameters, <code class="docutils literal"><span class="pre">table</span></code> and <code class="docutils literal"><span class="pre">string</span></code>.</p>
+<div class="section" id="table">
+<h4>7.3.33.4.1.1. <code class="docutils literal"><span class="pre">table</span></code><a class="headerlink" href="#table" title="Permalink to this headline">¶</a></h4>
+<p>It specifies the lexicon table. <code class="docutils literal"><span class="pre">table_tokenize</span></code> command uses the
+tokenizer, the normalizer, the token filters that is set the
+lexicon table.</p>
+</div>
+<div class="section" id="string">
+<h4>7.3.33.4.1.2. <code class="docutils literal"><span class="pre">string</span></code><a class="headerlink" href="#string" title="Permalink to this headline">¶</a></h4>
+<p>It specifies any string which you want to tokenize.</p>
+<p>See <a class="reference internal" href="tokenize.html#tokenize-string"><em>string</em></a> option in <a class="reference internal" href="tokenize.html"><em>tokenize</em></a> about details.</p>
+</div>
+</div>
+<div class="section" id="optional-parameters">
+<h3>7.3.33.4.2. Optional parameters<a class="headerlink" href="#optional-parameters" title="Permalink to this headline">¶</a></h3>
+<p>There are optional parameters.</p>
+<div class="section" id="flags">
+<h4>7.3.33.4.2.1. <code class="docutils literal"><span class="pre">flags</span></code><a class="headerlink" href="#flags" title="Permalink to this headline">¶</a></h4>
+<p>It specifies a tokenization customize options. You can specify
+multiple options separated by &quot;<code class="docutils literal"><span class="pre">|</span></code>&quot;.</p>
+<p>The default value is <code class="docutils literal"><span class="pre">NONE</span></code>.</p>
+<p>See <a class="reference internal" href="tokenize.html#tokenize-flags"><em>flags</em></a> option in <a class="reference internal" href="tokenize.html"><em>tokenize</em></a> about details.</p>
+</div>
+<div class="section" id="mode">
+<h4>7.3.33.4.2.2. <code class="docutils literal"><span class="pre">mode</span></code><a class="headerlink" href="#mode" title="Permalink to this headline">¶</a></h4>
+<p>It specifies a tokenize mode.</p>
+<p>The default value is <code class="docutils literal"><span class="pre">GET</span></code>.</p>
+<p>See <a class="reference internal" href="tokenize.html#tokenize-mode"><em>mode</em></a> option in <a class="reference internal" href="tokenize.html"><em>tokenize</em></a> about details.</p>
+</div>
+</div>
+</div>
+<div class="section" id="return-value">
+<h2>7.3.33.5. Return value<a class="headerlink" href="#return-value" title="Permalink to this headline">¶</a></h2>
+<p><code class="docutils literal"><span class="pre">table_tokenize</span></code> command returns tokenized tokens.</p>
+<p>See <a class="reference internal" href="tokenize.html#tokenize-return-value"><em>Return value</em></a> option in <a class="reference internal" href="tokenize.html"><em>tokenize</em></a> about details.</p>
+</div>
+<div class="section" id="see-also">
+<h2>7.3.33.6. See also<a class="headerlink" href="#see-also" title="Permalink to this headline">¶</a></h2>
+<ul class="simple">
+<li><a class="reference internal" href="../tokenizers.html"><em>Tokenizers</em></a></li>
+<li><a class="reference internal" href="tokenize.html"><em>tokenize</em></a></li>
+</ul>
+</div>
+</div>
+          </div>
+        </div>
+      </div>
+      <div class="sphinxsidebar" role="navigation" aria-label="main navigation">
+        <div class="sphinxsidebarwrapper">
+  <h3><a href="../../index.html">Table Of Contents</a></h3>
+  <ul>
+<li><a class="reference internal" href="#">7.3.33. <code class="docutils literal"><span class="pre">table_tokenize</span></code></a><ul>
+<li><a class="reference internal" href="#summary">7.3.33.1. Summary</a></li>
+<li><a class="reference internal" href="#syntax">7.3.33.2. Syntax</a></li>
+<li><a class="reference internal" href="#usage">7.3.33.3. Usage</a></li>
+<li><a class="reference internal" href="#parameters">7.3.33.4. Parameters</a><ul>
+<li><a class="reference internal" href="#required-parameters">7.3.33.4.1. Required parameters</a><ul>
+<li><a class="reference internal" href="#table">7.3.33.4.1.1. <code class="docutils literal"><span class="pre">table</span></code></a></li>
+<li><a class="reference internal" href="#string">7.3.33.4.1.2. <code class="docutils literal"><span class="pre">string</span></code></a></li>
+</ul>
+</li>
+<li><a class="reference internal" href="#optional-parameters">7.3.33.4.2. Optional parameters</a><ul>
+<li><a class="reference internal" href="#flags">7.3.33.4.2.1. <code class="docutils literal"><span class="pre">flags</span></code></a></li>
+<li><a class="reference internal" href="#mode">7.3.33.4.2.2. <code class="docutils literal"><span class="pre">mode</span></code></a></li>
+</ul>
+</li>
+</ul>
+</li>
+<li><a class="reference internal" href="#return-value">7.3.33.5. Return value</a></li>
+<li><a class="reference internal" href="#see-also">7.3.33.6. See also</a></li>
+</ul>
+</li>
+</ul>
+  <h4>Previous topic</h4>
+  <p class="topless"><a href="table_remove.html"
+                        title="previous chapter">7.3.32. <code class="docutils literal"><span class="pre">table_remove</span></code></a></p>
+  <h4>Next topic</h4>
+  <p class="topless"><a href="tokenize.html"
+                        title="next chapter">7.3.34. <code class="docutils literal"><span class="pre">tokenize</span></code></a></p>
+  <div role="note" aria-label="source link">
+    <h3>This Page</h3>
+    <ul class="this-page-menu">
+      <li><a href="../../_sources/reference/commands/table_tokenize.txt"
+            rel="nofollow">Show Source</a></li>
+    </ul>
+   </div>
+<div id="searchbox" style="display: none" role="search">
+  <h3>Quick search</h3>
+    <form class="search" action="../../search.html" method="get">
+      <input type="text" name="q" />
+      <input type="submit" value="Go" />
+      <input type="hidden" name="check_keywords" value="yes" />
+      <input type="hidden" name="area" value="default" />
+    </form>
+    <p class="searchtip" style="font-size: 90%">
+    Enter search terms or a module, class or function name.
+    </p>
+</div>
+<script type="text/javascript">$('#searchbox').show(0);</script>
+        </div>
+      </div>
+      <div class="clearer"></div>
+    </div>
+    <div class="related" role="navigation" aria-label="related navigation">
+      <h3>Navigation</h3>
+      <ul>
+        <li class="right" style="margin-right: 10px">
+          <a href="../../genindex.html" title="General Index"
+             >index</a></li>
+        <li class="right" >
+          <a href="tokenize.html" title="7.3.34. tokenize"
+             >next</a> |</li>
+        <li class="right" >
+          <a href="table_remove.html" title="7.3.32. table_remove"
+             >previous</a> |</li>
+        <li><a href="../../index.html">Groonga v4.0.8 documentation</a> &raquo;</li>
+          <li><a href="../../reference.html" >7. Reference manual</a> &raquo;</li>
+          <li><a href="../command.html" >7.3. Command</a> &raquo;</li>
+      </ul>
+    </div>
+    <div class="footer" role="contentinfo">
+        &copy; Copyright 2009-2014, Brazil, Inc.
+    </div>
+  </body>
+</html>