RubyGems - rroonga - Versions diffs - 4.0.6-x86-mingw32 → 4.0.7-x86-mingw32 - Mend

rroonga 4.0.6-x86-mingw32 → 4.0.7-x86-mingw32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (598) hide show

data/vendor/local/share/doc/groonga/en/html/reference/commands/tokenize.html CHANGED Viewed

@@ -7,7 +7,7 @@
   <head>
     <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
-    <title>7.3.33. tokenize &mdash; Groonga v4.0.6-211-g8c0bb0b documentation</title>
+    <title>7.3.34. tokenize &mdash; Groonga v4.0.8 documentation</title>
     <link rel="stylesheet" href="../../_static/groonga.css" type="text/css" />
     <link rel="stylesheet" href="../../_static/pygments.css" type="text/css" />
@@ -15,7 +15,7 @@
     <script type="text/javascript">
       var DOCUMENTATION_OPTIONS = {
         URL_ROOT:    '../../',
-        VERSION:     '4.0.6-211-g8c0bb0b',
+        VERSION:     '4.0.8',
         COLLAPSE_INDEX: false,
         FILE_SUFFIX: '.html',
         HAS_SOURCE:  true
@@ -25,12 +25,12 @@
     <script type="text/javascript" src="../../_static/underscore.js"></script>
     <script type="text/javascript" src="../../_static/doctools.js"></script>
     <link rel="shortcut icon" href="../../_static/favicon.ico"/>
-    <link rel="top" title="Groonga v4.0.6-211-g8c0bb0b documentation" href="../../index.html" />
+    <link rel="top" title="Groonga v4.0.8 documentation" href="../../index.html" />
     <link rel="up" title="7.3. Command" href="../command.html" />
-    <link rel="next" title="7.3.34. tokenizer_list" href="tokenizer_list.html" />
-    <link rel="prev" title="7.3.32. table_tokenize" href="table_tokenize.html" />
+    <link rel="next" title="7.3.35. tokenizer_list" href="tokenizer_list.html" />
+    <link rel="prev" title="7.3.33. table_tokenize" href="table_tokenize.html" />
   </head>
-  <body>
+  <body role="document">
 <div class="header">
   <h1 class="title">
     <a id="top-link" href="../../index.html">
@@ -48,19 +48,19 @@
 </div>
-    <div class="related">
+    <div class="related" role="navigation" aria-label="related navigation">
       <h3>Navigation</h3>
       <ul>
         <li class="right" style="margin-right: 10px">
           <a href="../../genindex.html" title="General Index"
              accesskey="I">index</a></li>
         <li class="right" >
-          <a href="tokenizer_list.html" title="7.3.34. tokenizer_list"
+          <a href="tokenizer_list.html" title="7.3.35. tokenizer_list"
              accesskey="N">next</a> |</li>
         <li class="right" >
-          <a href="table_tokenize.html" title="7.3.32. table_tokenize"
+          <a href="table_tokenize.html" title="7.3.33. table_tokenize"
              accesskey="P">previous</a> |</li>
-        <li><a href="../../index.html">Groonga v4.0.6-211-g8c0bb0b documentation</a> &raquo;</li>
+        <li><a href="../../index.html">Groonga v4.0.8 documentation</a> &raquo;</li>
           <li><a href="../../reference.html" >7. Reference manual</a> &raquo;</li>
           <li><a href="../command.html" accesskey="U">7.3. Command</a> &raquo;</li>
       </ul>
@@ -69,19 +69,19 @@
     <div class="document">
       <div class="documentwrapper">
         <div class="bodywrapper">
-          <div class="body">
+          <div class="body" role="main">
   <div class="section" id="tokenize">
-<h1>7.3.33. <tt class="docutils literal"><span class="pre">tokenize</span></tt><a class="headerlink" href="#tokenize" title="Permalink to this headline">¶</a></h1>
+<h1>7.3.34. <code class="docutils literal"><span class="pre">tokenize</span></code><a class="headerlink" href="#tokenize" title="Permalink to this headline">¶</a></h1>
 <div class="section" id="summary">
-<h2>7.3.33.1. Summary<a class="headerlink" href="#summary" title="Permalink to this headline">¶</a></h2>
-<p><tt class="docutils literal"><span class="pre">tokenize</span></tt> command tokenizes text by the specified tokenizer.
+<h2>7.3.34.1. Summary<a class="headerlink" href="#summary" title="Permalink to this headline">¶</a></h2>
+<p><code class="docutils literal"><span class="pre">tokenize</span></code> command tokenizes text by the specified tokenizer.
 It is useful to debug tokenization.</p>
 </div>
 <div class="section" id="syntax">
-<h2>7.3.33.2. Syntax<a class="headerlink" href="#syntax" title="Permalink to this headline">¶</a></h2>
-<p><tt class="docutils literal"><span class="pre">tokenize</span></tt> command has required parameters and optional parameters.
-<tt class="docutils literal"><span class="pre">tokenizer</span></tt> and <tt class="docutils literal"><span class="pre">string</span></tt> are required parameters. Others are
+<h2>7.3.34.2. Syntax<a class="headerlink" href="#syntax" title="Permalink to this headline">¶</a></h2>
+<p><code class="docutils literal"><span class="pre">tokenize</span></code> command has required parameters and optional parameters.
+<code class="docutils literal"><span class="pre">tokenizer</span></code> and <code class="docutils literal"><span class="pre">string</span></code> are required parameters. Others are
 optional:</p>
 <div class="highlight-none"><div class="highlight"><pre>tokenize tokenizer
          string
@@ -93,7 +93,7 @@ optional:</p>
 </div>
 </div>
 <div class="section" id="usage">
-<h2>7.3.33.3. Usage<a class="headerlink" href="#usage" title="Permalink to this headline">¶</a></h2>
+<h2>7.3.34.3. Usage<a class="headerlink" href="#usage" title="Permalink to this headline">¶</a></h2>
 <p>Here is a simple example.</p>
 <p>Execution example:</p>
 <div class="highlight-none"><div class="highlight"><pre>tokenize TokenBigram &quot;Fulltext Search&quot;
@@ -168,23 +168,23 @@ optional:</p>
 # ]
 </pre></div>
 </div>
-<p>It has only required parameters. <tt class="docutils literal"><span class="pre">tokenizer</span></tt> is <tt class="docutils literal"><span class="pre">TokenBigram</span></tt> and
-<tt class="docutils literal"><span class="pre">string</span></tt> is <tt class="docutils literal"><span class="pre">&quot;Fulltext</span> <span class="pre">Search&quot;</span></tt>. It returns tokens that is
-generated by tokenizing <tt class="docutils literal"><span class="pre">&quot;Fulltext</span> <span class="pre">Search&quot;</span></tt> with <tt class="docutils literal"><span class="pre">TokenBigram</span></tt>
-tokenizer. It doesn't normalize <tt class="docutils literal"><span class="pre">&quot;Fulltext</span> <span class="pre">Search&quot;</span></tt>.</p>
+<p>It has only required parameters. <code class="docutils literal"><span class="pre">tokenizer</span></code> is <code class="docutils literal"><span class="pre">TokenBigram</span></code> and
+<code class="docutils literal"><span class="pre">string</span></code> is <code class="docutils literal"><span class="pre">&quot;Fulltext</span> <span class="pre">Search&quot;</span></code>. It returns tokens that is
+generated by tokenizing <code class="docutils literal"><span class="pre">&quot;Fulltext</span> <span class="pre">Search&quot;</span></code> with <code class="docutils literal"><span class="pre">TokenBigram</span></code>
+tokenizer. It doesn't normalize <code class="docutils literal"><span class="pre">&quot;Fulltext</span> <span class="pre">Search&quot;</span></code>.</p>
 </div>
 <div class="section" id="parameters">
-<h2>7.3.33.4. Parameters<a class="headerlink" href="#parameters" title="Permalink to this headline">¶</a></h2>
+<h2>7.3.34.4. Parameters<a class="headerlink" href="#parameters" title="Permalink to this headline">¶</a></h2>
 <p>This section describes all parameters. Parameters are categorized.</p>
 <div class="section" id="required-parameters">
-<h3>7.3.33.4.1. Required parameters<a class="headerlink" href="#required-parameters" title="Permalink to this headline">¶</a></h3>
-<p>There are required parameters, <tt class="docutils literal"><span class="pre">tokenizer</span></tt> and <tt class="docutils literal"><span class="pre">string</span></tt>.</p>
+<h3>7.3.34.4.1. Required parameters<a class="headerlink" href="#required-parameters" title="Permalink to this headline">¶</a></h3>
+<p>There are required parameters, <code class="docutils literal"><span class="pre">tokenizer</span></code> and <code class="docutils literal"><span class="pre">string</span></code>.</p>
 <div class="section" id="tokenizer">
-<span id="tokenize-tokenizer"></span><h4>7.3.33.4.1.1. <tt class="docutils literal"><span class="pre">tokenizer</span></tt><a class="headerlink" href="#tokenizer" title="Permalink to this headline">¶</a></h4>
-<p>It specifies the tokenizer name. <tt class="docutils literal"><span class="pre">tokenize</span></tt> command uses the
-tokenizer that is named <tt class="docutils literal"><span class="pre">tokenizer</span></tt>.</p>
+<span id="tokenize-tokenizer"></span><h4>7.3.34.4.1.1. <code class="docutils literal"><span class="pre">tokenizer</span></code><a class="headerlink" href="#tokenizer" title="Permalink to this headline">¶</a></h4>
+<p>It specifies the tokenizer name. <code class="docutils literal"><span class="pre">tokenize</span></code> command uses the
+tokenizer that is named <code class="docutils literal"><span class="pre">tokenizer</span></code>.</p>
 <p>See <a class="reference internal" href="../tokenizers.html"><em>Tokenizers</em></a> about built-in tokenizers.</p>
-<p>Here is an example to use built-in <tt class="docutils literal"><span class="pre">TokenTrigram</span></tt> tokenizer.</p>
+<p>Here is an example to use built-in <code class="docutils literal"><span class="pre">TokenTrigram</span></code> tokenizer.</p>
 <p>Execution example:</p>
 <div class="highlight-none"><div class="highlight"><pre>tokenize TokenTrigram &quot;Fulltext Search&quot;
 # [
@@ -261,14 +261,14 @@ tokenizer that is named <tt class="docutils literal"><span class="pre">tokenizer
 <p>If you want to use other tokenizers, you need to register additional
 tokenizer plugin by <a class="reference internal" href="register.html"><em>register</em></a> command. For example, you can use
 <a class="reference external" href="http://www.phontron.com/kytea/">KyTea</a> based tokenizer by
-registering <tt class="docutils literal"><span class="pre">tokenizers/kytea</span></tt>.</p>
+registering <code class="docutils literal"><span class="pre">tokenizers/kytea</span></code>.</p>
 </div>
 <div class="section" id="string">
-<span id="tokenize-string"></span><h4>7.3.33.4.1.2. <tt class="docutils literal"><span class="pre">string</span></tt><a class="headerlink" href="#string" title="Permalink to this headline">¶</a></h4>
+<span id="tokenize-string"></span><h4>7.3.34.4.1.2. <code class="docutils literal"><span class="pre">string</span></code><a class="headerlink" href="#string" title="Permalink to this headline">¶</a></h4>
 <p>It specifies any string which you want to tokenize.</p>
-<p>If you want to include spaces in <tt class="docutils literal"><span class="pre">string</span></tt>, you need to quote
-<tt class="docutils literal"><span class="pre">string</span></tt> by single quotation (<tt class="docutils literal"><span class="pre">'</span></tt>) or double quotation (<tt class="docutils literal"><span class="pre">&quot;</span></tt>).</p>
-<p>Here is an example to use spaces in <tt class="docutils literal"><span class="pre">string</span></tt>.</p>
+<p>If you want to include spaces in <code class="docutils literal"><span class="pre">string</span></code>, you need to quote
+<code class="docutils literal"><span class="pre">string</span></code> by single quotation (<code class="docutils literal"><span class="pre">'</span></code>) or double quotation (<code class="docutils literal"><span class="pre">&quot;</span></code>).</p>
+<p>Here is an example to use spaces in <code class="docutils literal"><span class="pre">string</span></code>.</p>
 <p>Execution example:</p>
 <div class="highlight-none"><div class="highlight"><pre>tokenize TokenBigram &quot;Groonga is a fast fulltext earch engine!&quot;
 # [
@@ -445,13 +445,13 @@ registering <tt class="docutils literal"><span class="pre">tokenizers/kytea</spa
 </div>
 </div>
 <div class="section" id="optional-parameters">
-<h3>7.3.33.4.2. Optional parameters<a class="headerlink" href="#optional-parameters" title="Permalink to this headline">¶</a></h3>
+<h3>7.3.34.4.2. Optional parameters<a class="headerlink" href="#optional-parameters" title="Permalink to this headline">¶</a></h3>
 <p>There are optional parameters.</p>
 <div class="section" id="normalizer">
-<span id="tokenize-normalizer"></span><h4>7.3.33.4.2.1. <tt class="docutils literal"><span class="pre">normalizer</span></tt><a class="headerlink" href="#normalizer" title="Permalink to this headline">¶</a></h4>
-<p>It specifies the normalizer name. <tt class="docutils literal"><span class="pre">tokenize</span></tt> command uses the
-normalizer that is named <tt class="docutils literal"><span class="pre">normalizer</span></tt>. Normalizer is important for
-N-gram family tokenizers such as <tt class="docutils literal"><span class="pre">TokenBigram</span></tt>.</p>
+<span id="tokenize-normalizer"></span><h4>7.3.34.4.2.1. <code class="docutils literal"><span class="pre">normalizer</span></code><a class="headerlink" href="#normalizer" title="Permalink to this headline">¶</a></h4>
+<p>It specifies the normalizer name. <code class="docutils literal"><span class="pre">tokenize</span></code> command uses the
+normalizer that is named <code class="docutils literal"><span class="pre">normalizer</span></code>. Normalizer is important for
+N-gram family tokenizers such as <code class="docutils literal"><span class="pre">TokenBigram</span></code>.</p>
 <p>Normalizer detects character type for each character while
 normalizing. N-gram family tokenizers use character types while
 tokenizing.</p>
@@ -529,7 +529,7 @@ tokenizing.</p>
 # ]
 </pre></div>
 </div>
-<p>All alphabets are tokenized by two characters. For example, <tt class="docutils literal"><span class="pre">Fu</span></tt> is
+<p>All alphabets are tokenized by two characters. For example, <code class="docutils literal"><span class="pre">Fu</span></code> is
 a token.</p>
 <p>Here is an example that uses normalizer.</p>
 <p>Execution example:</p>
@@ -554,9 +554,9 @@ a token.</p>
 </pre></div>
 </div>
 <p>Continuous alphabets are tokenized as one token. For example,
-<tt class="docutils literal"><span class="pre">fulltext</span></tt> is a token.</p>
+<code class="docutils literal"><span class="pre">fulltext</span></code> is a token.</p>
 <p>If you want to tokenize by two characters with noramlizer, use
-<tt class="docutils literal"><span class="pre">TokenBigramSplitSymbolAlpha</span></tt>.</p>
+<code class="docutils literal"><span class="pre">TokenBigramSplitSymbolAlpha</span></code>.</p>
 <p>Execution example:</p>
 <div class="highlight-none"><div class="highlight"><pre>tokenize TokenBigramSplitSymbolAlpha &quot;Fulltext Search&quot; NormalizerAuto
 # [
@@ -627,13 +627,13 @@ a token.</p>
 </pre></div>
 </div>
 <p>All alphabets are tokenized by two characters. And they are normalized
-to lower case characters. For example, <tt class="docutils literal"><span class="pre">fu</span></tt> is a token.</p>
+to lower case characters. For example, <code class="docutils literal"><span class="pre">fu</span></code> is a token.</p>
 </div>
 <div class="section" id="flags">
-<span id="tokenize-flags"></span><h4>7.3.33.4.2.2. <tt class="docutils literal"><span class="pre">flags</span></tt><a class="headerlink" href="#flags" title="Permalink to this headline">¶</a></h4>
+<span id="tokenize-flags"></span><h4>7.3.34.4.2.2. <code class="docutils literal"><span class="pre">flags</span></code><a class="headerlink" href="#flags" title="Permalink to this headline">¶</a></h4>
 <p>It specifies a tokenization customize options. You can specify
-multiple options separated by &quot;<tt class="docutils literal"><span class="pre">|</span></tt>&quot;. For example,
-<tt class="docutils literal"><span class="pre">NONE|ENABLE_TOKENIZED_DELIMITER</span></tt>.</p>
+multiple options separated by &quot;<code class="docutils literal"><span class="pre">|</span></code>&quot;. For example,
+<code class="docutils literal"><span class="pre">NONE|ENABLE_TOKENIZED_DELIMITER</span></code>.</p>
 <p>Here are available flags.</p>
 <table border="1" class="docutils">
 <colgroup>
@@ -646,16 +646,16 @@ multiple options separated by &quot;<tt class="docutils literal"><span class="pr
 </tr>
 </thead>
 <tbody valign="top">
-<tr class="row-even"><td><tt class="docutils literal"><span class="pre">NONE</span></tt></td>
+<tr class="row-even"><td><code class="docutils literal"><span class="pre">NONE</span></code></td>
 <td>Just ignored.</td>
 </tr>
-<tr class="row-odd"><td><tt class="docutils literal"><span class="pre">ENABLE_TOKENIZED_DELIMITER</span></tt></td>
+<tr class="row-odd"><td><code class="docutils literal"><span class="pre">ENABLE_TOKENIZED_DELIMITER</span></code></td>
 <td>Enables tokenized delimiter. See <a class="reference internal" href="../tokenizers.html"><em>Tokenizers</em></a> about
 tokenized delimiter details.</td>
 </tr>
 </tbody>
 </table>
-<p>Here is an example that uses <tt class="docutils literal"><span class="pre">ENABLE_TOKENIZED_DELIMITER</span></tt>.</p>
+<p>Here is an example that uses <code class="docutils literal"><span class="pre">ENABLE_TOKENIZED_DELIMITER</span></code>.</p>
 <p>Execution example:</p>
 <div class="highlight-none"><div class="highlight"><pre>tokenize TokenDelimit &quot;Fulltext Seacrch&quot; NormalizerAuto ENABLE_TOKENIZED_DELIMITER
 # [
@@ -681,24 +681,24 @@ tokenized delimiter details.</td>
 # ]
 </pre></div>
 </div>
-<p><tt class="docutils literal"><span class="pre">TokenDelimit</span></tt> tokenizer is one of tokenized delimiter supported
-tokenizer. <tt class="docutils literal"><span class="pre">ENABLE_TOKENIZED_DELIMITER</span></tt> enables tokenized delimiter.
+<p><code class="docutils literal"><span class="pre">TokenDelimit</span></code> tokenizer is one of tokenized delimiter supported
+tokenizer. <code class="docutils literal"><span class="pre">ENABLE_TOKENIZED_DELIMITER</span></code> enables tokenized delimiter.
 Tokenized delimiter is special character that indicates token
-border. It is <tt class="docutils literal"><span class="pre">U+FFFE</span></tt>. The character is not assigned any
+border. It is <code class="docutils literal"><span class="pre">U+FFFE</span></code>. The character is not assigned any
 character. It means that the character is not appeared in normal
 string. So the character is good character for this puropose. If
-<tt class="docutils literal"><span class="pre">ENABLE_TOKENIZED_DELIMITER</span></tt> is enabled, the target string is
+<code class="docutils literal"><span class="pre">ENABLE_TOKENIZED_DELIMITER</span></code> is enabled, the target string is
 treated as already tokenized string. Tokenizer just tokenizes by
 tokenized delimiter.</p>
 </div>
 <div class="section" id="mode">
-<span id="tokenize-mode"></span><h4>7.3.33.4.2.3. <tt class="docutils literal"><span class="pre">mode</span></tt><a class="headerlink" href="#mode" title="Permalink to this headline">¶</a></h4>
-<p>It specifies a tokenize mode. If the mode is specified <tt class="docutils literal"><span class="pre">ADD</span></tt>, the text
+<span id="tokenize-mode"></span><h4>7.3.34.4.2.3. <code class="docutils literal"><span class="pre">mode</span></code><a class="headerlink" href="#mode" title="Permalink to this headline">¶</a></h4>
+<p>It specifies a tokenize mode. If the mode is specified <code class="docutils literal"><span class="pre">ADD</span></code>, the text
 is tokenized by the rule that adding a document. If the mode is specified
-<tt class="docutils literal"><span class="pre">GET</span></tt>, the text is tokenized by the rule that searching a document. If
-the mode is omitted, the text is tokenized by the <tt class="docutils literal"><span class="pre">ADD</span></tt> mode.</p>
-<p>The default mode is <tt class="docutils literal"><span class="pre">ADD</span></tt>.</p>
-<p>Here is an example to the <tt class="docutils literal"><span class="pre">ADD</span></tt> mode.</p>
+<code class="docutils literal"><span class="pre">GET</span></code>, the text is tokenized by the rule that searching a document. If
+the mode is omitted, the text is tokenized by the <code class="docutils literal"><span class="pre">ADD</span></code> mode.</p>
+<p>The default mode is <code class="docutils literal"><span class="pre">ADD</span></code>.</p>
+<p>Here is an example to the <code class="docutils literal"><span class="pre">ADD</span></code> mode.</p>
 <p>Execution example:</p>
 <div class="highlight-none"><div class="highlight"><pre>tokenize TokenBigram &quot;Fulltext Search&quot; --mode ADD
 # [
@@ -773,7 +773,7 @@ the mode is omitted, the text is tokenized by the <tt class="docutils literal"><
 </pre></div>
 </div>
 <p>The last alphabet is tokenized by one character.</p>
-<p>Here is an example to the <tt class="docutils literal"><span class="pre">GET</span></tt> mode.</p>
+<p>Here is an example to the <code class="docutils literal"><span class="pre">GET</span></code> mode.</p>
 <p>Execution example:</p>
 <div class="highlight-none"><div class="highlight"><pre>tokenize TokenBigram &quot;Fulltext Search&quot; --mode GET
 # [
@@ -846,27 +846,27 @@ the mode is omitted, the text is tokenized by the <tt class="docutils literal"><
 <p>The last alphabet is tokenized by two characters.</p>
 </div>
 <div class="section" id="token-filters">
-<span id="tokenize-token-filters"></span><h4>7.3.33.4.2.4. <tt class="docutils literal"><span class="pre">token_filters</span></tt><a class="headerlink" href="#token-filters" title="Permalink to this headline">¶</a></h4>
-<p>It specifies the token filter names. <tt class="docutils literal"><span class="pre">tokenize</span></tt> command uses the
-tokenizer that is named <tt class="docutils literal"><span class="pre">token_filters</span></tt>.</p>
+<span id="tokenize-token-filters"></span><h4>7.3.34.4.2.4. <code class="docutils literal"><span class="pre">token_filters</span></code><a class="headerlink" href="#token-filters" title="Permalink to this headline">¶</a></h4>
+<p>It specifies the token filter names. <code class="docutils literal"><span class="pre">tokenize</span></code> command uses the
+tokenizer that is named <code class="docutils literal"><span class="pre">token_filters</span></code>.</p>
 <p>See <a class="reference internal" href="../token_filters.html"><em>Token filters</em></a> about token filters.</p>
 </div>
 </div>
 </div>
 <div class="section" id="return-value">
-<span id="tokenize-return-value"></span><h2>7.3.33.5. Return value<a class="headerlink" href="#return-value" title="Permalink to this headline">¶</a></h2>
-<p><tt class="docutils literal"><span class="pre">tokenize</span></tt> command returns tokenized tokens. Each token has some
+<span id="tokenize-return-value"></span><h2>7.3.34.5. Return value<a class="headerlink" href="#return-value" title="Permalink to this headline">¶</a></h2>
+<p><code class="docutils literal"><span class="pre">tokenize</span></code> command returns tokenized tokens. Each token has some
 attributes except token itself. The attributes will be increased in
 the feature:</p>
 <div class="highlight-none"><div class="highlight"><pre>[HEADER, tokens]
 </pre></div>
 </div>
-<p><tt class="docutils literal"><span class="pre">HEADER</span></tt></p>
+<p><code class="docutils literal"><span class="pre">HEADER</span></code></p>
 <blockquote>
-<div>See <a class="reference internal" href="../command/output_format.html"><em>Output format</em></a> about <tt class="docutils literal"><span class="pre">HEADER</span></tt>.</div></blockquote>
-<p><tt class="docutils literal"><span class="pre">tokens</span></tt></p>
+<div>See <a class="reference internal" href="../command/output_format.html"><em>Output format</em></a> about <code class="docutils literal"><span class="pre">HEADER</span></code>.</div></blockquote>
+<p><code class="docutils literal"><span class="pre">tokens</span></code></p>
 <blockquote>
-<div><p><tt class="docutils literal"><span class="pre">tokens</span></tt> is an array of token. Token is an object that has the following
+<div><p><code class="docutils literal"><span class="pre">tokens</span></code> is an array of token. Token is an object that has the following
 attributes.</p>
 <table border="1" class="docutils">
 <colgroup>
@@ -879,10 +879,10 @@ attributes.</p>
 </tr>
 </thead>
 <tbody valign="top">
-<tr class="row-even"><td><tt class="docutils literal"><span class="pre">value</span></tt></td>
+<tr class="row-even"><td><code class="docutils literal"><span class="pre">value</span></code></td>
 <td>Token itself.</td>
 </tr>
-<tr class="row-odd"><td><tt class="docutils literal"><span class="pre">position</span></tt></td>
+<tr class="row-odd"><td><code class="docutils literal"><span class="pre">position</span></code></td>
 <td>The N-th token.</td>
 </tr>
 </tbody>
@@ -890,7 +890,7 @@ attributes.</p>
 </div></blockquote>
 </div>
 <div class="section" id="see-also">
-<h2>7.3.33.6. See also<a class="headerlink" href="#see-also" title="Permalink to this headline">¶</a></h2>
+<h2>7.3.34.6. See also<a class="headerlink" href="#see-also" title="Permalink to this headline">¶</a></h2>
 <ul class="simple">
 <li><a class="reference internal" href="../tokenizers.html"><em>Tokenizers</em></a></li>
 </ul>
@@ -901,47 +901,49 @@ attributes.</p>
           </div>
         </div>
       </div>
-      <div class="sphinxsidebar">
+      <div class="sphinxsidebar" role="navigation" aria-label="main navigation">
         <div class="sphinxsidebarwrapper">
   <h3><a href="../../index.html">Table Of Contents</a></h3>
   <ul>
-<li><a class="reference internal" href="#">7.3.33. <tt class="docutils literal"><span class="pre">tokenize</span></tt></a><ul>
-<li><a class="reference internal" href="#summary">7.3.33.1. Summary</a></li>
-<li><a class="reference internal" href="#syntax">7.3.33.2. Syntax</a></li>
-<li><a class="reference internal" href="#usage">7.3.33.3. Usage</a></li>
-<li><a class="reference internal" href="#parameters">7.3.33.4. Parameters</a><ul>
-<li><a class="reference internal" href="#required-parameters">7.3.33.4.1. Required parameters</a><ul>
-<li><a class="reference internal" href="#tokenizer">7.3.33.4.1.1. <tt class="docutils literal"><span class="pre">tokenizer</span></tt></a></li>
-<li><a class="reference internal" href="#string">7.3.33.4.1.2. <tt class="docutils literal"><span class="pre">string</span></tt></a></li>
+<li><a class="reference internal" href="#">7.3.34. <code class="docutils literal"><span class="pre">tokenize</span></code></a><ul>
+<li><a class="reference internal" href="#summary">7.3.34.1. Summary</a></li>
+<li><a class="reference internal" href="#syntax">7.3.34.2. Syntax</a></li>
+<li><a class="reference internal" href="#usage">7.3.34.3. Usage</a></li>
+<li><a class="reference internal" href="#parameters">7.3.34.4. Parameters</a><ul>
+<li><a class="reference internal" href="#required-parameters">7.3.34.4.1. Required parameters</a><ul>
+<li><a class="reference internal" href="#tokenizer">7.3.34.4.1.1. <code class="docutils literal"><span class="pre">tokenizer</span></code></a></li>
+<li><a class="reference internal" href="#string">7.3.34.4.1.2. <code class="docutils literal"><span class="pre">string</span></code></a></li>
 </ul>
 </li>
-<li><a class="reference internal" href="#optional-parameters">7.3.33.4.2. Optional parameters</a><ul>
-<li><a class="reference internal" href="#normalizer">7.3.33.4.2.1. <tt class="docutils literal"><span class="pre">normalizer</span></tt></a></li>
-<li><a class="reference internal" href="#flags">7.3.33.4.2.2. <tt class="docutils literal"><span class="pre">flags</span></tt></a></li>
-<li><a class="reference internal" href="#mode">7.3.33.4.2.3. <tt class="docutils literal"><span class="pre">mode</span></tt></a></li>
-<li><a class="reference internal" href="#token-filters">7.3.33.4.2.4. <tt class="docutils literal"><span class="pre">token_filters</span></tt></a></li>
+<li><a class="reference internal" href="#optional-parameters">7.3.34.4.2. Optional parameters</a><ul>
+<li><a class="reference internal" href="#normalizer">7.3.34.4.2.1. <code class="docutils literal"><span class="pre">normalizer</span></code></a></li>
+<li><a class="reference internal" href="#flags">7.3.34.4.2.2. <code class="docutils literal"><span class="pre">flags</span></code></a></li>
+<li><a class="reference internal" href="#mode">7.3.34.4.2.3. <code class="docutils literal"><span class="pre">mode</span></code></a></li>
+<li><a class="reference internal" href="#token-filters">7.3.34.4.2.4. <code class="docutils literal"><span class="pre">token_filters</span></code></a></li>
 </ul>
 </li>
 </ul>
 </li>
-<li><a class="reference internal" href="#return-value">7.3.33.5. Return value</a></li>
-<li><a class="reference internal" href="#see-also">7.3.33.6. See also</a></li>
+<li><a class="reference internal" href="#return-value">7.3.34.5. Return value</a></li>
+<li><a class="reference internal" href="#see-also">7.3.34.6. See also</a></li>
 </ul>
 </li>
 </ul>
   <h4>Previous topic</h4>
   <p class="topless"><a href="table_tokenize.html"
-                        title="previous chapter">7.3.32. <tt class="docutils literal"><span class="pre">table_tokenize</span></tt></a></p>
+                        title="previous chapter">7.3.33. <code class="docutils literal"><span class="pre">table_tokenize</span></code></a></p>
   <h4>Next topic</h4>
   <p class="topless"><a href="tokenizer_list.html"
-                        title="next chapter">7.3.34. <tt class="docutils literal"><span class="pre">tokenizer_list</span></tt></a></p>
-  <h3>This Page</h3>
-  <ul class="this-page-menu">
-    <li><a href="../../_sources/reference/commands/tokenize.txt"
-           rel="nofollow">Show Source</a></li>
-  </ul>
-<div id="searchbox" style="display: none">
+                        title="next chapter">7.3.35. <code class="docutils literal"><span class="pre">tokenizer_list</span></code></a></p>
+  <div role="note" aria-label="source link">
+    <h3>This Page</h3>
+    <ul class="this-page-menu">
+      <li><a href="../../_sources/reference/commands/tokenize.txt"
+            rel="nofollow">Show Source</a></li>
+    </ul>
+   </div>
+<div id="searchbox" style="display: none" role="search">
   <h3>Quick search</h3>
     <form class="search" action="../../search.html" method="get">
       <input type="text" name="q" />
@@ -958,24 +960,24 @@ attributes.</p>
       </div>
       <div class="clearer"></div>
     </div>
-    <div class="related">
+    <div class="related" role="navigation" aria-label="related navigation">
       <h3>Navigation</h3>
       <ul>
         <li class="right" style="margin-right: 10px">
           <a href="../../genindex.html" title="General Index"
              >index</a></li>
         <li class="right" >
-          <a href="tokenizer_list.html" title="7.3.34. tokenizer_list"
+          <a href="tokenizer_list.html" title="7.3.35. tokenizer_list"
              >next</a> |</li>
         <li class="right" >
-          <a href="table_tokenize.html" title="7.3.32. table_tokenize"
+          <a href="table_tokenize.html" title="7.3.33. table_tokenize"
              >previous</a> |</li>
-        <li><a href="../../index.html">Groonga v4.0.6-211-g8c0bb0b documentation</a> &raquo;</li>
+        <li><a href="../../index.html">Groonga v4.0.8 documentation</a> &raquo;</li>
           <li><a href="../../reference.html" >7. Reference manual</a> &raquo;</li>
           <li><a href="../command.html" >7.3. Command</a> &raquo;</li>
       </ul>
     </div>
-    <div class="footer">
+    <div class="footer" role="contentinfo">
         &copy; Copyright 2009-2014, Brazil, Inc.
     </div>
   </body>