RubyGems - rroonga - Versions diffs - 5.0.3-x64-mingw32 → 5.0.4-x64-mingw32 - Mend

rroonga 5.0.3-x64-mingw32 → 5.0.4-x64-mingw32

Files changed (541) hide show

data/vendor/local/share/doc/groonga/en/html/reference/commands/tokenize.html CHANGED Viewed

@@ -7,7 +7,7 @@
   <head>
     <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
-    <title>7.3.41. tokenize &mdash; Groonga v5.0.4 documentation</title>
+    <title>7.3.44. tokenize &mdash; Groonga v5.0.4-139-g6629adb documentation</title>
     <link rel="stylesheet" href="../../_static/groonga.css" type="text/css" />
     <link rel="stylesheet" href="../../_static/pygments.css" type="text/css" />
@@ -15,7 +15,7 @@
     <script type="text/javascript">
       var DOCUMENTATION_OPTIONS = {
         URL_ROOT:    '../../',
-        VERSION:     '5.0.4',
+        VERSION:     '5.0.4-139-g6629adb',
         COLLAPSE_INDEX: false,
         FILE_SUFFIX: '.html',
         HAS_SOURCE:  true
@@ -25,10 +25,10 @@
     <script type="text/javascript" src="../../_static/underscore.js"></script>
     <script type="text/javascript" src="../../_static/doctools.js"></script>
     <link rel="shortcut icon" href="../../_static/favicon.ico"/>
-    <link rel="top" title="Groonga v5.0.4 documentation" href="../../index.html" />
+    <link rel="top" title="Groonga v5.0.4-139-g6629adb documentation" href="../../index.html" />
     <link rel="up" title="7.3. Command" href="../command.html" />
-    <link rel="next" title="7.3.42. tokenizer_list" href="tokenizer_list.html" />
-    <link rel="prev" title="7.3.40. table_tokenize" href="table_tokenize.html" />
+    <link rel="next" title="7.3.45. tokenizer_list" href="tokenizer_list.html" />
+    <link rel="prev" title="7.3.43. table_tokenize" href="table_tokenize.html" />
   </head>
   <body>
 <div class="header">
@@ -55,12 +55,12 @@
           <a href="../../genindex.html" title="General Index"
              accesskey="I">index</a></li>
         <li class="right" >
-          <a href="tokenizer_list.html" title="7.3.42. tokenizer_list"
+          <a href="tokenizer_list.html" title="7.3.45. tokenizer_list"
              accesskey="N">next</a> |</li>
         <li class="right" >
-          <a href="table_tokenize.html" title="7.3.40. table_tokenize"
+          <a href="table_tokenize.html" title="7.3.43. table_tokenize"
              accesskey="P">previous</a> |</li>
-        <li><a href="../../index.html">Groonga v5.0.4 documentation</a> &raquo;</li>
+        <li><a href="../../index.html">Groonga v5.0.4-139-g6629adb documentation</a> &raquo;</li>
           <li><a href="../../reference.html" >7. Reference manual</a> &raquo;</li>
           <li><a href="../command.html" accesskey="U">7.3. Command</a> &raquo;</li>
       </ul>
@@ -72,14 +72,14 @@
           <div class="body">
   <div class="section" id="tokenize">
-<h1>7.3.41. <tt class="docutils literal"><span class="pre">tokenize</span></tt><a class="headerlink" href="#tokenize" title="Permalink to this headline">¶</a></h1>
+<h1>7.3.44. <tt class="docutils literal"><span class="pre">tokenize</span></tt><a class="headerlink" href="#tokenize" title="Permalink to this headline">¶</a></h1>
 <div class="section" id="summary">
-<h2>7.3.41.1. Summary<a class="headerlink" href="#summary" title="Permalink to this headline">¶</a></h2>
+<h2>7.3.44.1. Summary<a class="headerlink" href="#summary" title="Permalink to this headline">¶</a></h2>
 <p><tt class="docutils literal"><span class="pre">tokenize</span></tt> command tokenizes text by the specified tokenizer.
 It is useful to debug tokenization.</p>
 </div>
 <div class="section" id="syntax">
-<h2>7.3.41.2. Syntax<a class="headerlink" href="#syntax" title="Permalink to this headline">¶</a></h2>
+<h2>7.3.44.2. Syntax<a class="headerlink" href="#syntax" title="Permalink to this headline">¶</a></h2>
 <p><tt class="docutils literal"><span class="pre">tokenize</span></tt> command has required parameters and optional parameters.
 <tt class="docutils literal"><span class="pre">tokenizer</span></tt> and <tt class="docutils literal"><span class="pre">string</span></tt> are required parameters. Others are
 optional:</p>
@@ -93,7 +93,7 @@ optional:</p>
 </div>
 </div>
 <div class="section" id="usage">
-<h2>7.3.41.3. Usage<a class="headerlink" href="#usage" title="Permalink to this headline">¶</a></h2>
+<h2>7.3.44.3. Usage<a class="headerlink" href="#usage" title="Permalink to this headline">¶</a></h2>
 <p>Here is a simple example.</p>
 <p>Execution example:</p>
 <div class="highlight-none"><div class="highlight"><pre>tokenize TokenBigram &quot;Fulltext Search&quot;
@@ -106,62 +106,77 @@ optional:</p>
 #   [
 #     {
 #       &quot;position&quot;: 0,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;Fu&quot;
 #     },
 #     {
 #       &quot;position&quot;: 1,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;ul&quot;
 #     },
 #     {
 #       &quot;position&quot;: 2,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;ll&quot;
 #     },
 #     {
 #       &quot;position&quot;: 3,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;lt&quot;
 #     },
 #     {
 #       &quot;position&quot;: 4,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;te&quot;
 #     },
 #     {
 #       &quot;position&quot;: 5,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;ex&quot;
 #     },
 #     {
 #       &quot;position&quot;: 6,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;xt&quot;
 #     },
 #     {
 #       &quot;position&quot;: 7,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;t &quot;
 #     },
 #     {
 #       &quot;position&quot;: 8,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot; S&quot;
 #     },
 #     {
 #       &quot;position&quot;: 9,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;Se&quot;
 #     },
 #     {
 #       &quot;position&quot;: 10,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;ea&quot;
 #     },
 #     {
 #       &quot;position&quot;: 11,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;ar&quot;
 #     },
 #     {
 #       &quot;position&quot;: 12,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;rc&quot;
 #     },
 #     {
 #       &quot;position&quot;: 13,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;ch&quot;
 #     },
 #     {
 #       &quot;position&quot;: 14,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;h&quot;
 #     }
 #   ]
@@ -174,13 +189,13 @@ generated by tokenizing <tt class="docutils literal"><span class="pre">&quot;Ful
 tokenizer. It doesn't normalize <tt class="docutils literal"><span class="pre">&quot;Fulltext</span> <span class="pre">Search&quot;</span></tt>.</p>
 </div>
 <div class="section" id="parameters">
-<h2>7.3.41.4. Parameters<a class="headerlink" href="#parameters" title="Permalink to this headline">¶</a></h2>
+<h2>7.3.44.4. Parameters<a class="headerlink" href="#parameters" title="Permalink to this headline">¶</a></h2>
 <p>This section describes all parameters. Parameters are categorized.</p>
 <div class="section" id="required-parameters">
-<h3>7.3.41.4.1. Required parameters<a class="headerlink" href="#required-parameters" title="Permalink to this headline">¶</a></h3>
+<h3>7.3.44.4.1. Required parameters<a class="headerlink" href="#required-parameters" title="Permalink to this headline">¶</a></h3>
 <p>There are required parameters, <tt class="docutils literal"><span class="pre">tokenizer</span></tt> and <tt class="docutils literal"><span class="pre">string</span></tt>.</p>
 <div class="section" id="tokenizer">
-<span id="tokenize-tokenizer"></span><h4>7.3.41.4.1.1. <tt class="docutils literal"><span class="pre">tokenizer</span></tt><a class="headerlink" href="#tokenizer" title="Permalink to this headline">¶</a></h4>
+<span id="tokenize-tokenizer"></span><h4>7.3.44.4.1.1. <tt class="docutils literal"><span class="pre">tokenizer</span></tt><a class="headerlink" href="#tokenizer" title="Permalink to this headline">¶</a></h4>
 <p>Specifies the tokenizer name. <tt class="docutils literal"><span class="pre">tokenize</span></tt> command uses the
 tokenizer that is named <tt class="docutils literal"><span class="pre">tokenizer</span></tt>.</p>
 <p>See <a class="reference internal" href="../tokenizers.html"><em>Tokenizers</em></a> about built-in tokenizers.</p>
@@ -196,62 +211,77 @@ tokenizer that is named <tt class="docutils literal"><span class="pre">tokenizer
 #   [
 #     {
 #       &quot;position&quot;: 0,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;Ful&quot;
 #     },
 #     {
 #       &quot;position&quot;: 1,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;ull&quot;
 #     },
 #     {
 #       &quot;position&quot;: 2,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;llt&quot;
 #     },
 #     {
 #       &quot;position&quot;: 3,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;lte&quot;
 #     },
 #     {
 #       &quot;position&quot;: 4,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;tex&quot;
 #     },
 #     {
 #       &quot;position&quot;: 5,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;ext&quot;
 #     },
 #     {
 #       &quot;position&quot;: 6,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;xt &quot;
 #     },
 #     {
 #       &quot;position&quot;: 7,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;t S&quot;
 #     },
 #     {
 #       &quot;position&quot;: 8,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot; Se&quot;
 #     },
 #     {
 #       &quot;position&quot;: 9,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;Sea&quot;
 #     },
 #     {
 #       &quot;position&quot;: 10,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;ear&quot;
 #     },
 #     {
 #       &quot;position&quot;: 11,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;arc&quot;
 #     },
 #     {
 #       &quot;position&quot;: 12,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;rch&quot;
 #     },
 #     {
 #       &quot;position&quot;: 13,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;ch&quot;
 #     },
 #     {
 #       &quot;position&quot;: 14,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;h&quot;
 #     }
 #   ]
@@ -264,7 +294,7 @@ tokenizer plugin by <a class="reference internal" href="register.html"><em>regis
 registering <tt class="docutils literal"><span class="pre">tokenizers/kytea</span></tt>.</p>
 </div>
 <div class="section" id="string">
-<span id="tokenize-string"></span><h4>7.3.41.4.1.2. <tt class="docutils literal"><span class="pre">string</span></tt><a class="headerlink" href="#string" title="Permalink to this headline">¶</a></h4>
+<span id="tokenize-string"></span><h4>7.3.44.4.1.2. <tt class="docutils literal"><span class="pre">string</span></tt><a class="headerlink" href="#string" title="Permalink to this headline">¶</a></h4>
 <p>Specifies any string which you want to tokenize.</p>
 <p>If you want to include spaces in <tt class="docutils literal"><span class="pre">string</span></tt>, you need to quote
 <tt class="docutils literal"><span class="pre">string</span></tt> by single quotation (<tt class="docutils literal"><span class="pre">'</span></tt>) or double quotation (<tt class="docutils literal"><span class="pre">&quot;</span></tt>).</p>
@@ -280,162 +310,202 @@ registering <tt class="docutils literal"><span class="pre">tokenizers/kytea</spa
 #   [
 #     {
 #       &quot;position&quot;: 0,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;Gr&quot;
 #     },
 #     {
 #       &quot;position&quot;: 1,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;ro&quot;
 #     },
 #     {
 #       &quot;position&quot;: 2,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;oo&quot;
 #     },
 #     {
 #       &quot;position&quot;: 3,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;on&quot;
 #     },
 #     {
 #       &quot;position&quot;: 4,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;ng&quot;
 #     },
 #     {
 #       &quot;position&quot;: 5,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;ga&quot;
 #     },
 #     {
 #       &quot;position&quot;: 6,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;a &quot;
 #     },
 #     {
 #       &quot;position&quot;: 7,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot; i&quot;
 #     },
 #     {
 #       &quot;position&quot;: 8,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;is&quot;
 #     },
 #     {
 #       &quot;position&quot;: 9,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;s &quot;
 #     },
 #     {
 #       &quot;position&quot;: 10,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot; a&quot;
 #     },
 #     {
 #       &quot;position&quot;: 11,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;a &quot;
 #     },
 #     {
 #       &quot;position&quot;: 12,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot; f&quot;
 #     },
 #     {
 #       &quot;position&quot;: 13,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;fa&quot;
 #     },
 #     {
 #       &quot;position&quot;: 14,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;as&quot;
 #     },
 #     {
 #       &quot;position&quot;: 15,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;st&quot;
 #     },
 #     {
 #       &quot;position&quot;: 16,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;t &quot;
 #     },
 #     {
 #       &quot;position&quot;: 17,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot; f&quot;
 #     },
 #     {
 #       &quot;position&quot;: 18,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;fu&quot;
 #     },
 #     {
 #       &quot;position&quot;: 19,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;ul&quot;
 #     },
 #     {
 #       &quot;position&quot;: 20,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;ll&quot;
 #     },
 #     {
 #       &quot;position&quot;: 21,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;lt&quot;
 #     },
 #     {
 #       &quot;position&quot;: 22,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;te&quot;
 #     },
 #     {
 #       &quot;position&quot;: 23,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;ex&quot;
 #     },
 #     {
 #       &quot;position&quot;: 24,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;xt&quot;
 #     },
 #     {
 #       &quot;position&quot;: 25,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;t &quot;
 #     },
 #     {
 #       &quot;position&quot;: 26,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot; e&quot;
 #     },
 #     {
 #       &quot;position&quot;: 27,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;ea&quot;
 #     },
 #     {
 #       &quot;position&quot;: 28,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;ar&quot;
 #     },
 #     {
 #       &quot;position&quot;: 29,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;rc&quot;
 #     },
 #     {
 #       &quot;position&quot;: 30,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;ch&quot;
 #     },
 #     {
 #       &quot;position&quot;: 31,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;h &quot;
 #     },
 #     {
 #       &quot;position&quot;: 32,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot; e&quot;
 #     },
 #     {
 #       &quot;position&quot;: 33,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;en&quot;
 #     },
 #     {
 #       &quot;position&quot;: 34,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;ng&quot;
 #     },
 #     {
 #       &quot;position&quot;: 35,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;gi&quot;
 #     },
 #     {
 #       &quot;position&quot;: 36,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;in&quot;
 #     },
 #     {
 #       &quot;position&quot;: 37,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;ne&quot;
 #     },
 #     {
 #       &quot;position&quot;: 38,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;e!&quot;
 #     },
 #     {
 #       &quot;position&quot;: 39,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;!&quot;
 #     }
 #   ]
@@ -445,10 +515,10 @@ registering <tt class="docutils literal"><span class="pre">tokenizers/kytea</spa
 </div>
 </div>
 <div class="section" id="optional-parameters">
-<h3>7.3.41.4.2. Optional parameters<a class="headerlink" href="#optional-parameters" title="Permalink to this headline">¶</a></h3>
+<h3>7.3.44.4.2. Optional parameters<a class="headerlink" href="#optional-parameters" title="Permalink to this headline">¶</a></h3>
 <p>There are optional parameters.</p>
 <div class="section" id="normalizer">
-<span id="tokenize-normalizer"></span><h4>7.3.41.4.2.1. <tt class="docutils literal"><span class="pre">normalizer</span></tt><a class="headerlink" href="#normalizer" title="Permalink to this headline">¶</a></h4>
+<span id="tokenize-normalizer"></span><h4>7.3.44.4.2.1. <tt class="docutils literal"><span class="pre">normalizer</span></tt><a class="headerlink" href="#normalizer" title="Permalink to this headline">¶</a></h4>
 <p>Specifies the normalizer name. <tt class="docutils literal"><span class="pre">tokenize</span></tt> command uses the
 normalizer that is named <tt class="docutils literal"><span class="pre">normalizer</span></tt>. Normalizer is important for
 N-gram family tokenizers such as <tt class="docutils literal"><span class="pre">TokenBigram</span></tt>.</p>
@@ -467,62 +537,77 @@ tokenizing.</p>
 #   [
 #     {
 #       &quot;position&quot;: 0,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;Fu&quot;
 #     },
 #     {
 #       &quot;position&quot;: 1,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;ul&quot;
 #     },
 #     {
 #       &quot;position&quot;: 2,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;ll&quot;
 #     },
 #     {
 #       &quot;position&quot;: 3,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;lt&quot;
 #     },
 #     {
 #       &quot;position&quot;: 4,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;te&quot;
 #     },
 #     {
 #       &quot;position&quot;: 5,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;ex&quot;
 #     },
 #     {
 #       &quot;position&quot;: 6,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;xt&quot;
 #     },
 #     {
 #       &quot;position&quot;: 7,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;t &quot;
 #     },
 #     {
 #       &quot;position&quot;: 8,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot; S&quot;
 #     },
 #     {
 #       &quot;position&quot;: 9,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;Se&quot;
 #     },
 #     {
 #       &quot;position&quot;: 10,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;ea&quot;
 #     },
 #     {
 #       &quot;position&quot;: 11,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;ar&quot;
 #     },
 #     {
 #       &quot;position&quot;: 12,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;rc&quot;
 #     },
 #     {
 #       &quot;position&quot;: 13,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;ch&quot;
 #     },
 #     {
 #       &quot;position&quot;: 14,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;h&quot;
 #     }
 #   ]
@@ -543,10 +628,12 @@ a token.</p>
 #   [
 #     {
 #       &quot;position&quot;: 0,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;fulltext&quot;
 #     },
 #     {
 #       &quot;position&quot;: 1,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;search&quot;
 #     }
 #   ]
@@ -568,58 +655,72 @@ a token.</p>
 #   [
 #     {
 #       &quot;position&quot;: 0,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;fu&quot;
 #     },
 #     {
 #       &quot;position&quot;: 1,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;ul&quot;
 #     },
 #     {
 #       &quot;position&quot;: 2,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;ll&quot;
 #     },
 #     {
 #       &quot;position&quot;: 3,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;lt&quot;
 #     },
 #     {
 #       &quot;position&quot;: 4,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;te&quot;
 #     },
 #     {
 #       &quot;position&quot;: 5,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;ex&quot;
 #     },
 #     {
 #       &quot;position&quot;: 6,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;xt&quot;
 #     },
 #     {
 #       &quot;position&quot;: 7,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;t&quot;
 #     },
 #     {
 #       &quot;position&quot;: 8,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;se&quot;
 #     },
 #     {
 #       &quot;position&quot;: 9,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;ea&quot;
 #     },
 #     {
 #       &quot;position&quot;: 10,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;ar&quot;
 #     },
 #     {
 #       &quot;position&quot;: 11,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;rc&quot;
 #     },
 #     {
 #       &quot;position&quot;: 12,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;ch&quot;
 #     },
 #     {
 #       &quot;position&quot;: 13,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;h&quot;
 #     }
 #   ]
@@ -630,7 +731,7 @@ a token.</p>
 to lower case characters. For example, <tt class="docutils literal"><span class="pre">fu</span></tt> is a token.</p>
 </div>
 <div class="section" id="flags">
-<span id="tokenize-flags"></span><h4>7.3.41.4.2.2. <tt class="docutils literal"><span class="pre">flags</span></tt><a class="headerlink" href="#flags" title="Permalink to this headline">¶</a></h4>
+<span id="tokenize-flags"></span><h4>7.3.44.4.2.2. <tt class="docutils literal"><span class="pre">flags</span></tt><a class="headerlink" href="#flags" title="Permalink to this headline">¶</a></h4>
 <p>Specifies a tokenization customize options. You can specify
 multiple options separated by &quot;<tt class="docutils literal"><span class="pre">|</span></tt>&quot;. For example,
 <tt class="docutils literal"><span class="pre">NONE|ENABLE_TOKENIZED_DELIMITER</span></tt>.</p>
@@ -667,14 +768,17 @@ tokenized delimiter details.</td>
 #   [
 #     {
 #       &quot;position&quot;: 0,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;full&quot;
 #     },
 #     {
 #       &quot;position&quot;: 1,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;text sea&quot;
 #     },
 #     {
 #       &quot;position&quot;: 2,
+#       &quot;force_prefix&quot;: false,
 #       &quot;value&quot;: &quot;crch&quot;
 #     }
 #   ]
@@ -692,7 +796,7 @@ treated as already tokenized string. Tokenizer just tokenizes by
 tokenized delimiter.</p>
 </div>
 <div class="section" id="mode">
-<span id="tokenize-mode"></span><h4>7.3.41.4.2.3. <tt class="docutils literal"><span class="pre">mode</span></tt><a class="headerlink" href="#mode" title="Permalink to this headline">¶</a></h4>
+<span id="tokenize-mode"></span><h4>7.3.44.4.2.3. <tt class="docutils literal"><span class="pre">mode</span></tt><a class="headerlink" href="#mode" title="Permalink to this headline">¶</a></h4>
 <p>Specifies a tokenize mode. If the mode is specified <tt class="docutils literal"><span class="pre">ADD</span></tt>, the text
 is tokenized by the rule that adding a document. If the mode is specified
 <tt class="docutils literal"><span class="pre">GET</span></tt>, the text is tokenized by the rule that searching a document. If
@@ -704,69 +808,84 @@ the mode is omitted, the text is tokenized by the <tt class="docutils literal"><
 # [
 #   [
 #     0,
-#     1408017697.66886,
-#     0.00126171112060547
+#     1337566253.89858,
+#     0.000355720520019531
 #   ],
 #   [
 #     {
-#       &quot;value&quot;: &quot;Fu&quot;,
-#       &quot;position&quot;: 0
+#       &quot;position&quot;: 0,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;Fu&quot;
 #     },
 #     {
-#       &quot;value&quot;: &quot;ul&quot;,
-#       &quot;position&quot;: 1
+#       &quot;position&quot;: 1,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;ul&quot;
 #     },
 #     {
-#       &quot;value&quot;: &quot;ll&quot;,
-#       &quot;position&quot;: 2
+#       &quot;position&quot;: 2,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;ll&quot;
 #     },
 #     {
-#       &quot;value&quot;: &quot;lt&quot;,
-#       &quot;position&quot;: 3
+#       &quot;position&quot;: 3,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;lt&quot;
 #     },
 #     {
-#       &quot;value&quot;: &quot;te&quot;,
-#       &quot;position&quot;: 4
+#       &quot;position&quot;: 4,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;te&quot;
 #     },
 #     {
-#       &quot;value&quot;: &quot;ex&quot;,
-#       &quot;position&quot;: 5
+#       &quot;position&quot;: 5,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;ex&quot;
 #     },
 #     {
-#       &quot;value&quot;: &quot;xt&quot;,
-#       &quot;position&quot;: 6
+#       &quot;position&quot;: 6,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;xt&quot;
 #     },
 #     {
-#       &quot;value&quot;: &quot;t &quot;,
-#       &quot;position&quot;: 7
+#       &quot;position&quot;: 7,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;t &quot;
 #     },
 #     {
-#       &quot;value&quot;: &quot; S&quot;,
-#       &quot;position&quot;: 8
+#       &quot;position&quot;: 8,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot; S&quot;
 #     },
 #     {
-#       &quot;value&quot;: &quot;Se&quot;,
-#       &quot;position&quot;: 9
+#       &quot;position&quot;: 9,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;Se&quot;
 #     },
 #     {
-#       &quot;value&quot;: &quot;ea&quot;,
-#       &quot;position&quot;: 10
+#       &quot;position&quot;: 10,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;ea&quot;
 #     },
 #     {
-#       &quot;value&quot;: &quot;ar&quot;,
-#       &quot;position&quot;: 11
+#       &quot;position&quot;: 11,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;ar&quot;
 #     },
 #     {
-#       &quot;value&quot;: &quot;rc&quot;,
-#       &quot;position&quot;: 12
+#       &quot;position&quot;: 12,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;rc&quot;
 #     },
 #     {
-#       &quot;value&quot;: &quot;ch&quot;,
-#       &quot;position&quot;: 13
+#       &quot;position&quot;: 13,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;ch&quot;
 #     },
 #     {
-#       &quot;value&quot;: &quot;h&quot;,
-#       &quot;position&quot;: 14
+#       &quot;position&quot;: 14,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;h&quot;
 #     }
 #   ]
 # ]
@@ -779,65 +898,79 @@ the mode is omitted, the text is tokenized by the <tt class="docutils literal"><
 # [
 #   [
 #     0,
-#     1408017732.62883,
-#     0.000665903091430664
+#     1337566253.89858,
+#     0.000355720520019531
 #   ],
 #   [
 #     {
-#       &quot;value&quot;: &quot;Fu&quot;,
-#       &quot;position&quot;: 0
+#       &quot;position&quot;: 0,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;Fu&quot;
 #     },
 #     {
-#       &quot;value&quot;: &quot;ul&quot;,
-#       &quot;position&quot;: 1
+#       &quot;position&quot;: 1,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;ul&quot;
 #     },
 #     {
-#       &quot;value&quot;: &quot;ll&quot;,
-#       &quot;position&quot;: 2
+#       &quot;position&quot;: 2,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;ll&quot;
 #     },
 #     {
-#       &quot;value&quot;: &quot;lt&quot;,
-#       &quot;position&quot;: 3
+#       &quot;position&quot;: 3,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;lt&quot;
 #     },
 #     {
-#       &quot;value&quot;: &quot;te&quot;,
-#       &quot;position&quot;: 4
+#       &quot;position&quot;: 4,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;te&quot;
 #     },
 #     {
-#       &quot;value&quot;: &quot;ex&quot;,
-#       &quot;position&quot;: 5
+#       &quot;position&quot;: 5,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;ex&quot;
 #     },
 #     {
-#       &quot;value&quot;: &quot;xt&quot;,
-#       &quot;position&quot;: 6
+#       &quot;position&quot;: 6,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;xt&quot;
 #     },
 #     {
-#       &quot;value&quot;: &quot;t &quot;,
-#       &quot;position&quot;: 7
+#       &quot;position&quot;: 7,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;t &quot;
 #     },
 #     {
-#       &quot;value&quot;: &quot; S&quot;,
-#       &quot;position&quot;: 8
+#       &quot;position&quot;: 8,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot; S&quot;
 #     },
 #     {
-#       &quot;value&quot;: &quot;Se&quot;,
-#       &quot;position&quot;: 9
+#       &quot;position&quot;: 9,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;Se&quot;
 #     },
 #     {
-#       &quot;value&quot;: &quot;ea&quot;,
-#       &quot;position&quot;: 10
+#       &quot;position&quot;: 10,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;ea&quot;
 #     },
 #     {
-#       &quot;value&quot;: &quot;ar&quot;,
-#       &quot;position&quot;: 11
+#       &quot;position&quot;: 11,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;ar&quot;
 #     },
 #     {
-#       &quot;value&quot;: &quot;rc&quot;,
-#       &quot;position&quot;: 12
+#       &quot;position&quot;: 12,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;rc&quot;
 #     },
 #     {
-#       &quot;value&quot;: &quot;ch&quot;,
-#       &quot;position&quot;: 13
+#       &quot;position&quot;: 13,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;ch&quot;
 #     }
 #   ]
 # ]
@@ -846,7 +979,7 @@ the mode is omitted, the text is tokenized by the <tt class="docutils literal"><
 <p>The last alphabet is tokenized by two characters.</p>
 </div>
 <div class="section" id="token-filters">
-<span id="tokenize-token-filters"></span><h4>7.3.41.4.2.4. <tt class="docutils literal"><span class="pre">token_filters</span></tt><a class="headerlink" href="#token-filters" title="Permalink to this headline">¶</a></h4>
+<span id="tokenize-token-filters"></span><h4>7.3.44.4.2.4. <tt class="docutils literal"><span class="pre">token_filters</span></tt><a class="headerlink" href="#token-filters" title="Permalink to this headline">¶</a></h4>
 <p>Specifies the token filter names. <tt class="docutils literal"><span class="pre">tokenize</span></tt> command uses the
 tokenizer that is named <tt class="docutils literal"><span class="pre">token_filters</span></tt>.</p>
 <p>See <a class="reference internal" href="../token_filters.html"><em>Token filters</em></a> about token filters.</p>
@@ -854,7 +987,7 @@ tokenizer that is named <tt class="docutils literal"><span class="pre">token_fil
 </div>
 </div>
 <div class="section" id="return-value">
-<span id="tokenize-return-value"></span><h2>7.3.41.5. Return value<a class="headerlink" href="#return-value" title="Permalink to this headline">¶</a></h2>
+<span id="tokenize-return-value"></span><h2>7.3.44.5. Return value<a class="headerlink" href="#return-value" title="Permalink to this headline">¶</a></h2>
 <p><tt class="docutils literal"><span class="pre">tokenize</span></tt> command returns tokenized tokens. Each token has some
 attributes except token itself. The attributes will be increased in
 the feature:</p>
@@ -890,7 +1023,7 @@ attributes.</p>
 </div></blockquote>
 </div>
 <div class="section" id="see-also">
-<h2>7.3.41.6. See also<a class="headerlink" href="#see-also" title="Permalink to this headline">¶</a></h2>
+<h2>7.3.44.6. See also<a class="headerlink" href="#see-also" title="Permalink to this headline">¶</a></h2>
 <ul class="simple">
 <li><a class="reference internal" href="../tokenizers.html"><em>Tokenizers</em></a></li>
 </ul>
@@ -905,37 +1038,37 @@ attributes.</p>
         <div class="sphinxsidebarwrapper">
   <h3><a href="../../index.html">Table Of Contents</a></h3>
   <ul>
-<li><a class="reference internal" href="#">7.3.41. <tt class="docutils literal"><span class="pre">tokenize</span></tt></a><ul>
-<li><a class="reference internal" href="#summary">7.3.41.1. Summary</a></li>
-<li><a class="reference internal" href="#syntax">7.3.41.2. Syntax</a></li>
-<li><a class="reference internal" href="#usage">7.3.41.3. Usage</a></li>
-<li><a class="reference internal" href="#parameters">7.3.41.4. Parameters</a><ul>
-<li><a class="reference internal" href="#required-parameters">7.3.41.4.1. Required parameters</a><ul>
-<li><a class="reference internal" href="#tokenizer">7.3.41.4.1.1. <tt class="docutils literal"><span class="pre">tokenizer</span></tt></a></li>
-<li><a class="reference internal" href="#string">7.3.41.4.1.2. <tt class="docutils literal"><span class="pre">string</span></tt></a></li>
+<li><a class="reference internal" href="#">7.3.44. <tt class="docutils literal"><span class="pre">tokenize</span></tt></a><ul>
+<li><a class="reference internal" href="#summary">7.3.44.1. Summary</a></li>
+<li><a class="reference internal" href="#syntax">7.3.44.2. Syntax</a></li>
+<li><a class="reference internal" href="#usage">7.3.44.3. Usage</a></li>
+<li><a class="reference internal" href="#parameters">7.3.44.4. Parameters</a><ul>
+<li><a class="reference internal" href="#required-parameters">7.3.44.4.1. Required parameters</a><ul>
+<li><a class="reference internal" href="#tokenizer">7.3.44.4.1.1. <tt class="docutils literal"><span class="pre">tokenizer</span></tt></a></li>
+<li><a class="reference internal" href="#string">7.3.44.4.1.2. <tt class="docutils literal"><span class="pre">string</span></tt></a></li>
 </ul>
 </li>
-<li><a class="reference internal" href="#optional-parameters">7.3.41.4.2. Optional parameters</a><ul>
-<li><a class="reference internal" href="#normalizer">7.3.41.4.2.1. <tt class="docutils literal"><span class="pre">normalizer</span></tt></a></li>
-<li><a class="reference internal" href="#flags">7.3.41.4.2.2. <tt class="docutils literal"><span class="pre">flags</span></tt></a></li>
-<li><a class="reference internal" href="#mode">7.3.41.4.2.3. <tt class="docutils literal"><span class="pre">mode</span></tt></a></li>
-<li><a class="reference internal" href="#token-filters">7.3.41.4.2.4. <tt class="docutils literal"><span class="pre">token_filters</span></tt></a></li>
+<li><a class="reference internal" href="#optional-parameters">7.3.44.4.2. Optional parameters</a><ul>
+<li><a class="reference internal" href="#normalizer">7.3.44.4.2.1. <tt class="docutils literal"><span class="pre">normalizer</span></tt></a></li>
+<li><a class="reference internal" href="#flags">7.3.44.4.2.2. <tt class="docutils literal"><span class="pre">flags</span></tt></a></li>
+<li><a class="reference internal" href="#mode">7.3.44.4.2.3. <tt class="docutils literal"><span class="pre">mode</span></tt></a></li>
+<li><a class="reference internal" href="#token-filters">7.3.44.4.2.4. <tt class="docutils literal"><span class="pre">token_filters</span></tt></a></li>
 </ul>
 </li>
 </ul>
 </li>
-<li><a class="reference internal" href="#return-value">7.3.41.5. Return value</a></li>
-<li><a class="reference internal" href="#see-also">7.3.41.6. See also</a></li>
+<li><a class="reference internal" href="#return-value">7.3.44.5. Return value</a></li>
+<li><a class="reference internal" href="#see-also">7.3.44.6. See also</a></li>
 </ul>
 </li>
 </ul>
   <h4>Previous topic</h4>
   <p class="topless"><a href="table_tokenize.html"
-                        title="previous chapter">7.3.40. <tt class="docutils literal"><span class="pre">table_tokenize</span></tt></a></p>
+                        title="previous chapter">7.3.43. <tt class="docutils literal"><span class="pre">table_tokenize</span></tt></a></p>
   <h4>Next topic</h4>
   <p class="topless"><a href="tokenizer_list.html"
-                        title="next chapter">7.3.42. <tt class="docutils literal"><span class="pre">tokenizer_list</span></tt></a></p>
+                        title="next chapter">7.3.45. <tt class="docutils literal"><span class="pre">tokenizer_list</span></tt></a></p>
   <h3>This Page</h3>
   <ul class="this-page-menu">
     <li><a href="../../_sources/reference/commands/tokenize.txt"
@@ -965,12 +1098,12 @@ attributes.</p>
           <a href="../../genindex.html" title="General Index"
              >index</a></li>
         <li class="right" >
-          <a href="tokenizer_list.html" title="7.3.42. tokenizer_list"
+          <a href="tokenizer_list.html" title="7.3.45. tokenizer_list"
              >next</a> |</li>
         <li class="right" >
-          <a href="table_tokenize.html" title="7.3.40. table_tokenize"
+          <a href="table_tokenize.html" title="7.3.43. table_tokenize"
              >previous</a> |</li>
-        <li><a href="../../index.html">Groonga v5.0.4 documentation</a> &raquo;</li>
+        <li><a href="../../index.html">Groonga v5.0.4-139-g6629adb documentation</a> &raquo;</li>
           <li><a href="../../reference.html" >7. Reference manual</a> &raquo;</li>
           <li><a href="../command.html" >7.3. Command</a> &raquo;</li>
       </ul>