RubyGems - rroonga - Versions diffs - 7.1.1-x86-mingw32 → 9.0.2-x86-mingw32 - Mend

rroonga 7.1.1-x86-mingw32 → 9.0.2-x86-mingw32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (980) hide show

data/vendor/local/share/doc/groonga/en/html/reference/tokenizers/token_bigram_split_symbol_alpha.html ADDED Viewed

@@ -0,0 +1,200 @@
+<!DOCTYPE html>
+<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <title>7.8.8. TokenBigramSplitSymbolAlpha &#8212; Groonga v9.0.2 documentation</title>
+    <link rel="stylesheet" href="../../_static/groonga.css" type="text/css" />
+    <link rel="stylesheet" href="../../_static/pygments.css" type="text/css" />
+    <script type="text/javascript" id="documentation_options" data-url_root="../../" src="../../_static/documentation_options.js"></script>
+    <script type="text/javascript" src="../../_static/jquery.js"></script>
+    <script type="text/javascript" src="../../_static/underscore.js"></script>
+    <script type="text/javascript" src="../../_static/doctools.js"></script>
+    <script type="text/javascript" src="../../_static/language_data.js"></script>
+    <link rel="shortcut icon" href="../../_static/favicon.ico"/>
+    <link rel="index" title="Index" href="../../genindex.html" />
+    <link rel="search" title="Search" href="../../search.html" />
+    <link rel="next" title="7.8.9. TokenBigramSplitSymbolAlphaDigit" href="token_bigram_split_symbol_alpha_digit.html" />
+    <link rel="prev" title="7.8.7. TokenBigramSplitSymbol" href="token_bigram_split_symbol.html" />
+  </head><body>
+<div class="header">
+  <h1 class="title">
+    <a id="top-link" href="../../index.html">
+      <span class="project">groonga</span>
+      <span class="separator">-</span>
+      <span class="description">An open-source fulltext search engine and column store.</span>
+    </a>
+  </h1>
+  <div class="other-language-links">
+    <ul>
+      <li><a href="../../../../ja/html/reference/tokenizers/token_bigram_split_symbol_alpha.html">日本語</a></li>
+    </ul>
+  </div>
+</div>
+    <div class="related" role="navigation" aria-label="related navigation">
+      <h3>Navigation</h3>
+      <ul>
+        <li class="right" style="margin-right: 10px">
+          <a href="../../genindex.html" title="General Index"
+             accesskey="I">index</a></li>
+        <li class="right" >
+          <a href="token_bigram_split_symbol_alpha_digit.html" title="7.8.9. TokenBigramSplitSymbolAlphaDigit"
+             accesskey="N">next</a> |</li>
+        <li class="right" >
+          <a href="token_bigram_split_symbol.html" title="7.8.7. TokenBigramSplitSymbol"
+             accesskey="P">previous</a> |</li>
+        <li class="nav-item nav-item-0"><a href="../../index.html">Groonga v9.0.2 documentation</a> &#187;</li>
+          <li class="nav-item nav-item-1"><a href="../../reference.html" >7. Reference manual</a> &#187;</li>
+          <li class="nav-item nav-item-2"><a href="../tokenizers.html" accesskey="U">7.8. Tokenizers</a> &#187;</li>
+      </ul>
+    </div>
+    <div class="document">
+      <div class="documentwrapper">
+        <div class="bodywrapper">
+          <div class="body" role="main">
+  <div class="section" id="tokenbigramsplitsymbolalpha">
+<span id="token-bigram-split-symbol-alpha"></span><h1>7.8.8. <code class="docutils literal notranslate"><span class="pre">TokenBigramSplitSymbolAlpha</span></code><a class="headerlink" href="#tokenbigramsplitsymbolalpha" title="Permalink to this headline">¶</a></h1>
+<div class="section" id="summary">
+<h2>7.8.8.1. Summary<a class="headerlink" href="#summary" title="Permalink to this headline">¶</a></h2>
+<p><code class="docutils literal notranslate"><span class="pre">TokenBigramSplitSymbolAlpha</span></code> is similar to <a class="reference internal" href="token_bigram.html#token-bigram"><span class="std std-ref">TokenBigram</span></a>. The
+difference between them is symbol and alphabet handling.</p>
+</div>
+<div class="section" id="syntax">
+<h2>7.8.8.2. Syntax<a class="headerlink" href="#syntax" title="Permalink to this headline">¶</a></h2>
+<p><code class="docutils literal notranslate"><span class="pre">TokenBigramSplitSymbolAlpha</span></code> hasn’t parameter:</p>
+<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>TokenBigramSplitSymbolAlpha
+</pre></div>
+</div>
+</div>
+<div class="section" id="usage">
+<h2>7.8.8.3. Usage<a class="headerlink" href="#usage" title="Permalink to this headline">¶</a></h2>
+<p><code class="docutils literal notranslate"><span class="pre">TokenBigramSplitSymbolAlpha</span></code> tokenizes symbols and
+alphabets by bigram tokenize method:</p>
+<p>Execution example:</p>
+<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>tokenize TokenBigramSplitSymbolAlpha &quot;100cents!!!&quot; NormalizerAuto
+# [
+#   [
+#     0,
+#     1337566253.89858,
+#     0.000355720520019531
+#   ],
+#   [
+#     {
+#       &quot;position&quot;: 0,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;100&quot;
+#     },
+#     {
+#       &quot;position&quot;: 1,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;ce&quot;
+#     },
+#     {
+#       &quot;position&quot;: 2,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;en&quot;
+#     },
+#     {
+#       &quot;position&quot;: 3,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;nt&quot;
+#     },
+#     {
+#       &quot;position&quot;: 4,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;ts&quot;
+#     },
+#     {
+#       &quot;position&quot;: 5,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;s!&quot;
+#     },
+#     {
+#       &quot;position&quot;: 6,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;!!&quot;
+#     },
+#     {
+#       &quot;position&quot;: 7,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;!!&quot;
+#     },
+#     {
+#       &quot;position&quot;: 8,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;!&quot;
+#     }
+#   ]
+# ]
+</pre></div>
+</div>
+</div>
+</div>
+          </div>
+        </div>
+      </div>
+      <div class="sphinxsidebar" role="navigation" aria-label="main navigation">
+        <div class="sphinxsidebarwrapper">
+  <h3><a href="../../index.html">Table of Contents</a></h3>
+  <ul>
+<li><a class="reference internal" href="#">7.8.8. <code class="docutils literal notranslate"><span class="pre">TokenBigramSplitSymbolAlpha</span></code></a><ul>
+<li><a class="reference internal" href="#summary">7.8.8.1. Summary</a></li>
+<li><a class="reference internal" href="#syntax">7.8.8.2. Syntax</a></li>
+<li><a class="reference internal" href="#usage">7.8.8.3. Usage</a></li>
+</ul>
+</li>
+</ul>
+  <h4>Previous topic</h4>
+  <p class="topless"><a href="token_bigram_split_symbol.html"
+                        title="previous chapter">7.8.7. <code class="docutils literal notranslate"><span class="pre">TokenBigramSplitSymbol</span></code></a></p>
+  <h4>Next topic</h4>
+  <p class="topless"><a href="token_bigram_split_symbol_alpha_digit.html"
+                        title="next chapter">7.8.9. <code class="docutils literal notranslate"><span class="pre">TokenBigramSplitSymbolAlphaDigit</span></code></a></p>
+<div id="searchbox" style="display: none" role="search">
+  <h3>Quick search</h3>
+    <div class="searchformwrapper">
+    <form class="search" action="../../search.html" method="get">
+      <input type="text" name="q" />
+      <input type="submit" value="Go" />
+    </form>
+    </div>
+</div>
+<script type="text/javascript">$('#searchbox').show(0);</script>
+        </div>
+      </div>
+      <div class="clearer"></div>
+    </div>
+    <div class="related" role="navigation" aria-label="related navigation">
+      <h3>Navigation</h3>
+      <ul>
+        <li class="right" style="margin-right: 10px">
+          <a href="../../genindex.html" title="General Index"
+             >index</a></li>
+        <li class="right" >
+          <a href="token_bigram_split_symbol_alpha_digit.html" title="7.8.9. TokenBigramSplitSymbolAlphaDigit"
+             >next</a> |</li>
+        <li class="right" >
+          <a href="token_bigram_split_symbol.html" title="7.8.7. TokenBigramSplitSymbol"
+             >previous</a> |</li>
+        <li class="nav-item nav-item-0"><a href="../../index.html">Groonga v9.0.2 documentation</a> &#187;</li>
+          <li class="nav-item nav-item-1"><a href="../../reference.html" >7. Reference manual</a> &#187;</li>
+          <li class="nav-item nav-item-2"><a href="../tokenizers.html" >7.8. Tokenizers</a> &#187;</li>
+      </ul>
+    </div>
+    <div class="footer" role="contentinfo">
+        &#169; Copyright 2009-2019, Brazil, Inc.
+    </div>
+  </body>
+</html>

data/vendor/local/share/doc/groonga/en/html/reference/tokenizers/token_bigram_split_symbol_alpha_digit.html ADDED Viewed

@@ -0,0 +1,212 @@
+<!DOCTYPE html>
+<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <title>7.8.9. TokenBigramSplitSymbolAlphaDigit &#8212; Groonga v9.0.2 documentation</title>
+    <link rel="stylesheet" href="../../_static/groonga.css" type="text/css" />
+    <link rel="stylesheet" href="../../_static/pygments.css" type="text/css" />
+    <script type="text/javascript" id="documentation_options" data-url_root="../../" src="../../_static/documentation_options.js"></script>
+    <script type="text/javascript" src="../../_static/jquery.js"></script>
+    <script type="text/javascript" src="../../_static/underscore.js"></script>
+    <script type="text/javascript" src="../../_static/doctools.js"></script>
+    <script type="text/javascript" src="../../_static/language_data.js"></script>
+    <link rel="shortcut icon" href="../../_static/favicon.ico"/>
+    <link rel="index" title="Index" href="../../genindex.html" />
+    <link rel="search" title="Search" href="../../search.html" />
+    <link rel="next" title="7.8.10. TokenDelimit" href="token_delimit.html" />
+    <link rel="prev" title="7.8.8. TokenBigramSplitSymbolAlpha" href="token_bigram_split_symbol_alpha.html" />
+  </head><body>
+<div class="header">
+  <h1 class="title">
+    <a id="top-link" href="../../index.html">
+      <span class="project">groonga</span>
+      <span class="separator">-</span>
+      <span class="description">An open-source fulltext search engine and column store.</span>
+    </a>
+  </h1>
+  <div class="other-language-links">
+    <ul>
+      <li><a href="../../../../ja/html/reference/tokenizers/token_bigram_split_symbol_alpha_digit.html">日本語</a></li>
+    </ul>
+  </div>
+</div>
+    <div class="related" role="navigation" aria-label="related navigation">
+      <h3>Navigation</h3>
+      <ul>
+        <li class="right" style="margin-right: 10px">
+          <a href="../../genindex.html" title="General Index"
+             accesskey="I">index</a></li>
+        <li class="right" >
+          <a href="token_delimit.html" title="7.8.10. TokenDelimit"
+             accesskey="N">next</a> |</li>
+        <li class="right" >
+          <a href="token_bigram_split_symbol_alpha.html" title="7.8.8. TokenBigramSplitSymbolAlpha"
+             accesskey="P">previous</a> |</li>
+        <li class="nav-item nav-item-0"><a href="../../index.html">Groonga v9.0.2 documentation</a> &#187;</li>
+          <li class="nav-item nav-item-1"><a href="../../reference.html" >7. Reference manual</a> &#187;</li>
+          <li class="nav-item nav-item-2"><a href="../tokenizers.html" accesskey="U">7.8. Tokenizers</a> &#187;</li>
+      </ul>
+    </div>
+    <div class="document">
+      <div class="documentwrapper">
+        <div class="bodywrapper">
+          <div class="body" role="main">
+  <div class="section" id="tokenbigramsplitsymbolalphadigit">
+<span id="token-bigram-split-symbol-alpha-digit"></span><h1>7.8.9. <code class="docutils literal notranslate"><span class="pre">TokenBigramSplitSymbolAlphaDigit</span></code><a class="headerlink" href="#tokenbigramsplitsymbolalphadigit" title="Permalink to this headline">¶</a></h1>
+<div class="section" id="summary">
+<h2>7.8.9.1. Summary<a class="headerlink" href="#summary" title="Permalink to this headline">¶</a></h2>
+<p><code class="docutils literal notranslate"><span class="pre">TokenBigramSplitSymbolAlphaDigit</span></code> is similar to
+<a class="reference internal" href="token_bigram.html#token-bigram"><span class="std std-ref">TokenBigram</span></a>. The difference between them is symbol, alphabet
+and digit handling.</p>
+</div>
+<div class="section" id="syntax">
+<h2>7.8.9.2. Syntax<a class="headerlink" href="#syntax" title="Permalink to this headline">¶</a></h2>
+<p><code class="docutils literal notranslate"><span class="pre">TokenBigramSplitSymbolAlphaDigit</span></code> hasn’t parameter:</p>
+<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>TokenBigramSplitSymbolAlphaDigit
+</pre></div>
+</div>
+</div>
+<div class="section" id="usage">
+<h2>7.8.9.3. Usage<a class="headerlink" href="#usage" title="Permalink to this headline">¶</a></h2>
+<p><code class="docutils literal notranslate"><span class="pre">TokenBigramSplitSymbolAlphaDigit</span></code> tokenizes
+symbols, alphabets and digits by bigram tokenize method. It means that
+all characters are tokenized by bigram tokenize method:</p>
+<p>Execution example:</p>
+<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>tokenize TokenBigramSplitSymbolAlphaDigit &quot;100cents!!!&quot; NormalizerAuto
+# [
+#   [
+#     0,
+#     1337566253.89858,
+#     0.000355720520019531
+#   ],
+#   [
+#     {
+#       &quot;position&quot;: 0,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;10&quot;
+#     },
+#     {
+#       &quot;position&quot;: 1,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;00&quot;
+#     },
+#     {
+#       &quot;position&quot;: 2,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;0c&quot;
+#     },
+#     {
+#       &quot;position&quot;: 3,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;ce&quot;
+#     },
+#     {
+#       &quot;position&quot;: 4,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;en&quot;
+#     },
+#     {
+#       &quot;position&quot;: 5,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;nt&quot;
+#     },
+#     {
+#       &quot;position&quot;: 6,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;ts&quot;
+#     },
+#     {
+#       &quot;position&quot;: 7,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;s!&quot;
+#     },
+#     {
+#       &quot;position&quot;: 8,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;!!&quot;
+#     },
+#     {
+#       &quot;position&quot;: 9,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;!!&quot;
+#     },
+#     {
+#       &quot;position&quot;: 10,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;!&quot;
+#     }
+#   ]
+# ]
+</pre></div>
+</div>
+</div>
+</div>
+          </div>
+        </div>
+      </div>
+      <div class="sphinxsidebar" role="navigation" aria-label="main navigation">
+        <div class="sphinxsidebarwrapper">
+  <h3><a href="../../index.html">Table of Contents</a></h3>
+  <ul>
+<li><a class="reference internal" href="#">7.8.9. <code class="docutils literal notranslate"><span class="pre">TokenBigramSplitSymbolAlphaDigit</span></code></a><ul>
+<li><a class="reference internal" href="#summary">7.8.9.1. Summary</a></li>
+<li><a class="reference internal" href="#syntax">7.8.9.2. Syntax</a></li>
+<li><a class="reference internal" href="#usage">7.8.9.3. Usage</a></li>
+</ul>
+</li>
+</ul>
+  <h4>Previous topic</h4>
+  <p class="topless"><a href="token_bigram_split_symbol_alpha.html"
+                        title="previous chapter">7.8.8. <code class="docutils literal notranslate"><span class="pre">TokenBigramSplitSymbolAlpha</span></code></a></p>
+  <h4>Next topic</h4>
+  <p class="topless"><a href="token_delimit.html"
+                        title="next chapter">7.8.10. <code class="docutils literal notranslate"><span class="pre">TokenDelimit</span></code></a></p>
+<div id="searchbox" style="display: none" role="search">
+  <h3>Quick search</h3>
+    <div class="searchformwrapper">
+    <form class="search" action="../../search.html" method="get">
+      <input type="text" name="q" />
+      <input type="submit" value="Go" />
+    </form>
+    </div>
+</div>
+<script type="text/javascript">$('#searchbox').show(0);</script>
+        </div>
+      </div>
+      <div class="clearer"></div>
+    </div>
+    <div class="related" role="navigation" aria-label="related navigation">
+      <h3>Navigation</h3>
+      <ul>
+        <li class="right" style="margin-right: 10px">
+          <a href="../../genindex.html" title="General Index"
+             >index</a></li>
+        <li class="right" >
+          <a href="token_delimit.html" title="7.8.10. TokenDelimit"
+             >next</a> |</li>
+        <li class="right" >
+          <a href="token_bigram_split_symbol_alpha.html" title="7.8.8. TokenBigramSplitSymbolAlpha"
+             >previous</a> |</li>
+        <li class="nav-item nav-item-0"><a href="../../index.html">Groonga v9.0.2 documentation</a> &#187;</li>
+          <li class="nav-item nav-item-1"><a href="../../reference.html" >7. Reference manual</a> &#187;</li>
+          <li class="nav-item nav-item-2"><a href="../tokenizers.html" >7.8. Tokenizers</a> &#187;</li>
+      </ul>
+    </div>
+    <div class="footer" role="contentinfo">
+        &#169; Copyright 2009-2019, Brazil, Inc.
+    </div>
+  </body>
+</html>

data/vendor/local/share/doc/groonga/en/html/reference/tokenizers/token_delimit.html ADDED Viewed

@@ -0,0 +1,357 @@
+<!DOCTYPE html>
+<html xmlns="http://www.w3.org/1999/xhtml" lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <title>7.8.10. TokenDelimit &#8212; Groonga v9.0.2 documentation</title>
+    <link rel="stylesheet" href="../../_static/groonga.css" type="text/css" />
+    <link rel="stylesheet" href="../../_static/pygments.css" type="text/css" />
+    <script type="text/javascript" id="documentation_options" data-url_root="../../" src="../../_static/documentation_options.js"></script>
+    <script type="text/javascript" src="../../_static/jquery.js"></script>
+    <script type="text/javascript" src="../../_static/underscore.js"></script>
+    <script type="text/javascript" src="../../_static/doctools.js"></script>
+    <script type="text/javascript" src="../../_static/language_data.js"></script>
+    <link rel="shortcut icon" href="../../_static/favicon.ico"/>
+    <link rel="index" title="Index" href="../../genindex.html" />
+    <link rel="search" title="Search" href="../../search.html" />
+    <link rel="next" title="7.8.11. TokenDelimitNull" href="token_delimit_null.html" />
+    <link rel="prev" title="7.8.9. TokenBigramSplitSymbolAlphaDigit" href="token_bigram_split_symbol_alpha_digit.html" />
+  </head><body>
+<div class="header">
+  <h1 class="title">
+    <a id="top-link" href="../../index.html">
+      <span class="project">groonga</span>
+      <span class="separator">-</span>
+      <span class="description">An open-source fulltext search engine and column store.</span>
+    </a>
+  </h1>
+  <div class="other-language-links">
+    <ul>
+      <li><a href="../../../../ja/html/reference/tokenizers/token_delimit.html">日本語</a></li>
+    </ul>
+  </div>
+</div>
+    <div class="related" role="navigation" aria-label="related navigation">
+      <h3>Navigation</h3>
+      <ul>
+        <li class="right" style="margin-right: 10px">
+          <a href="../../genindex.html" title="General Index"
+             accesskey="I">index</a></li>
+        <li class="right" >
+          <a href="token_delimit_null.html" title="7.8.11. TokenDelimitNull"
+             accesskey="N">next</a> |</li>
+        <li class="right" >
+          <a href="token_bigram_split_symbol_alpha_digit.html" title="7.8.9. TokenBigramSplitSymbolAlphaDigit"
+             accesskey="P">previous</a> |</li>
+        <li class="nav-item nav-item-0"><a href="../../index.html">Groonga v9.0.2 documentation</a> &#187;</li>
+          <li class="nav-item nav-item-1"><a href="../../reference.html" >7. Reference manual</a> &#187;</li>
+          <li class="nav-item nav-item-2"><a href="../tokenizers.html" accesskey="U">7.8. Tokenizers</a> &#187;</li>
+      </ul>
+    </div>
+    <div class="document">
+      <div class="documentwrapper">
+        <div class="bodywrapper">
+          <div class="body" role="main">
+  <div class="section" id="tokendelimit">
+<span id="token-delimit"></span><h1>7.8.10. <code class="docutils literal notranslate"><span class="pre">TokenDelimit</span></code><a class="headerlink" href="#tokendelimit" title="Permalink to this headline">¶</a></h1>
+<div class="section" id="summary">
+<h2>7.8.10.1. Summary<a class="headerlink" href="#summary" title="Permalink to this headline">¶</a></h2>
+<p><code class="docutils literal notranslate"><span class="pre">TokenDelimit</span></code> extracts token by splitting one or more space
+characters (<code class="docutils literal notranslate"><span class="pre">U+0020</span></code>). For example, <code class="docutils literal notranslate"><span class="pre">Hello</span> <span class="pre">World</span></code> is tokenized to
+<code class="docutils literal notranslate"><span class="pre">Hello</span></code> and <code class="docutils literal notranslate"><span class="pre">World</span></code>.</p>
+<p><code class="docutils literal notranslate"><span class="pre">TokenDelimit</span></code> is suitable for tag text. You can extract <code class="docutils literal notranslate"><span class="pre">groonga</span></code>
+and <code class="docutils literal notranslate"><span class="pre">full-text-search</span></code> and <code class="docutils literal notranslate"><span class="pre">http</span></code> as tags from <code class="docutils literal notranslate"><span class="pre">groonga</span>
+<span class="pre">full-text-search</span> <span class="pre">http</span></code>.</p>
+</div>
+<div class="section" id="syntax">
+<h2>7.8.10.2. Syntax<a class="headerlink" href="#syntax" title="Permalink to this headline">¶</a></h2>
+<p><code class="docutils literal notranslate"><span class="pre">TokenDelimit</span></code> has optional parameter.</p>
+<p>No options(Extracts token by splitting one or more space characters (<code class="docutils literal notranslate"><span class="pre">U+0020</span></code>)):</p>
+<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>TokenDelimit
+</pre></div>
+</div>
+<p>Specify delimiter:</p>
+<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>TokenDelimit(&quot;delimiter&quot;,  &quot;delimiter1&quot;, delimiter&quot;, &quot;delimiter2&quot;, ...)
+</pre></div>
+</div>
+<p>Specify delimiter with regular expression:</p>
+<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>TokenDelimit(&quot;pattern&quot;, pattern)
+</pre></div>
+</div>
+<p>The <code class="docutils literal notranslate"><span class="pre">delimiter</span></code> option and a <code class="docutils literal notranslate"><span class="pre">pattern</span></code> option are not use at the same time.</p>
+</div>
+<div class="section" id="usage">
+<h2>7.8.10.3. Usage<a class="headerlink" href="#usage" title="Permalink to this headline">¶</a></h2>
+</div>
+<div class="section" id="simple-usage">
+<h2>7.8.10.4. Simple usage<a class="headerlink" href="#simple-usage" title="Permalink to this headline">¶</a></h2>
+<p>Here is an example of <code class="docutils literal notranslate"><span class="pre">TokenDelimit</span></code>:</p>
+<p>Execution example:</p>
+<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>tokenize TokenDelimit &quot;Groonga full-text-search HTTP&quot; NormalizerAuto
+# [
+#   [
+#     0,
+#     1337566253.89858,
+#     0.000355720520019531
+#   ],
+#   [
+#     {
+#       &quot;position&quot;: 0,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;groonga&quot;
+#     },
+#     {
+#       &quot;position&quot;: 1,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;full-text-search&quot;
+#     },
+#     {
+#       &quot;position&quot;: 2,
+#       &quot;force_prefix&quot;: false,
+#       &quot;value&quot;: &quot;http&quot;
+#     }
+#   ]
+# ]
+</pre></div>
+</div>
+<p><code class="docutils literal notranslate"><span class="pre">TokenDelimit</span></code> can also specify options.
+<code class="docutils literal notranslate"><span class="pre">TokenDelimit</span></code> has <code class="docutils literal notranslate"><span class="pre">delimiter</span></code> option and <code class="docutils literal notranslate"><span class="pre">pattern</span></code> option.</p>
+<p><code class="docutils literal notranslate"><span class="pre">delimiter</span></code> option can split token with a specified character.</p>
+<p>For example, <code class="docutils literal notranslate"><span class="pre">Hello,World</span></code> is tokenized to <code class="docutils literal notranslate"><span class="pre">Hello</span></code> and <code class="docutils literal notranslate"><span class="pre">World</span></code>
+with <code class="docutils literal notranslate"><span class="pre">delimiter</span></code> option as below.</p>
+<p>Execution example:</p>
+<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>tokenize &#39;TokenDelimit(&quot;delimiter&quot;, &quot;,&quot;)&#39; &quot;Hello,World&quot;
+# [
+#   [
+#     0,
+#     1337566253.89858,
+#     0.000355720520019531
+#   ],
+#   [
+#     {
+#       &quot;value&quot;: &quot;Hello&quot;,
+#       &quot;position&quot;: 0,
+#       &quot;force_prefix&quot;: false,
+#       &quot;force_prefix_search&quot;: false
+#     },
+#     {
+#       &quot;value&quot;: &quot;World&quot;,
+#       &quot;position&quot;: 1,
+#       &quot;force_prefix&quot;: false,
+#       &quot;force_prefix_search&quot;: false
+#     }
+#   ]
+# ]
+</pre></div>
+</div>
+<p><code class="docutils literal notranslate"><span class="pre">pattern</span></code> option can split token with a regular expression.
+You can except needless space by <code class="docutils literal notranslate"><span class="pre">pattern</span></code> option.</p>
+<p>For example, <code class="docutils literal notranslate"><span class="pre">This</span> <span class="pre">is</span> <span class="pre">a</span> <span class="pre">pen.</span> <span class="pre">This</span> <span class="pre">is</span> <span class="pre">an</span> <span class="pre">apple</span></code> is tokenized to <code class="docutils literal notranslate"><span class="pre">This</span> <span class="pre">is</span> <span class="pre">a</span> <span class="pre">pen</span></code> and
+<code class="docutils literal notranslate"><span class="pre">This</span> <span class="pre">is</span> <span class="pre">an</span> <span class="pre">apple</span></code> with <code class="docutils literal notranslate"><span class="pre">pattern</span></code> option as below.</p>
+<p>Normally, when <code class="docutils literal notranslate"><span class="pre">This</span> <span class="pre">is</span> <span class="pre">a</span> <span class="pre">pen.</span> <span class="pre">This</span> <span class="pre">is</span> <span class="pre">an</span> <span class="pre">apple.</span></code> is splitted by <code class="docutils literal notranslate"><span class="pre">.</span></code>,
+needless spaces are included at the beginning of “This is an apple.”.</p>
+<p>You can except the needless spaces by a <code class="docutils literal notranslate"><span class="pre">pattern</span></code> option as below example.</p>
+<p>Execution example:</p>
+<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>tokenize &#39;TokenDelimit(&quot;pattern&quot;, &quot;\\.\\s*&quot;)&#39; &quot;This is a pen. This is an apple.&quot;
+# [
+#   [
+#     0,
+#     1337566253.89858,
+#     0.000355720520019531
+#   ],
+#   [
+#     {
+#       &quot;value&quot;: &quot;This is a pen.&quot;,
+#       &quot;position&quot;: 0,
+#       &quot;force_prefix&quot;: false,
+#       &quot;force_prefix_search&quot;: false
+#     },
+#     {
+#       &quot;value&quot;: &quot;This is an apple.&quot;,
+#       &quot;position&quot;: 1,
+#       &quot;force_prefix&quot;: false,
+#       &quot;force_prefix_search&quot;: false
+#     }
+#   ]
+# ]
+</pre></div>
+</div>
+</div>
+<div class="section" id="advanced-usage">
+<h2>7.8.10.5. Advanced usage<a class="headerlink" href="#advanced-usage" title="Permalink to this headline">¶</a></h2>
+<p><code class="docutils literal notranslate"><span class="pre">delimiter</span></code> option can also specify multiple delimiters.</p>
+<p>For example, <code class="docutils literal notranslate"><span class="pre">Hello,</span> <span class="pre">World</span></code> is tokenized to <code class="docutils literal notranslate"><span class="pre">Hello</span></code> and <code class="docutils literal notranslate"><span class="pre">World</span></code>.
+<code class="docutils literal notranslate"><span class="pre">,</span></code> and `` `` are delimiters in below example.</p>
+<p>Execution example:</p>
+<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>tokenize &#39;TokenDelimit(&quot;delimiter&quot;, &quot;,&quot;, &quot;delimiter&quot;, &quot; &quot;)&#39; &quot;Hello, World&quot;
+# [
+#   [
+#     0,
+#     1337566253.89858,
+#     0.000355720520019531
+#   ],
+#   [
+#     {
+#       &quot;value&quot;: &quot;Hello&quot;,
+#       &quot;position&quot;: 0,
+#       &quot;force_prefix&quot;: false,
+#       &quot;force_prefix_search&quot;: false
+#     },
+#     {
+#       &quot;value&quot;: &quot;World&quot;,
+#       &quot;position&quot;: 1,
+#       &quot;force_prefix&quot;: false,
+#       &quot;force_prefix_search&quot;: false
+#     }
+#   ]
+# ]
+</pre></div>
+</div>
+<p>You can extract token in complex conditions by <code class="docutils literal notranslate"><span class="pre">pattern</span></code> option.</p>
+<p>For example, <code class="docutils literal notranslate"><span class="pre">これはペンですか！？リンゴですか？「リンゴです。」</span></code> is tokenize to <code class="docutils literal notranslate"><span class="pre">これはペンですか</span></code> and <code class="docutils literal notranslate"><span class="pre">リンゴですか</span></code>, <code class="docutils literal notranslate"><span class="pre">「リンゴです。」</span></code> with <code class="docutils literal notranslate"><span class="pre">delimiter</span></code> option as below.</p>
+<p>Execution example:</p>
+<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>tokenize &#39;TokenDelimit(&quot;pattern&quot;, &quot;([。！？]+(?![）」])|[\\r\\n]+)\\s*&quot;)&#39; &quot;これはペンですか！？リンゴですか？「リンゴです。」&quot;
+# [
+#   [
+#     0,
+#     1545179416.22277,
+#     0.0002887248992919922
+#   ],
+#   [
+#     {
+#       &quot;value&quot;: &quot;これはペンですか&quot;,
+#       &quot;position&quot;: 0,
+#       &quot;force_prefix&quot;: false,
+#       &quot;force_prefix_search&quot;: false
+#     },
+#     {
+#       &quot;value&quot;: &quot;リンゴですか&quot;,
+#       &quot;position&quot;: 1,
+#       &quot;force_prefix&quot;: false,
+#       &quot;force_prefix_search&quot;: false
+#     },
+#     {
+#       &quot;value&quot;: &quot;「リンゴです。」&quot;,
+#       &quot;position&quot;: 2,
+#       &quot;force_prefix&quot;: false,
+#       &quot;force_prefix_search&quot;: false
+#     }
+#   ]
+# ]
+</pre></div>
+</div>
+<p><code class="docutils literal notranslate"><span class="pre">\\s*</span></code> of the end of above regular expression match 0 or more spaces after a delimiter.</p>
+<p><code class="docutils literal notranslate"><span class="pre">[。！？]+</span></code> matches 1 or more <code class="docutils literal notranslate"><span class="pre">。</span></code> or <code class="docutils literal notranslate"><span class="pre">！</span></code>, <code class="docutils literal notranslate"><span class="pre">？</span></code>.
+For example, <code class="docutils literal notranslate"><span class="pre">[。！？]+</span></code> matches <code class="docutils literal notranslate"><span class="pre">！？</span></code> of <code class="docutils literal notranslate"><span class="pre">これはペンですか！？</span></code>.</p>
+<p><code class="docutils literal notranslate"><span class="pre">(?![）」])</span></code> is negative lookahead.
+<code class="docutils literal notranslate"><span class="pre">(?![）」])</span></code> matches if a character is not matched <code class="docutils literal notranslate"><span class="pre">）</span></code> or <code class="docutils literal notranslate"><span class="pre">」</span></code>.
+negative lookahead interprets in combination regular expression of just before.</p>
+<p>Therefore it interprets <code class="docutils literal notranslate"><span class="pre">[。！？]+(?![）」])</span></code>.</p>
+<p><code class="docutils literal notranslate"><span class="pre">[。！？]+(?![）」])</span></code> matches if there are not <code class="docutils literal notranslate"><span class="pre">）</span></code> or <code class="docutils literal notranslate"><span class="pre">」</span></code> after <code class="docutils literal notranslate"><span class="pre">。</span></code> or <code class="docutils literal notranslate"><span class="pre">！</span></code>, <code class="docutils literal notranslate"><span class="pre">？</span></code>.</p>
+<p>In other words, <code class="docutils literal notranslate"><span class="pre">[。！？]+(?![）」])</span></code> matches <code class="docutils literal notranslate"><span class="pre">。</span></code> of <code class="docutils literal notranslate"><span class="pre">これはペンですか。</span></code>. But <code class="docutils literal notranslate"><span class="pre">[。！？]+(?![）」])</span></code> doesn’t match <code class="docutils literal notranslate"><span class="pre">。</span></code> of <code class="docutils literal notranslate"><span class="pre">「リンゴです。」</span></code>.
+Because there is <code class="docutils literal notranslate"><span class="pre">」</span></code> after <code class="docutils literal notranslate"><span class="pre">。</span></code>.</p>
+<p><code class="docutils literal notranslate"><span class="pre">[\\r\\n]+</span></code> match 1 or more newline character.</p>
+<p>In conclusion, <code class="docutils literal notranslate"><span class="pre">([。！？]+(?![）」])|[\\r\\n]+)\\s*</span></code> uses <code class="docutils literal notranslate"><span class="pre">。</span></code> and <code class="docutils literal notranslate"><span class="pre">！</span></code> and <code class="docutils literal notranslate"><span class="pre">？</span></code>, newline character as delimiter. However, <code class="docutils literal notranslate"><span class="pre">。</span></code> and <code class="docutils literal notranslate"><span class="pre">!</span></code>, <code class="docutils literal notranslate"><span class="pre">？</span></code> are not delimiters if there is <code class="docutils literal notranslate"><span class="pre">）</span></code> or <code class="docutils literal notranslate"><span class="pre">」</span></code> after <code class="docutils literal notranslate"><span class="pre">。</span></code> or <code class="docutils literal notranslate"><span class="pre">！</span></code>, <code class="docutils literal notranslate"><span class="pre">？</span></code>.</p>
+</div>
+<div class="section" id="parameters">
+<h2>7.8.10.6. Parameters<a class="headerlink" href="#parameters" title="Permalink to this headline">¶</a></h2>
+<div class="section" id="optional-parameter">
+<h3>7.8.10.6.1. Optional parameter<a class="headerlink" href="#optional-parameter" title="Permalink to this headline">¶</a></h3>
+<p>There are two optional parameters <code class="docutils literal notranslate"><span class="pre">delimiter</span></code> and <code class="docutils literal notranslate"><span class="pre">pattern</span></code>.</p>
+<div class="section" id="delimiter">
+<h4>7.8.10.6.1.1. <code class="docutils literal notranslate"><span class="pre">delimiter</span></code><a class="headerlink" href="#delimiter" title="Permalink to this headline">¶</a></h4>
+<p>Split token with a specified one or more characters.</p>
+<p>You can use one or more characters for a delimiter.</p>
+</div>
+<div class="section" id="pattern">
+<h4>7.8.10.6.1.2. <code class="docutils literal notranslate"><span class="pre">pattern</span></code><a class="headerlink" href="#pattern" title="Permalink to this headline">¶</a></h4>
+<p>Split token with a regular expression.</p>
+</div>
+</div>
+</div>
+<div class="section" id="see-also">
+<h2>7.8.10.7. See also<a class="headerlink" href="#see-also" title="Permalink to this headline">¶</a></h2>
+<ul class="simple">
+<li><p><a class="reference internal" href="../commands/tokenize.html"><span class="doc">tokenize</span></a></p></li>
+</ul>
+</div>
+</div>
+          </div>
+        </div>
+      </div>
+      <div class="sphinxsidebar" role="navigation" aria-label="main navigation">
+        <div class="sphinxsidebarwrapper">
+  <h3><a href="../../index.html">Table of Contents</a></h3>
+  <ul>
+<li><a class="reference internal" href="#">7.8.10. <code class="docutils literal notranslate"><span class="pre">TokenDelimit</span></code></a><ul>
+<li><a class="reference internal" href="#summary">7.8.10.1. Summary</a></li>
+<li><a class="reference internal" href="#syntax">7.8.10.2. Syntax</a></li>
+<li><a class="reference internal" href="#usage">7.8.10.3. Usage</a></li>
+<li><a class="reference internal" href="#simple-usage">7.8.10.4. Simple usage</a></li>
+<li><a class="reference internal" href="#advanced-usage">7.8.10.5. Advanced usage</a></li>
+<li><a class="reference internal" href="#parameters">7.8.10.6. Parameters</a><ul>
+<li><a class="reference internal" href="#optional-parameter">7.8.10.6.1. Optional parameter</a><ul>
+<li><a class="reference internal" href="#delimiter">7.8.10.6.1.1. <code class="docutils literal notranslate"><span class="pre">delimiter</span></code></a></li>
+<li><a class="reference internal" href="#pattern">7.8.10.6.1.2. <code class="docutils literal notranslate"><span class="pre">pattern</span></code></a></li>
+</ul>
+</li>
+</ul>
+</li>
+<li><a class="reference internal" href="#see-also">7.8.10.7. See also</a></li>
+</ul>
+</li>
+</ul>
+  <h4>Previous topic</h4>
+  <p class="topless"><a href="token_bigram_split_symbol_alpha_digit.html"
+                        title="previous chapter">7.8.9. <code class="docutils literal notranslate"><span class="pre">TokenBigramSplitSymbolAlphaDigit</span></code></a></p>
+  <h4>Next topic</h4>
+  <p class="topless"><a href="token_delimit_null.html"
+                        title="next chapter">7.8.11. <code class="docutils literal notranslate"><span class="pre">TokenDelimitNull</span></code></a></p>
+<div id="searchbox" style="display: none" role="search">
+  <h3>Quick search</h3>
+    <div class="searchformwrapper">
+    <form class="search" action="../../search.html" method="get">
+      <input type="text" name="q" />
+      <input type="submit" value="Go" />
+    </form>
+    </div>
+</div>
+<script type="text/javascript">$('#searchbox').show(0);</script>
+        </div>
+      </div>
+      <div class="clearer"></div>
+    </div>
+    <div class="related" role="navigation" aria-label="related navigation">
+      <h3>Navigation</h3>
+      <ul>
+        <li class="right" style="margin-right: 10px">
+          <a href="../../genindex.html" title="General Index"
+             >index</a></li>
+        <li class="right" >
+          <a href="token_delimit_null.html" title="7.8.11. TokenDelimitNull"
+             >next</a> |</li>
+        <li class="right" >
+          <a href="token_bigram_split_symbol_alpha_digit.html" title="7.8.9. TokenBigramSplitSymbolAlphaDigit"
+             >previous</a> |</li>
+        <li class="nav-item nav-item-0"><a href="../../index.html">Groonga v9.0.2 documentation</a> &#187;</li>
+          <li class="nav-item nav-item-1"><a href="../../reference.html" >7. Reference manual</a> &#187;</li>
+          <li class="nav-item nav-item-2"><a href="../tokenizers.html" >7.8. Tokenizers</a> &#187;</li>
+      </ul>
+    </div>
+    <div class="footer" role="contentinfo">
+        &#169; Copyright 2009-2019, Brazil, Inc.
+    </div>
+  </body>
+</html>