native-vector-store 0.3.8 → 0.4.0

package/src/english_punctuations.h

@@ -0,0 +1,42 @@
+#pragma once
+#include <unordered_set>
+#include <string>
+
+class EnglishPunctuations {
+public:
+    /// Returns the singleton instance
+    static const EnglishPunctuations& getInstance() {
+        static EnglishPunctuations instance;  // Thread-safe in C++11+
+        return instance;
+    }
+
+    /// Test membership
+    bool contains(const std::string& mark) const noexcept {
+        return dict_.find(mark) != dict_.end();
+    }
+
+    /// Number of punctuation marks
+    std::size_t size() const noexcept {
+        return dict_.size();
+    }
+
+    /// Iteration support
+    auto begin() const noexcept { return dict_.begin(); }
+    auto end()   const noexcept { return dict_.end();   }
+
+private:
+    // Private ctor builds the fixed set
+    EnglishPunctuations()
+      : dict_{
+          "[", "]", "(", ")", "{", "}", "<", ">", ":",
+          ",", ";", "-", "--", "---", "!", "?", ".",
+          "...", "`", "'", "\"", "/"
+        }
+    {}
+
+    // Non-copyable, non-movable
+    EnglishPunctuations(const EnglishPunctuations&) = delete;
+    EnglishPunctuations& operator=(const EnglishPunctuations&) = delete;
+
+    const std::unordered_set<std::string> dict_;
+};

package/src/english_stop_words.h

@@ -0,0 +1,434 @@
+// EnglishStopWords.h
+#pragma once
+
+#include <string>
+#include <unordered_set>
+#include <sstream>
+
+/// Provides access to a single, compiled-in list of English stop words.
+/// The set is built once (thread-safe in C++11+) from the raw string literal below.
+class EnglishStopWords {
+public:
+    /// Returns the singleton stop-word set.
+    static const std::unordered_set<std::string>& instance() {
+        static const std::unordered_set<std::string> dict = []{
+            // Paste your newline-separated stop-words list between the delimiters:
+            static constexpr const char* blob = R"STOPWORDS(
+*
+•
+
+'ll
+'re
+'ve
+n't
+'s
+'m
+'da
+'n
+'ye
+i'm
+you're
+he's
+she's
+it's
+we're
+they're
+i've
+you've
+we've
+they've
+i'd
+you'd
+he'd
+she'd
+we'd
+they'd
+i'll
+you'll
+he'll
+she'll
+we'll
+they'll
+isn't
+aren't
+wasn't
+weren't
+hasn't
+haven't
+hadn't
+doesn't
+don't
+didn't
+won't
+wouldn't
+shan't
+shouldn't
+can't
+cannot
+couldn't
+mustn't
+let's
+that's
+who's
+what's
+here's
+there's
+when's
+where's
+why's
+how's
+daren't
+needn't
+oughtn't
+mightn't
+a
+able
+about
+above
+across
+after
+afterwards
+again
+against
+all
+almost
+alone
+along
+already
+also
+although
+always
+am
+among
+amongst
+amoungst
+amount
+an
+and
+another
+any
+anyhow
+anyone
+anything
+anyway
+anywhere
+are
+around
+as
+at
+back
+be
+became
+because
+become
+becomes
+becoming
+been
+before
+beforehand
+behind
+being
+below
+beside
+besides
+between
+beyond
+bill
+both
+bottom
+but
+by
+call
+can
+cannot
+cant
+co
+con
+could
+couldnt
+cry
+de
+describe
+detail
+do
+done
+down
+due
+during
+each
+eg
+eight
+either
+eleven
+else
+elsewhere
+empty
+enough
+etc
+even
+ever
+every
+everyone
+everything
+everywhere
+except
+few
+fifteen
+fify
+fill
+find
+fire
+first
+five
+for
+former
+formerly
+forty
+found
+four
+from
+front
+full
+further
+get
+give
+go
+had
+has
+hasnt
+have
+he
+hence
+her
+here
+hereafter
+hereby
+herein
+hereupon
+hers
+herself
+him
+himself
+his
+how
+however
+hundred
+i
+ie
+if
+in
+inc
+indeed
+interest
+into
+is
+it
+its
+itself
+keep
+last
+latter
+latterly
+least
+less
+ltd
+made
+many
+may
+me
+meanwhile
+might
+mill
+mine
+more
+moreover
+most
+mostly
+move
+much
+must
+my
+myself
+name
+namely
+neither
+never
+nevertheless
+next
+nine
+no
+nobody
+none
+noone
+nor
+not
+nothing
+now
+nowhere
+of
+off
+often
+on
+once
+one
+only
+onto
+or
+other
+others
+otherwise
+our
+ours
+ourselves
+out
+over
+own
+part
+per
+perhaps
+please
+put
+rather
+re
+same
+see
+seem
+seemed
+seeming
+seems
+serious
+several
+she
+should
+show
+side
+since
+sincere
+six
+sixty
+so
+some
+somehow
+someone
+something
+sometime
+sometimes
+somewhere
+still
+such
+system
+take
+ten
+than
+that
+the
+their
+them
+themselves
+then
+thence
+there
+thereafter
+thereby
+therefore
+therein
+thereupon
+these
+they
+thick
+thin
+third
+this
+those
+though
+three
+through
+throughout
+thru
+thus
+to
+together
+too
+top
+toward
+towards
+twelve
+twenty
+two
+un
+under
+until
+up
+upon
+us
+very
+via
+was
+we
+well
+were
+what
+whatever
+when
+whence
+whenever
+where
+whereafter
+whereas
+whereby
+wherein
+whereupon
+wherever
+whether
+which
+while
+whither
+who
+whoever
+whole
+whom
+whose
+why
+will
+with
+within
+without
+would
+yet
+you
+your
+yours
+yourself
+yourselves
+)STOPWORDS";
+
+            std::unordered_set<std::string> tmp;
+            std::istringstream in{blob};
+            for (std::string w; std::getline(in, w); ) {
+                if (!w.empty()) {
+                    tmp.insert(w);
+                }
+            }
+            return tmp;
+        }();
+        return dict;
+    }
+
+    /// Returns true if `word` is a stop word.
+    static bool contains(const std::string& word) {
+        return instance().count(word) > 0;
+    }
+
+    /// Number of stop words loaded.
+    static std::size_t size() {
+        return instance().size();
+    }
+
+private:
+    // Prevent instantiation or copying.
+    EnglishStopWords() = delete;
+    ~EnglishStopWords() = delete;
+    EnglishStopWords(const EnglishStopWords&) = delete;
+    EnglishStopWords& operator=(const EnglishStopWords&) = delete;
+};
+

package/src/simple_sentence_splitter.h

@@ -0,0 +1,218 @@
+// SimpleSentenceSplitter.h
+#pragma once
+
+#include <string>
+#include <vector>
+#include <regex>
+#include <algorithm>
+#include <cctype>
+
+// You’ll need C++ ports of these:
+//   • EnglishAbbreviations::contains(const std::string&)
+//   • EnglishDictionary::instance().count(const std::string&)
+#include "english_abbreviations.h"
+#include "english_dictionary.h"
+
+class SimpleSentenceSplitter {
+public:
+    /// Singleton accessor
+    static SimpleSentenceSplitter& getInstance() {
+        static SimpleSentenceSplitter instance;
+        return instance;
+    }
+
+    /// Split text into sentences.
+    std::vector<std::string> split(const std::string& input) {
+        std::vector<std::string> sentences;
+        int len = 0;
+        std::string text = input;
+
+        // 1) Normalize carriage returns to spaces
+        text = std::regex_replace(text, regexCarriageReturn(), " ");
+
+        // 2) Clear any stray 0x19 markers
+        for (char& c : text) if (c == '\x19') c = ' ';
+
+        // 3) Insert 0x19 where a space was likely forgotten after .!? 
+        text = std::regex_replace(text, regexForgottenSpace(), "$1$2\x19$3");
+
+        // 4) Add a newline so regex can match the final sentence
+        text.push_back('\n');
+
+        auto begin = text.cbegin();
+        std::smatch m;
+        std::string current;
+
+        // 5) Loop over sentence-boundary matches
+        while (std::regex_search(begin, text.cend(), m, regexSentence())) {
+            // Extract groups
+            std::string sent = m[1].str();
+            std::string punct = m[2].str();
+
+            // Determine which “after” group matched, and compute its end offset
+            std::string after;
+            size_t offsetBase = begin - text.cbegin();
+            size_t newEnd;
+            if (m[3].matched) {
+                after = m[3].str();
+                newEnd = m.position(3) + m.length(3) + offsetBase;
+            }
+            else if (m[5].matched) {
+                after = m[5].str();
+                newEnd = m.position(5) + m.length(5) + offsetBase;
+            }
+            else {
+                after.clear();
+                newEnd = m.position(0) + m.length(0) + offsetBase;
+            }
+
+            // Count words in 'sent'
+            len += countWords(sent);
+
+            std::string nextWord = m[4].matched ? m[4].str() : "";
+
+            // Decide if this is a true break
+            bool isBreak = false;
+            if (punct == ".") {
+                if (!isAbbreviation(sent, nextWord, len)) isBreak = true;
+            }
+            else if (punct == "!" || punct == "?" || (punct == ":" && len > 6)) {
+                isBreak = true;
+            }
+
+            // Append appropriately
+            if (isBreak) {
+                appendSentence(sentences, current, sent, punct, after);
+                len = 0;
+            } else {
+                appendContinuation(current, sent, punct, after);
+            }
+
+            // Move search cursor forward
+            begin = text.cbegin() + newEnd;
+        }
+
+        // Capture any trailing text
+        size_t consumed = begin - text.cbegin();
+        if (consumed < text.size()) {
+            current += text.substr(consumed);
+        }
+        if (!current.empty()) {
+            sentences.push_back(cleanOutput(current));
+        }
+
+        return sentences;
+    }
+
+private:
+    SimpleSentenceSplitter() = default;
+    SimpleSentenceSplitter(const SimpleSentenceSplitter&) = delete;
+    SimpleSentenceSplitter& operator=(const SimpleSentenceSplitter&) = delete;
+
+    // Regex factories (thread‐safe init)
+    static const std::regex& regexCarriageReturn() {
+        static const std::regex r{"[\\n\\r]+"};
+        return r;
+    }
+    static const std::regex& regexForgottenSpace() {
+        static const std::regex r{"(.)([\\.!?])([^0-9\\s\\.\"'`\\)\\}\\]])"};
+        return r;
+    }
+    static const std::regex& regexSentence() {
+        static const std::regex r{
+            R"((['\"`]*[\(\{\[]?[A-Za-z0-9]+.*?)([\.!\?:])"
+            R"(?:(?=([\(\[\{\"'`<>]*[ \x19]+)[\(\[\{\"'`\)\}\] ]*([A-Z0-9][a-z]*))"
+            R"(|(?=([\(\)\"'`<\}\] \x19]+)\s)))"
+        };
+        return r;
+    }
+    static const std::regex& regexWhitespace() {
+        static const std::regex r{"\\s+"};
+        return r;
+    }
+    static const std::regex& regexLastWord() {
+        static const std::regex r{"\\b([\\w0-9\\.']+)$"};
+        return r;
+    }
+
+    // Helpers
+    static size_t countWords(const std::string& s) {
+        return std::distance(
+            std::sregex_token_iterator(s.begin(), s.end(), regexWhitespace(), -1),
+            std::sregex_token_iterator{}
+        );
+    }
+
+    static std::string extractLastWord(const std::string& s) {
+        std::smatch m2;
+        if (std::regex_search(s, m2, regexLastWord())) return m2[1].str();
+        return "";
+    }
+
+    static bool isAbbreviation(const std::string& sentence,
+                               const std::string& nextWord,
+                               int wordCount) 
+    {
+        std::string last = extractLastWord(sentence);
+        // Check vowel presence, letter patterns, single-letter
+        static const std::regex hasVowel{"[AEIOUaeiou]"};
+        static const std::regex hasLower{"[a-z]"};
+        static const std::regex hasY{"y"};
+        static const std::regex letterDot{"([A-Za-z]\\.)+"};
+
+        bool cond1 = !std::regex_search(last, hasVowel)
+                     && std::regex_search(last, hasLower)
+                     && !std::regex_search(last, hasY);
+        bool cond2 = std::regex_match(last, letterDot);
+        bool cond3 = (last.size()==1 && std::isalpha(last[0]) && last!="I");
+        bool cond4 = EnglishAbbreviations::contains(toLower(last));
+
+        if (cond1||cond2||cond3||cond4) {
+            if (EnglishDictionary::instance().count(nextWord) && wordCount>6) {
+                return false; // actually a sentence break
+            }
+            return true; // abbreviation = no break
+        }
+        return false;
+    }
+
+    static std::string toLower(const std::string& s) {
+        std::string out; out.reserve(s.size());
+        for (char c: s) out.push_back(std::tolower((unsigned char)c));
+        return out;
+    }
+
+    static void appendSentence(std::vector<std::string>& v,
+                               std::string& curr,
+                               const std::string& sent,
+                               const std::string& punct,
+                               const std::string& after)
+    {
+        curr += sent + punct + after;
+        v.push_back(cleanOutput(curr));
+        curr.clear();
+    }
+
+    static void appendContinuation(std::string& curr,
+                                   const std::string& sent,
+                                   const std::string& punct,
+                                   const std::string& after)
+    {
+        curr += sent + punct;
+        if (after.find('\x19')==std::string::npos) curr.push_back(' ');
+    }
+
+    static std::string cleanOutput(const std::string& s) {
+        // Remove markers and trim whitespace
+        std::string tmp;
+        tmp.reserve(s.size());
+        for (char c: s) if (c!='\x19') tmp.push_back(c);
+        // Trim
+        auto ws = " \t\n\r";
+        auto start = tmp.find_first_not_of(ws);
+        if (start==std::string::npos) return "";
+        auto end = tmp.find_last_not_of(ws);
+        return tmp.substr(start, end-start+1);
+    }
+};
+