unf_ext 0.0.8.2.beta-x64-mingw-ucrt

data/ext/unf_ext/unf/trie/char_stream.hh

@@ -0,0 +1,150 @@
+#ifndef UNF_TRIE_CHAR_STREAM_HH
+#define UNF_TRIE_CHAR_STREAM_HH
+
+#include <vector>
+#include <string>
+#include "../util.hh"
+
+namespace UNF {
+  namespace Trie {
+    class CharStream {
+    public:
+      CharStream(const char* str) : cur_(str) {}
+      unsigned char read() { return eos() ? '\0' : *cur_++; }
+      unsigned char prev() const { return cur_[-1]; }
+      unsigned char peek() const { return *cur_; } 
+      const char*   cur() const { return cur_; }
+      bool          eos() const { return *cur_ == '\0'; }
+      void          setCur(const char* new_cur) { cur_ = new_cur; }
+
+    private:
+      const char* cur_;
+    };
+
+    class RangeCharStream {
+    public:
+      RangeCharStream(const char* beg, const char* end) : cur_(beg), end_(end) {}
+      unsigned char read() { return eos() ? '\0' : *cur_++; }
+      unsigned char prev() const { return cur_[-1]; }
+      unsigned char peek() const { return *cur_; } 
+      const char*   cur() const { return cur_; }
+      const char*   end() const { return end_; }
+      bool          eos() const { return cur_ == end_; }
+
+    private:
+      const char* cur_;
+      const char* end_;
+    };
+
+    class CompoundCharStream {
+    public:
+      CompoundCharStream(const char* first, const char* second) 
+	: beg1(first), beg2(second), cur1(beg1), cur2(beg2) {}
+
+      unsigned char read() { return !eos1() ? read1() : read2(); }
+      unsigned char peek() const { return !eos1() ? *cur1 : *cur2; }
+      unsigned char prev() const { return !eos1() || beg2==cur2 ? cur1[-1] : cur2[-1]; }
+
+      const char* cur() const { return !eos1() ? cur1 : cur2; }
+      bool eos() const { return eos1() && eos2(); }
+      bool within_first() const { return !eos1(); }
+
+      unsigned offset() const { return cur1-beg1 + cur2-beg2; }
+      void setCur(const char* p) { 
+	if(beg1 <= p && p <= cur1) {
+	  cur1=p;
+	  cur2=beg2;
+	} else {
+	  cur2=p;
+	}
+      }
+
+    protected:
+      unsigned char read1() { return eos1() ? '\0' : *cur1++; }
+      unsigned char read2() { return eos2() ? '\0' : *cur2++; }
+      bool eos1() const { return *cur1=='\0'; }
+      bool eos2() const { return *cur2=='\0'; }
+      
+    protected:
+      const char* beg1;
+      const char* beg2;
+      const char* cur1;
+      const char* cur2;
+    };
+
+    class CharStreamForComposition : public CompoundCharStream {
+    public:
+      CharStreamForComposition (const char* first, const char* second, 
+				const std::vector<unsigned char>& canonical_classes, 
+				std::string& buf)
+	: CompoundCharStream(first, second), classes(canonical_classes), skipped(buf) 
+      {}
+      
+      void init_skipinfo() { 
+	skipped.clear();
+	skipped_tail = 0;
+      }
+
+      void mark_as_last_valid_point() {
+	skipped_tail = skipped.size();
+	marked_point = cur();
+      }
+
+      void reset_at_marked_point() {
+	setCur(marked_point);
+      }
+
+      void append_read_char_to_str(std::string& s, const char* beg) const {
+	if(eos1()==false) {
+	  s.append(beg, cur());
+	} else {
+	  s.append(beg,  cur1);
+	  s.append(beg2, cur());
+	}
+      }
+
+      void append_skipped_chars_to_str(std::string& s) const {
+	s.append(skipped.begin(), skipped.begin()+skipped_tail);
+      }
+
+      unsigned char get_canonical_class() const { 
+	return offset() < classes.size() ? classes[offset()] : 0;
+      }
+      
+      bool next_combining_char(unsigned char prev_class, const char* ppp) {
+	while(Util::is_utf8_char_start_byte(peek()) == false)
+	  read();
+	
+	unsigned char mid_class = get_prev_canonical_class();
+	unsigned char cur_class = get_canonical_class();
+	
+	if(prev_class==0 && mid_class==0 && cur_class!=0)
+	  return false;
+
+	if(prev_class < cur_class && mid_class < cur_class) {
+	  skipped.append(ppp, cur());
+	  return true;
+	} else {
+	  if(cur_class != 0) {
+	    read();
+	    return next_combining_char(prev_class,ppp);
+	  }
+	  return false;
+	}
+      }
+
+    private:
+      unsigned char get_prev_canonical_class() const { 
+	return offset()-1 < classes.size() ? classes[offset()-1] : 0;
+      }
+
+    private:
+      const std::vector<unsigned char>& classes;
+      std::string& skipped;
+      unsigned skipped_tail;
+      const char* marked_point;
+    };
+  }
+}
+
+#endif

data/ext/unf_ext/unf/trie/node.hh

@@ -0,0 +1,25 @@
+#ifndef UNF_TRIE_NODE_HH
+#define UNF_TRIE_NODE_HH
+
+namespace UNF {
+  namespace Trie {
+    class Node {
+    public:
+      unsigned jump(unsigned char ch) const { return base() + ch; }
+      unsigned value() const { return base(); }
+      unsigned check_char() const { return data>>24; }
+      unsigned to_uint() const { return data; }
+
+      static const Node* from_uint_array(const unsigned* node_uints)
+      { return reinterpret_cast<const Node*>(node_uints); }
+
+    private:
+      unsigned base() const { return data & 0xFFFFFF; }
+
+    private:
+      unsigned data;
+    };
+  }
+}
+
+#endif

data/ext/unf_ext/unf/trie/searcher.hh

@@ -0,0 +1,194 @@
+#ifndef UNF_TRIE_SEARCHER_HH
+#define UNF_TRIE_SEARCHER_HH
+
+#include "char_stream.hh"
+#include "node.hh"
+#include "../util.hh"
+
+namespace UNF {
+  namespace Trie {
+    class Searcher {
+    public:
+      Searcher(const Node* nodes, unsigned root, const char* value=NULL)
+	: nodes(nodes), root(root), value(value) {}
+
+      unsigned find_value(const char* key, int default_value) const {
+	unsigned node_index=root;
+	for(CharStream in(key);; in.read()) {
+	  node_index = nodes[node_index].jump(in.peek());
+	  if(nodes[node_index].check_char()==in.peek()) {
+	    unsigned terminal_index = nodes[node_index].jump('\0'); 
+	    if(nodes[terminal_index].check_char()=='\0') {
+	      return nodes[terminal_index].value();
+            }
+	  } else
+	    return default_value;
+	}
+      }     
+
+    protected:
+      const Node* nodes;
+      const unsigned root;
+      const char* value;
+    }; 
+    
+    class CanonicalCombiningClass : private Searcher {
+    public:
+      CanonicalCombiningClass(const unsigned* node_uints, unsigned root)
+	: Searcher(Node::from_uint_array(node_uints), root) {}
+      
+      unsigned get_class(const char* str) const { return find_value(str,0); }
+
+      void sort(char* str, std::vector<unsigned char>& classes) const {
+	CharStream in(str);
+	unsigned sort_beg=0;
+	unsigned sort_end=0;
+	unsigned unicode_char_count=0;
+
+      loop_head:
+	unsigned beg = in.cur()-str;
+	
+	for(unsigned node_index=root;;){
+	  node_index = nodes[node_index].jump(in.read());
+	  
+	  if(nodes[node_index].check_char()==in.prev()) {
+	    unsigned terminal_index = nodes[node_index].jump('\0');
+	    if(nodes[terminal_index].check_char()=='\0') {
+	      if((unicode_char_count++)==0)
+		sort_beg = beg;
+	      sort_end = in.cur()-str;
+	      
+	      unsigned char klass = nodes[terminal_index].value();
+	      for(unsigned i=beg; i < sort_end; i++) 
+		classes[i] = klass;
+ 	      break;
+	    }
+	  } else {
+	    if(unicode_char_count > 1)
+	      bubble_sort(str, classes, sort_beg, sort_end);
+	    unicode_char_count = 0;
+	    break;
+	  }
+	} 
+	Util::eat_until_utf8_char_start_point(in);
+
+	if(in.eos()==false)
+	  goto loop_head;
+
+	if(unicode_char_count > 1)
+	  bubble_sort(str, classes, sort_beg, sort_end);
+      }      
+
+    private:
+      void bubble_sort(char* str, std::vector<unsigned char>& canonical_classes, unsigned beg, unsigned end) const {
+	for(unsigned limit=beg, next=end; limit != next;) {
+	  limit = next;
+	  for(unsigned i=beg+1; i < limit; i++)
+	    if(canonical_classes[i-1] > canonical_classes[i]) {
+	      std::swap(canonical_classes[i-1], canonical_classes[i]);
+	      std::swap(str[i-1], str[i]);
+	      next = i;
+	    }
+	}
+      }
+    };
+
+    class NormalizationForm : private Searcher {
+    public:
+      NormalizationForm(const unsigned* node_uints, unsigned root, const char* value=NULL)
+	: Searcher(Node::from_uint_array(node_uints), root, value) {} 
+
+      bool quick_check(const char* key) const { return find_value(key,0xFFFFFFFF)==0xFFFFFFFF; }
+
+      void decompose(RangeCharStream in, std::string& buffer) const {
+      loop_head:
+	const char* beg = in.cur();
+
+	for(unsigned node_index=root;;) {
+	  node_index = nodes[node_index].jump(in.read());
+	  if(nodes[node_index].check_char()==in.prev()) {
+	    unsigned terminal_index = nodes[node_index].jump('\0');
+	    if(nodes[terminal_index].check_char()=='\0') {
+              word_append(buffer, value, nodes[terminal_index].value());
+	      beg = in.cur();
+	      break;
+	    }
+	  } else {
+	    Util::eat_until_utf8_char_start_point(in);
+	    buffer.append(beg, in.cur());
+	    break;
+	  }
+	}  
+
+	if(in.eos()==false)
+	  goto loop_head;
+      }
+
+      void compose(CharStreamForComposition& in, std::string& buf) const {
+	in.init_skipinfo();
+
+	const char* const beg = in.cur();
+	const char* current_char_head = in.cur();
+	unsigned composed_char_info = 0;
+	
+	unsigned node_index = root;
+	unsigned retry_root_node = root;
+	unsigned char retry_root_class = 0;
+
+	for(bool first=true;;) {
+	  if(Util::is_utf8_char_start_byte(in.peek())) {
+	    if(node_index != root)
+	      first=false;
+	    current_char_head = in.cur();
+
+	    retry_root_node = node_index;
+	    retry_root_class = in.get_canonical_class();
+	  }
+
+	retry:
+	  unsigned next_index = nodes[node_index].jump(in.peek());
+	  if(nodes[next_index].check_char()==in.read()) {
+	    // succeeded
+	    node_index = next_index;
+	    unsigned terminal_index = nodes[node_index].jump('\0');
+	    if(nodes[terminal_index].check_char()=='\0') {
+	      composed_char_info = nodes[terminal_index].value();
+              
+	      in.mark_as_last_valid_point();
+	      if(in.eos() || retry_root_class > in.get_canonical_class())
+		break;
+	    }
+	  } else if (first==true) {
+	    // no retry if current point is a part of first starter
+	    break;
+	  } else if (in.next_combining_char(retry_root_class, current_char_head)==true) { 
+	    // back previous code-point and retry
+	    node_index = retry_root_node;
+	    current_char_head = in.cur();
+	    goto retry;
+	  } else {
+	    break;
+	  }
+	}  
+	
+	if(composed_char_info != 0) {
+	  // append composed unicode-character and skipped combining-characters
+	  word_append(buf, value, composed_char_info);
+	  in.append_skipped_chars_to_str(buf);
+	  in.reset_at_marked_point();
+	} else {
+	  // append one unicode-character
+	  in.setCur(Util::nearest_utf8_char_start_point(beg+1));
+	  in.append_read_char_to_str(buf, beg);
+	}
+      }
+      
+    private:
+      static void word_append(std::string& buffer, const char* base, unsigned pos_info) {
+        buffer.append(base+(pos_info&0x3FFFF), pos_info>>18);
+      }
+    };
+  }
+}
+
+#endif

data/ext/unf_ext/unf/util.hh

@@ -0,0 +1,24 @@
+#ifndef UNF_UTIL_HH
+#define UNF_UTIL_HH
+
+namespace UNF {
+  namespace Util {
+    inline bool is_utf8_char_start_byte(char byte) {
+      if(!(byte&0x80))    return true; // ascii
+      else if (byte&0x40) return true; // start of a UTF-8 character byte sequence
+      return false;
+    }
+
+    inline const char* nearest_utf8_char_start_point(const char* s) {
+      for(; is_utf8_char_start_byte(*s)==false; s++);
+      return s;
+    }
+
+    template <class CharStream>
+    inline void eat_until_utf8_char_start_point(CharStream& in) {
+      for(; is_utf8_char_start_byte(in.peek())==false; in.read());
+    }
+  }
+}
+
+#endif

data/ext/unf_ext/unf.cc

@@ -0,0 +1,75 @@
+#include "unf/normalizer.hh"
+
+#include <ruby.h>
+#if defined(HAVE_RUBY_ENCODING_H)
+#include <ruby/encoding.h>
+#endif
+
+extern "C" {
+  VALUE unf_allocate(VALUE klass);
+  VALUE unf_initialize(VALUE self);
+  void unf_delete(UNF::Normalizer* ptr);
+  VALUE unf_normalize(VALUE self, VALUE source, VALUE normalization_form);
+
+  ID FORM_NFD;
+  ID FORM_NFC;
+  ID FORM_NFKD;
+  ID FORM_NFKC;
+
+  void Init_unf_ext() {
+    VALUE mdl = rb_define_module("UNF");
+
+    VALUE cls = rb_define_class_under(mdl, "Normalizer", rb_cObject);
+    rb_define_alloc_func(cls, unf_allocate);
+    rb_define_method(cls, "initialize", (VALUE (*)(...))unf_initialize, 0);
+    rb_define_method(cls, "normalize", (VALUE (*)(...))unf_normalize, 2);
+
+    FORM_NFD = rb_intern("nfd");
+    FORM_NFC = rb_intern("nfc");
+    FORM_NFKD= rb_intern("nfkd");
+    FORM_NFKC= rb_intern("nfkc");
+  }
+
+
+  VALUE unf_allocate(VALUE klass) {
+    UNF::Normalizer* ptr;
+    VALUE obj = Data_Make_Struct(klass, UNF::Normalizer, NULL, unf_delete, ptr);
+    new ((void*)ptr) UNF::Normalizer;
+    return obj;
+  }
+
+  VALUE unf_initialize(VALUE self) {
+    return self;
+  }
+
+  void unf_delete(UNF::Normalizer* ptr) {
+    ptr->~Normalizer();
+    ruby_xfree(ptr);
+  }
+
+  VALUE unf_normalize(VALUE self, VALUE source, VALUE normalization_form) {
+    UNF::Normalizer* ptr;
+    Data_Get_Struct(self, UNF::Normalizer, ptr);
+
+    const char* src = StringValueCStr(source);
+    const char* rlt;
+    ID form_id = SYM2ID(normalization_form);
+
+    if(form_id == FORM_NFD)
+      rlt = ptr->normalize(src, UNF::Normalizer::FORM_NFD);
+    else if(form_id == FORM_NFC)
+      rlt = ptr->normalize(src, UNF::Normalizer::FORM_NFC);
+    else if(form_id == FORM_NFKD)
+      rlt = ptr->normalize(src, UNF::Normalizer::FORM_NFKD);
+    else if(form_id == FORM_NFKC)
+      rlt = ptr->normalize(src, UNF::Normalizer::FORM_NFKC);
+    else
+      rb_raise(rb_eArgError, "Specified Normalization-Form is unknown. Please select one from among :nfc, :nfd, :nfkc, :nfkd.");
+
+#if defined(HAVE_RUBY_ENCODING_H)
+    return rb_enc_str_new(rlt, strlen(rlt), rb_utf8_encoding());
+#else
+    return rb_str_new2(rlt);
+#endif
+  }
+}

data/lib/unf_ext/version.rb

@@ -0,0 +1,5 @@
+module UNF
+  class Normalizer
+    VERSION = "0.0.8.2.beta"
+  end
+end

data/lib/unf_ext.rb

@@ -0,0 +1,5 @@
+begin
+  require "#{RUBY_VERSION[/\A[0-9]+\.[0-9]+/]}/unf_ext.so"
+rescue LoadError
+  require "unf_ext.so"
+end

data/test/helper.rb

@@ -0,0 +1,18 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'unf_ext'
+
+class Test::Unit::TestCase
+end