unf_ext 0.0.3

data/unf/trie/char_stream.hh

@@ -0,0 +1,150 @@
+#ifndef UNF_TRIE_CHAR_STREAM_HH
+#define UNF_TRIE_CHAR_STREAM_HH
+
+#include <vector>
+#include <string>
+#include "../util.hh"
+
+namespace UNF {
+  namespace Trie {
+    class CharStream {
+    public:
+      CharStream(const char* str) : cur_(str) {}
+      unsigned char read() { return eos() ? '\0' : *cur_++; }
+      unsigned char prev() const { return cur_[-1]; }
+      unsigned char peek() const { return *cur_; } 
+      const char*   cur() const { return cur_; }
+      bool          eos() const { return *cur_ == '\0'; }
+      void          setCur(const char* new_cur) { cur_ = new_cur; }
+
+    private:
+      const char* cur_;
+    };
+
+    class RangeCharStream {
+    public:
+      RangeCharStream(const char* beg, const char* end) : cur_(beg), end_(end) {}
+      unsigned char read() { return eos() ? '\0' : *cur_++; }
+      unsigned char prev() const { return cur_[-1]; }
+      unsigned char peek() const { return *cur_; } 
+      const char*   cur() const { return cur_; }
+      const char*   end() const { return end_; }
+      bool          eos() const { return cur_ == end_; }
+
+    private:
+      const char* cur_;
+      const char* end_;
+    };
+
+    class CompoundCharStream {
+    public:
+      CompoundCharStream(const char* first, const char* second) 
+	: beg1(first), beg2(second), cur1(beg1), cur2(beg2) {}
+
+      unsigned char read() { return !eos1() ? read1() : read2(); }
+      unsigned char peek() const { return !eos1() ? *cur1 : *cur2; }
+      unsigned char prev() const { return !eos1() || beg2==cur2 ? cur1[-1] : cur2[-1]; }
+
+      const char* cur() const { return !eos1() ? cur1 : cur2; }
+      bool eos() const { return eos1() && eos2(); }
+      bool within_first() const { return !eos1(); }
+
+      unsigned offset() const { return cur1-beg1 + cur2-beg2; }
+      void setCur(const char* p) { 
+	if(beg1 <= p && p <= cur1) {
+	  cur1=p;
+	  cur2=beg2;
+	} else {
+	  cur2=p;
+	}
+      }
+
+    protected:
+      unsigned char read1() { return eos1() ? '\0' : *cur1++; }
+      unsigned char read2() { return eos2() ? '\0' : *cur2++; }
+      bool eos1() const { return *cur1=='\0'; }
+      bool eos2() const { return *cur2=='\0'; }
+      
+    protected:
+      const char* beg1;
+      const char* beg2;
+      const char* cur1;
+      const char* cur2;
+    };
+
+    class CharStreamForComposition : public CompoundCharStream {
+    public:
+      CharStreamForComposition (const char* first, const char* second, 
+				const std::vector<unsigned char>& canonical_classes, 
+				std::string& buf)
+	: CompoundCharStream(first, second), classes(canonical_classes), skipped(buf) 
+      {}
+      
+      void init_skipinfo() { 
+	skipped.clear();
+	skipped_tail = 0;
+      }
+
+      void mark_as_last_valid_point() {
+	skipped_tail = skipped.size();
+	marked_point = cur();
+      }
+
+      void reset_at_marked_point() {
+	setCur(marked_point);
+      }
+
+      void append_read_char_to_str(std::string& s, const char* beg) const {
+	if(eos1()==false) {
+	  s.append(beg, cur());
+	} else {
+	  s.append(beg,  cur1);
+	  s.append(beg2, cur());
+	}
+      }
+
+      void append_skipped_chars_to_str(std::string& s) const {
+	s.append(skipped.begin(), skipped.begin()+skipped_tail);
+      }
+
+      unsigned char get_canonical_class() const { 
+	return offset() < classes.size() ? classes[offset()] : 0;
+      }
+      
+      bool next_combining_char(unsigned char prev_class, const char* ppp) {
+	while(Util::is_utf8_char_start_byte(peek()) == false)
+	  read();
+	
+	unsigned char mid_class = get_prev_canonical_class();
+	unsigned char cur_class = get_canonical_class();
+	
+	if(prev_class==0 && mid_class==0 && cur_class!=0)
+	  return false;
+
+	if(prev_class < cur_class && mid_class < cur_class) {
+	  skipped.append(ppp, cur());
+	  return true;
+	} else {
+	  if(cur_class != 0) {
+	    read();
+	    return next_combining_char(prev_class,ppp);
+	  }
+	  return false;
+	}
+      }
+
+    private:
+      unsigned char get_prev_canonical_class() const { 
+	return offset()-1 < classes.size() ? classes[offset()-1] : 0;
+      }
+
+    private:
+      const std::vector<unsigned char>& classes;
+      std::string& skipped;
+      unsigned skipped_tail;
+      const char* marked_point;
+    };
+  }
+}
+
+#endif

data/unf/trie/node.hh

@@ -0,0 +1,33 @@
+#ifndef UNF_TRIE_NODE_HH
+#define UNF_TRIE_NODE_HH
+
+namespace UNF {
+  namespace Trie {
+    class Node {
+    public:
+      Node() : data(0xFFFFFFFF) {}
+      
+      void set_base_index(unsigned base_index) { data = (data&0xFF000000)+(base_index&0x00FFFFFF); }
+      void set_value(unsigned value) { set_base_index(value); }
+      void set_check_char(unsigned char ch) { data = (ch << 24) + base(); }
+
+      bool is_unused() const { return data==0xFFFFFFFF; }
+
+      unsigned jump(unsigned char ch) const { return base() + ch; }
+      unsigned value() const { return base(); }
+      unsigned check_char() const { return data>>24; }
+      unsigned to_uint() const { return data; }
+
+      static const Node* from_uint_array(const unsigned* node_uints)
+      { return reinterpret_cast<const Node*>(node_uints); }
+
+    private:
+      unsigned base() const { return data & 0xFFFFFF; }
+
+    private:
+      unsigned data;
+    };
+  }
+}
+
+#endif

data/unf/trie/searcher.hh

@@ -0,0 +1,186 @@
+#ifndef UNF_TRIE_SEARCHER_HH
+#define UNF_TRIE_SEARCHER_HH
+
+#include "char_stream.hh"
+#include "node.hh"
+#include "../util.hh"
+
+namespace UNF {
+  namespace Trie {
+    class Searcher {
+    public:
+      Searcher(const Node* nodes, const char* value=NULL)
+	: nodes(nodes), value(value) {}
+
+      unsigned find_value(const char* key, int default_value) const {
+	unsigned node_index=0;
+	for(CharStream in(key);; in.read()) {
+	  node_index = nodes[node_index].jump(in.peek());
+	  if(nodes[node_index].check_char()==in.peek()) {
+	    unsigned terminal_index = nodes[node_index].jump('\0');
+	    if(nodes[terminal_index].check_char()=='\0')
+	      return nodes[terminal_index].value();
+	  } else
+	    return default_value;
+	}
+      }     
+
+    protected:
+      const Node* nodes;
+      const char* value;
+    }; 
+    
+    class CanonicalCombiningClass : private Searcher {
+    public:
+      CanonicalCombiningClass(const unsigned* node_uints)
+	: Searcher(Node::from_uint_array(node_uints)) {}
+      
+      unsigned get_class(const char* str) const { return find_value(str,0); }
+
+      void sort(char* str, std::vector<unsigned char>& classes) const {
+	CharStream in(str);
+	unsigned sort_beg=0;
+	unsigned sort_end=0;
+	unsigned unicode_char_count=0;
+
+      loop_head:
+	unsigned beg = in.cur()-str;
+	
+	for(unsigned node_index=0;;){
+	  node_index = nodes[node_index].jump(in.read());
+	  
+	  if(nodes[node_index].check_char()==in.prev()) {
+	    unsigned terminal_index = nodes[node_index].jump('\0');
+	    if(nodes[terminal_index].check_char()=='\0') {
+	      if((unicode_char_count++)==0)
+		sort_beg = beg;
+	      sort_end = in.cur()-str;
+	      
+	      unsigned char klass = nodes[terminal_index].value();
+	      for(unsigned i=beg; i < sort_end; i++) 
+		classes[i] = klass;
+ 	      break;
+	    }
+	  } else {
+	    if(unicode_char_count > 1)
+	      bubble_sort(str, classes, sort_beg, sort_end);
+	    unicode_char_count = 0;
+	    break;
+	  }
+	} 
+	Util::eat_until_utf8_char_start_point(in);
+
+	if(in.eos()==false)
+	  goto loop_head;
+
+	if(unicode_char_count > 1)
+	  bubble_sort(str, classes, sort_beg, sort_end);
+      }      
+
+    private:
+      void bubble_sort(char* str, std::vector<unsigned char>& canonical_classes, unsigned beg, unsigned end) const {
+	for(unsigned limit=beg, next=end; limit != next;) {
+	  limit = next;
+	  for(unsigned i=beg+1; i < limit; i++)
+	    if(canonical_classes[i-1] > canonical_classes[i]) {
+	      std::swap(canonical_classes[i-1], canonical_classes[i]);
+	      std::swap(str[i-1], str[i]);
+	      next = i;
+	    }
+	}
+      }
+    };
+
+    class NormalizationForm : private Searcher {
+    public:
+      NormalizationForm(const unsigned* node_uints, const char* value=NULL)
+	: Searcher(Node::from_uint_array(node_uints), value) {} 
+
+      bool quick_check(const char* key) const { return find_value(key,0xFFFFFFFF)==0xFFFFFFFF; }
+
+      void decompose(RangeCharStream in, std::string& buffer) const {
+      loop_head:
+	const char* beg = in.cur();
+
+	for(unsigned node_index=0;;) {
+	  node_index = nodes[node_index].jump(in.read());
+	  if(nodes[node_index].check_char()==in.prev()) {
+	    unsigned terminal_index = nodes[node_index].jump('\0');
+	    if(nodes[terminal_index].check_char()=='\0') {
+	      buffer.append(value+nodes[terminal_index].value());
+	      beg = in.cur();
+	      break;
+	    }
+	  } else {
+	    Util::eat_until_utf8_char_start_point(in);
+	    buffer.append(beg, in.cur());
+	    break;
+	  }
+	}  
+
+	if(in.eos()==false)
+	  goto loop_head;
+      }
+
+      void compose(CharStreamForComposition& in, std::string& buf) const {
+	in.init_skipinfo();
+
+	const char* const beg = in.cur();
+	const char* current_char_head = in.cur();
+	const char* composed_char = NULL;
+	
+	unsigned node_index = 0;
+	unsigned retry_root_node = 0;
+	unsigned char retry_root_class = 0;
+
+	for(bool first=true;;) {
+	  if(Util::is_utf8_char_start_byte(in.peek())) {
+	    if(node_index != 0)
+	      first=false;
+	    current_char_head = in.cur();
+
+	    retry_root_node = node_index;
+	    retry_root_class = in.get_canonical_class();
+	  }
+
+	retry:
+	  unsigned next_index = nodes[node_index].jump(in.read());
+	  if(nodes[next_index].check_char()==in.prev()) {
+	    // succeeded
+	    node_index = next_index;
+	    unsigned terminal_index = nodes[node_index].jump('\0');
+	    if(nodes[terminal_index].check_char()=='\0') {
+	      composed_char = value+nodes[terminal_index].value();
+	      in.mark_as_last_valid_point();
+	      if(in.eos() || retry_root_class > in.get_canonical_class())
+		break;
+	    }
+	  } else if (first==true) {
+	    // no retry if current point is a part of first starter
+	    break;
+	  } else if (in.next_combining_char(retry_root_class, current_char_head)==true) { 
+	    // back previous code-point and retry
+	    node_index = retry_root_node;
+	    current_char_head = in.cur();
+	    goto retry;
+	  } else {
+	    break;
+	  }
+	}  
+	
+	if(composed_char) {
+	  // append composed unicode-character and skipped combining-characters
+	  buf.append(composed_char);
+	  in.append_skipped_chars_to_str(buf);
+	  in.reset_at_marked_point();
+	} else {
+	  // append one unicode-character
+	  in.setCur(Util::nearest_utf8_char_start_point(beg+1));
+	  in.append_read_char_to_str(buf, beg);
+	}
+      }
+    };
+  }
+}
+
+#endif

data/unf/util.hh

@@ -0,0 +1,24 @@
+#ifndef UNF_UTIL_HH
+#define UNF_UTIL_HH
+
+namespace UNF {
+  namespace Util {
+    inline bool is_utf8_char_start_byte(char byte) {
+      if(!(byte&0x80))    return true; // ascii
+      else if (byte&0x40) return true; // start of a UTF-8 character byte sequence
+      return false;
+    }
+
+    inline const char* nearest_utf8_char_start_point(const char* s) {
+      for(; is_utf8_char_start_byte(*s)==false; s++);
+      return s;
+    }
+
+    template <class CharStream>
+    inline void eat_until_utf8_char_start_point(CharStream& in) {
+      for(; is_utf8_char_start_byte(in.peek())==false; in.read());
+    }
+  }
+}
+
+#endif

data/unf_ext.gemspec

@@ -0,0 +1,67 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{unf_ext}
+  s.version = "0.0.3"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = [%q{Takeru Ohta}, %q{Akinori MUSHA}]
+  s.date = %q{2011-10-24}
+  s.description = %q{Unicode Normalization Form support library for CRuby}
+  s.email = %q{knu@idaemons.org}
+  s.extensions = [%q{extconf.rb}]
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.md"
+  ]
+  s.files = [
+    ".document",
+    "Gemfile",
+    "LICENSE.txt",
+    "README.md",
+    "Rakefile",
+    "VERSION",
+    "extconf.rb",
+    "test/helper.rb",
+    "test/normalization-test.txt",
+    "test/test_unf_ext.rb",
+    "unf.cc",
+    "unf/normalizer.hh",
+    "unf/table.hh",
+    "unf/trie/char_stream.hh",
+    "unf/trie/node.hh",
+    "unf/trie/searcher.hh",
+    "unf/util.hh",
+    "unf_ext.gemspec"
+  ]
+  s.homepage = %q{http://github.com/knu/ruby-unf_ext}
+  s.licenses = [%q{MIT}]
+  s.require_paths = [%q{lib}]
+  s.rubygems_version = %q{1.8.5}
+  s.summary = %q{Unicode Normalization Form support library for CRuby}
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_development_dependency(%q<shoulda>, [">= 0"])
+      s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_development_dependency(%q<jeweler>, ["~> 1.6.4"])
+      s.add_development_dependency(%q<rcov>, [">= 0"])
+    else
+      s.add_dependency(%q<shoulda>, [">= 0"])
+      s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
+      s.add_dependency(%q<rcov>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<shoulda>, [">= 0"])
+    s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+    s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
+    s.add_dependency(%q<rcov>, [">= 0"])
+  end
+end
+