unf_ext 0.0.3 → 0.0.4

data/{unf → ext/unf_ext/unf}/trie/node.hh

@@ -5,14 +5,6 @@ namespace UNF {
   namespace Trie {
     class Node {
     public:
-      Node() : data(0xFFFFFFFF) {}
-      
-      void set_base_index(unsigned base_index) { data = (data&0xFF000000)+(base_index&0x00FFFFFF); }
-      void set_value(unsigned value) { set_base_index(value); }
-      void set_check_char(unsigned char ch) { data = (ch << 24) + base(); }
-
-      bool is_unused() const { return data==0xFFFFFFFF; }
-
       unsigned jump(unsigned char ch) const { return base() + ch; }
       unsigned value() const { return base(); }
       unsigned check_char() const { return data>>24; }

data/{unf → ext/unf_ext/unf}/trie/searcher.hh

@@ -9,17 +9,18 @@ namespace UNF {
   namespace Trie {
     class Searcher {
     public:
-      Searcher(const Node* nodes, const char* value=NULL)
-	: nodes(nodes), value(value) {}
+      Searcher(const Node* nodes, unsigned root, const char* value=NULL)
+	: nodes(nodes), root(root), value(value) {}
 
       unsigned find_value(const char* key, int default_value) const {
-	unsigned node_index=0;
+	unsigned node_index=root;
 	for(CharStream in(key);; in.read()) {
 	  node_index = nodes[node_index].jump(in.peek());
 	  if(nodes[node_index].check_char()==in.peek()) {
-	    unsigned terminal_index = nodes[node_index].jump('\0');
-	    if(nodes[terminal_index].check_char()=='\0')
+	    unsigned terminal_index = nodes[node_index].jump('\0'); 
+	    if(nodes[terminal_index].check_char()=='\0') {
 	      return nodes[terminal_index].value();
+            }
 	  } else
 	    return default_value;
 	}
@@ -27,13 +28,14 @@ namespace UNF {
 
     protected:
       const Node* nodes;
+      const unsigned root;
       const char* value;
     }; 
     
     class CanonicalCombiningClass : private Searcher {
     public:
-      CanonicalCombiningClass(const unsigned* node_uints)
-	: Searcher(Node::from_uint_array(node_uints)) {}
+      CanonicalCombiningClass(const unsigned* node_uints, unsigned root)
+	: Searcher(Node::from_uint_array(node_uints), root) {}
       
       unsigned get_class(const char* str) const { return find_value(str,0); }
 
@@ -46,7 +48,7 @@ namespace UNF {
       loop_head:
 	unsigned beg = in.cur()-str;
 	
-	for(unsigned node_index=0;;){
+	for(unsigned node_index=root;;){
 	  node_index = nodes[node_index].jump(in.read());
 	  
 	  if(nodes[node_index].check_char()==in.prev()) {
@@ -93,8 +95,8 @@ namespace UNF {
 
     class NormalizationForm : private Searcher {
     public:
-      NormalizationForm(const unsigned* node_uints, const char* value=NULL)
-	: Searcher(Node::from_uint_array(node_uints), value) {} 
+      NormalizationForm(const unsigned* node_uints, unsigned root, const char* value=NULL)
+	: Searcher(Node::from_uint_array(node_uints), root, value) {} 
 
       bool quick_check(const char* key) const { return find_value(key,0xFFFFFFFF)==0xFFFFFFFF; }
 
@@ -102,12 +104,12 @@ namespace UNF {
       loop_head:
 	const char* beg = in.cur();
 
-	for(unsigned node_index=0;;) {
+	for(unsigned node_index=root;;) {
 	  node_index = nodes[node_index].jump(in.read());
 	  if(nodes[node_index].check_char()==in.prev()) {
 	    unsigned terminal_index = nodes[node_index].jump('\0');
 	    if(nodes[terminal_index].check_char()=='\0') {
-	      buffer.append(value+nodes[terminal_index].value());
+              word_append(buffer, value, nodes[terminal_index].value());
 	      beg = in.cur();
 	      break;
 	    }
@@ -127,15 +129,15 @@ namespace UNF {
 
 	const char* const beg = in.cur();
 	const char* current_char_head = in.cur();
-	const char* composed_char = NULL;
+	unsigned composed_char_info = 0;
 	
-	unsigned node_index = 0;
-	unsigned retry_root_node = 0;
+	unsigned node_index = root;
+	unsigned retry_root_node = root;
 	unsigned char retry_root_class = 0;
 
 	for(bool first=true;;) {
 	  if(Util::is_utf8_char_start_byte(in.peek())) {
-	    if(node_index != 0)
+	    if(node_index != root)
 	      first=false;
 	    current_char_head = in.cur();
 
@@ -144,13 +146,14 @@ namespace UNF {
 	  }
 
 	retry:
-	  unsigned next_index = nodes[node_index].jump(in.read());
-	  if(nodes[next_index].check_char()==in.prev()) {
+	  unsigned next_index = nodes[node_index].jump(in.peek());
+	  if(nodes[next_index].check_char()==in.read()) {
 	    // succeeded
 	    node_index = next_index;
 	    unsigned terminal_index = nodes[node_index].jump('\0');
 	    if(nodes[terminal_index].check_char()=='\0') {
-	      composed_char = value+nodes[terminal_index].value();
+	      composed_char_info = nodes[terminal_index].value();
+              
 	      in.mark_as_last_valid_point();
 	      if(in.eos() || retry_root_class > in.get_canonical_class())
 		break;
@@ -168,9 +171,9 @@ namespace UNF {
 	  }
 	}  
 	
-	if(composed_char) {
+	if(composed_char_info != 0) {
 	  // append composed unicode-character and skipped combining-characters
-	  buf.append(composed_char);
+	  word_append(buf, value, composed_char_info);
 	  in.append_skipped_chars_to_str(buf);
 	  in.reset_at_marked_point();
 	} else {
@@ -179,6 +182,11 @@ namespace UNF {
 	  in.append_read_char_to_str(buf, beg);
 	}
       }
+      
+    private:
+      static void word_append(std::string& buffer, const char* base, unsigned pos_info) {
+        buffer.append(base+(pos_info&0x3FFFF), pos_info>>18);
+      }
     };
   }
 }

data/{unf.cc → ext/unf_ext/unf.cc}

@@ -1,10 +1,10 @@
+#include "unf/normalizer.hh"
+
 #include <ruby.h>
 #if defined(HAVE_RUBY_ENCODING_H)
 #include <ruby/encoding.h>
 #endif
 
-#include "unf/normalizer.hh"
-
 extern "C" {
   VALUE unf_allocate(VALUE klass);
   VALUE unf_initialize(VALUE self);

data/lib/unf_ext.rb

@@ -0,0 +1,5 @@
+begin
+  require "#{RUBY_VERSION[/\A[0-9]+\.[0-9]+/]}/unf_ext.so"
+rescue LoadError
+  require "unf_ext.so"
+end

data/unf_ext.gemspec

@@ -5,14 +5,14 @@
 
 Gem::Specification.new do |s|
   s.name = %q{unf_ext}
-  s.version = "0.0.3"
+  s.version = "0.0.4"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = [%q{Takeru Ohta}, %q{Akinori MUSHA}]
-  s.date = %q{2011-10-24}
+  s.date = %q{2011-12-08}
   s.description = %q{Unicode Normalization Form support library for CRuby}
   s.email = %q{knu@idaemons.org}
-  s.extensions = [%q{extconf.rb}]
+  s.extensions = [%q{ext/unf_ext/extconf.rb}]
   s.extra_rdoc_files = [
     "LICENSE.txt",
     "README.md"
@@ -24,17 +24,18 @@ Gem::Specification.new do |s|
     "README.md",
     "Rakefile",
     "VERSION",
-    "extconf.rb",
+    "ext/unf_ext/extconf.rb",
+    "ext/unf_ext/unf.cc",
+    "ext/unf_ext/unf/normalizer.hh",
+    "ext/unf_ext/unf/table.hh",
+    "ext/unf_ext/unf/trie/char_stream.hh",
+    "ext/unf_ext/unf/trie/node.hh",
+    "ext/unf_ext/unf/trie/searcher.hh",
+    "ext/unf_ext/unf/util.hh",
+    "lib/unf_ext.rb",
     "test/helper.rb",
     "test/normalization-test.txt",
     "test/test_unf_ext.rb",
-    "unf.cc",
-    "unf/normalizer.hh",
-    "unf/table.hh",
-    "unf/trie/char_stream.hh",
-    "unf/trie/node.hh",
-    "unf/trie/searcher.hh",
-    "unf/util.hh",
     "unf_ext.gemspec"
   ]
   s.homepage = %q{http://github.com/knu/ruby-unf_ext}
@@ -51,17 +52,20 @@ Gem::Specification.new do |s|
       s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
       s.add_development_dependency(%q<jeweler>, ["~> 1.6.4"])
       s.add_development_dependency(%q<rcov>, [">= 0"])
+      s.add_development_dependency(%q<rake-compiler>, [">= 0.7.9"])
     else
       s.add_dependency(%q<shoulda>, [">= 0"])
       s.add_dependency(%q<bundler>, ["~> 1.0.0"])
       s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
       s.add_dependency(%q<rcov>, [">= 0"])
+      s.add_dependency(%q<rake-compiler>, [">= 0.7.9"])
     end
   else
     s.add_dependency(%q<shoulda>, [">= 0"])
     s.add_dependency(%q<bundler>, ["~> 1.0.0"])
     s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
     s.add_dependency(%q<rcov>, [">= 0"])
+    s.add_dependency(%q<rake-compiler>, [">= 0.7.9"])
   end
 end
 

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: unf_ext
 version: !ruby/object:Gem::Version 
-  hash: 25
+  hash: 23
   prerelease: 
   segments: 
   - 0
   - 0
-  - 3
-  version: 0.0.3
+  - 4
+  version: 0.0.4
 platform: ruby
 authors: 
 - Takeru Ohta
@@ -16,7 +16,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-10-24 00:00:00 Z
+date: 2011-12-08 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: shoulda
@@ -78,12 +78,28 @@ dependencies:
   type: :development
   requirement: *id004
   prerelease: false
+- !ruby/object:Gem::Dependency 
+  name: rake-compiler
+  version_requirements: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 17
+        segments: 
+        - 0
+        - 7
+        - 9
+        version: 0.7.9
+  type: :development
+  requirement: *id005
+  prerelease: false
 description: Unicode Normalization Form support library for CRuby
 email: knu@idaemons.org
 executables: []
 
 extensions: 
-- extconf.rb
+- ext/unf_ext/extconf.rb
 extra_rdoc_files: 
 - LICENSE.txt
 - README.md
@@ -94,17 +110,18 @@ files:
 - README.md
 - Rakefile
 - VERSION
-- extconf.rb
+- ext/unf_ext/extconf.rb
+- ext/unf_ext/unf.cc
+- ext/unf_ext/unf/normalizer.hh
+- ext/unf_ext/unf/table.hh
+- ext/unf_ext/unf/trie/char_stream.hh
+- ext/unf_ext/unf/trie/node.hh
+- ext/unf_ext/unf/trie/searcher.hh
+- ext/unf_ext/unf/util.hh
+- lib/unf_ext.rb
 - test/helper.rb
 - test/normalization-test.txt
 - test/test_unf_ext.rb
-- unf.cc
-- unf/normalizer.hh
-- unf/table.hh
-- unf/trie/char_stream.hh
-- unf/trie/node.hh
-- unf/trie/searcher.hh
-- unf/util.hh
 - unf_ext.gemspec
 homepage: http://github.com/knu/ruby-unf_ext
 licenses: 

data/extconf.rb

@@ -1,4 +0,0 @@
-require 'mkmf'
-have_library('stdc++')
-have_header('ruby/encoding.h')
-create_makefile 'unf_ext'