analects 0.2.0

data/lib/analects/chise_ids_loader.rb

@@ -0,0 +1,34 @@
+module Analects
+  class ChiseIdsLoader
+    include Enumerable
+
+    attr_accessor :only_unicode
+
+    class MultiFile < Struct.new(:files)
+      def each_line(&blk)
+        return to_enum(__method__) unless block_given?
+        files.each do |file|
+          file.each_line(&blk)
+        end
+        self
+      end
+    end
+
+    def initialize(pathname, only_unicode = true)
+      @contents = MultiFile.new(pathname.children.select{|ch| ch.to_s =~ /IDS-.*\.txt/})
+      @only_unicode = only_unicode
+    end
+
+    def field_names
+      [:name, :representation, :ids]
+    end
+
+    def each(&blk)
+      return to_enum(__method__) unless block_given?
+      @entries ||= @contents.each_line
+        .reject {|line| line !~ /\t/ || (only_unicode && line !~ /^U/) }
+        .map    {|line| line.strip.split("\t")[0..2] }
+      @entries.each(&blk)
+    end
+  end
+end

data/lib/analects/cli/progress.rb

@@ -0,0 +1,37 @@
+module Analects
+  module CLI
+    # Command line progress bar
+    class Progress
+      attr_accessor :length, :count
+      
+      def initialize(total, accuracy = 1000, prefix = '')
+        @total = total
+        @current = 0
+        @length = 60
+        @count = 100
+        @accuracy = accuracy
+        @prefix = prefix
+      end
+      
+      def next
+        @current += 1
+        draw if (@current % (Float(@total)/@accuracy).ceil) == 0 || @current == @total
+      end
+      
+      def draw
+        return unless 
+        x = pos(@length).floor
+        total_count = @count == 100 ? '%' : "/#{@count}"
+        print "\e[%dD\e[32m%s[\e[31m%s%s\e[32m]\e[34m %d%s\e[0m" % [@length+10+@prefix.length, @prefix, '='*x, ' '*(@length-x), pos(@count), total_count] 
+      end
+      
+      def pos(scale)
+        if @current == @total
+          scale
+        else
+          Float(@current)/@total * scale 
+        end
+      end
+    end
+  end
+end

data/lib/analects/encoding.rb

@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*-
+
+module Analects
+  module Encoding
+    extend self
+
+    GB   = ::Encoding::GB18030
+    BIG5 = ::Encoding::BIG5_UAO
+
+    def recode(enc, str)
+      str.force_encoding(enc).encode('UTF-8')
+    end
+
+    def from_gb(str)
+      recode(GB, str)
+    end
+
+    def from_big5(str)
+      recode(BIG5, str)
+    end
+
+    def valid_cjk(str)
+      [GB, BIG5].map do |enc|
+        begin
+          recode(enc, str)
+          enc
+        rescue ::Encoding::UndefinedConversionError
+        rescue ::Encoding::InvalidByteSequenceError
+        end
+      end.compact
+    end
+
+    # Crude way to guess which encoding it is
+    def ratings(str)
+      all_valid_cjk(str).map do |enc|
+        [
+          enc,
+          recode(enc, str).codepoints.map do |point|
+            Analects::Models::Zi.codepoint_ranges.map.with_index do |range, idx|
+              next 6-idx if range.include?(point)
+              0
+            end.inject(:+)
+          end.inject(:+)
+        ]
+      end.sort_by(&:last).reverse
+    end
+
+  end
+end
+
+# For info on Taiwanese Big5 variants + Ruby
+# * https://bugs.ruby-lang.org/issues/1784
+# * http://lists.gnu.org/archive/html/bug-gnu-libiconv/2010-11/msg00007.html
+
+# Wikipedia pages of GB (国家标准) encodings (chronological?)
+# * http://en.wikipedia.org/wiki/GB_2312
+# * http://en.wikipedia.org/wiki/GBK
+# * http://en.wikipedia.org/wiki/GB18030
+
+# Ruby also knows about this one, but can't convert it to UTF-8
+# * http://en.wikipedia.org/wiki/Extended_Unix_Code#EUC-TW

data/lib/analects/library.rb

@@ -0,0 +1,68 @@
+module Analects
+  CEDICT_URL      = 'http://www.mdbg.net/chindict/export/cedict/cedict_1_0_ts_utf-8_mdbg.txt.gz'
+  CHISE_IDS_URL   = 'http://git.chise.org/git/chise/ids.git'
+  UNIHAN_URL      = ''
+  HSK_URL         = ''
+
+  class Library
+    attr_reader :options
+
+    def initialize(options = {})
+      @options = options.freeze
+    end
+
+    def data_dir
+      if options[:data_dir]
+        Dir.mkdir(options[:data_dir]) unless File.exist?(options[:data_dir])
+        return options[:data_dir]
+      end
+      File.join(Dir.home, '.analects').tap do |dir|
+        unless File.exist? dir
+          Dir.mkdir dir
+        end
+      end
+    end
+
+    def cedict
+      @cedict ||= Source.new(
+        {
+          data_file: 'cedict_1_0_ts_utf-8_mdbg.txt',
+          retrieval: [ :http, :gunzip, :save ]
+        }.merge(options_for :cedict)
+      )
+    end
+
+    def chise_ids
+      @chise_ids ||= Source.new(
+        {
+          retrieval: :git
+        }.merge(options_for :chise_ids)
+      )
+    end
+
+    def unihan
+      @unihan ||= Source.new(
+        {
+          data_file: ''
+        }.merge(options_for :chise_ids)
+      )
+    end
+    # def hsk
+    #   @hsk ||= Source.new(
+
+    #   ).merge(options_for :hsk)
+    # end
+
+    private
+
+    def options_for(name)
+      {
+        name: name,
+        url: Analects.const_get("#{name.to_s.upcase}_URL"),
+        loader: Analects.const_get("#{Inflecto.camelize name}Loader"),
+        data_dir: data_dir
+      }.merge(options.fetch(name, {}))
+    end
+
+  end
+end

data/lib/analects/models/kangxi_radical.rb

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+module Analects
+  module Models
+    class KangxiRadical
+      # Mapping of Kangxi radical (Unicode Symbol) to the compatibility character that is
+      # composed of that radical (Unicode Letter). The former are not full characters
+      # and not normally used in text unless explicitly referring to only their
+      # usage as radicals.
+      def self.compat(radical)
+        {"⼀"=>"一", "⼁"=>"丨", "⼂"=>"丶", "⼃"=>"丿", "⼄"=>"乙", "⼅"=>"亅", "⼆"=>"二", "⼇"=>"亠", "⼈"=>"人", "⼉"=>"儿", "⼊"=>"入", "⼋"=>"八", "⼌"=>"冂", "⼍"=>"冖", "⼎"=>"冫", "⼏"=>"几", "⼐"=>"凵", "⼑"=>"刀", "⼒"=>"力", "⼓"=>"勹", "⼔"=>"匕", "⼕"=>"匚", "⼖"=>"匸", "⼗"=>"十", "⼘"=>"卜", "⼙"=>"卩", "⼚"=>"厂", "⼛"=>"厶", "⼜"=>"又", "⼝"=>"口", "⼞"=>"囗", "⼟"=>"土", "⼠"=>"士", "⼡"=>"夂", "⼢"=>"夊", "⼣"=>"夕", "⼤"=>"大", "⼥"=>"女", "⼦"=>"子", "⼧"=>"宀", "⼨"=>"寸", "⼩"=>"小", "⼪"=>"尢", "⼫"=>"尸", "⼬"=>"屮", "⼭"=>"山", "⼮"=>"巛", "⼯"=>"工", "⼰"=>"己", "⼱"=>"巾", "⼲"=>"干", "⼳"=>"幺", "⼴"=>"广", "⼵"=>"廴", "⼶"=>"廾", "⼷"=>"弋", "⼸"=>"弓", "⼹"=>"彐", "⼺"=>"彡", "⼻"=>"彳", "⼼"=>"心", "⼽"=>"戈", "⼾"=>"戶", "⼿"=>"手", "⽀"=>"支", "⽁"=>"攴", "⽂"=>"文", "⽃"=>"斗", "⽄"=>"斤", "⽅"=>"方", "⽆"=>"无", "⽇"=>"日", "⽈"=>"曰", "⽉"=>"月", "⽊"=>"木", "⽋"=>"欠", "⽌"=>"止", "⽍"=>"歹", "⽎"=>"殳", "⽏"=>"毋", "⽐"=>"比", "⽑"=>"毛", "⽒"=>"氏", "⽓"=>"气", "⽔"=>"水", "⽕"=>"火", "⽖"=>"爪", "⽗"=>"父", "⽘"=>"爻", "⽙"=>"爿", "⽚"=>"片", "⽛"=>"牙", "⽜"=>"牛", "⽝"=>"犬", "⽞"=>"玄", "⽟"=>"玉", "⽠"=>"瓜", "⽡"=>"瓦", "⽢"=>"甘", "⽣"=>"生", "⽤"=>"用", "⽥"=>"田", "⽦"=>"疋", "⽧"=>"疒", "⽨"=>"癶", "⽩"=>"白", "⽪"=>"皮", "⽫"=>"皿", "⽬"=>"目", "⽭"=>"矛", "⽮"=>"矢", "⽯"=>"石", "⽰"=>"示", "⽱"=>"禸", "⽲"=>"禾", "⽳"=>"穴", "⽴"=>"立", "⽵"=>"竹", "⽶"=>"米", "⽷"=>"糸", "⽸"=>"缶", "⽹"=>"网", "⽺"=>"羊", "⽻"=>"羽", "⽼"=>"老", "⽽"=>"而", "⽾"=>"耒", "⽿"=>"耳", "⾀"=>"聿", "⾁"=>"肉", "⾂"=>"臣", "⾃"=>"自", "⾄"=>"至", "⾅"=>"臼", "⾆"=>"舌", "⾇"=>"舛", "⾈"=>"舟", "⾉"=>"艮", "⾊"=>"色", "⾋"=>"艸", "⾌"=>"虍", "⾍"=>"虫", "⾎"=>"血", "⾏"=>"行", "⾐"=>"衣", "⾑"=>"襾", "⾒"=>"見", "⾓"=>"角", "⾔"=>"言", "⾕"=>"谷", "⾖"=>"豆", "⾗"=>"豕", "⾘"=>"豸", "⾙"=>"貝", "⾚"=>"赤", "⾛"=>"走", "⾜"=>"足", "⾝"=>"身", "⾞"=>"車", "⾟"=>"辛", "⾠"=>"辰", "⾡"=>"辵", "⾢"=>"邑", "⾣"=>"酉", "⾤"=>"釆", "⾥"=>"里", "⾦"=>"金", "⾧"=>"長", "⾨"=>"門", "⾩"=>"阜", "⾪"=>"隶", "⾫"=>"隹", "⾬"=>"雨", "⾭"=>"靑", "⾮"=>"非", "⾯"=>"面", "⾰"=>"革", "⾱"=>"韋", "⾲"=>"韭", "⾳"=>"音", "⾴"=>"頁", "⾵"=>"風", "⾶"=>"飛", "⾷"=>"食", "⾸"=>"首", "⾹"=>"香", "⾺"=>"馬", "⾻"=>"骨", "⾼"=>"高", "⾽"=>"髟", "⾾"=>"鬥", "⾿"=>"鬯", "⿀"=>"鬲", "⿁"=>"鬼", "⿂"=>"魚", "⿃"=>"鳥", "⿄"=>"鹵", "⿅"=>"鹿", "⿆"=>"麥", "⿇"=>"麻", "⿈"=>"黃", "⿉"=>"黍", "⿊"=>"黑", "⿋"=>"黹", "⿌"=>"黽", "⿍"=>"鼎", "⿎"=>"鼓", "⿏"=>"鼠", "⿐"=>"鼻", "⿑"=>"齊", "⿒"=>"齒", "⿓"=>"龍", "⿔"=>"龜", "⿕"=>"龠"}[radical]
+      end
+    end
+  end
+end

data/lib/analects/models/zi.rb

@@ -0,0 +1,64 @@
+module Analects
+  module Models
+    class Zi
+      RANGES = IceNine.deep_freeze(
+        unified:
+        { :name => "CJK Unified Ideographs",
+          :range => 0x4E00 .. 0x9FFF, # Ox9FC4..9FFF have no Unihan data
+          :sort_diff => -0x4E00
+        },
+        extension_A:
+        { :name => "CJK Unified Ideographs Extension A",
+          :range => 0x3400 .. 0x4DBF, # 0x4DB6..4DBF have no Unihan data
+          :sort_diff => 0x1E00
+        },
+        extension_B:
+        { :name => "CJK Unified Ideographs Extension B",
+          :range => 0x20000 .. 0x2A6DF, # 0x2A6D7..2A6DF have no Unihan data
+          :sort_diff => -0x19400
+        },
+        compatibility:
+        { :name => "CJK Compatibility Ideographs",
+          :range => 0xF900 .. 0xFAFF, # 0xFADA..FAFF; 0xFA2E..0xFA2F; 0xFA6B..0xFA6F have no Unihan data
+          :sort_diff => 0xFD00
+        },
+        supplement:
+        { :name => "CJK Compatibility Ideographs Supplement",
+          :range => 0x2F800 .. 0x2FA1F, # 0x2FA1E..0x2FA1F have no Unihan data
+          :sort_diff => -0x10000
+        },
+        radicals_supplement:
+        { name: "CJK Radicals supplement",
+          range: 0x2E80 .. 0x2EFF
+        },
+        kangxi_radicals:
+        { name: "Kangxi Radicals",
+          range: 0x2F00 .. 0x2FDF
+        }
+      )
+
+      def self.codepoint_ranges
+        RANGES.values.map{|v| v[:range]}
+      end
+
+      # Regexp that matches a single CJK character
+      REGEXP = Regexp.union(
+        codepoint_ranges.map { |range|
+          Regexp.new('[\u{%s}-\u{%s}]' % [ range.begin.to_s(16), range.end.to_s(16) ])
+        }
+      )
+
+      ANTIREGEXP = Regexp.new(
+        '[^'+
+        codepoint_ranges.map { |range| '\u{%s}-\u{%s}' % [ range.begin.to_s(16), range.end.to_s(16) ] }.join +
+        ']'
+      )
+
+      def self.each_radical(&block)
+        RANGES[:kangxi_radicals][:range].each do |codepoint|
+          block.([codepoint].pack('U'))
+        end
+      end
+    end
+  end
+end

data/lib/analects/rake_tasks.rb

@@ -0,0 +1,49 @@
+require 'rake/tasklib'
+
+module Analects
+  class RakeTasks < Rake::TaskLib
+    def initialize(name = :analects, &blk)
+      @name = name
+      if block_given?
+        if blk.arity == 0
+          self.instance_eval(&blk)
+        else
+          yield self
+        end
+      end
+      define
+    end
+
+    def library
+      @library ||= Analects::Library.new(options)
+    end
+
+    def options
+      @options ||= {}
+    end
+
+    def data_dir(dir)
+      options[:data_dir] = dir
+    end
+
+    def define
+      namespace @name do
+        namespace :download do
+          desc 'download CC-CEDICT'
+          task :cedict do
+            library.cedict.retrieve!
+          end
+
+          desc 'download Chise-IDS'
+          task :chise_ids do
+            library.chise_ids.retrieve!
+          end
+
+          desc 'download all sources'
+          task :all => [:cedict, :chise_ids]
+        end
+      end
+
+    end
+  end
+end

data/lib/analects/source.rb

@@ -0,0 +1,70 @@
+module Analects
+  class Source
+    include Enumerable
+    attr_reader :options
+
+    def initialize(options = {})
+      @options = options
+    end
+
+    def name      ; options[:name]              ; end
+    def url       ; options[:url]               ; end
+    def retrieval ; Array(options[:retrieval])  ; end
+
+    def loader
+      @loader ||= options[:loader].new(Pathname(location))
+    end
+
+    def data_dir
+      options[:data_dir]
+    end
+
+    def location
+      options[:data_file] ? File.join( data_dir, options[:data_file] ) : File.join( data_dir, options[:name].to_s )
+    end
+
+    def data_file_present?
+      File.exist? location
+    end
+
+    def retrieve
+      retrieve! unless data_file_present?
+    end
+
+    def retrieve!
+      retrieval.inject( url ) do | result, method |
+        self.send( "retrieve_#{method}", result )
+      end
+    end
+
+    # url -> stream
+    def retrieve_http( url )
+      require 'open-uri'
+      open( url )
+    end
+
+    # gzipped stream -> uncompressed stream
+    def retrieve_gunzip( stream )
+      require 'zlib'
+      Zlib::GzipReader.new( stream )
+    end
+
+    # stream|string -> create data file
+    def retrieve_save( data )
+      File.open( location, 'w' ) do |f|
+        f << ( data.respond_to?(:read) ? data.read : data )
+      end
+    end
+
+    # url -> clones repo
+    def retrieve_git( url )
+      `git clone #{url} #{data_dir}/#{name}` # Admittedly crude
+    end
+
+    def each(&block)
+      return to_enum unless block_given?
+      loader.each(&block)
+    end
+
+  end
+end

data/lib/analects/tokenizer.rb

@@ -0,0 +1,54 @@
+module Analects
+  class Tokenizer
+    #ALGO = RMMSeg::Algorithm
+    ALGO = RMMSeg::SimpleAlgorithm
+
+    def initialize(chars_dic = '/tmp/chars.dic', words_dic = '/tmp/words.dic')
+      unless File.exist?(chars_dic) && File.exist?(words_dic)
+        create_dict_from_cedict( chars_dic, words_dic )
+      end
+      #RMMSeg::Dictionary.dictionaries = [[:chars, chars_dic], [:words, words_dic]]
+      RMMSeg::Config.dictionaries = [[chars_dic, true], [words_dic, false]]
+    end
+
+    def library
+      @library ||= Analects::Library.new
+    end
+
+    def cedict( fn = '/tmp/cedict.json' )
+      require 'json'
+      unless File.exist?( fn )
+        library.cedict.retrieve
+        File.write( fn, library.cedict.to_a.to_json )
+      end
+      @cedict ||= JSON.parse IO.read( fn )
+    end
+
+    def create_dict_from_cedict(chars_dic, words_dic)
+      words = Set.new
+      histo = Hash.new(0)
+
+      cedict.each do |c|
+        words << c[0]
+        words << c[1]
+        (c[0] + c[1]).chars.each do |c|
+          histo[c] += 1
+        end
+      end
+
+      File.write(words_dic, words.sort.join("\n"))
+      File.write(chars_dic, histo.map {|ch, cnt| "%s %d\n" % [ ch, cnt ]}.join )
+    end
+
+    def tokenize( str )
+      [].tap do |result|
+        ALGO.new( str ).tap do |alg|
+          until (tok = alg.next_token).nil?
+            result << tok.text.force_encoding('UTF-8')
+          end
+        end
+      end
+    end
+    alias call tokenize
+  end
+end