analects 0.2.0

data/README.md

@@ -0,0 +1,81 @@
+analects.rb
+===========
+
+[![Gem Version](https://badge.fury.io/rb/analects.png)][gem]
+[![Build Status](https://secure.travis-ci.org/plexus/analects.png?branch=master)][travis]
+[![Dependency Status](https://gemnasium.com/plexus/analects.png)][gemnasium]
+[![Code Climate](https://codeclimate.com/github/plexus/analects.png)][codeclimate]
+
+[gem]: https://rubygems.org/gems/analects
+[travis]: https://travis-ci.org/plexus/analects
+[gemnasium]: https://gemnasium.com/plexus/analects
+[codeclimate]: https://codeclimate.com/github/plexus/analects
+
+Public datasets on the Chinese language, accessible from Ruby
+
+## Download the data
+
+With Rake
+
+```ruby
+# Rakefile
+require 'analects/rake_tasks'
+
+Analects.init_rake_tasks do
+  data_dir '/tmp/analects' # defaults to ~/.analects
+
+  task :import_cedict do
+    library.cedict.each do |entry|
+      # ..
+    end
+  end
+end
+```
+
+```sh
+rake analects:download:all        # download all sources
+rake analects:download:cedict     # download CC-CEDICT
+rake analects:download:chise_ids  # download Chise-IDS
+```
+
+Or from Ruby
+
+```ruby
+analects = Analects::Library.new(data_dir: '/tmp/analects')
+analects.cedict.retrieve
+analects.chise_ids.retrieve
+```
+
+## Use the data
+
+```ruby
+analects = Analects::Library.new(data_dir: '/tmp/analects')
+analects.cedict.take(3)
+# => [["AA制", "AA制", "A A zhi4", "/to split the bill/to go Dutch/"], ["A咖", "A咖", "A ka1", "/class \"A\"/top grade/"], ["A片", "A片", "A pian4", "/adult movie/pornography/"]]
+
+analects.chise_ids.to_a.sample(3)
+# [["U+59BF", "妿", "⿱加女"], ["U-0002441B", "𤐛", "⿰火閙"], ["U+83A1", "莡", "⿱艹足"]]
+```
+
+## Other stuff
+
+Analects wraps RMMSeg for easy segmenting of Chinese text
+
+```ruby
+Analects::Tokenizer.new.tokenize("为待那个朋友拿哟出来，咿呀噢哎…")
+# => ["为", "待", "那个", "朋友", "拿", "哟", "出来", "，", "咿", "呀", "噢", "哎", "…"]
+```
+
+If you have Chinese text in GB or BIG5 encoding, you can do stuff like this
+
+```ruby
+Analects::Encoding.valid_cjk(str)
+Analects::Encoding.from_gb(str)   # returns UTF-8
+Analects::Encoding.from_big5(str) # returns UTF-8
+```
+
+## License
+
+Copyright ⓒ Arne Brasseur 2012-2014
+
+Licensed as GPL-v3

data/Rakefile

@@ -0,0 +1,26 @@
+require 'rspec/core/rake_task'
+require 'devtools'
+require 'rubygems/tasks'
+require 'rubygems/package_task'
+
+$LOAD_PATH.unshift File.expand_path('../lib', __FILE__)
+require 'analects'
+
+Devtools.init_rake_tasks
+Analects.init_rake_tasks
+
+RSpec::Core::RakeTask.new(:spec)
+task :default => :spec
+
+Gem::Tasks.new
+
+spec = Gem::Specification.load(File.expand_path('../analects.gemspec', __FILE__))
+gem = Gem::PackageTask.new(spec)
+gem.define
+
+desc "Push gem to rubygems.org"
+task :push => :gem do
+  sh "git tag v#{Analects::VERSION}"
+  sh "git push --tags"
+  sh "gem push pkg/analects-#{Analects::VERSION}.gem"
+end

data/SOURCES.md

@@ -0,0 +1,17 @@
+# More sources to add
+
+* [CJK Decomposition data](http://cjkdecomp.codeplex.com/)
+* [Jun-Da Character Frequenct Lists](http://lingua.mtsu.edu/chinese-computing/)
+* Unihan
+** [On line lookup](http://unicode.org/charts/unihan.html)
+** [Raw Data](http://www.unicode.org/Public/UNIDATA/)
+** [Single Zip](http://www.unicode.org/Public/UNIDATA/Unihan.zip)
+* [KanjiVG](https://github.com/kanjivg/kanjivg)
+* [Wikipedia : Ancient Chinese characters project](http://commons.wikimedia.org/wiki/Commons:Ancient_Chinese_characters_project)
+* [Hanzim Data](http://interstitiality.net/hanziData.html)
+
+## Corpora
+
+* [Leiden Weibo Corpus](http://lwc.daanvanesch.nl/)
+* [The Lancaster Corpus of Mandarin Chinese](http://www.ota.ox.ac.uk/headers/2474.xml)
+* [Blog post: Top 5 "Language data consortium" corpora for Mandarin](http://corplinguistics.wordpress.com/2011/10/30/top-five-ldc-corpora/)

data/analects.gemspec

@@ -0,0 +1,29 @@
+require File.expand_path('../lib/analects/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = 'analects'
+  gem.version     = Analects::VERSION
+  gem.platform    = Gem::Platform::RUBY
+  gem.authors     = ['Arne Brasseur']
+  gem.email       = ['arne.brasseur@gmail.com']
+  gem.homepage    = 'https://github.com/arnebrasseur/analects.rb'
+  gem.license     = 'GPL-3.0'
+  gem.summary     = 'Toolkit for Mandarin language learning apps'
+  gem.description = gem.summary
+
+  gem.require_paths    = %w[lib]
+  gem.files            = `git ls-files`.split($/)
+  gem.test_files       = `git ls-files -- spec`.split($/)
+  gem.extra_rdoc_files = %w[README.md]
+
+  gem.add_development_dependency 'rspec'
+  gem.add_development_dependency 'simplecov'
+  gem.add_development_dependency 'rake'
+  gem.add_development_dependency 'rubygems-tasks'
+  gem.add_development_dependency 'pry'
+
+  gem.add_runtime_dependency 'inflecto', '~> 0.0.2'
+  gem.add_runtime_dependency 'rmmseg'
+  gem.add_runtime_dependency 'ting', '~> 0.9.0'
+  gem.add_runtime_dependency 'ice_nine'
+end

data/bin/wp_hsk_filter

@@ -0,0 +1,36 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+
+require 'analects'
+
+RXP   = Regexp.union(Analects::Models::Zi::REGEXP, '…', '.')
+
+hanyu = Ting.writer(:hanyu, :accents)
+all_pinyin = Ting.all_syllables.map {|s| hanyu << s }.sort_by(&:length).reverse
+PY = Regexp.union(*all_pinyin.map{|p| p+'r'}, *all_pinyin, 'ng', '.', ' ', '\'')
+
+def compile(str)
+  Regexp.new(
+    Regexp.escape(str.gsub(/\s+/, 'WW'))
+        .gsub('ZZ', "(#{RXP}+)")
+        .gsub('PY', "(#{PY}+)")
+        .gsub('WW','\s+'))
+end
+
+csv = CSV.generate do |csv|
+  $stdin.read.lines.each do |line|
+    case line
+    when compile('{{l|cmn|ZZ|sc=Hani}} ({{l|cmn|ZZ|sc=Hani}}) {{l|cmn|PY|sc=Hani}} (also: {{l|cmn|PY|sc=Hani}})'), compile('[[ZZ]] [[PY]], [[PY]]')
+      csv << [$1, $2, $3]
+      csv << [$1, $2, $4]
+    when compile('{{l|cmn|ZZ|sc=Hani}} ({{l|cmn|ZZ|sc=Hani}}) {{l|cmn|PY|sc=Hani}}'), compile('[[ZZ]] ([[ZZ]]) [[PY]]')
+      csv << [$1, $2, $3]
+    when compile('{{l|cmn|ZZ|sc=Hani}} {{l|cmn|PY|sc=Hani}}'), compile('{{l|cmn|ZZ|sc=Hani}} {{l|cmn|PY|sc= Hani }}'), compile('[[ZZ]] [[PY]]')
+      csv << [$1, $1, $2]
+    else
+      $stderr << line
+    end
+  end
+end
+
+puts csv

data/config/devtools.yml

@@ -0,0 +1,2 @@
+---
+unit_test_timeout: 0.1

data/config/flay.yml

@@ -0,0 +1,3 @@
+---
+threshold: 11
+total_score: 184

data/config/flog.yml

@@ -0,0 +1,2 @@
+---
+threshold: 0

data/config/mutant.yml

@@ -0,0 +1,3 @@
+---
+name: your_lib
+namespace: YourLib

data/config/reek.yml

@@ -0,0 +1,103 @@
+---
+Attribute:
+  enabled: true
+  exclude: []
+BooleanParameter:
+  enabled: true
+  exclude: []
+ClassVariable:
+  enabled: true
+  exclude: []
+ControlParameter:
+  enabled: true
+  exclude: []
+DataClump:
+  enabled: true
+  exclude: []
+  max_copies: 2
+  min_clump_size: 2
+DuplicateMethodCall:
+  enabled: true
+  exclude: []
+  max_calls: 1
+  allow_calls: []
+FeatureEnvy:
+  enabled: true
+  exclude: []
+IrresponsibleModule:
+  enabled: true
+  exclude: []
+LongParameterList:
+  enabled: true
+  exclude: []
+  max_params: 2
+  overrides:
+    initialize:
+      max_params: 3
+LongYieldList:
+  enabled: true
+  exclude: []
+  max_params: 2
+NestedIterators:
+  enabled: true
+  exclude: []
+  max_allowed_nesting: 1
+  ignore_iterators: []
+NilCheck:
+  enabled: true
+  exclude: []
+RepeatedConditional:
+  enabled: true
+  exclude: []
+  max_ifs: 1
+TooManyInstanceVariables:
+  enabled: true
+  exclude: []
+  max_instance_variables: 3
+TooManyMethods:
+  enabled: true
+  exclude: []
+  max_methods: 10
+TooManyStatements:
+  enabled: true
+  exclude:
+  - each
+  max_statements: 2
+UncommunicativeMethodName:
+  enabled: true
+  exclude: []
+  reject:
+  - !ruby/regexp /^[a-z]$/
+  - !ruby/regexp /[0-9]$/
+  - !ruby/regexp /[A-Z]/
+  accept: []
+UncommunicativeModuleName:
+  enabled: true
+  exclude: []
+  reject:
+  - !ruby/regexp /^.$/
+  - !ruby/regexp /[0-9]$/
+  accept: []
+UncommunicativeParameterName:
+  enabled: true
+  exclude: []
+  reject:
+  - !ruby/regexp /^.$/
+  - !ruby/regexp /[0-9]$/
+  - !ruby/regexp /[A-Z]/
+  accept: []
+UncommunicativeVariableName:
+  enabled: true
+  exclude: []
+  reject:
+  - !ruby/regexp /^.$/
+  - !ruby/regexp /[0-9]$/
+  - !ruby/regexp /[A-Z]/
+  accept: []
+UnusedParameters:
+  enabled: true
+  exclude: []
+UtilityFunction:
+  enabled: true
+  exclude: []
+  max_helper_calls: 0

data/config/rubocop.yml

@@ -0,0 +1,58 @@
+AllCops:
+  Includes:
+    - '**/*.rake'
+    - 'Gemfile'
+    - 'Gemfile.devtools'
+  Excludes:
+    - '**/vendor/**'
+    - '**/benchmarks/**'
+
+# Avoid parameter lists longer than five parameters.
+ParameterLists:
+  Max: 3
+  CountKeywordArgs: true
+
+# Avoid more than `Max` levels of nesting.
+BlockNesting:
+  Max: 3
+
+# Align with the style guide.
+CollectionMethods:
+  PreferredMethods:
+    collect:  'map'
+    inject:   'reduce'
+    find:     'detect'
+    find_all: 'select'
+
+# Do not force public/protected/private keyword to be indented at the same
+# level as the def keyword. My personal preference is to outdent these keywords
+# because I think when scanning code it makes it easier to identify the
+# sections of code and visually separate them. When the keyword is at the same
+# level I think it sort of blends in with the def keywords and makes it harder
+# to scan the code and see where the sections are.
+AccessControl:
+  Enabled: false
+
+# Limit line length
+LineLength:
+  Max: 79
+
+# Disable documentation checking until a class needs to be documented once
+Documentation:
+  Enabled: false
+
+# Do not favor modifier if/unless usage when you have a single-line body
+IfUnlessModifier:
+  Enabled: false
+
+# Allow case equality operator (in limited use within the specs)
+CaseEquality:
+  Enabled: false
+
+# Constants do not always have to use SCREAMING_SNAKE_CASE
+ConstantName:
+  Enabled: false
+
+# Not all trivial readers/writers can be defined with attr_* methods
+TrivialAccessors:
+  Enabled: false

data/config/yardstick.yml

@@ -0,0 +1,2 @@
+---
+threshold: 53.3

data/lib/analects.rb

@@ -0,0 +1,37 @@
+require 'tmpdir'
+require 'pathname'
+require 'delegate'
+
+require 'inflecto'
+require 'ice_nine'
+require 'rmmseg'
+require 'ting'
+
+module Analects
+  ROOT = Pathname(__FILE__).dirname.parent
+
+  def self.init_rake_tasks(*args, &blk)
+    require 'analects/rake_tasks'
+    Analects::RakeTasks.new(*args, &blk)
+  end
+
+  def self.cjk?(str)
+    str.codepoints.all? do |cp|
+      Analects::Models::Zi.codepoint_ranges.any? {|range| range.include?(cp)}
+    end
+  end
+end
+
+require 'cjk_string'
+
+require 'analects/version'
+require 'analects/encoding'
+require 'analects/cli/progress'
+require 'analects/cedict_loader'
+require 'analects/chise_ids_loader'
+require 'analects/source'
+require 'analects/library'
+require 'analects/tokenizer'
+
+require 'analects/models/zi'
+require 'analects/models/kangxi_radical'

data/lib/analects/cedict_loader.rb

@@ -0,0 +1,44 @@
+# encoding: UTF-8
+
+module Analects
+  class CedictLoader
+    include Enumerable
+
+    attr_reader :headers
+
+    def initialize(io)
+      @contents = io.read
+      @headers = {}
+      @contents.each_line do |line|
+        if line =~ /^#! (.*)=(.*)/
+          @headers[$1.strip] = $2.strip
+        end
+        break unless line =~ /^#/
+      end
+    end
+
+    def field_names
+      [:traditional, :simplified, :pinyin, :definitions]
+    end
+
+    def each
+      if block_given?
+        @contents.each_line do |line|
+          yield process_contents(line) if line !~ /^#/
+        end
+      else
+        enum_for(:each)
+      end
+    end
+
+    private
+
+    def process_contents(line)
+      if line.strip =~ /^([^\s]*) ([^\s]*) \[([\w\d:,· ]+)\](.*)/
+        [$1,$2,$3,$4].map{|x| x.strip}
+      else
+        raise "Unexpected contents : #{line.inspect}"
+      end
+    end
+  end
+end