analects 0.2.0

data/lib/analects/version.rb

@@ -0,0 +1,3 @@
+module Analects
+  VERSION = '0.2.0'
+end

data/lib/cjk_string.rb

@@ -0,0 +1,56 @@
+class CJKString < DelegateClass(String)
+  def cjk_chars
+    @cjk_chars ||= scan(Analects::Models::Zi::REGEXP)
+  end
+
+  def one_cjk?
+    cjk_chars.length == 1
+  end
+
+  def all_cjk?
+    length == cjk_chars.length
+  end
+
+  def any_cjk?
+    cjk_chars.length > 1
+  end
+end
+
+class CJKChar < DelegateClass(String)
+  def unicode_range
+    Analects::Models::Zi::RANGES.each do |name, info|
+      return name if info[:range].include? codepoint
+    end
+  end
+
+  def unicode_range_name
+    Analects::Models::Zi::RANGES[unicode_range][:name]
+  end
+
+  def codepoint
+    codepoints.first
+  end
+end
+
+def CJKChar(str)
+  return str if str.is_a? CJKChar
+
+  if str.length > 1
+    if str =~ /^(U\+)?([0-9A-Fa-f]+)/
+      str = [$2].pack('U')
+    else
+      raise ArgumentError, 'CJKChar must have length one'
+    end
+  end
+
+  CJKChar.new(str)
+end
+
+def CJKString(str)
+  if str.is_a? CJKString
+    return str
+  elsif str.respond_to? :to_cjk
+    str = str.to_cjk
+  end
+  CJKString.new(str.freeze)
+end

data/lib/generators/analects.rb

@@ -0,0 +1,20 @@
+require 'rails/generators/named_base'
+require 'rails/generators/migration'
+require 'rails/generators/active_record'
+
+module Analects
+  module Generators
+    class Base < ::Rails::Generators::Base
+      include Rails::Generators::Migration
+
+      def self.next_migration_number(dirname)
+        ActiveRecord::Generators::Base.next_migration_number(dirname)
+      end
+
+      def self.source_root
+        @_analects_source_root ||= File.expand_path(File.join(File.dirname(__FILE__), 'analects', generator_name, 'templates'))
+      end
+
+    end
+  end
+end

data/lib/generators/analects/cedict/cedict_generator.rb

@@ -0,0 +1,22 @@
+require 'generators/analects'
+
+module Analects
+  module Generators
+    class CedictGenerator < Analects::Generators::Base
+
+      desc %{Description:\n    Copy analects CC-CEDICT files to your application.\n}
+
+      def analects_create_cedict_table
+        migration_template "create_cedict_table.rb", "db/migrate/create_cedict_table.rb"
+      end
+
+      def analects_populate_cedict_table
+        migration_template "populate_cedict_table.rb", "db/migrate/populate_cedict_table.rb"
+      end
+
+      def analects_cedict_model
+        template "model.rb", "app/models/cedict.rb"
+      end
+    end
+  end
+end

data/lib/generators/analects/cedict/templates/create_cedict_table.rb

@@ -0,0 +1,12 @@
+class CreateCedictTable < ActiveRecord::Migration
+  def change
+    create_table :cedicts do |t|
+      t.string :simplified
+      t.string :traditional
+      t.string :pinyin
+      t.string :english
+      
+      t.timestamps
+    end
+  end
+end

data/lib/generators/analects/cedict/templates/model.rb

@@ -0,0 +1,3 @@
+class Cedict < ActiveRecord::Base
+  attr_accessible :simplified, :traditional, :pinyin, :english
+end

data/lib/generators/analects/cedict/templates/populate_cedict_table.rb

@@ -0,0 +1,41 @@
+require 'analects/cedict'
+require 'analects/cli/progress'
+
+class PopulateCedictTable < ActiveRecord::Migration
+  def up
+    path = ENV['CEDICT_PATH'] || Analects::CedictLoader::LOCAL
+    unless File.exist? path
+      puts "-- cedict file not found, downloading"
+      Analects::CedictLoader.download!
+    end
+
+    if File.exist? path
+      f = File.open path
+      l = Analects::CedictLoader.new(f)
+      puts "-- Inserting CC-CEDICT"
+      l.headers.each do |k,v|
+        puts "     #{k}=#{v}"
+      end
+      p = Analects::CLI::Progress.new(Integer(l.headers['entries'])-1, 5000, '   ')
+      Cedict.transaction do
+        l.each do |traditional, simplified, pinyin, english|
+          p.next
+          Cedict.create!(
+                         :traditional => traditional,
+                         :simplified => simplified,
+                         :pinyin => pinyin,
+                         :english => english
+                         )
+        end
+      end
+      f.close
+      puts
+    else
+      raise "CC-Cedict file not found and failed to download"
+    end
+  end
+
+  def down
+    Cedict.delete_all
+  end
+end

data/spec/analects/cedict_loader_spec.rb

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+require 'spec_helper'
+
+describe Analects::CedictLoader do
+  let ( :contents ) do
+    '# CC-CEDICT
+# Community maintained free Chinese-English dictionary.
+#
+#! charset=UTF-8
+#! entries=104941
+佰 佰 [bai3] /hundred (banker\'s anti-fraud numeral)/
+佱 佱 [fa3] /old variant of 法[fa3]/law/
+佳 佳 [jia1] /beautiful/fine/good/
+'
+  end
+
+  let ( :cedict_loader ) { Analects::CedictLoader.new( StringIO.new(contents) ) }
+
+  it "should parse headers" do
+    cedict_loader.headers.should == { 'charset' => 'UTF-8', 'entries' => '104941' }
+  end
+
+  it "should parse entries" do
+    cedict_loader.take(1).should == [ ['佰', '佰', 'bai3', '/hundred (banker\'s anti-fraud numeral)/' ] ]
+  end
+
+  it "can be enumerated multiple times" do
+    2.times do
+      cedict_loader.each do |x|
+        x.should == ['佰', '佰', 'bai3', '/hundred (banker\'s anti-fraud numeral)/' ]
+        break;
+      end
+    end
+  end
+
+  it "is an enumerable" do
+    cedict_loader.count.should === 3
+    cedict_loader.to_a.should be_instance_of( Array )
+    cedict_loader.map {|x| x.size.should === 4 }
+  end
+
+  if RUBY_VERSION.split('.').take(2).join('.').to_f >= 1.9
+    it "returns an enumerator when each is called without block" do
+      cedict_loader.each.is_a? Enumerator
+    end
+  end
+
+end

data/spec/analects/chise_ids_loader_spec.rb

@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*-
+require 'spec_helper'
+
+describe Analects::ChiseIdsLoader do
+  subject(:loader) do
+    Analects::ChiseIdsLoader.new(Analects::ROOT.join('spec/test_data/chise_ids'), only_unicode)
+  end
+
+  let(:only_unicode) { false }
+  let(:entries) { loader.each.to_a }
+
+  describe '#field_names' do
+    it 'should return names for the fields in an IDS record' do
+      expect(loader.field_names).to eq([:name, :representation, :ids])
+    end
+  end
+
+  context 'with a loader that only returns data for unicode characters' do
+    let(:only_unicode) { true }
+
+    it 'should still return the unicode entries' do
+      expect(entries.first).to eq(['U+4E0D', '不', '不'])
+    end
+
+    it 'should filter out the non-unicode entries' do
+      entries.each do |entry|
+        expect(entry.first).to match /^U\+[0-9A-F]{4}/
+      end
+    end
+  end
+
+  describe '#each' do
+    it 'should return an enumerator when no block is given' do
+      expect(loader.each).to be_instance_of(Enumerator)
+    end
+
+    it 'should loop over all entries' do
+      expect(entries.first).to eq(['U+4E0D', '不', '不'])
+      expect(entries.last).to  eq(['CB00003', '&CB00003;', '⿱㓛&GT-47348;'])
+    end
+
+    it 'should filter out entries without a tab' do
+      entry_no_tab = entries.detect do |entry|
+        entry.join(' ') =~ /Entry without a tab/
+      end
+      expect(entry_no_tab).to be_nil
+    end
+  end
+
+end

data/spec/analects/library_spec.rb

@@ -0,0 +1,50 @@
+require 'tempfile'
+require 'spec_helper'
+require 'securerandom'
+
+describe Analects::Library do
+  subject(:library) {
+    described_class.new(options)
+  }
+  let(:options) { {} }
+
+  context 'with a data_dir specified' do
+    let(:data_dir) { File.join(Dir.tmpdir, 'analects-' + SecureRandom.hex(16)) }
+    let(:options) {
+      { data_dir: data_dir }
+    }
+
+    it 'should set that data dir on the sources' do
+      subject.cedict.data_dir.should == data_dir
+    end
+  end
+
+  describe "#cedict" do
+    subject(:cedict) { library.cedict }
+
+    its(:name)     { should == :cedict }
+    its(:location) { should == File.join(Dir.home, '.analects/cedict_1_0_ts_utf-8_mdbg.txt') }
+
+    it "should download and unpack the CEDICT archive" do
+      cedict.should_receive(:retrieve_http).once.with(Analects::CEDICT_URL).and_return(:a_stream)
+      cedict.should_receive(:retrieve_gunzip).once.with(:a_stream).and_return(:an_unzipped_stream)
+
+      cedict.retrieve!
+    end
+  end
+
+  it "should have a CHISE IDS source" do
+    library.chise_ids.name.should == :chise_ids
+  end
+
+  describe "#chise_ids" do
+    subject (:chise_ids) { library.chise_ids }
+
+    its( :name )      { should == :chise_ids }
+    its( :retrieval ) { should == [ :git ] }
+    its(:location   ) { should == File.join(Dir.home, '.analects/chise_ids') }
+    its( :url       ) { should == Analects::CHISE_IDS_URL}
+  end
+
+
+end

data/spec/analects/source_spec.rb

@@ -0,0 +1,18 @@
+require 'spec_helper'
+
+describe Analects::Source do
+  let( :url    ) { 'a_url' }
+  let( :source ) { Analects::Source.new( :retrieval => [ :step1, :step2, :step3 ], :url => url ) }
+
+  it "should do retrieve by pipelining the retrieve methods" do
+    source.should_receive(:retrieve_step1).with(url).once.and_return(:intermediary_result_1)
+    source.should_receive(:retrieve_step2).with(:intermediary_result_1).once.and_return(:intermediary_result_2)
+    source.should_receive(:retrieve_step3).with(:intermediary_result_2).once.and_return(:result)
+    source.retrieve!
+  end
+
+  it "should accept both arrays or single values as retrieval methods" do
+    Analects::Source.new( :retrieval =>   :step1   ).retrieval.should == [ :step1 ]
+    Analects::Source.new( :retrieval => [ :step1 ] ).retrieval.should == [ :step1 ]
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,19 @@
+# encoding: utf-8
+
+if ENV['COVERAGE'] == 'true'
+  require 'simplecov'
+  require 'coveralls'
+
+  SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter[
+    SimpleCov::Formatter::HTMLFormatter,
+    Coveralls::SimpleCov::Formatter
+  ]
+
+  SimpleCov.start do
+    command_name 'spec:unit'
+    minimum_coverage 72.64
+  end
+end
+
+require 'devtools/spec_helper'
+require 'analects'

data/spec/test_data/chise_ids/IDS-foo.txt

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+U+4E0D	不	不
+U+4E0E	与	⿹&CDP-8BBF;一
+U+4E12	丒	⿱刃一
+U+4E15	丕	⿱不一
+Entry without a tab
+U+4E19	丙	⿱一内
+CB00001	&I-CB00001;	⿰𠤕欠
+CB00002	&CB00002;	⿰⿱匕示頁
+CB00003	&CB00003;	⿱㓛&GT-47348;

metadata

@@ -0,0 +1,221 @@
+--- !ruby/object:Gem::Specification
+name: analects
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+platform: ruby
+authors:
+- Arne Brasseur
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-03-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubygems-tasks
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: inflecto
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.0.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.0.2
+- !ruby/object:Gem::Dependency
+  name: rmmseg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ting
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.0
+- !ruby/object:Gem::Dependency
+  name: ice_nine
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Toolkit for Mandarin language learning apps
+email:
+- arne.brasseur@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- README.md
+files:
+- .gitignore
+- .rvmrc
+- .travis.yml
+- Gemfile
+- Gemfile.devtools
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- SOURCES.md
+- analects.gemspec
+- bin/wp_hsk_filter
+- config/devtools.yml
+- config/flay.yml
+- config/flog.yml
+- config/mutant.yml
+- config/reek.yml
+- config/rubocop.yml
+- config/yardstick.yml
+- data/.gitkeep
+- lib/analects.rb
+- lib/analects/cedict_loader.rb
+- lib/analects/chise_ids_loader.rb
+- lib/analects/cli/progress.rb
+- lib/analects/encoding.rb
+- lib/analects/library.rb
+- lib/analects/models/kangxi_radical.rb
+- lib/analects/models/zi.rb
+- lib/analects/rake_tasks.rb
+- lib/analects/source.rb
+- lib/analects/tokenizer.rb
+- lib/analects/version.rb
+- lib/cjk_string.rb
+- lib/generators/analects.rb
+- lib/generators/analects/cedict/cedict_generator.rb
+- lib/generators/analects/cedict/templates/create_cedict_table.rb
+- lib/generators/analects/cedict/templates/model.rb
+- lib/generators/analects/cedict/templates/populate_cedict_table.rb
+- spec/analects/cedict_loader_spec.rb
+- spec/analects/chise_ids_loader_spec.rb
+- spec/analects/library_spec.rb
+- spec/analects/source_spec.rb
+- spec/spec_helper.rb
+- spec/test_data/chise_ids/IDS-foo.txt
+homepage: https://github.com/arnebrasseur/analects.rb
+licenses:
+- GPL-3.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.1
+signing_key: 
+specification_version: 4
+summary: Toolkit for Mandarin language learning apps
+test_files:
+- spec/analects/cedict_loader_spec.rb
+- spec/analects/chise_ids_loader_spec.rb
+- spec/analects/library_spec.rb
+- spec/analects/source_spec.rb
+- spec/spec_helper.rb
+- spec/test_data/chise_ids/IDS-foo.txt