analects 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c11afafa3e6efdbad98b84a73d3a190e2dc7fb52
-  data.tar.gz: 74b50dbb99b0fbb20182e682d1a5b044930e089a
+  metadata.gz: 9765c8c83a2eaee1072a513d296bb558b28dfe7e
+  data.tar.gz: 388599cfc445653f210dda46a5c1b26253ca37d6
 SHA512:
-  metadata.gz: bf0ef56017cdf8a29b8931ca7b6e838c0c0255b7e9cfa9f878e23a531bbb43c176f4ffc1840308c0a82427e2c00b764e625226d20b912b7eb4731a8ba8d4f834
-  data.tar.gz: ad3972c46c16426698709ecf2a1cf86246b7221333398ec5d8a2d9b51e9b3dd0532e66754c173bb3e04c05cb288bf475e20229dc844b5a8e0991d00101cceccf
+  metadata.gz: 4f11f282ce2684189ba63300a030481149e06a3e86835836443b171cffcfca9cbd37af360f54f24d0612fa7a0e5c3f20ca9f55d39fab2fe7cd8f5925f8bba2ec
+  data.tar.gz: 2049a14e1c204438ba004a92f69127d221c452d9b04bdd243dbfebb99f070a918df2e17e1e5dba151e44128120ab07f47639363cc1fbf1652cbe215a215276fc

data/Gemfile

@@ -5,4 +5,6 @@ group :development, :test do
   eval_gemfile 'Gemfile.devtools'
 end
 
+gem 'rubyzip' , path: '/home/arne/github/rubyzip'
+
 gemspec

data/Gemfile.lock

@@ -7,12 +7,18 @@ GIT
 PATH
   remote: .
   specs:
-    analects (0.3.1)
+    analects (0.4.0)
       ice_nine (~> 0.11.0)
       inflecto (~> 0.0.2)
       plexus-rmmseg (~> 0.1.6)
+      rubyzip (~> 1.1)
       ting (~> 0.9.0)
 
+PATH
+  remote: /home/arne/github/rubyzip
+  specs:
+    rubyzip (1.1.3)
+
 GEM
   remote: https://rubygems.org/
   specs:
@@ -149,7 +155,6 @@ GEM
       sexp_processor (~> 4.0)
     ruby_parser (3.4.1)
       sexp_processor (~> 4.1)
-    rubygems-tasks (0.2.4)
     rubysl-logger (2.0.0)
     rubysl-open-uri (2.0.0)
     rubysl-prettyprint (2.0.3)
@@ -212,11 +217,11 @@ DEPENDENCIES
   rspec (~> 2.14.1)
   rspec-core (~> 2.14.8)
   rubocop (~> 0.18.1)
-  rubygems-tasks
   rubysl-logger (~> 2.0.0)
   rubysl-open-uri (~> 2.0.0)
   rubysl-prettyprint (~> 2.0.2)
   rubysl-singleton (~> 2.0.0)
+  rubyzip!
   simplecov (~> 0.8.2)
   terminal-notifier-guard (~> 1.5.3)
   yard (~> 0.8.7)

data/README.md

@@ -36,6 +36,8 @@ end
 rake analects:download:all        # download all sources
 rake analects:download:cedict     # download CC-CEDICT
 rake analects:download:chise_ids  # download Chise-IDS
+rake analects:download:hsk        # download HSK data
+rake analects:download:unihan     # download Unihan database
 ```
 
 Or from Ruby

data/Rakefile

@@ -1,6 +1,5 @@
 require 'rspec/core/rake_task'
 require 'devtools'
-require 'rubygems/tasks'
 require 'rubygems/package_task'
 
 $LOAD_PATH.unshift File.expand_path('../lib', __FILE__)
@@ -12,8 +11,6 @@ Analects.init_rake_tasks
 RSpec::Core::RakeTask.new(:spec)
 task :default => :spec
 
-Gem::Tasks.new
-
 spec = Gem::Specification.load(File.expand_path('../analects.gemspec', __FILE__))
 gem = Gem::PackageTask.new(spec)
 gem.define

data/analects.gemspec

@@ -16,14 +16,13 @@ Gem::Specification.new do |gem|
   gem.test_files       = `git ls-files -- spec`.split($/)
   gem.extra_rdoc_files = %w[README.md]
 
-  gem.add_development_dependency 'rspec'
-  gem.add_development_dependency 'simplecov'
-  gem.add_development_dependency 'rake'
-  gem.add_development_dependency 'rubygems-tasks'
-  gem.add_development_dependency 'pry'
-
   gem.add_runtime_dependency 'inflecto'      , '~> 0.0.2'
   gem.add_runtime_dependency 'plexus-rmmseg' , '~> 0.1.6'
   gem.add_runtime_dependency 'ting'          , '~> 0.9.0'
   gem.add_runtime_dependency 'ice_nine'      , '~> 0.11.0'
+  gem.add_runtime_dependency 'rubyzip'       , '~> 1.1'
+
+  gem.add_development_dependency 'rspec'
+  gem.add_development_dependency 'rake'
+  gem.add_development_dependency 'pry'
 end

data/lib/analects.rb

@@ -30,6 +30,7 @@ require 'analects/cli/progress'
 require 'analects/cedict_loader'
 require 'analects/hsk_loader'
 require 'analects/chise_ids_loader'
+require 'analects/unihan_loader'
 require 'analects/source'
 require 'analects/library'
 require 'analects/tokenizer'

data/lib/analects/library.rb

@@ -1,8 +1,9 @@
 module Analects
-  CEDICT_URL      = 'http://www.mdbg.net/chindict/export/cedict/cedict_1_0_ts_utf-8_mdbg.txt.gz'
-  CHISE_IDS_URL   = 'http://git.chise.org/git/chise/ids.git'
-  UNIHAN_URL      = ''
-  HSK_URL         = ''
+  CEDICT_URL        = 'http://www.mdbg.net/chindict/export/cedict/cedict_1_0_ts_utf-8_mdbg.txt.gz'
+  CHISE_IDS_URL     = 'http://git.chise.org/git/chise/ids.git'
+  UNIHAN_URL        = 'http://www.unicode.org/Public/UCD/latest/ucd/Unihan.zip'
+  HSK_URL           = 'https://raw.githubusercontent.com/plexus/analects-data/master/hsk/hsk.csv'
+  TW_CURRICULUM_URL = 'https://raw.githubusercontent.com/plexus/analects-data/master/taiwan_school_curriculum.txt'
 
   class Library
     attr_reader :options
@@ -23,48 +24,59 @@ module Analects
       end
     end
 
+    def sources
+      [
+        cedict,
+        chise_ids,
+        unihan,
+        hsk
+      ]
+    end
+
     def cedict
-      @cedict ||= Source.new(
-        {
-          data_file: 'cedict_1_0_ts_utf-8_mdbg.txt',
-          retrieval: [ :http, :gunzip, :save ]
-        }.merge(options_for :cedict)
+      @cedict ||= create_source(
+        :cedict,
+        data_file: 'cedict_1_0_ts_utf-8_mdbg.txt',
+        retrieval: [ :http, :gunzip, :save ]
       )
     end
 
     def chise_ids
-      @chise_ids ||= Source.new(
-        {
-          retrieval: :git
-        }.merge(options_for :chise_ids)
+      @chise_ids ||= create_source(
+        :chise_ids,
+        retrieval: :git
       )
     end
 
     def unihan
-      @unihan ||= Source.new(
-        {
-          data_file: ''
-        }.merge(options_for :chise_ids)
+      @unihan ||= create_source(
+        :unihan,
+        retrieval: [ :http, :unzip ]
       )
     end
 
     def hsk
-      @hsk ||= Source.new( {
-          data_file: 'hsk.csv'
-        }.merge(options_for :hsk)
+      @hsk ||= create_source(
+        :hsk,
+        data_file: 'hsk.csv',
+        retrieval: [ :http, :save ]
       )
     end
 
     private
 
-    def options_for(name)
-      {
-        name: name,
-        library: self,
-        url: Analects.const_get("#{name.to_s.upcase}_URL"),
-        loader: Analects.const_get("#{Inflecto.camelize name}Loader"),
-        data_dir: data_dir
-      }.merge(options.fetch(name, {}))
+    def create_source(name, source_options)
+      Source.new(
+        source_options.merge(
+          {
+            name: name,
+            library: self,
+            url: Analects.const_get("#{name.to_s.upcase}_URL"),
+            loader: Analects.const_get("#{Inflecto.camelize name}Loader"),
+            data_dir: data_dir
+          }
+        ).merge(options.fetch(name, {}))
+      )
     end
 
   end

data/lib/analects/rake_tasks.rb

@@ -18,6 +18,10 @@ module Analects
       @library ||= Analects::Library.new(options)
     end
 
+    def sources
+      library.sources
+    end
+
     def options
       @options ||= {}
     end
@@ -29,18 +33,15 @@ module Analects
     def define
       namespace @name do
         namespace :download do
-          desc 'download CC-CEDICT'
-          task :cedict do
-            library.cedict.retrieve!
-          end
-
-          desc 'download Chise-IDS'
-          task :chise_ids do
-            library.chise_ids.retrieve!
+          sources.each do |source|
+            desc "download #{source.name}"
+            task source.name do
+              source.retrieve!
+            end
           end
 
           desc 'download all sources'
-          task :all => [:cedict, :chise_ids]
+          task :all => sources.map(&:name)
         end
       end
 

data/lib/analects/source.rb

@@ -17,15 +17,17 @@ module Analects
     end
 
     def data_dir
-      options[:data_dir]
+      Pathname(options[:data_dir])
     end
 
     def location
-      options[:data_file] ? File.join( data_dir, options[:data_file] ) : File.join( data_dir, options[:name].to_s )
+      options[:data_file] ?
+        data_dir.join(options[:data_file]) :
+        data_dir.join(options[:name].to_s)
     end
 
     def data_file_present?
-      File.exist? location
+      location.exist?
     end
 
     def retrieve
@@ -33,32 +35,45 @@ module Analects
     end
 
     def retrieve!
-      retrieval.inject( url ) do | result, method |
+      retrieval.inject(url) do | result, method |
         self.send( "retrieve_#{method}", result )
       end
     end
 
     # url -> stream
-    def retrieve_http( url )
+    def retrieve_http(url)
       require 'open-uri'
-      open( url )
+      StringIO.new(open(url).read)
     end
 
     # gzipped stream -> uncompressed stream
-    def retrieve_gunzip( stream )
+    def retrieve_gunzip(stream)
       require 'zlib'
-      Zlib::GzipReader.new( stream )
+      Zlib::GzipReader.new(stream)
+    end
+
+    def retrieve_unzip(stream)
+      require 'zip'
+      location.mkdir unless location.exist?
+      Zip::InputStream.open(stream) do |io|
+        while (entry = io.get_next_entry)
+          next if entry.ftype == :symlink
+          loc = location.join(entry.name)
+          loc.delete if loc.exist?
+          entry.extract(loc)
+        end
+      end
     end
 
     # stream|string -> create data file
-    def retrieve_save( data )
+    def retrieve_save(data)
       File.open( location, 'w' ) do |f|
         f << ( data.respond_to?(:read) ? data.read : data )
       end
     end
 
     # url -> clones repo
-    def retrieve_git( url )
+    def retrieve_git(url)
       `git clone #{url} #{data_dir}/#{name}` # Admittedly crude
     end
 

data/lib/analects/unihan_loader.rb

@@ -0,0 +1,7 @@
+# encoding: UTF-8
+
+module Analects
+  class UnihanLoader
+    include Enumerable
+  end
+end

data/lib/analects/version.rb

@@ -1,3 +1,3 @@
 module Analects
-  VERSION = '0.3.1'
+  VERSION = '0.4.0'
 end

data/spec/analects/library_spec.rb

@@ -8,7 +8,7 @@ describe Analects::Library do
   subject(:library) {
     described_class.new(options)
   }
-  let(:data_dir) { File.join(Dir.tmpdir, 'analects-' + SecureRandom.hex(16)) }
+  let(:data_dir) { Pathname(Dir.tmpdir).join('analects-' + SecureRandom.hex(16)) }
   let(:options) {
     { data_dir: data_dir }
   }

metadata

@@ -1,141 +1,127 @@
 --- !ruby/object:Gem::Specification
 name: analects
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
 platform: ruby
 authors:
 - Arne Brasseur
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-10 00:00:00.000000000 Z
+date: 2014-04-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: rspec
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: simplecov
+  name: inflecto
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
+        version: 0.0.2
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.0.2
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: plexus-rmmseg
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
+        version: 0.1.6
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.1.6
 - !ruby/object:Gem::Dependency
-  name: rubygems-tasks
+  name: ting
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
+        version: 0.9.0
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.9.0
 - !ruby/object:Gem::Dependency
-  name: pry
+  name: ice_nine
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
+        version: 0.11.0
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.11.0
 - !ruby/object:Gem::Dependency
-  name: inflecto
+  name: rubyzip
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.0.2
+        version: '1.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.0.2
+        version: '1.1'
 - !ruby/object:Gem::Dependency
-  name: plexus-rmmseg
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 0.1.6
-  type: :runtime
+        version: '0'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 0.1.6
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: ting
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 0.9.0
-  type: :runtime
+        version: '0'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 0.9.0
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: ice_nine
+  name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 0.11.0
-  type: :runtime
+        version: '0'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 0.11.0
+        version: '0'
 description: Toolkit for Mandarin language learning apps
 email:
 - arne.brasseur@gmail.com
@@ -176,6 +162,7 @@ files:
 - lib/analects/rake_tasks.rb
 - lib/analects/source.rb
 - lib/analects/tokenizer.rb
+- lib/analects/unihan_loader.rb
 - lib/analects/version.rb
 - lib/cjk_string.rb
 - lib/generators/analects.rb