iarea 0.3.0 → 0.4.0

data/Gemfile.lock

@@ -2,13 +2,13 @@ PATH
   remote: .
   specs:
     iarea (0.3.0)
-      leveldb-ruby
+      msgpack
 
 GEM
   remote: http://rubygems.org/
   specs:
     diff-lcs (1.1.2)
-    leveldb-ruby (0.7)
+    msgpack (0.4.5)
     rake (0.9.2)
     rdoc (3.8)
     rspec (2.6.0)

data/iarea.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |s|
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
 
-  s.add_runtime_dependency 'leveldb-ruby'
+  s.add_runtime_dependency 'msgpack'
   s.add_development_dependency 'rspec'
   s.add_development_dependency 'rdoc'
   s.add_development_dependency 'rake'

data/lib/iarea.rb

@@ -1,10 +1,11 @@
-require 'leveldb'
 require 'json'
+require 'msgpack'
 require 'ostruct'
 require_relative 'iarea/version'
 
 module Iarea # :nodoc:
-  DB = LevelDB::DB.load(File.expand_path("../../db/iareadata", __FILE__)) # :nodoc:
+  DB = MessagePack::unpack(File.read File.expand_path("../../db/meta.msgpack", __FILE__)) # :nodoc:
+  MESH = MessagePack::unpack(File.read File.expand_path("../../db/mesh.msgpack", __FILE__)) # :nodoc:
   autoload :Area, File.expand_path('../iarea/area', __FILE__)
   autoload :Zone, File.expand_path('../iarea/zone', __FILE__)
   autoload :Prefecture, File.expand_path('../iarea/prefecture', __FILE__)

data/lib/iarea/area.rb

@@ -15,20 +15,20 @@ module Iarea
 
     # Neighbor areas of the area
     def neighbors
-      JSON.parse(DB['n:'+self.areacode]).map{|areacode| Area.find areacode}
+      @table[:neighbors].map{|areacode| Area.find areacode}
     end
 
     class << self
       # Find area by <tt>areacode</tt>
       def find(areacode)
-        @@areas[areacode] ||= new JSON.parse(DB['a:'+areacode])
+        @@areas[areacode] ||= new DB['area'][areacode]
       end
 
       # Find area by latitude and longitude in degrees
       def find_by_lat_lng(lat, lng)
         meshcodes = Utils.expand_meshcode(Utils.lat_lng_to_meshcode(lat, lng))
         meshcodes.each do |meshcode|
-          if areacode = DB['m:'+meshcode]
+          if areacode = MESH[meshcode]
             return find(areacode)
           end
         end

data/lib/iarea/prefecture.rb

@@ -16,13 +16,12 @@ module Iarea
     class << self
       # Find a prefecture by <tt>id</tt>
       def find(id)
-        id = id.to_s
-        @@prefectures[id] ||= new JSON.parse(DB['p:'+id])
+        @@prefectures[id.to_i] ||= new DB['prefecture'][id.to_i]
       end
 
       # All prefectures
       def all
-        JSON.parse(DB['p']).map{ |id| find(id) }
+        DB['prefecture_ids'].map{ |id| find(id) }
       end
     end
   end

data/lib/iarea/version.rb

@@ -1,3 +1,3 @@
 module Iarea
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

data/lib/iarea/zone.rb

@@ -15,13 +15,12 @@ module Iarea
     class << self
       # Find a zone by <tt>id</tt>
       def find(id)
-        id = id.to_s
-        @@zones[id] ||= new JSON.parse(DB['z:'+id])
+        @@zones[id.to_i] ||= new DB['zone'][id.to_i]
       end
 
       # All zones
       def all
-        JSON.parse(DB['z']).map{ |id| find(id) }
+        DB['zone_ids'].map{|id| Zone.find id}
       end
     end
   end

data/lib/tasks/iarea.rake

@@ -4,33 +4,27 @@ require 'tmpdir'
 require 'tempfile'
 
 namespace :iarea do
-  desc "generate database"
-  task :generate => ["fetch:def", "fetch:data"]
+  desc "generate data"
+  task :generate do
+    # fetch iarea metadata
+    uri = "http://www.nttdocomo.co.jp/service/imode/make/content/iarea/domestic/map/js/iarea_def.js"
+    tf = Tempfile.open("iarea")
+    tf.write URI(uri).read
+    tf.close
 
-  namespace :fetch do
-    desc "fetch iarea metadata"
-    task :def do
-      uri = "http://www.nttdocomo.co.jp/service/imode/make/content/iarea/domestic/map/js/iarea_def.js"
-      tf = Tempfile.open("iarea")
-      tf.write URI(uri).read
-      tf.close
-      ruby "./tools/import_areas.rb", tf.path, "db/iareadata"
-    end
-
-    desc "fetch iareadata.lzh"
-    task :data do
-      Dir.mktmpdir("iareadata") do |dir|
-        Dir.chdir(dir) do
-          uri = "http://www.nttdocomo.co.jp/binary/archive/service/imode/make/content/iarea/domestic/iareadata.lzh"
-          File.open("iareadata.lzh", "w") do |f|
-            f.write URI(uri).read
-          end
-          unless system("lha", "xf", "iareadata.lzh")
-            raise "lha failed"
-          end
+    # fetch and extract iareadata.lzh
+    Dir.mktmpdir("iareadata") do |dir|
+      Dir.chdir(dir) do
+        uri = "http://www.nttdocomo.co.jp/binary/archive/service/imode/make/content/iarea/domestic/iareadata.lzh"
+        File.open("iareadata.lzh", "w") do |f|
+          f.write URI(uri).read
+        end
+        unless system("lha", "xf", "iareadata.lzh")
+          raise "lha failed"
         end
-        ruby "./tools/import_meshes.rb", dir, "db/iareadata"
       end
+
+      ruby './tools/generate.rb', 'db', tf.path, dir
     end
   end
 end

data/tools/generate.rb

@@ -0,0 +1,122 @@
+#!/usr/bin/env ruby
+# coding: utf-8
+
+require 'csv'
+require 'json'
+
+require 'rubygems'
+require 'bundler/setup'
+require 'msgpack'
+
+def import_meshes(src_path)
+  mesh_to_iarea_hash = {}
+  Dir[File.join(src_path, "iareadata/iarea*.txt")].each do |path|
+    data = CSV.open(path, "r:cp932")
+    data.each do |row|
+      areaid, subareaid, name, west, south, east, north, *mesh_data = row
+      areacode = areaid + subareaid
+
+      name.encode!("UTF-8")
+
+      mesh_data.pop if mesh_data.last.nil?
+      $stderr.puts "  importing meshes %s %s" % [areacode, name]
+      num_meshes = mesh_data.shift(7).map(&:to_i)
+
+      expected_num_meshes = num_meshes.inject(&:+)
+      if expected_num_meshes != mesh_data.size
+        raise "number of meshes mismatch: expected %d, actual %d" % [
+          expected_num_meshes, mesh_data.size
+        ]
+      end
+
+      # insert meshes
+      mesh_data.each do |mesh|
+        mesh_to_iarea_hash[mesh] = areacode
+      end
+    end
+  end
+  mesh_to_iarea_hash
+end
+
+def import_metadata(src_path)
+  js = open(src_path, "r:cp932").read
+  js.sub!(%r|^var IAreaDef=new function\(\)\{|, "")
+  data = {}
+  js.scan %r|this\.(.+?)=(.+?);|m do |key, json|
+    data[key] = JSON.parse(json)
+  end
+
+  db = Hash.new{|h,k| h[k] = {}}
+
+  # zones
+  data['zones'].each_with_index do |zone, i|
+    id = i + 1
+    $stderr.puts "  importing zone %d %s" % [id, zone['name']]
+    areacodes = data['areas'].select {|areacode, area| area['zone'] == i}.keys
+    prefecture_ids = data['prefs'].each.with_index.select {|pref, j| pref['zone'] == i}.map{|pref, j| j+1 }
+
+    db['zone'][id] = {
+      :id => id,
+      :name => zone['name'],
+      :areacodes => areacodes,
+      :prefecture_ids => prefecture_ids
+    }
+  end
+
+  db['zone_ids'] = data['zones'].map.with_index{|z, i| i + 1}
+
+  # prefs
+  data['prefs'].each_with_index do |pref, i|
+    id = i + 1
+    $stderr.puts "  importing prefecture %d %s" % [id, pref['name']]
+    areacodes = data['areas'].select {|areacode, area| area['pref'] == i}.keys
+    db['prefecture'][id] = {
+      :id => id,
+      :name => pref['name'],
+      :zone_id => pref['zone'] + 1,
+      :areacodes => areacodes
+    }
+  end
+  db['prefecture_ids'] = data['prefs'].map.with_index{|z, i| i + 1}
+
+  # areas
+  data['areas'].each do |areacode, area|
+    $stderr.puts "  importing area %s %s" % [areacode, area['name']]
+    # insert area
+    hash = {
+      :areaid => areacode[0..2],
+      :subareaid => areacode[3..4],
+      :areacode => areacode,
+      :name => area['name'],
+      :west => area['msWest'],
+      :south => area['msSouth'],
+      :east => area['msEast'],
+      :north => area['msNorth'],
+      :zone_id => area['zone'] + 1,
+      :prefecture_id => area['pref'] + 1,
+      :neighbors => area['neighbors']
+    }
+    # fix .js problem
+    if hash[:name] == '新宿１?２丁目'
+      hash[:name] = '新宿１〜２丁目'
+    end
+    db['area'][areacode] = hash
+  end
+  db
+end
+
+
+db_dir = ARGV.shift
+js_path = ARGV.shift
+iareadata_dir = ARGV.shift
+
+meta = import_metadata(js_path)
+File.open(File.join(db_dir, 'meta.msgpack'), 'w') do |f|
+  f.write meta.to_msgpack
+end
+
+mesh_to_iarea_hash = import_meshes(iareadata_dir)
+File.open(File.join(db_dir, 'mesh.msgpack'), 'w') do |f|
+  f.write mesh_to_iarea_hash.to_msgpack
+end
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: iarea
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,12 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-08-01 00:00:00.000000000 +09:00
-default_executable: 
+date: 2011-08-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: leveldb-ruby
-  requirement: &2157498120 !ruby/object:Gem::Requirement
+  name: msgpack
+  requirement: &2157922240 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -22,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2157498120
+  version_requirements: *2157922240
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &2157497700 !ruby/object:Gem::Requirement
+  requirement: &2157921820 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -33,10 +32,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2157497700
+  version_requirements: *2157921820
 - !ruby/object:Gem::Dependency
   name: rdoc
-  requirement: &2157497280 !ruby/object:Gem::Requirement
+  requirement: &2157921400 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -44,10 +43,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2157497280
+  version_requirements: *2157921400
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &2157496860 !ruby/object:Gem::Requirement
+  requirement: &2157920980 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -55,10 +54,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2157496860
+  version_requirements: *2157920980
 - !ruby/object:Gem::Dependency
   name: bundler
-  requirement: &2157496440 !ruby/object:Gem::Requirement
+  requirement: &2157920560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -66,7 +65,7 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2157496440
+  version_requirements: *2157920560
 description: A library for DoCoMo Open Iarea.
 email:
 - dara@shidara.net
@@ -83,16 +82,8 @@ files:
 - README.rdoc
 - Rakefile
 - VERSION
-- db/iareadata/000013.sst
-- db/iareadata/000014.sst
-- db/iareadata/000015.sst
-- db/iareadata/000016.log
-- db/iareadata/000017.sst
-- db/iareadata/CURRENT
-- db/iareadata/LOCK
-- db/iareadata/LOG
-- db/iareadata/LOG.old
-- db/iareadata/MANIFEST-000004
+- db/mesh.msgpack
+- db/meta.msgpack
 - iarea.gemspec
 - lib/iarea.rb
 - lib/iarea/area.rb
@@ -106,9 +97,7 @@ files:
 - spec/iarea/util_spec.rb
 - spec/iarea/zone_spec.rb
 - spec/spec_helper.rb
-- tools/import_areas.rb
-- tools/import_meshes.rb
-has_rdoc: true
+- tools/generate.rb
 homepage: http://github.com/darashi/iarea
 licenses: []
 post_install_message: 
@@ -129,7 +118,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: iarea
-rubygems_version: 1.6.1
+rubygems_version: 1.8.3
 signing_key: 
 specification_version: 3
 summary: A library for DoCoMo Open Iarea.

data/db/iareadata/CURRENT

@@ -1 +0,0 @@
-MANIFEST-000004

data/db/iareadata/LOG

@@ -1,27 +0,0 @@
-2011/08/01-06:56:10.243731 7fff7e529960 Recovering log #3
-2011/08/01-06:56:10.245217 7fff7e529960 Level-0 table #5: started
-2011/08/01-06:56:10.247597 7fff7e529960 Level-0 table #5: 137776 bytes OK
-2011/08/01-06:56:10.248902 7fff7e529960 Delete type=0 #3
-2011/08/01-06:56:10.248995 7fff7e529960 Delete type=3 #2
-2011/08/01-06:56:11.052004 1007f2000 Level-0 table #8: started
-2011/08/01-06:56:11.087336 1007f2000 Level-0 table #8: 1540152 bytes OK
-2011/08/01-06:56:11.093042 1007f2000 Delete type=0 #6
-2011/08/01-06:56:11.913693 1007f2000 Level-0 table #10: started
-2011/08/01-06:56:11.961970 1007f2000 Level-0 table #10: 1538749 bytes OK
-2011/08/01-06:56:11.968064 1007f2000 Delete type=0 #7
-2011/08/01-06:56:12.808704 1007f2000 Level-0 table #12: started
-2011/08/01-06:56:12.841190 1007f2000 Level-0 table #12: 1532571 bytes OK
-2011/08/01-06:56:12.850141 1007f2000 Delete type=0 #9
-2011/08/01-06:56:12.850476 1007f2000 Compacting 4@0 + 0@1 files
-2011/08/01-06:56:12.972097 1007f2000 Generated table #13: 109414 keys, 2116092 bytes
-2011/08/01-06:56:13.060578 1007f2000 Generated table #14: 114250 keys, 2115011 bytes
-2011/08/01-06:56:13.077859 1007f2000 Generated table #15: 26061 keys, 513678 bytes
-2011/08/01-06:56:13.077922 1007f2000 Compacted 4@0 + 0@1 files => 4744781 bytes
-2011/08/01-06:56:13.081068 1007f2000 Delete type=2 #5
-2011/08/01-06:56:13.081286 1007f2000 Delete type=2 #8
-2011/08/01-06:56:13.081596 1007f2000 Delete type=2 #10
-2011/08/01-06:56:13.081880 1007f2000 Delete type=2 #12
-2011/08/01-06:56:13.082217 1007f2000 compacted to: files[ 0 3 0 0 0 0 0 ]
-2011/08/01-06:56:13.662966 1007f2000 Level-0 table #17: started
-2011/08/01-06:56:13.700564 1007f2000 Level-0 table #17: 1535854 bytes OK
-2011/08/01-06:56:13.705899 1007f2000 Delete type=0 #11

data/db/iareadata/LOG.old

@@ -1 +0,0 @@
-2011/08/01-06:56:08.209457 7fff7e529960 Delete type=3 #1

data/tools/import_areas.rb

@@ -1,79 +0,0 @@
-#!/usr/bin/env ruby
-# coding: utf-8
-
-require 'json'
-
-require 'rubygems'
-require 'bundler/setup'
-require 'leveldb'
-
-if ARGV.size != 2
-  puts "usage #$0 [iarea_def.js] [dest_lebeldb_file]"
-  exit -1
-end
-
-src_path = ARGV.shift
-dest_path = ARGV.shift
-
-db = LevelDB::DB.new dest_path
-
-js = open(src_path, "r:cp932").read
-js.sub!(%r|^var IAreaDef=new function\(\)\{|, "")
-data = {}
-js.scan %r|this\.(.+?)=(.+?);|m do |key, json|
-  data[key] = JSON.parse(json)
-end
-# zones
-data['zones'].each_with_index do |zone, i|
-  id = i + 1
-  $stderr.puts "  importing zone %d %s" % [id, zone['name']]
-  areacodes = data['areas'].select {|areacode, area| area['zone'] == i}.keys
-  prefecture_ids = data['prefs'].each.with_index.select {|pref, j| pref['zone'] == i}.map{|pref, j| j+1 }
-
-  db['z:%d' % id] = JSON.dump(
-    :id => id,
-    :name => zone['name'],
-    :areacodes => areacodes,
-    :prefecture_ids => prefecture_ids
-  )
-end
-
-db['z'] = JSON.dump(data['zones'].map.with_index{|z, i| i + 1})
-
-# prefs
-data['prefs'].each_with_index do |pref, i|
-  id = i + 1
-  $stderr.puts "  importing prefecture %d %s" % [id, pref['name']]
-  areacodes = data['areas'].select {|areacode, area| area['pref'] == i}.keys
-  db['p:%d' % id] = JSON.dump(
-    :id => id,
-    :name => pref['name'],
-    :zone_id => pref['zone'] + 1,
-    :areacodes => areacodes
-  )
-end
-db['p'] = JSON.dump(data['prefs'].map.with_index{|z, i| i + 1})
-
-# areas
-data['areas'].each do |areacode, area|
-  $stderr.puts "  importing area %s %s" % [areacode, area['name']]
-  # insert area
-  hash = {
-    :areaid => areacode[0..2],
-    :subareaid => areacode[3..4],
-    :areacode => areacode,
-    :name => area['name'],
-    :west => area['msWest'],
-    :south => area['msSouth'],
-    :east => area['msEast'],
-    :north => area['msNorth'],
-    :zone_id => area['zone'] + 1,
-    :prefecture_id => area['pref'] + 1
-  }
-  # fix .js problem
-  if hash[:name] == '新宿１?２丁目'
-    hash[:name] = '新宿１〜２丁目'
-  end
-  db['a:%s' % areacode] = JSON.dump(hash)
-  db['n:%s' % areacode] = JSON.dump(area['neighbors'])
-end

data/tools/import_meshes.rb

@@ -1,44 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'csv'
-
-require 'rubygems'
-require 'bundler/setup'
-require 'leveldb'
-
-
-if ARGV.size != 2
-  puts "usage #$0 [src_dir_contains_iareadata_dir] [dest_leveldb_file]"
-  exit -1
-end
-
-src_path = ARGV.shift
-dest_path = ARGV.shift
-
-db = LevelDB::DB.new dest_path
-
-Dir[File.join(src_path, "iareadata/iarea*.txt")].each do |path|
-  data = CSV.open(path, "r:cp932")
-  data.each do |row|
-    areaid, subareaid, name, west, south, east, north, *mesh_data = row
-    areacode = areaid + subareaid
-
-    name.encode!("UTF-8")
-
-    mesh_data.pop if mesh_data.last.nil?
-    $stderr.puts "  importing meshes %s %s" % [areacode, name]
-    num_meshes = mesh_data.shift(7).map(&:to_i)
-
-    expected_num_meshes = num_meshes.inject(&:+)
-    if expected_num_meshes != mesh_data.size
-      raise "number of meshes mismatch: expected %d, actual %d" % [
-        expected_num_meshes, mesh_data.size
-      ]
-    end
-
-    # insert meshes
-    mesh_data.each do |mesh|
-      db['m:%s' % mesh] = areacode
-    end
-  end
-end