iarea 0.2.0 → 0.3.0

data/Gemfile.lock

@@ -1,14 +1,14 @@
 PATH
   remote: .
   specs:
-    iarea (0.1.4)
-      sequel
-      sqlite3-ruby
+    iarea (0.3.0)
+      leveldb-ruby
 
 GEM
   remote: http://rubygems.org/
   specs:
     diff-lcs (1.1.2)
+    leveldb-ruby (0.7)
     rake (0.9.2)
     rdoc (3.8)
     rspec (2.6.0)
@@ -19,10 +19,6 @@ GEM
     rspec-expectations (2.6.0)
       diff-lcs (~> 1.1.2)
     rspec-mocks (2.6.0)
-    sequel (3.25.0)
-    sqlite3 (1.3.3)
-    sqlite3-ruby (1.3.3)
-      sqlite3 (>= 1.3.3)
 
 PLATFORMS
   ruby

data/README.rdoc

@@ -31,4 +31,4 @@ on your Gemfile.
 
 http://www.nttdocomo.co.jp/service/imode/make/content/iarea/domestic/index.html
 
-db/iareadata.sqlite3 contains the data extracted from http://www.nttdocomo.co.jp/binary/archive/service/imode/make/content/iarea/domestic/iareadata.lzh and http://www.nttdocomo.co.jp/service/imode/make/content/iarea/domestic/map/js/iarea_def.js.
+db/iareadata contains the data extracted from http://www.nttdocomo.co.jp/binary/archive/service/imode/make/content/iarea/domestic/iareadata.lzh and http://www.nttdocomo.co.jp/service/imode/make/content/iarea/domestic/map/js/iarea_def.js.

data/db/iareadata/CURRENT

@@ -0,0 +1 @@
+MANIFEST-000004

data/db/iareadata/LOG

@@ -0,0 +1,27 @@
+2011/08/01-06:56:10.243731 7fff7e529960 Recovering log #3
+2011/08/01-06:56:10.245217 7fff7e529960 Level-0 table #5: started
+2011/08/01-06:56:10.247597 7fff7e529960 Level-0 table #5: 137776 bytes OK
+2011/08/01-06:56:10.248902 7fff7e529960 Delete type=0 #3
+2011/08/01-06:56:10.248995 7fff7e529960 Delete type=3 #2
+2011/08/01-06:56:11.052004 1007f2000 Level-0 table #8: started
+2011/08/01-06:56:11.087336 1007f2000 Level-0 table #8: 1540152 bytes OK
+2011/08/01-06:56:11.093042 1007f2000 Delete type=0 #6
+2011/08/01-06:56:11.913693 1007f2000 Level-0 table #10: started
+2011/08/01-06:56:11.961970 1007f2000 Level-0 table #10: 1538749 bytes OK
+2011/08/01-06:56:11.968064 1007f2000 Delete type=0 #7
+2011/08/01-06:56:12.808704 1007f2000 Level-0 table #12: started
+2011/08/01-06:56:12.841190 1007f2000 Level-0 table #12: 1532571 bytes OK
+2011/08/01-06:56:12.850141 1007f2000 Delete type=0 #9
+2011/08/01-06:56:12.850476 1007f2000 Compacting 4@0 + 0@1 files
+2011/08/01-06:56:12.972097 1007f2000 Generated table #13: 109414 keys, 2116092 bytes
+2011/08/01-06:56:13.060578 1007f2000 Generated table #14: 114250 keys, 2115011 bytes
+2011/08/01-06:56:13.077859 1007f2000 Generated table #15: 26061 keys, 513678 bytes
+2011/08/01-06:56:13.077922 1007f2000 Compacted 4@0 + 0@1 files => 4744781 bytes
+2011/08/01-06:56:13.081068 1007f2000 Delete type=2 #5
+2011/08/01-06:56:13.081286 1007f2000 Delete type=2 #8
+2011/08/01-06:56:13.081596 1007f2000 Delete type=2 #10
+2011/08/01-06:56:13.081880 1007f2000 Delete type=2 #12
+2011/08/01-06:56:13.082217 1007f2000 compacted to: files[ 0 3 0 0 0 0 0 ]
+2011/08/01-06:56:13.662966 1007f2000 Level-0 table #17: started
+2011/08/01-06:56:13.700564 1007f2000 Level-0 table #17: 1535854 bytes OK
+2011/08/01-06:56:13.705899 1007f2000 Delete type=0 #11

data/db/iareadata/LOG.old

@@ -0,0 +1 @@
+2011/08/01-06:56:08.209457 7fff7e529960 Delete type=3 #1

data/iarea.gemspec

@@ -18,8 +18,7 @@ Gem::Specification.new do |s|
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
 
-  s.add_runtime_dependency 'sequel'
-  s.add_runtime_dependency 'sqlite3-ruby'
+  s.add_runtime_dependency 'leveldb-ruby'
   s.add_development_dependency 'rspec'
   s.add_development_dependency 'rdoc'
   s.add_development_dependency 'rake'

data/lib/iarea/area.rb

@@ -15,23 +15,24 @@ module Iarea
 
     # Neighbor areas of the area
     def neighbors
-      DB[:neighbors].where(:areacode => areacode).select(:neighbor_areacode).map do |neighbor|
-        Area.find neighbor[:neighbor_areacode]
-      end
+      JSON.parse(DB['n:'+self.areacode]).map{|areacode| Area.find areacode}
     end
 
     class << self
       # Find area by <tt>areacode</tt>
       def find(areacode)
-        @@areas[areacode] ||= new DB[:areas].where(:areacode => areacode).first
+        @@areas[areacode] ||= new JSON.parse(DB['a:'+areacode])
       end
 
       # Find area by latitude and longitude in degrees
       def find_by_lat_lng(lat, lng)
         meshcodes = Utils.expand_meshcode(Utils.lat_lng_to_meshcode(lat, lng))
-        area = DB[:meshes].where(:meshcode => meshcodes).select(:areacode).first
-        return nil if area.nil?
-        find(area[:areacode])
+        meshcodes.each do |meshcode|
+          if areacode = DB['m:'+meshcode]
+            return find(areacode)
+          end
+        end
+        return nil
       end
     end
   end

data/lib/iarea/prefecture.rb

@@ -1,7 +1,7 @@
 module Iarea
   # Prefecture
   class Prefecture < OpenStruct
-    @@prefectures = []
+    @@prefectures = {}
 
     # Zone of the prefecture
     def zone
@@ -10,20 +10,19 @@ module Iarea
 
     # Areas in the prefecture
     def areas
-      DB[:areas].where(:prefecture_id => self.id).select(:areacode).order(:areacode).map do |area|
-        Area.find(area[:areacode])
-      end
+      self.areacodes.map{|areacode| Area.find areacode}
     end
 
     class << self
       # Find a prefecture by <tt>id</tt>
       def find(id)
-        @@prefectures[id] ||= new DB[:prefectures].where(:id => id).first
+        id = id.to_s
+        @@prefectures[id] ||= new JSON.parse(DB['p:'+id])
       end
 
       # All prefectures
       def all
-        DB[:prefectures].select(:id).order(:id).all.map{ |pr| find(pr[:id]) }
+        JSON.parse(DB['p']).map{ |id| find(id) }
       end
     end
   end

data/lib/iarea/version.rb

@@ -1,3 +1,3 @@
 module Iarea
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

data/lib/iarea/zone.rb

@@ -1,30 +1,27 @@
 module Iarea
   class Zone < OpenStruct
-    @@zones = []
+    @@zones = {}
 
     # Prefectures in the zone
     def prefectures
-      DB[:prefectures].where(:zone_id => self.id).select(:id).order(:id).map do |prefecture|
-        Prefecture.find(prefecture[:id])
-      end
+      self.prefecture_ids.map {|prefecture_id| Prefecture.find prefecture_id}
     end
 
     # Areas in the zone
     def areas
-      DB[:areas].where(:zone_id => self.id).select(:areacode).order(:areacode).map do |area|
-        Area.find(area[:areacode])
-      end
+      self.areacodes.map {|areacode| Area.find areacode}
     end
 
     class << self
       # Find a zone by <tt>id</tt>
       def find(id)
-        @@zones[id] ||= new DB[:zones].where(:id => id).first
+        id = id.to_s
+        @@zones[id] ||= new JSON.parse(DB['z:'+id])
       end
 
       # All zones
       def all
-        DB[:zones].select(:id).order(:id).all.map{ |z| find(z[:id]) }
+        JSON.parse(DB['z']).map{ |id| find(id) }
       end
     end
   end

data/lib/iarea.rb

@@ -1,9 +1,10 @@
-require 'sequel'
+require 'leveldb'
+require 'json'
 require 'ostruct'
 require_relative 'iarea/version'
 
 module Iarea # :nodoc:
-  DB = Sequel.sqlite(File.expand_path("../../db/iareadata.sqlite3", __FILE__)) # :nodoc:
+  DB = LevelDB::DB.load(File.expand_path("../../db/iareadata", __FILE__)) # :nodoc:
   autoload :Area, File.expand_path('../iarea/area', __FILE__)
   autoload :Zone, File.expand_path('../iarea/zone', __FILE__)
   autoload :Prefecture, File.expand_path('../iarea/prefecture', __FILE__)

data/lib/tasks/iarea.rake

@@ -14,7 +14,7 @@ namespace :iarea do
       tf = Tempfile.open("iarea")
       tf.write URI(uri).read
       tf.close
-      ruby "./tools/import_areas.rb", tf.path, "db/iareadata.sqlite3"
+      ruby "./tools/import_areas.rb", tf.path, "db/iareadata"
     end
 
     desc "fetch iareadata.lzh"
@@ -29,7 +29,7 @@ namespace :iarea do
             raise "lha failed"
           end
         end
-        ruby "./tools/import_meshes.rb", dir, "db/iareadata.sqlite3"
+        ruby "./tools/import_meshes.rb", dir, "db/iareadata"
       end
     end
   end

data/tools/import_areas.rb

@@ -1,103 +1,79 @@
 #!/usr/bin/env ruby
 # coding: utf-8
 
-require 'csv'
 require 'json'
-require 'sequel'
+
+require 'rubygems'
+require 'bundler/setup'
+require 'leveldb'
 
 if ARGV.size != 2
-  puts "usage #$0 [iarea_def.js] [dest_sqlite3_file]"
+  puts "usage #$0 [iarea_def.js] [dest_lebeldb_file]"
   exit -1
 end
 
 src_path = ARGV.shift
 dest_path = ARGV.shift
 
-db = Sequel.sqlite(dest_path)
+db = LevelDB::DB.new dest_path
 
-db.drop_table :zones if db.table_exists?(:zones)
-db.create_table :zones do
-  primary_key :id
-  String :name
+js = open(src_path, "r:cp932").read
+js.sub!(%r|^var IAreaDef=new function\(\)\{|, "")
+data = {}
+js.scan %r|this\.(.+?)=(.+?);|m do |key, json|
+  data[key] = JSON.parse(json)
 end
+# zones
+data['zones'].each_with_index do |zone, i|
+  id = i + 1
+  $stderr.puts "  importing zone %d %s" % [id, zone['name']]
+  areacodes = data['areas'].select {|areacode, area| area['zone'] == i}.keys
+  prefecture_ids = data['prefs'].each.with_index.select {|pref, j| pref['zone'] == i}.map{|pref, j| j+1 }
 
-db.drop_table :prefectures if db.table_exists?(:prefectures)
-db.create_table :prefectures do
-  primary_key :id
-  String :name
-  Integer :zone_id
-  index :zone_id
+  db['z:%d' % id] = JSON.dump(
+    :id => id,
+    :name => zone['name'],
+    :areacodes => areacodes,
+    :prefecture_ids => prefecture_ids
+  )
 end
 
-db.drop_table :areas if db.table_exists?(:areas)
-db.create_table :areas do
-  String :areaid
-  String :subareaid
-  String :areacode
-  String :name
-  Integer :west
-  Integer :south
-  Integer :east
-  Integer :north
-  Integer :zone_id
-  Integer :prefecture_id
-  index :areaid
-  index :subareaid
-  index :areacode, :unique => true
-  index :zone_id
-  index :prefecture_id
-end
+db['z'] = JSON.dump(data['zones'].map.with_index{|z, i| i + 1})
 
-db.drop_table :neighbors if db.table_exists?(:neighbors)
-db.create_table :neighbors do
-  String :areacode
-  String :neighbor_areacode
-  index :areacode
+# prefs
+data['prefs'].each_with_index do |pref, i|
+  id = i + 1
+  $stderr.puts "  importing prefecture %d %s" % [id, pref['name']]
+  areacodes = data['areas'].select {|areacode, area| area['pref'] == i}.keys
+  db['p:%d' % id] = JSON.dump(
+    :id => id,
+    :name => pref['name'],
+    :zone_id => pref['zone'] + 1,
+    :areacodes => areacodes
+  )
 end
+db['p'] = JSON.dump(data['prefs'].map.with_index{|z, i| i + 1})
 
-db.transaction do
-  js = open(src_path, "r:cp932").read
-  js.sub!(%r|^var IAreaDef=new function\(\)\{|, "")
-  data = {}
-  js.scan %r|this\.(.+?)=(.+?);|m do |key, json|
-    data[key] = JSON.parse(json)
-  end
-  # zones
-  data['zones'].each_with_index do |zone, i|
-    id = i + 1
-    $stderr.puts "  importing zone %d %s" % [id, zone['name']]
-    db[:zones].insert(:id => id, :name => zone['name'])
-  end
-  # prefs
-  data['prefs'].each_with_index do |pref, i|
-    id = i + 1
-    $stderr.puts "  importing prefecture %d %s" % [id, pref['name']]
-    db[:prefectures].insert(:id => id, :name => pref['name'], :zone_id => pref['zone'] + 1)
-  end
-  # areas
-  data['areas'].each do |areacode, area|
-    $stderr.puts "  importing area %s %s" % [areacode, area['name']]
-    # insert area
-    hash = {
-      :areaid => areacode[0..2],
-      :subareaid => areacode[3..4],
-      :areacode => areacode,
-      :name => area['name'],
-      :west => area['msWest'],
-      :south => area['msSouth'],
-      :east => area['msEast'],
-      :north => area['msNorth'],
-      :zone_id => area['zone'] + 1,
-      :prefecture_id => area['pref'] + 1
-    }
-    # fix .js problem
-    if hash[:name] == '新宿１?２丁目'
-      hash[:name] = '新宿１〜２丁目'
-    end
-    db[:areas].insert(hash)
-
-    area['neighbors'].each do |neighbor|
-      db[:neighbors].insert({:areacode => areacode, :neighbor_areacode => neighbor})
-    end
+# areas
+data['areas'].each do |areacode, area|
+  $stderr.puts "  importing area %s %s" % [areacode, area['name']]
+  # insert area
+  hash = {
+    :areaid => areacode[0..2],
+    :subareaid => areacode[3..4],
+    :areacode => areacode,
+    :name => area['name'],
+    :west => area['msWest'],
+    :south => area['msSouth'],
+    :east => area['msEast'],
+    :north => area['msNorth'],
+    :zone_id => area['zone'] + 1,
+    :prefecture_id => area['pref'] + 1
+  }
+  # fix .js problem
+  if hash[:name] == '新宿１?２丁目'
+    hash[:name] = '新宿１〜２丁目'
   end
+  db['a:%s' % areacode] = JSON.dump(hash)
+  db['n:%s' % areacode] = JSON.dump(area['neighbors'])
 end

data/tools/import_meshes.rb

@@ -1,53 +1,44 @@
 #!/usr/bin/env ruby
 
 require 'csv'
-require 'json'
-require 'sequel'
+
+require 'rubygems'
+require 'bundler/setup'
+require 'leveldb'
+
 
 if ARGV.size != 2
-  puts "usage #$0 [src_dir_contains_iareadata_dir] [dest_sqlite3_file]"
+  puts "usage #$0 [src_dir_contains_iareadata_dir] [dest_leveldb_file]"
   exit -1
 end
 
 src_path = ARGV.shift
 dest_path = ARGV.shift
 
-db = Sequel.sqlite(dest_path)
+db = LevelDB::DB.new dest_path
 
-db.drop_table :meshes if db.table_exists?(:meshes)
-db.create_table :meshes do
-  String :meshcode
-  String :areacode
-  index :meshcode
-  index :areacode
-end
+Dir[File.join(src_path, "iareadata/iarea*.txt")].each do |path|
+  data = CSV.open(path, "r:cp932")
+  data.each do |row|
+    areaid, subareaid, name, west, south, east, north, *mesh_data = row
+    areacode = areaid + subareaid
+
+    name.encode!("UTF-8")
+
+    mesh_data.pop if mesh_data.last.nil?
+    $stderr.puts "  importing meshes %s %s" % [areacode, name]
+    num_meshes = mesh_data.shift(7).map(&:to_i)
+
+    expected_num_meshes = num_meshes.inject(&:+)
+    if expected_num_meshes != mesh_data.size
+      raise "number of meshes mismatch: expected %d, actual %d" % [
+        expected_num_meshes, mesh_data.size
+      ]
+    end
 
-db.transaction do
-  meshes = db[:meshes]
-  areas = db[:areas]
-  Dir[File.join(src_path, "iareadata/iarea*.txt")].each do |path|
-    data = CSV.open(path, "r:cp932")
-    data.each do |row|
-      areaid, subareaid, name, west, south, east, north, *mesh_data = row
-      areacode = areaid + subareaid
-
-      name.encode!("UTF-8")
-
-      mesh_data.pop if mesh_data.last.nil?
-      $stderr.puts "  importing meshes %s %s" % [areacode, name]
-      num_meshes = mesh_data.shift(7).map(&:to_i)
-
-      expected_num_meshes = num_meshes.inject(&:+)
-      if expected_num_meshes != mesh_data.size
-        raise "number of meshes mismatch: expected %d, actual %d" % [
-          expected_num_meshes, mesh_data.size
-        ]
-      end
-
-      # insert meshes
-      mesh_data.each do |mesh|
-        meshes.insert(:meshcode => mesh, :areacode => areacode)
-      end
+    # insert meshes
+    mesh_data.each do |mesh|
+      db['m:%s' % mesh] = areacode
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: iarea
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
   prerelease: 
 platform: ruby
 authors:
@@ -13,8 +13,8 @@ date: 2011-08-01 00:00:00.000000000 +09:00
 default_executable: 
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: sequel
-  requirement: &2158352220 !ruby/object:Gem::Requirement
+  name: leveldb-ruby
+  requirement: &2157498120 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -22,21 +22,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2158352220
-- !ruby/object:Gem::Dependency
-  name: sqlite3-ruby
-  requirement: &2158351740 !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: *2158351740
+  version_requirements: *2157498120
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &2158351260 !ruby/object:Gem::Requirement
+  requirement: &2157497700 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -44,10 +33,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2158351260
+  version_requirements: *2157497700
 - !ruby/object:Gem::Dependency
   name: rdoc
-  requirement: &2158350760 !ruby/object:Gem::Requirement
+  requirement: &2157497280 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -55,10 +44,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2158350760
+  version_requirements: *2157497280
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &2158350300 !ruby/object:Gem::Requirement
+  requirement: &2157496860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -66,10 +55,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2158350300
+  version_requirements: *2157496860
 - !ruby/object:Gem::Dependency
   name: bundler
-  requirement: &2158349800 !ruby/object:Gem::Requirement
+  requirement: &2157496440 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -77,7 +66,7 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2158349800
+  version_requirements: *2157496440
 description: A library for DoCoMo Open Iarea.
 email:
 - dara@shidara.net
@@ -94,7 +83,16 @@ files:
 - README.rdoc
 - Rakefile
 - VERSION
-- db/iareadata.sqlite3
+- db/iareadata/000013.sst
+- db/iareadata/000014.sst
+- db/iareadata/000015.sst
+- db/iareadata/000016.log
+- db/iareadata/000017.sst
+- db/iareadata/CURRENT
+- db/iareadata/LOCK
+- db/iareadata/LOG
+- db/iareadata/LOG.old
+- db/iareadata/MANIFEST-000004
 - iarea.gemspec
 - lib/iarea.rb
 - lib/iarea/area.rb