jp_address 1.0.2 → 2.0.0

data/app/models/jp_address/zipcode.rb

@@ -1,68 +1,116 @@
 module JpAddress
-  class Zipcode < ActiveRecord::Base
+  class Zipcode < ApplicationRecord
 
-    MASTER_CSV_PATH = 'tmp/ken_all.csv'
+    # 日本郵便からダウンロードした CSV の一時保存先。
+    MASTER_CSV_PATH = 'tmp/ken_all.csv'.freeze
+
+    # 郵便番号データ（住所の郵便番号／小書き）の配布 URL。
+    MASTER_FILE_URL = 'https://www.post.japanpost.jp/zipcode/dl/kogaki/zip/ken_all.zip'.freeze
+
+    # insert_all で一度に流し込む件数。
+    INSERT_BATCH_SIZE = 1_000
+
+    # リダイレクトを追う上限。
+    MAX_REDIRECTS = 5
+
+    class DownloadError < StandardError; end
 
     def self.download_master_file_from_japanpost
       _setup_directory
-      zip = _save_zip(_request_to_japanpost.body)
-      _save_csv(zip.path)
+      tmp_zip = _save_zip(_request_to_japanpost)
+      begin
+        _save_csv(tmp_zip.path)
+      ensure
+        tmp_zip.close!
+      end
       :success if File.exist?(MASTER_CSV_PATH)
     end
 
     def self._setup_directory
-      FileUtils.mkdir_p 'tmp'
+      FileUtils.mkdir_p File.dirname(MASTER_CSV_PATH)
       _remove_csv
     end
     private_class_method :_setup_directory
 
-    def self._request_to_japanpost
-      res = nil
-      https = Net::HTTP.new('www.post.japanpost.jp',443)
-      https.use_ssl = true
-      https.start {
-        res = https.get('/zipcode/dl/kogaki/zip/ken_all.zip')
-      }
-      res
+    def self._request_to_japanpost(url = MASTER_FILE_URL, redirect_limit = MAX_REDIRECTS)
+      raise DownloadError, "too many redirects while downloading #{MASTER_FILE_URL}" if redirect_limit <= 0
+
+      uri = URI.parse(url)
+      res = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https') do |https|
+        https.get(uri.request_uri)
+      end
+
+      case res
+      when Net::HTTPSuccess
+        res.body
+      when Net::HTTPRedirection
+        location = res['location']
+        raise DownloadError, "got a redirect without a location from #{url}" if location.nil?
+
+        _request_to_japanpost(URI.join(url, location).to_s, redirect_limit - 1)
+      else
+        raise DownloadError, "failed to download #{url} (#{res.code} #{res.message})"
+      end
     end
     private_class_method :_request_to_japanpost
 
     def self._save_zip(binary)
-      tmp_zip = Tempfile.open('ken_all.zip') do |f|
-        f.binmode
-        f.write binary
-        f
-      end
+      tmp_zip = Tempfile.new(['ken_all', '.zip'])
+      tmp_zip.binmode
+      tmp_zip.write binary
+      tmp_zip.flush
+      tmp_zip.close
       tmp_zip
     end
     private_class_method :_save_zip
 
     def self._save_csv(zip_path)
       Zip::File.open(zip_path) do |zip_file|
-        zip_file.each do |entry|
-          entry.extract(MASTER_CSV_PATH)
-        end
+        entry = zip_file.find { |e| e.file? && File.extname(e.name).downcase == '.csv' }
+        raise DownloadError, "no CSV file found in the archive from #{MASTER_FILE_URL}" if entry.nil?
+
+        entry.extract(MASTER_CSV_PATH)
       end
     end
     private_class_method :_save_csv
 
+    # CSV を読み込んでテーブルを作り直します。
+    #
+    # 同じ郵便番号を持つ行は１レコードに統合してから insert するので、
+    # 郵便番号はテーブル内で一意になります。
     def self.load_master_data(csv_path = MASTER_CSV_PATH)
-      download_master_file_from_japanpost if !File.exist?(csv_path)
+      unless File.exist?(csv_path)
+        raise ArgumentError, "#{csv_path} does not exist" unless csv_path == MASTER_CSV_PATH
+
+        download_master_file_from_japanpost
+      end
+
       _clear_table
-      CSV.foreach(csv_path, encoding:'SJIS:UTF-8') do |row|
-        connection.execute(
-          sanitize_sql(
-            ["insert into jp_address_zipcodes (zip, prefecture, city, town)
-              values ('%s', '%s', '%s', '%s')", row[2], row[6], row[7], _remove_needless_words(row[8])
-            ]
-          )
-        )
+      transaction do
+        _each_record_batch_from(csv_path) { |batch| insert_all!(batch) }
       end
-      _merge_same_zip_addresses
       _remove_csv
+      nil
+    end
+
+    # CSV を郵便番号ごとにまとめ、insert_all に渡せる Hash の配列を
+    # INSERT_BATCH_SIZE 件ずつ yield します。
+    def self._each_record_batch_from(csv_path)
+      _read_master_csv(csv_path).each_value.each_slice(INSERT_BATCH_SIZE) do |slice|
+        yield slice.map { |rec|
+          {
+            zip:        rec[:zip],
+            prefecture: rec[:prefecture],
+            city:       rec[:city],
+            town:       _find_shared_name_from(rec[:towns])
+          }
+        }
+      end
     end
+    private_class_method :_each_record_batch_from
 
-    # 同じ郵便番号を持つレコードを統合します。
+    # 郵便番号をキーにして CSV の行をまとめます。
+    # 都道府県名・市区町村名は同じ郵便番号の最初の行のものを採用します。
     #
     # 例：9896712
     #   "宮城県","大崎市","鳴子温泉水沼"
@@ -72,23 +120,45 @@ module JpAddress
     # これらは
     #   "宮城県","大崎市","鳴子温泉" として１つのレコードにします。
     #   共通する地名が抜き出せない場合は空の町名にします。
-    def self._merge_same_zip_addresses
-      group(:zip).having('count(*) > 1').pluck(:zip).each do |dup_zip|
-        buf = nil
-        town_names = []
-        where(zip: dup_zip).order(:id).each_with_index do |rec, i|
-          town_names << rec.town if rec.town.present?
-          if i == 0
-            buf = rec.dup
-          end
-          rec.destroy
+    #
+    # ken_all.csv では町域名が長いと括弧の途中で改行され、次の行に続きます。
+    #   "米出（１２４、１２５、１２７、"
+    #   "１４８、１４９、１５４番地）"
+    # 続きの行は町域名ではないので、括弧が閉じるまでの行は読み飛ばします。
+    def self._read_master_csv(csv_path)
+      records = {}
+      # 郵便番号ごとの「閉じていない括弧の数」。0 より大きい間は継続行。
+      open_parens = Hash.new(0)
+
+      # ken_all.csv の文字コードは CP932（Ruby では SJIS が CP932 の別名）。
+      CSV.foreach(csv_path, encoding: 'CP932:UTF-8') do |row|
+        zip = row[2].to_s.strip
+        next if zip.empty?
+
+        raw_town = row[8].to_s
+        if open_parens[zip] > 0
+          open_parens[zip] += _paren_balance(raw_town)
+          next
         end
-        shared_town_name = _find_shared_name_from(town_names)
-        buf.town = shared_town_name
-        buf.save!
+        open_parens[zip] = [_paren_balance(raw_town), 0].max
+
+        record = (records[zip] ||= {
+          zip:        zip,
+          prefecture: row[6].to_s.strip,
+          city:       row[7].to_s.strip,
+          towns:      []
+        })
+        town = _remove_needless_words(raw_town)
+        record[:towns] << town if town.present?
       end
-      nil
+      records
+    end
+    private_class_method :_read_master_csv
+
+    def self._paren_balance(text)
+      text.count('（') - text.count('）')
     end
+    private_class_method :_paren_balance
 
     # 引数に渡された地名群から、先頭から見て共通となる地名を返します。
     #
@@ -98,51 +168,45 @@ module JpAddress
     #         ]
     #
     # return => 鳴子温泉
+    #
+    # 共通部分が１文字しかない場合（例：大分／大阪）は、地名として意味を
+    # なさないので空文字を返します。ただし短い方の地名がまるごと共通部分に
+    # なる場合（例：東中島／東中島本町）は、その地名をそのまま返します。
     def self._find_shared_name_from(names)
-      return '' if names.blank?
-
-      name_length_min = names.map{ |n| n.length }.min
-      diff_pos        = nil
-      chars           = []
-
-      (0..(name_length_min - 1)).each do |pos|
-        break if diff_pos.present?
-        char = nil
-        names.each do |name|
-          if char.nil?
-            char  = name.each_char.to_a[pos]
-            chars << char
-          elsif char != name.each_char.to_a[pos]
-            diff_pos = pos
-            break
-          end
-        end
-      end
+      names = Array(names).reject { |name| name.blank? }
+      return '' if names.empty?
 
-      if diff_pos.present? && diff_pos > 1
-        ret = chars[0, diff_pos].join
-        return ret
+      shortest      = names.min_by { |name| name.length }
+      shared_length = 0
+      shortest.each_char.with_index do |char, pos|
+        break unless names.all? { |name| name[pos] == char }
+
+        shared_length = pos + 1
       end
 
-      ''
+      return shortest if shared_length == shortest.length
+
+      shared_length > 1 ? shortest[0, shared_length] : ''
     end
 
     def self._clear_table
-      begin
-        connection.truncate 'jp_address_zipcodes'
-      rescue NotImplementedError
-        connection.execute 'delete from jp_address_zipcodes'
+      connection_pool.with_connection do |conn|
+        begin
+          conn.truncate table_name
+        rescue NotImplementedError
+          conn.delete "delete from #{conn.quote_table_name(table_name)}"
+        end
       end
     end
     private_class_method :_clear_table
 
     def self._remove_needless_words(base)
-      base.sub(/以下に掲載がない場合/, '').sub(/（.*/, '')
+      base.to_s.sub(/以下に掲載がない場合/, '').sub(/（.*/, '').strip
     end
     private_class_method :_remove_needless_words
 
     def self._remove_csv
-      FileUtils.rm_rf MASTER_CSV_PATH
+      FileUtils.rm_f MASTER_CSV_PATH
     end
     private_class_method :_remove_csv
 

data/db/migrate/20160312045953_create_jp_address_zipcodes.rb

@@ -1,4 +1,4 @@
-class CreateJpAddressZipcodes < ActiveRecord::Migration[5.0]
+class CreateJpAddressZipcodes < ActiveRecord::Migration[7.1]
   def change
     create_table :jp_address_zipcodes do |t|
       t.string :zip       , null: false, index: true

data/lib/jp_address/version.rb

@@ -1,3 +1,3 @@
 module JpAddress
-  VERSION = "1.0.2"
+  VERSION = "2.0.0"
 end

data/lib/jp_address.rb

@@ -1,9 +1,11 @@
 require 'net/https'
 require 'uri'
 require 'tempfile'
+require 'fileutils'
 require 'csv'
 require 'zip'
-require "jp_address/engine"
+require 'jp_address/version'
+require 'jp_address/engine'
 
 module JpAddress
 end

metadata

@@ -1,113 +1,94 @@
 --- !ruby/object:Gem::Specification
 name: jp_address
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 2.0.0
 platform: ruby
 authors:
 - Tad Kam
-autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-06-11 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: rubyzip
+  name: railties
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '7.1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '9.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rails
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
+        version: '7.1'
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
+        version: '9.0'
 - !ruby/object:Gem::Dependency
-  name: sqlite3
+  name: activerecord
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rspec-rails
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
+        version: '7.1'
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
+        version: '9.0'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: factory_bot_rails
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
+        version: '7.1'
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
+        version: '9.0'
 - !ruby/object:Gem::Dependency
-  name: webmock
+  name: rubyzip
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
+        version: '2.3'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.3'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
 - !ruby/object:Gem::Dependency
-  name: vcr
+  name: csv
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
+        version: '3.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
 description: JpAddress is simple japan-zipcode-address-search API. You can load master-data
   from JapanPost and mount address-search-api to your rails application.
 email:
@@ -116,72 +97,28 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- CHANGELOG.md
 - MIT-LICENSE
 - README.md
 - Rakefile
-- app/assets/javascripts/jp_address/application.js
-- app/assets/stylesheets/jp_address/application.css
 - app/controllers/jp_address/application_controller.rb
 - app/controllers/jp_address/zipcodes_controller.rb
-- app/helpers/jp_address/application_helper.rb
+- app/models/jp_address/application_record.rb
 - app/models/jp_address/zipcode.rb
-- app/views/layouts/jp_address/application.html.erb
 - config/routes.rb
 - db/migrate/20160312045953_create_jp_address_zipcodes.rb
 - lib/jp_address.rb
 - lib/jp_address/engine.rb
 - lib/jp_address/version.rb
 - lib/tasks/jp_address_tasks.rake
-- spec/controllers/jp_address/zipcodes_controller_spec.rb
-- spec/dummy/README.rdoc
-- spec/dummy/Rakefile
-- spec/dummy/app/assets/config/manifest.js
-- spec/dummy/app/assets/javascripts/application.js
-- spec/dummy/app/assets/stylesheets/application.css
-- spec/dummy/app/controllers/application_controller.rb
-- spec/dummy/app/helpers/application_helper.rb
-- spec/dummy/app/views/layouts/application.html.erb
-- spec/dummy/bin/bundle
-- spec/dummy/bin/rails
-- spec/dummy/bin/rake
-- spec/dummy/bin/setup
-- spec/dummy/config.ru
-- spec/dummy/config/application.rb
-- spec/dummy/config/boot.rb
-- spec/dummy/config/database.yml
-- spec/dummy/config/environment.rb
-- spec/dummy/config/environments/development.rb
-- spec/dummy/config/environments/production.rb
-- spec/dummy/config/environments/test.rb
-- spec/dummy/config/initializers/assets.rb
-- spec/dummy/config/initializers/backtrace_silencers.rb
-- spec/dummy/config/initializers/cookies_serializer.rb
-- spec/dummy/config/initializers/filter_parameter_logging.rb
-- spec/dummy/config/initializers/inflections.rb
-- spec/dummy/config/initializers/mime_types.rb
-- spec/dummy/config/initializers/session_store.rb
-- spec/dummy/config/initializers/wrap_parameters.rb
-- spec/dummy/config/locales/en.yml
-- spec/dummy/config/routes.rb
-- spec/dummy/config/secrets.yml
-- spec/dummy/db/schema.rb
-- spec/dummy/db/test.sqlite3
-- spec/dummy/log/test.log
-- spec/dummy/public/404.html
-- spec/dummy/public/422.html
-- spec/dummy/public/500.html
-- spec/dummy/public/favicon.ico
-- spec/factories/jp_address_zipcodes.rb
-- spec/models/jp_address/zipcode_spec.rb
-- spec/rails_helper.rb
-- spec/spec_helper.rb
-- spec/support/files/sample_ken.csv
-- spec/vcr/download_master_file_from_japanpost.yml
 homepage: https://github.com/densya203/jp_address
 licenses:
 - MIT
-metadata: {}
-post_install_message: 
+metadata:
+  homepage_uri: https://github.com/densya203/jp_address
+  source_code_uri: https://github.com/densya203/jp_address/tree/v2.0.0
+  bug_tracker_uri: https://github.com/densya203/jp_address/issues
+  changelog_uri: https://github.com/densya203/jp_address/blob/master/CHANGELOG.md
 rdoc_options: []
 require_paths:
 - lib
@@ -189,60 +126,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 3.1.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
-signing_key: 
+rubygems_version: 3.6.9
 specification_version: 4
 summary: Simple japan-zipcode-addresses API
-test_files:
-- spec/controllers/jp_address/zipcodes_controller_spec.rb
-- spec/dummy/README.rdoc
-- spec/dummy/Rakefile
-- spec/dummy/app/assets/config/manifest.js
-- spec/dummy/app/assets/javascripts/application.js
-- spec/dummy/app/assets/stylesheets/application.css
-- spec/dummy/app/controllers/application_controller.rb
-- spec/dummy/app/helpers/application_helper.rb
-- spec/dummy/app/views/layouts/application.html.erb
-- spec/dummy/bin/bundle
-- spec/dummy/bin/rails
-- spec/dummy/bin/rake
-- spec/dummy/bin/setup
-- spec/dummy/config/application.rb
-- spec/dummy/config/boot.rb
-- spec/dummy/config/database.yml
-- spec/dummy/config/environment.rb
-- spec/dummy/config/environments/development.rb
-- spec/dummy/config/environments/production.rb
-- spec/dummy/config/environments/test.rb
-- spec/dummy/config/initializers/assets.rb
-- spec/dummy/config/initializers/backtrace_silencers.rb
-- spec/dummy/config/initializers/cookies_serializer.rb
-- spec/dummy/config/initializers/filter_parameter_logging.rb
-- spec/dummy/config/initializers/inflections.rb
-- spec/dummy/config/initializers/mime_types.rb
-- spec/dummy/config/initializers/session_store.rb
-- spec/dummy/config/initializers/wrap_parameters.rb
-- spec/dummy/config/locales/en.yml
-- spec/dummy/config/routes.rb
-- spec/dummy/config/secrets.yml
-- spec/dummy/config.ru
-- spec/dummy/db/schema.rb
-- spec/dummy/db/test.sqlite3
-- spec/dummy/log/test.log
-- spec/dummy/public/404.html
-- spec/dummy/public/422.html
-- spec/dummy/public/500.html
-- spec/dummy/public/favicon.ico
-- spec/factories/jp_address_zipcodes.rb
-- spec/models/jp_address/zipcode_spec.rb
-- spec/rails_helper.rb
-- spec/spec_helper.rb
-- spec/support/files/sample_ken.csv
-- spec/vcr/download_master_file_from_japanpost.yml
+test_files: []

data/app/assets/javascripts/jp_address/application.js

@@ -1,13 +0,0 @@
-// This is a manifest file that'll be compiled into application.js, which will include all the files
-// listed below.
-//
-// Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts,
-// or any plugin's vendor/assets/javascripts directory can be referenced here using a relative path.
-//
-// It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the
-// compiled file.
-//
-// Read Sprockets README (https://github.com/rails/sprockets#sprockets-directives) for details
-// about supported directives.
-//
-//= require_tree .

data/app/assets/stylesheets/jp_address/application.css

@@ -1,15 +0,0 @@
-/*
- * This is a manifest file that'll be compiled into application.css, which will include all the files
- * listed below.
- *
- * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
- * or any plugin's vendor/assets/stylesheets directory can be referenced here using a relative path.
- *
- * You're free to add application-wide styles to this file and they'll appear at the bottom of the
- * compiled file so the styles you add here take precedence over styles defined in any styles
- * defined in the other CSS/SCSS files in this directory. It is generally better to create a new
- * file per style scope.
- *
- *= require_tree .
- *= require_self
- */

data/app/helpers/jp_address/application_helper.rb

@@ -1,4 +0,0 @@
-module JpAddress
-  module ApplicationHelper
-  end
-end

data/app/views/layouts/jp_address/application.html.erb

@@ -1,14 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-  <title>JpAddress</title>
-  <%= stylesheet_link_tag    "jp_address/application", media: "all" %>
-  <%= javascript_include_tag "jp_address/application" %>
-  <%= csrf_meta_tags %>
-</head>
-<body>
-
-<%= yield %>
-
-</body>
-</html>