kamome 0.1.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eabc3d7c8fab14f88254defa7d52769a9d674951c3af50f5332e1f5af678fa77
-  data.tar.gz: ae51e66cd3b662dc93ea8cd31223281004db1b097acb07bb07c123d8e0a8ab00
+  metadata.gz: 580db2ebe5e52ce51d8014ad886247ef89ae3ab9f8bccb82397a3c3ce0a15e75
+  data.tar.gz: 0605ca8f9f680d15fb87b2396e4f8db0930a8d3984ee18eaa5b3b95d51ec9a8b
 SHA512:
-  metadata.gz: f354384ca83990e9eddf6367c7d4453bff84729067a137aa1ebae59398f8d4dae94e1a21657e9830ba8289d8d671caec13314ec27235b0db935c95a1c90d2572
-  data.tar.gz: 705d96a73c879e96c18fa7afc6e186913fab28308bf9e4f560d39d25f564712302e1af941382468786a1b42f2b10bda2f7f8043f9b824898efc17f12bec18424
+  metadata.gz: 6a2cd80afb342ff0ca2f68ff769a6f11b478dc06e5514d836142763102cff983a9760fa9bea9334d59fd76744a09c766a727cad3a5f79e52719a5c89330b9fa1
+  data.tar.gz: 2e17669f0e652e42ba98f8429ca034d5d770b0b7ba5bd5e0380402b05a7ad7d7d3883aa3c57738cbb638e376722eef40e656fc9f56d92f2e0ce98aa9318194e2

data/.rubocop.yml

@@ -23,3 +23,6 @@ Metrics/LineLength:
 
 RSpec/NestedGroups:
   Max: 4
+
+RSpec/MessageSpies:
+  EnforcedStyle: receive

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    kamome (0.1.0)
+    kamome (0.8.0)
       rubyzip (~> 1.2)
 
 GEM

data/README.md

@@ -12,6 +12,15 @@ Kamome.import_general_all do |model, lineno|
 end
 ```
 
+対応CSV
+
+* 全国一括 (全件, 差分)
+* 事業所(全件, 差分)
+
+各CSVは共通のモデルにマッピングされます(type オプションで詳細なモデル or Hashに変更可能)
+
+詳細は [Kamome::Models::Address](https://github.com/kengos/kamome/blob/master/lib/kamome/models/address.rb) を参照してください
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -43,7 +52,7 @@ end
 ```rb
 date = Time.local(2018, 11) # 2018/11 更新分の取り込み
 Kamome.import_general_diff(date: date) do |model, lineno|
-  if model.changed?
+  if model.update?
     puts "追加データ: #{model.zipcode} => #{model.prefecture}#{model.city}#{model.town}"
   else
     puts "削除データ: #{model.zipcode} => #{model.prefecture}#{model.city}#{model.town}"
@@ -64,7 +73,7 @@ end
 ```rb
 date = Time.local(2018, 11) # 2018/11 更新分の取り込み
 Kamome.import_jigyosho_diff(date: date) do |model, lineno|
-  if model.changed?
+  if model.update?
     puts "追加データ: #{model.zipcode} => #{model.prefecture}#{model.city}#{model.town}"
   else
     puts "削除データ: #{model.zipcode} => #{model.prefecture}#{model.city}#{model.town}"
@@ -104,6 +113,13 @@ Kamome.import_general_all(type: :detail or Kamome::Operation::TYPE_DETAIL)
 |XXXXX（YYYY階）|XXXXX（YYYY階）|元の値を維持しています|
 |XXXXX（YYYYY）|XXXXX|() の部分が除去されます|
 
+※ 以下のような削除データが発生する可能性があり、 `town` が正確に取得できないことがあります
+
+```
+26106,"600  ","6008093","ｷｮｳﾄﾌ","ｷｮｳﾄｼｼﾓｷﾞｮｳｸ","ﾀｹﾔﾁｮｳ","京都府","京都市下京区","竹屋町（高倉通綾小路下る、高倉通仏光寺上る、堺町通綾小路下",0,0,0,0,2,6
+26106,"600  ","6008093","ｷｮｳﾄﾌ","ｷｮｳﾄｼｼﾓｷﾞｮｳｸ","ﾀｹﾔﾁｮｳ","京都府","京都市下京区","る西入）",0,0,0,0,2,6
+```
+
 #### `Kamome::Models::General#town_kana`
 
 |元データ|変換後|備考|
@@ -126,8 +142,6 @@ Kamome.import_general_all(type: :detail or Kamome::Operation::TYPE_DETAIL)
 
 として設定されます
 
-#### `Kamome::Models::General#town_kana`
-
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `make` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/lib/kamome/configuration.rb

@@ -2,17 +2,30 @@
 
 module Kamome
   class Configuration
-    require 'securerandom'
     require 'tmpdir'
 
+    # @!attribute [rw] tmp_path
+    #   @return [String]
     attr_accessor :tmp_path
+
+    # @!attribute [rw] open_uri_options
+    #   OpenURI options
+    #   @see https://docs.ruby-lang.org/ja/latest/class/OpenURI.html
+    #   @return [Hash]
     attr_accessor :open_uri_options
 
+    # @!attribute [rw] cleanup
+    #   If you do not want to delete the downloaded csv, please specify `false`
+    #   @return [Bolean]
+    attr_accessor :cleanup
+
     def initialize
-      @tmp_path = ::File.join(Dir.tmpdir, 'kamome', SecureRandom.hex(8))
+      @tmp_path = ::File.join(Dir.tmpdir, 'kamome')
       @open_uri_options = {}
+      @cleanup = true
     end
 
+    # @return [Pathname]
     def working_directory
       @working_directory ||= ::Pathname.new(@tmp_path.to_s)
     end

data/lib/kamome/csv_handler.rb

@@ -13,7 +13,7 @@ module Kamome
     def call(csv_path)
       ::CSV.foreach(csv_path) do |row|
         obj = transform(row)
-        yield(obj, $INPUT_LINE_NUMBER)
+        yield(obj, $INPUT_LINE_NUMBER) if block_given?
       end
     end
 

data/lib/kamome/downloader.rb

@@ -26,7 +26,7 @@ module Kamome
 
     # @return [Pathname] downloaded file
     def download(url)
-      dest = @config.working_directory.join(SecureRandom.hex(8) + '.zip')
+      dest = @config.working_directory.join(download_filename)
       ::FileUtils.mkdir_p(dest.dirname)
 
       ::OpenURI.open_uri(url, @config.open_uri_options) do |response|
@@ -35,11 +35,13 @@ module Kamome
       dest
     end
 
-    def unpack(src)
-      # convert encoding Shift_JIS to UTF-8 (and half width Katakana to full width Katakana)
-      content = ::NKF.nkf('-w -Lu', take_csv_entry(src).get_input_stream.read)
+    def download_filename
+      ::Time.now.strftime("%Y%m%d%H%M%S_#{SecureRandom.hex(3)}.zip")
+    end
 
+    def unpack(src)
       dest = src.sub('.zip', '.csv')
+      content = convert_encoding(take_csv_entry(src))
       ::File.binwrite(dest, content)
       dest
     end
@@ -48,9 +50,14 @@ module Kamome
       entry = ::Zip::File.open(src) do |zipfile|
         zipfile.select { |o| o.name =~ /.*\.csv/i }.first
       end
-      raise ::Kamome::DownloadError, 'Not found csvfile in zipfile' if entry.nil?
+      raise 'Not found csvfile in zipfile' if entry.nil?
+
+      entry.get_input_stream.read
+    end
 
-      entry
+    # convert encoding Shift_JIS to UTF-8 (and half width Katakana to full width Katakana)
+    def convert_encoding(content)
+      ::NKF.nkf('-w -Lu', content)
     end
   end
 end

data/lib/kamome/loader.rb

@@ -2,6 +2,7 @@
 
 require 'kamome/downloader'
 require 'kamome/csv_handler'
+require 'digest'
 
 module Kamome
   class Loader
@@ -9,13 +10,11 @@ module Kamome
       @config = config
     end
 
+    # @return [Array] csv digest
     def call(operation:, &block)
-      csv_handler = build_csv_handler(operation)
-
-      operation.urls.each do |url|
+      operation.urls.map do |url|
         csv_path = download(url)
-        csv_handler.call(csv_path, &block)
-        csv_path.delete
+        call_csv_handler(operation, csv_path, &block)
       end
     end
 
@@ -29,8 +28,11 @@ module Kamome
       @downloader ||= ::Kamome::Downloader.new(@config)
     end
 
-    def build_csv_handler(operation)
-      ::Kamome::CsvHandler.new(operation)
+    def call_csv_handler(operation, csv_path, &block)
+      ::Kamome::CsvHandler.new(operation).call(csv_path, &block)
+      ::Digest::SHA256.file(csv_path).to_s
+    ensure
+      csv_path.delete if @config.cleanup
     end
   end
 end

data/lib/kamome/models/address.rb

@@ -51,12 +51,18 @@ module Kamome
       #   @return [Integer]
       attr_accessor :state
 
+      STATE_NOT_CHANGE = 0
+      STATE_UPDATE = 1
+      STATE_DELETE = 2
+
+      # @params [Hash] model attributes
       def initialize(params = {})
         params.each do |key, value|
           public_send("#{key}=", value)
         end
       end
 
+      # @return [Hash] model attributes
       def attributes
         {
           code: code,
@@ -71,12 +77,22 @@ module Kamome
         }
       end
 
-      def changed?
-        state == 1
+      # Returns true when state is 1 or 2
+      # @return [Boolean]
+      def change?
+        updated? || deleted?
+      end
+
+      # Returns true when state is 1
+      # @return [Boolean]
+      def update?
+        state == STATE_UPDATE
       end
 
+      # Returns true when state is 2
+      # @return [Boolean]
       def delete?
-        state == 2
+        state == STATE_DELETE
       end
     end
   end

data/lib/kamome/operation.rb

@@ -75,5 +75,9 @@ module Kamome
     def genre_jigyosho?
       genre == GENRE_JIGYOSHO
     end
+
+    def ==(other)
+      other.class == self.class && other.type == type && other.genre == genre && other.urls == urls
+    end
   end
 end

data/lib/kamome/transformations/cleaner.rb

@@ -10,7 +10,7 @@ module Kamome
         value.sub(TOWN_KAN_SUB_RULE, '')
       end
 
-      TOWN_SUB_RULE = /(（(?!.+階）).+）|以下に掲載がない場合|.+の次に番地がくる場合|.+一円)$/.freeze
+      TOWN_SUB_RULE = /(（(?!.+階）).+|以下に掲載がない場合|.+の次に番地がくる場合|.+一円|)$/.freeze
       def clean_town(value)
         value.sub(TOWN_SUB_RULE, '')
       end

data/lib/kamome/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kamome
-  VERSION = '0.1.0'
+  VERSION = '0.8.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kamome
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.8.0
 platform: ruby
 authors:
 - kengos
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-11-30 00:00:00.000000000 Z
+date: 2018-12-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -184,5 +184,5 @@ rubyforge_project:
 rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
-summary: kamome-0.1.0
+summary: kamome-0.8.0
 test_files: []