pumi 0.13.0 → 0.13.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4b0cfc831082d67eb960f4bef137ff0f6ae67ea60eee36aeaae1edeb7f8fc945
-  data.tar.gz: 7b3fb1bc47eb838c8aa75c0467bbc518e83f09a2473e75d2a70c53c76fdce492
+  metadata.gz: c7df0e8620fb33e07db08efc84310967c1ce09c545de619b98c7b8294baab835
+  data.tar.gz: d752627f28b601a273508e7868de577dc2498b61278955152a88a7890552c992
 SHA512:
-  metadata.gz: bb6d74bd6d8b7192c8da709c30e2be05591cfe80f06f703cd765a35058458dca553f31a3cd5ffe870f2a3875054162ad9b71fa10aac47ac38a3755b2d0f42912
-  data.tar.gz: aa3d20e100720654a696d03b82470954fe1fb2691e01e17cd93961a8a39eaff99dd0475931fc0ccae8de2f4ac26e02bb96265ec923039128922240facc0fee74
+  metadata.gz: abebda567ad01aceb9ab68279f37456870dedb498c89e2b8465b215fb453da437a1963924fd1342a5f55a904a4ae76943ad76fd523f6af1f29d2f722509a3d20
+  data.tar.gz: 420fd418789d76c2eef6ce332c75d92c08fead8d99b902b8ccf43816e26f1bfa9ebc4d75cf3d58579906e0a1ca6202d85a5d6d4869bf9c0bf194a3a1a134c4e8

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## [0.13.1](https://github.com/dwilkie/pumi/compare/v0.13.0...v0.13.1) (2023-03-02)
+
+
+### Bug Fixes
+
+* Improve data parser ([#35](https://github.com/dwilkie/pumi/issues/35)) ([f17ebf5](https://github.com/dwilkie/pumi/commit/f17ebf5eba26d88f4fff4fb5bb2461d1817a0829))
+
 ## [0.13.0](https://github.com/dwilkie/pumi/compare/v0.12.2...v0.13.0) (2023-02-06)
 
 

data/README.md

@@ -8,13 +8,13 @@ Pumi (ភូមិ pronounced Poom, which means Village in Khmer) is an Open Sou
 
 ## Demo and API
 
-A [JSON API](https://pumiapp.herokuapp.com) is available to if you're not using Ruby or if you just don't want to install a local copy of the data. The [API Start Page](https://pumiapp.herokuapp.com) also shows a Demo of a UI for for entering any location in Cambodia.
+A [JSON API](https://pumi.onrender.com/) is available to if you're not using Ruby or if you just don't want to install a local copy of the data. The [API Start Page](https://pumi.onrender.com/) also shows a Demo of a UI for for entering any location in Cambodia.
 
 ## Usage
 
 ### Rails
 
-Using Pumi with Rails gives you some javascript helpers as well as an API to filter and select Provinces (ខេត្ត), Districts (ស្រុក / ខណ្ឌ), Communes (ឃុំ / សង្កាត់) and Villages (ភូមិ) in both latin and Khmer as seen below and in the [Pumi API Start Page](https://pumiapp.herokuapp.com)
+Using Pumi with Rails gives you some javascript helpers as well as an API to filter and select Provinces (ខេត្ត), Districts (ស្រុក / ខណ្ឌ), Communes (ឃុំ / សង្កាត់) and Villages (ភូមិ) in both latin and Khmer as seen below and in the [Pumi API Start Page](https://pumi.onrender.com/)
 
 ![Pumi UI Latin](https://raw.githubusercontent.com/dwilkie/pumi/master/pumi_ui_en.png)
 ![Pumi UI Khmer](https://raw.githubusercontent.com/dwilkie/pumi/master/pumi_ui_km.png)

data/bin/parse_data

@@ -4,63 +4,9 @@
 
 # $ parse_data
 
-require "pathname"
-require "csv"
-require "yaml"
+require "bundler/setup"
+require "pumi"
 
-DEFAULT_DATA_LOCATION = "tmp/*.csv".freeze
-HEADERS = %w[type code name_km name_latin reference note1 note2].freeze
-CODE_LENGTHS = {
-  4 => "districts",
-  6 => "communes",
-  8 => "villages"
-}.freeze
-
-# https://en.wikipedia.org/wiki/Administrative_divisions_of_Cambodia
-# https://en.wikipedia.org/wiki/Romanization_of_Khmer
-# https://en.wikipedia.org/wiki/United_Nations_Group_of_Experts_on_Geographical_Names
-
-AdministrativeUnit = Struct.new(:en, :km, :latin, keyword_init: true)
-ADMINISTRATIVE_UNITS = {
-  "ស្រុក" => AdministrativeUnit.new(en: "District", km: "ស្រុក", latin: "Srok"),
-  "ខណ្ឌ" => AdministrativeUnit.new(en: "Section", km: "ខណ្ឌ", latin: "Khan"),
-  "ឃុំ" => AdministrativeUnit.new(en: "Commune", km: "ឃុំ", latin: "Khum"),
-  "សង្កាត់" => AdministrativeUnit.new(en: "Quarter", km: "សង្កាត់", latin: "Sangkat"),
-  "ភូមិ" => AdministrativeUnit.new(en: "Village", km: "ភូមិ", latin: "Phum"),
-  "ក្រុង" => AdministrativeUnit.new(en: "Municipality", km: "ក្រុង", latin: "Krong")
-}.freeze
-
-data = {}
-
-Pathname.glob(DEFAULT_DATA_LOCATION).select(&:file?).each do |file|
-  CSV.read(file, headers: HEADERS).each do |row|
-    code = row.fetch("code")
-    next if code.to_s.gsub(/\D/, "").empty?
-
-    code = code.rjust(code.length + 1, "0") if code.length.odd?
-    location_type = CODE_LENGTHS.fetch(code.length)
-    administrative_unit = ADMINISTRATIVE_UNITS.fetch(row.fetch("type"))
-
-    data[location_type] ||= {}
-    data[location_type][code] = {
-      "name" => {
-        "km" => row.fetch("name_km"),
-        "latin" => row.fetch("name_latin")
-      },
-      "administrative_unit" => {
-        "km" => administrative_unit.km,
-        "latin" => administrative_unit.latin,
-        "en" => administrative_unit.en
-      }
-    }
-  end
-end
-
-if data.any?
-  CODE_LENGTHS.each do |_, type|
-    File.write(
-      "data/#{type}.yml",
-      { type => data.fetch(type).sort.to_h }.to_yaml
-    )
-  end
-end
+data_parser = Pumi::DataParser.new
+data = data_parser.load_data!
+data_parser.write_data!(data)

data/lib/pumi/data_parser.rb

@@ -0,0 +1,75 @@
+require "pathname"
+require "csv"
+require "yaml"
+
+# https://en.wikipedia.org/wiki/Administrative_divisions_of_Cambodia
+# https://en.wikipedia.org/wiki/Romanization_of_Khmer
+# https://en.wikipedia.org/wiki/United_Nations_Group_of_Experts_on_Geographical_Names
+
+module Pumi
+  class DataParser
+    CSV_HEADERS = %w[type code name_km name_latin reference note1 note2].freeze
+
+    AdministrativeUnit = Struct.new(:en, :km, :latin, :code_length, :group, keyword_init: true)
+    ADMINISTRATIVE_UNITS = {
+      "ស្រុក" => AdministrativeUnit.new(en: "District", km: "ស្រុក", latin: "Srok", code_length: 4, group: "districts"),
+      "ខណ្ឌ" => AdministrativeUnit.new(en: "Section", km: "ខណ្ឌ", latin: "Khan", code_length: 4, group: "districts"),
+      "ក្រុង" => AdministrativeUnit.new(en: "Municipality", km: "ក្រុង", latin: "Krong", code_length: 4, group: "districts"),
+      "ឃុំ" => AdministrativeUnit.new(en: "Commune", km: "ឃុំ", latin: "Khum", code_length: 6, group: "communes"),
+      "សង្កាត់" => AdministrativeUnit.new(en: "Quarter", km: "សង្កាត់", latin: "Sangkat", code_length: 6, group: "communes"),
+      "ភូមិ" => AdministrativeUnit.new(en: "Village", km: "ភូមិ", latin: "Phum", code_length: 8, group: "villages")
+    }.freeze
+
+    def load_data!(source_dir: "tmp")
+      data = {}
+
+      source_files(source_dir).each do |file|
+        CSV.read(file, headers: CSV_HEADERS).each do |row|
+          code = row.fetch("code")
+          next if code.to_s.gsub(/\D/, "").empty?
+
+          code = code.rjust(code.length + 1, "0") if code.length.odd?
+          administrative_unit = ADMINISTRATIVE_UNITS.fetch(row.fetch("type"))
+
+          next if administrative_unit.code_length != code.length
+
+          data[administrative_unit.group] ||= {}
+          data[administrative_unit.group][code] = {
+            "name" => {
+              "km" => row.fetch("name_km"),
+              "latin" => row.fetch("name_latin")
+            },
+            "administrative_unit" => {
+              "km" => administrative_unit.km,
+              "latin" => administrative_unit.latin,
+              "en" => administrative_unit.en
+            }
+          }
+        end
+      end
+
+      data
+    end
+
+    def write_data!(data, destination_dir: "data")
+      return if data.empty?
+
+      data_groups.each do |group|
+        File.write(
+          "#{destination_dir}/#{group}.yml",
+          { group => data.fetch(group).sort.to_h }.to_yaml
+        )
+      end
+    end
+
+    private
+
+    def source_files(source_dir)
+      Pathname.glob("#{source_dir}/*.csv").select(&:file?)
+    end
+
+    def data_groups
+      ADMINISTRATIVE_UNITS.values.map(&:group).uniq
+    end
+  end
+end

data/lib/pumi/version.rb

@@ -1,3 +1,3 @@
 module Pumi
-  VERSION = "0.13.0".freeze
+  VERSION = "0.13.1".freeze
 end

data/lib/pumi.rb

@@ -23,3 +23,4 @@ require_relative "pumi/district"
 require_relative "pumi/commune"
 require_relative "pumi/village"
 require_relative "pumi/parser"
+require_relative "pumi/data_parser"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pumi
 version: !ruby/object:Gem::Version
-  version: 0.13.0
+  version: 0.13.1
 platform: ruby
 authors:
 - David Wilkie
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-02-06 00:00:00.000000000 Z
+date: 2023-03-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bump
@@ -274,6 +274,7 @@ files:
 - lib/pumi.rb
 - lib/pumi/administrative_unit.rb
 - lib/pumi/commune.rb
+- lib/pumi/data_parser.rb
 - lib/pumi/data_store.rb
 - lib/pumi/district.rb
 - lib/pumi/location.rb