pumi 0.17.0 → 0.18.0

data/lib/pumi/parser.rb

@@ -3,9 +3,12 @@ require "pathname"
 
 module Pumi
   class Parser
-    DATA_DIRECTORY = Pathname.new(File.expand_path("..", File.dirname(__dir__))).join("data")
+    DEFAULT_DATA_DIRECTORY = File.join(File.expand_path("..", File.dirname(__dir__)), "data")
+
+    AdministrativeDivision = Struct.new(
+      :type, :name, :data_key, :id_length, :parent_divisions, keyword_init: true
+    )
 
-    AdministrativeDivision = Struct.new(:type, :name, :data_key, :id_length, :parent_divisions, keyword_init: true)
     PROVINCE = AdministrativeDivision.new(
       type: Province,
       name: :province,
@@ -48,19 +51,16 @@ module Pumi
       AddressType.new(locale: :en, default_delimiter: ", ")
     ].freeze
 
-    attr_reader :type, :administrative_division
+    attr_reader :type, :data_directory, :administrative_division
 
-    def initialize(type)
+    def initialize(type, data_directory: DEFAULT_DATA_DIRECTORY)
       @type = type
+      @data_directory = Pathname(data_directory)
       @administrative_division = ADMINISTRATIVE_DIVISIONS.fetch(type)
     end
 
     def load
-      data = YAML.load_file(
-        DATA_DIRECTORY.join("#{administrative_division.data_key}.yml")
-      ).fetch(administrative_division.data_key.to_s)
-
-      data.each_with_object({}) do |(id, attributes), result|
+      raw_data.each_with_object({}) do |(id, attributes), result|
         location_data = build_location_data(id, attributes)
         add_parent_divisions(location_data)
         add_addresses(location_data)
@@ -71,6 +71,12 @@ module Pumi
 
     private
 
+    def raw_data
+      @raw_data ||= YAML.load_file(
+        data_directory.join("#{administrative_division.data_key}.yml")
+      ).fetch(administrative_division.data_key.to_s)
+    end
+
     def build_location_data(id, attributes)
       name = attributes.fetch("name")
       name_km = name.fetch("km")
@@ -80,10 +86,11 @@ module Pumi
       )
 
       {
-        id: id,
-        administrative_unit: administrative_unit,
-        name_km: name_km,
-        name_latin: name_latin,
+        id:,
+        administrative_unit:,
+        name_km:,
+        name_latin:,
+        links: attributes.fetch("links", {}).transform_keys(&:to_sym),
         name_en: name_latin,
         full_name_km: [
           administrative_unit_name(name_km, administrative_unit.name_km),

data/lib/pumi/scraper/result.rb

@@ -0,0 +1,5 @@
+module Pumi
+  module Scraper
+    Result = Struct.new(:code, :wikipedia, keyword_init: true)
+  end
+end

data/lib/pumi/version.rb

@@ -1,3 +1,3 @@
 module Pumi
-  VERSION = "0.17.0".freeze
+  VERSION = "0.18.0".freeze
 end

data/lib/pumi.rb

@@ -23,4 +23,5 @@ require_relative "pumi/district"
 require_relative "pumi/commune"
 require_relative "pumi/village"
 require_relative "pumi/parser"
-require_relative "pumi/data_parser"
+require_relative "pumi/data_file"
+require_relative "pumi/data_source"

data/pumi.gemspec

@@ -26,6 +26,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "capybara"
   spec.add_development_dependency "coffee-rails"
   spec.add_development_dependency "jquery-rails"
+  spec.add_development_dependency "nokogiri"
   spec.add_development_dependency "pry"
   spec.add_development_dependency "puma"
   spec.add_development_dependency "rails"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pumi
 version: !ruby/object:Gem::Version
-  version: 0.17.0
+  version: 0.18.0
 platform: ruby
 authors:
 - David Wilkie
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-05-26 00:00:00.000000000 Z
+date: 2023-05-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -66,6 +66,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
@@ -262,7 +276,10 @@ files:
 - lib/pumi.rb
 - lib/pumi/administrative_unit.rb
 - lib/pumi/commune.rb
-- lib/pumi/data_parser.rb
+- lib/pumi/data_file.rb
+- lib/pumi/data_source.rb
+- lib/pumi/data_source/ncdd.rb
+- lib/pumi/data_source/wikipedia.rb
 - lib/pumi/data_store.rb
 - lib/pumi/district.rb
 - lib/pumi/location.rb
@@ -270,6 +287,7 @@ files:
 - lib/pumi/province.rb
 - lib/pumi/rails.rb
 - lib/pumi/rails/engine.rb
+- lib/pumi/scraper/result.rb
 - lib/pumi/store_cache.rb
 - lib/pumi/version.rb
 - lib/pumi/village.rb

data/lib/pumi/data_parser.rb

@@ -1,75 +0,0 @@
-require "pathname"
-require "csv"
-require "yaml"
-
-# https://en.wikipedia.org/wiki/Administrative_divisions_of_Cambodia
-# https://en.wikipedia.org/wiki/Romanization_of_Khmer
-# https://en.wikipedia.org/wiki/United_Nations_Group_of_Experts_on_Geographical_Names
-
-module Pumi
-  class DataParser
-    CSV_HEADERS = %w[type code name_km name_latin reference note1 note2].freeze
-
-    AdministrativeUnit = Struct.new(:en, :km, :latin, :code_length, :group, keyword_init: true)
-    ADMINISTRATIVE_UNITS = {
-      "ស្រុក" => AdministrativeUnit.new(en: "District", km: "ស្រុក", latin: "Srok", code_length: 4, group: "districts"),
-      "ខណ្ឌ" => AdministrativeUnit.new(en: "Section", km: "ខណ្ឌ", latin: "Khan", code_length: 4, group: "districts"),
-      "ក្រុង" => AdministrativeUnit.new(en: "Municipality", km: "ក្រុង", latin: "Krong", code_length: 4, group: "districts"),
-      "ឃុំ" => AdministrativeUnit.new(en: "Commune", km: "ឃុំ", latin: "Khum", code_length: 6, group: "communes"),
-      "សង្កាត់" => AdministrativeUnit.new(en: "Quarter", km: "សង្កាត់", latin: "Sangkat", code_length: 6, group: "communes"),
-      "ភូមិ" => AdministrativeUnit.new(en: "Village", km: "ភូមិ", latin: "Phum", code_length: 8, group: "villages")
-    }.freeze
-
-    def load_data!(source_dir: "tmp")
-      data = {}
-
-      source_files(source_dir).each do |file|
-        CSV.read(file, headers: CSV_HEADERS).each do |row|
-          code = row.fetch("code")
-          next if code.to_s.gsub(/\D/, "").empty?
-
-          code = code.rjust(code.length + 1, "0") if code.length.odd?
-          administrative_unit = ADMINISTRATIVE_UNITS.fetch(row.fetch("type"))
-
-          next if administrative_unit.code_length != code.length
-
-          data[administrative_unit.group] ||= {}
-          data[administrative_unit.group][code] = {
-            "name" => {
-              "km" => row.fetch("name_km"),
-              "latin" => row.fetch("name_latin")
-            },
-            "administrative_unit" => {
-              "km" => administrative_unit.km,
-              "latin" => administrative_unit.latin,
-              "en" => administrative_unit.en
-            }
-          }
-        end
-      end
-
-      data
-    end
-
-    def write_data!(data, destination_dir: "data")
-      return if data.empty?
-
-      data_groups.each do |group|
-        File.write(
-          "#{destination_dir}/#{group}.yml",
-          { group => data.fetch(group).sort.to_h }.to_yaml
-        )
-      end
-    end
-
-    private
-
-    def source_files(source_dir)
-      Pathname.glob("#{source_dir}/*.csv").select(&:file?)
-    end
-
-    def data_groups
-      ADMINISTRATIVE_UNITS.values.map(&:group).uniq
-    end
-  end
-end