pumi 0.25.0 → 0.26.0

data/lib/pumi/administrative_unit.rb

@@ -1,3 +1,3 @@
 module Pumi
-  AdministrativeUnit = Struct.new(:name_km, :name_latin, :name_en, keyword_init: true)
+  AdministrativeUnit = Struct.new(:name_km, :name_latin, :name_en, :name_ungegn, keyword_init: true)
 end

data/lib/pumi/bot/wikipedia/communes_in_cambodia_article.rb

@@ -1,4 +1,5 @@
 require "ostruct"
+require "nokogiri"
 
 module Pumi
   module Bot
@@ -105,7 +106,7 @@ module Pumi
         def replace_communes_list
           provinces = Pumi::Province.all.map { |province| Province.new(province) }
           data = OpenStruct.new(provinces:)
-          communes_list = ERB.new(TEMPLATE).result(data.instance_eval { binding })
+          communes_list = ERB.new(TEMPLATE, trim_mode: "-").result(data.instance_eval { binding })
           source.sub!(communes_list_section.to_html, communes_list)
         end
 

data/lib/pumi/bot/wikipedia/districts_in_cambodia_article.rb

@@ -1,4 +1,5 @@
 require "ostruct"
+require "erb"
 
 module Pumi
   module Bot
@@ -12,15 +13,10 @@ module Pumi
         Misspelling = Struct.new(:incorrect_text, :correct_text, keyword_init: true)
 
         MISSPELLINGS = [
-          Misspelling.new(incorrect_text: "Kratié", correct_text: "Kratie"),
-          Misspelling.new(incorrect_text: "Mondulkiri", correct_text: "Mondul Kiri"),
           Misspelling.new(
             incorrect_text: "Phnom Penh (autonomous municipality)",
             correct_text: "Phnom Penh"
-          ),
-          Misspelling.new(incorrect_text: "Ratanakiri", correct_text: "Ratanak Kiri"),
-          Misspelling.new(incorrect_text: "Siem Reap", correct_text: "Siemreap"),
-          Misspelling.new(incorrect_text: "Takéo", correct_text: "Takeo")
+          )
         ].freeze
 
         def publish
@@ -49,7 +45,7 @@ module Pumi
             districts:,
             districts_summary: generate_districts_summary(districts:)
           )
-          result = ERB.new(DISTRICTS_TEMPLATE).result(data.instance_eval { binding })
+          result = ERB.new(DISTRICTS_TEMPLATE, trim_mode: "-").result(data.instance_eval { binding })
           "\n\n#{result}\n"
         end
 

data/lib/pumi/bot/wikipedia/templates/commune_list.wikitext.erb

@@ -2,7 +2,7 @@
 <% provinces.each do |province| %>
 <% province_page = URI.parse(province.links[:wikipedia]).path.split("/").last %>
 ==[[<%= province_page %>|<%= province.full_name_en %>]]==
-<div id=province-communes-<%= province.id %>>
+<div id="province-communes-<%= province.id %>">
 <%= province.name_en %> contains <%= province.communes_summary %>.<ref>{{cite web|url=http://db.ncdd.gov.kh/gazetteer/view/province.castle?pv=<%= province.id %> |title=<%= province.name_en %> |publisher=National Committee for Sub-National Democratic Development }}</ref>
 
 <% province.districts.each do |district| %>
@@ -13,7 +13,7 @@
 ===<%= district.full_name_en %>===
 <% end %>
 
-<div id=district-communes-<%= district.id %>>
+<div id="district-communes-<%= district.id %>">
 <%= district.name_en %> contains <%= district.communes_summary %>.<ref>{{cite web|url=http://db.ncdd.gov.kh/gazetteer/view/district.castle?ds=<%= district.id %> |title=<%= district.name_en %> |publisher=National Committee for Sub-National Democratic Development }}</ref>
 
 {| class="wikitable sortable"
@@ -21,26 +21,35 @@
 ! #
 ! Name
 ! Khmer
+! [[Romanization of Khmer#UNGEGN|UNGEGN]]
 ! Administrative Unit
 ! Geocode
 |-
-
 <% district.communes.each_with_index do |commune, index| %>
 | <%= index + 1 %>
-<% if commune.links[:wikipedia] %>
-<% commune_page = URI.parse(commune.links[:wikipedia]).path.split("/").last.gsub("_", " ") %>
+<% if commune.links[:wikipedia] -%>
+<% commune_page = URI.parse(commune.links[:wikipedia]).path.split("/").last -%>
+<% if commune_page.gsub("_", " ") == commune.name_en -%>
+| [[<%= commune.name_en %>]]
+<% else -%>
 | [[<%= commune_page %>|<%= commune.name_en %>]]
-<% else %>
+<% end -%>
+<% else -%>
 | <%= commune.name_en %>
-<% end %>
+<% end -%>
 | <%= commune.name_km %>
+<% if commune.name_ungegn -%>
+| {{transliteration|km|<%= commune.name_ungegn %>}}
+<% else -%>
+|
+<% end -%>
 | <%= "#{commune.administrative_unit.name_en} (#{commune.administrative_unit.name_km} #{commune.administrative_unit.name_latin})" %>
 | <%= commune.id %>
 |-
-<% end %>
+<% end -%>
 |}
 </div>
-<% end %>
+<% end -%>
 </div>
-<% end %>
+<% end -%>
 </div>

data/lib/pumi/bot/wikipedia/templates/district_list.wikitext.erb

@@ -1,27 +1,36 @@
-<div id=province-districts-<%= province.id %>>
-<%= province.name_en %> contains <%= districts_summary %>. <ref>{{cite web|url=http://db.ncdd.gov.kh/gazetteer/view/province.castle?pv=<%= province.id %> |title=<%= province.name_en %> |publisher=National Committee for Sub-National Democratic Development }}</ref>
+<div id="province-districts-<%= province.id %>">
+<%= province.name_en %> contains <%= districts_summary %>.<ref>{{cite web|url=http://db.ncdd.gov.kh/gazetteer/view/province.castle?pv=<%= province.id %> |title=<%= province.name_en %> |publisher=National Committee for Sub-National Democratic Development }}</ref>
 
 {| class="wikitable sortable"
 |-
 ! #
 ! Name
 ! Khmer
+! [[Romanization of Khmer#UNGEGN|UNGEGN]]
 ! Administrative Unit
 ! Geocode
 |-
-
 <% districts.each_with_index do |district, index| %>
 | <%= index + 1 %>
-<% if district.links[:wikipedia] %>
-<% district_page = URI.parse(district.links[:wikipedia]).path.split("/").last.gsub("_", " ") %>
+<% if district.links[:wikipedia] -%>
+<% district_page = URI.parse(district.links[:wikipedia]).path.split("/").last -%>
+<% if district_page.gsub("_", " ") == district.name_en -%>
+| [[<%= district.name_en %>]]
+<% else -%>
 | [[<%= district_page %>|<%= district.name_en %>]]
-<% else %>
+<% end -%>
+<% else -%>
 | <%= district.name_en %>
-<% end %>
+<% end -%>
 | <%= district.name_km %>
+<% if district.name_ungegn -%>
+| {{transliteration|km|<%= district.name_ungegn %>}}
+<% else -%>
+|
+<% end -%>
 | <%= "#{district.administrative_unit.name_en} (#{district.administrative_unit.name_km} #{district.administrative_unit.name_latin})" %>
 | <%= district.id %>
 |-
-<% end %>
+<% end -%>
 |}
 </div>

data/lib/pumi/data_source/ncdd.rb

@@ -14,7 +14,14 @@ module Pumi
         "1715" => { type: "ក្រុង" }
       }.freeze
 
-      AdministrativeUnit = Struct.new(:en, :km, :latin, :code_length, :group, :type, keyword_init: true)
+      Misspelling = Struct.new(:incorrect_text, :correct_text, keyword_init: true)
+
+      MISSPELLINGS = [
+        Misspelling.new(incorrect_text: "Siem Reab", correct_text: "Siem Reap"),
+        Misspelling.new(incorrect_text: "Aoral", correct_text: "Aural")
+      ].freeze
+
+      AdministrativeUnit = Struct.new(:en, :km, :latin, :ungegn, :code_length, :group, :type, keyword_init: true)
       Row = Struct.new(:code, :name_km, :name_latin, :type, keyword_init: true) do
         def administrative_unit
           ADMINISTRATIVE_UNITS.fetch(type)
@@ -22,12 +29,54 @@ module Pumi
       end
 
       ADMINISTRATIVE_UNITS = {
-        "ស្រុក" => AdministrativeUnit.new(en: "District", km: "ស្រុក", latin: "Srok", code_length: 4, group: "districts"),
-        "ខណ្ឌ" => AdministrativeUnit.new(en: "Section", km: "ខណ្ឌ", latin: "Khan", code_length: 4, group: "districts"),
-        "ក្រុង" => AdministrativeUnit.new(en: "Municipality", km: "ក្រុង", latin: "Krong", code_length: 4, group: "districts"),
-        "ឃុំ" => AdministrativeUnit.new(en: "Commune", km: "ឃុំ", latin: "Khum", code_length: 6, group: "communes"),
-        "សង្កាត់" => AdministrativeUnit.new(en: "Quarter", km: "សង្កាត់", latin: "Sangkat", code_length: 6, group: "communes"),
-        "ភូមិ" => AdministrativeUnit.new(en: "Village", km: "ភូមិ", latin: "Phum", code_length: 8, group: "villages")
+        "ស្រុក" => AdministrativeUnit.new(
+          en: "District",
+          km: "ស្រុក",
+          latin: "Srok",
+          ungegn: "Srŏk",
+          code_length: 4,
+          group: "districts"
+        ),
+        "ខណ្ឌ" => AdministrativeUnit.new(
+          en: "Section",
+          km: "ខណ្ឌ",
+          latin: "Khan",
+          ungegn: "Khând",
+          code_length: 4,
+          group: "districts"
+        ),
+        "ក្រុង" => AdministrativeUnit.new(
+          en: "Municipality",
+          km: "ក្រុង",
+          latin: "Krong",
+          ungegn: "Krŏng",
+          code_length: 4,
+          group: "districts"
+        ),
+        "ឃុំ" => AdministrativeUnit.new(
+          en: "Commune",
+          km: "ឃុំ",
+          latin: "Khum",
+          ungegn: "Khŭm",
+          code_length: 6,
+          group: "communes"
+        ),
+        "សង្កាត់" => AdministrativeUnit.new(
+          en: "Quarter",
+          km: "សង្កាត់",
+          latin: "Sangkat",
+          ungegn: "Sângkéat",
+          code_length: 6,
+          group: "communes"
+        ),
+        "ភូមិ" => AdministrativeUnit.new(
+          en: "Village",
+          km: "ភូមិ",
+          latin: "Phum",
+          ungegn: "Phum",
+          code_length: 8,
+          group: "villages"
+        )
       }.freeze
 
       attr_accessor :existing_data
@@ -66,10 +115,13 @@ module Pumi
       def build_row(row)
         code = parse_location_code(row)
 
+        name_latin = row.fetch("name_latin")
+        name_latin = MISSPELLINGS.find { |m| m.incorrect_text == name_latin }&.correct_text || name_latin
+
         Row.new(
           code:,
           name_km: row.fetch("name_km"),
-          name_latin: row.fetch("name_latin"),
+          name_latin:,
           type: row.fetch("type") || MISSING_DATA.dig(code, :type)
         )
       end
@@ -85,15 +137,17 @@ module Pumi
       def add_data(row)
         data[row.administrative_unit.group] ||= {}
         data[row.administrative_unit.group][row.code] = existing_data.dig(row.administrative_unit.group, row.code) || {}
+        data[row.administrative_unit.group][row.code]["name"] = existing_data.dig(row.administrative_unit.group, row.code, "name") || {}
+        data[row.administrative_unit.group][row.code]["name"].merge!(
+          "km" => row.name_km,
+          "latin" => row.name_latin
+        )
         data[row.administrative_unit.group][row.code].merge!(
-          "name" => {
-            "km" => row.name_km,
-            "latin" => row.name_latin
-          },
           "administrative_unit" => {
             "km" => row.administrative_unit.km,
             "latin" => row.administrative_unit.latin,
-            "en" => row.administrative_unit.en
+            "en" => row.administrative_unit.en,
+            "ungegn" => row.administrative_unit.ungegn
           }
         )
       end

data/lib/pumi/data_source/wikipedia.rb

@@ -16,8 +16,12 @@ module Pumi
           location_data = scraped_data.find { |location| location.code == code }
           next unless location_data
 
-          attributes["links"] ||= {}
-          attributes["links"]["wikipedia"] = location_data.wikipedia
+          if location_data.wikipedia
+            attributes["links"] ||= {}
+            attributes["links"]["wikipedia"] = location_data.wikipedia
+          end
+
+          attributes["name"]["ungegn"] = location_data.name_ungegn if location_data.name_ungegn
         end
 
         write_data!(output_dir)
@@ -37,7 +41,7 @@ module Pumi
         data_file.write(data, data_directory:)
       end
 
-      ScraperResult = Struct.new(:code, :wikipedia, keyword_init: true)
+      ScraperResult = Struct.new(:code, :wikipedia, :name_ungegn, keyword_init: true)
 
       class WebScraper
         class ElementNotFoundError < StandardError; end
@@ -58,7 +62,11 @@ module Pumi
 
         def scrape!
           Province.all.each_with_object([]) do |province, result|
-            result << ScraperResult.new(code: province.id, wikipedia: find_url(province))
+            result << ScraperResult.new(
+              code: province.id,
+              wikipedia: find_url(province),
+              name_ungegn: find_ungegn(province)
+            )
           end
         end
 
@@ -69,14 +77,25 @@ module Pumi
         end
 
         def find_url(province)
+          td = find_khmer_name_td(province)
+          link = td.at_xpath("preceding-sibling::td/a[contains(@href, '/wiki/')]")
+          URI.join(URL, link[:href]).to_s
+        end
+
+        def find_ungegn(province)
+          td = find_khmer_name_td(province)
+          td.at_xpath("following-sibling::td/span[contains(@title, 'Khmer-language romanization')]")&.text
+        end
+
+        def find_khmer_name_td(province)
           td = province_table_rows.at_xpath("child::td[contains(., '#{province.name_km}')]")
+
           if td.nil?
             raise WebScraper::ElementNotFoundError,
                   "No cell containing '#{province.name_km}' was found in a table on #{URL}"
           end
 
-          link = td.at_xpath("preceding-sibling::td/a[contains(@href, '/wiki/')]")
-          URI.join(URL, link[:href]).to_s
+          td
         end
 
         def province_table_rows
@@ -99,10 +118,11 @@ module Pumi
 
         def scrape!
           District.all.each_with_object([]) do |district, result|
-            url = find_url(district)
-            next unless url
-
-            result << ScraperResult.new(code: district.id, wikipedia: url)
+            result << ScraperResult.new(
+              code: district.id,
+              wikipedia: find_url(district),
+              name_ungegn: find_ungegn(district)
+            )
           end
         end
 
@@ -113,16 +133,28 @@ module Pumi
         end
 
         def find_url(district)
-          geocode = scraper.page.at_xpath("//td[text()[contains(., '#{district.id}')]]")
+          geocode_td = find_geocode_td(district)
 
-          return if geocode.nil?
+          return if geocode_td.nil?
 
-          link = geocode.at_xpath("preceding-sibling::td/a[contains(@href, '/wiki/')]")
+          link = geocode_td.at_xpath("preceding-sibling::td/a[contains(@href, '/wiki/')]")
 
           return if link.nil?
 
           URI.join(URL, link[:href]).to_s
         end
+
+        def find_ungegn(district)
+          geocode_td = find_geocode_td(district)
+
+          return if geocode_td.nil?
+
+          geocode_td.at_xpath("preceding-sibling::td/span[contains(@title, 'Khmer-language romanization')]")&.text
+        end
+
+        def find_geocode_td(district)
+          scraper.page.at_xpath("//td[text()[contains(., '#{district.id}')]]")
+        end
       end
 
       class CambodianCommunesScraper
@@ -130,27 +162,40 @@ module Pumi
 
         def scrape!
           Commune.all.each_with_object([]) do |commune, result|
-            url = find_url(commune)
-            next if url.nil?
-
-            result << ScraperResult.new(code: commune.id, wikipedia: url)
+            result << ScraperResult.new(
+              code: commune.id,
+              wikipedia: find_url(commune),
+              name_ungegn: find_ungegn(commune)
+            )
           end
         end
 
         private
 
         def find_url(commune)
-          geocode = scraper.page.at_xpath("//td[text()[contains(., '#{commune.id}')]]")
+          geocode_td = find_geocode_td(commune)
 
-          return if geocode.nil?
+          return if geocode_td.nil?
 
-          link = geocode.at_xpath("preceding-sibling::td/a[contains(@href, '/wiki/')]")
+          link = geocode_td.at_xpath("preceding-sibling::td/a[contains(@href, '/wiki/')]")
 
           return if link.nil?
 
           URI.join(URL, link[:href]).to_s
         end
 
+        def find_ungegn(commune)
+          geocode_td = find_geocode_td(commune)
+
+          return if geocode_td.nil?
+
+          geocode_td.at_xpath("preceding-sibling::td/span[contains(@title, 'Khmer-language romanization')]")&.text
+        end
+
+        def find_geocode_td(commune)
+          scraper.page.at_xpath("//td[text()[contains(., '#{commune.id}')]]")
+        end
+
         def scraper
           @scraper ||= WebScraper.new(URL)
         end

data/lib/pumi/location.rb

@@ -5,6 +5,7 @@ module Pumi
     :name_km, :full_name_km,
     :name_latin, :full_name_latin,
     :name_en, :full_name_en,
+    :name_ungegn, :full_name_ungegn,
     :address_km, :address_latin, :address_en,
     :administrative_unit,
     :links,

data/lib/pumi/parser.rb

@@ -82,6 +82,7 @@ module Pumi
       name = attributes.fetch("name")
       name_km = name.fetch("km")
       name_latin = name.fetch("latin")
+      name_ungegn = name["ungegn"]
       administrative_unit = build_administrative_unit(
         attributes.fetch("administrative_unit")
       )
@@ -94,20 +95,25 @@ module Pumi
         id:,
         administrative_unit:,
         name_km:,
+        name_en: name_latin,
         name_latin:,
+        name_ungegn:,
         geodata:,
         iso3166_2: attributes["iso3166_2"],
         links: attributes.fetch("links", {}).transform_keys(&:to_sym),
-        name_en: name_latin,
         full_name_km: [
           administrative_unit_name(name_km, administrative_unit.name_km),
           name_km
         ].compact.join,
+        full_name_en: [name_latin, administrative_unit.name_en].join(" "),
         full_name_latin: [
           administrative_unit_name(name_latin, administrative_unit.name_latin),
           name_latin
         ].compact.join(" "),
-        full_name_en: [name_latin, administrative_unit.name_en].join(" ")
+        full_name_ungegn: name_ungegn && [
+          administrative_unit_name(name_ungegn, administrative_unit.name_ungegn),
+          name_ungegn
+        ].compact.join(" ")
       }
     end
 
@@ -138,7 +144,8 @@ module Pumi
       AdministrativeUnit.new(
         name_km: attributes.fetch("km"),
         name_latin: attributes.fetch("latin"),
-        name_en: attributes.fetch("en")
+        name_en: attributes.fetch("en"),
+        name_ungegn: attributes.fetch("ungegn")
       )
     end
 

data/lib/pumi/version.rb

@@ -1,3 +1,3 @@
 module Pumi
-  VERSION = "0.25.0".freeze
+  VERSION = "0.26.0".freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pumi
 version: !ruby/object:Gem::Version
-  version: 0.25.0
+  version: 0.26.0
 platform: ruby
 authors:
 - David Wilkie
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-05-19 00:00:00.000000000 Z
+date: 2024-05-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler