red-datasets 0.1.8 → 0.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: deff1c4d13294030c25c06691e272881eb049274decd9e2a958d0486c792ef26
-  data.tar.gz: e48151e045fbf343291a5f5770409dea7672ff39c75bcc617152b52a39890f31
+  metadata.gz: 01ddaa57da3c64de47cfd9eb2ca9ae2ec3cbcb5d35138fdd74f74009f062358f
+  data.tar.gz: 431dba2c0e41bc25a4e2716ed20936ee2022c2b04c49683bb7d0d2e2aaa2f99e
 SHA512:
-  metadata.gz: 7ae5a39a716bfa719f937c6ff139bd90ccf625e8a7ce72298507506f1a2c6a100e81c1273bf395a40d0ae8f1a13c1b1fb554e123e49dff718607a734ffd6c67b
-  data.tar.gz: 72e664ba5f2a569a92d9d4237588f92f7398621642486e4c9e6c930bade4c17e4a68d30da14cb277fcd602025f959c42284462bf4370872f9826c9634ff78aca
+  metadata.gz: ab12e9783e4a23b81f9bd1be22c31704f9095026cb27185a6fe985e106320982fb999e1cf2348f6b18b509a7f1a6a5b58d405ece5d541e8ddbe43cd08f252a80
+  data.tar.gz: 157df5fffd3ba8fd021cdef3933c0a9e99a0fa7f173771e2177d1a86e79788c3250b12453f06084e24afcf624ec3283e702e6bd7595f08e2cf2b5a7dd404065c

data/doc/text/news.md

@@ -1,5 +1,12 @@
 # News
 
+## 0.1.9 - 2025-04-08
+
+### Improvements
+
+  * `Datasets::Penguins`: Changed to use `POST` for downloading data
+    from EDI.
+
 ## 0.1.8 - 2025-02-07
 
 ### Improvements

data/lib/datasets/dataset.rb

@@ -33,8 +33,8 @@ module Datasets
       @cache_path ||= CachePath.new(@metadata.id)
     end
 
-    def download(output_path, url, *fallback_urls, &block)
-      downloader = Downloader.new(url, *fallback_urls)
+    def download(output_path, url, *fallback_urls, **options, &block)
+      downloader = Downloader.new(url, *fallback_urls, **options)
       downloader.download(output_path, &block)
     end
 

data/lib/datasets/downloader.rb

@@ -12,9 +12,11 @@ module Datasets
   class Downloader
     class TooManyRedirects < Error; end
 
-    def initialize(url, *fallback_urls)
+    def initialize(url, *fallback_urls, http_method: nil, http_parameters: nil)
       @url = normalize_url(url)
       @fallback_urls = fallback_urls.collect { |fallback_url| normalize_url(fallback_url) }
+      @http_method = http_method
+      @http_parameters = http_parameters
     end
 
     def download(output_path, &block)
@@ -151,7 +153,21 @@ module Datasets
       http.start do
         path = url.path
         path += "?#{url.query}" if url.query
-        request = Net::HTTP::Get.new(path, headers)
+        if @http_method == :post
+          # TODO: We may want to add @http_content_type, @http_body
+          # and so on.
+          if @http_parameters
+            body = URI.encode_www_form(@http_parameters)
+            content_type = "application/x-www-form-urlencoded"
+            headers = {"Content-Type" => content_type}.merge(headers)
+          else
+            body = ""
+          end
+          request = Net::HTTP::Post.new(path, headers)
+          request.body = body
+        else
+          request = Net::HTTP::Get.new(path, headers)
+        end
         http.request(request) do |response|
           case response
           when Net::HTTPSuccess, Net::HTTPPartialContent

data/lib/datasets/penguins.rb

@@ -26,7 +26,9 @@ module Datasets
         super
         species = self.class.name.split("::").last.downcase
         @metadata.id = "palmerpenguins-#{species}"
-        @metadata.url = self.class::URL
+        package_id = http_parameters["packageid"]
+        @metadata.url = "https://portal.edirepository.org/nis/mapbrowse" +
+                        "?packageid=#{package_id}"
         @metadata.licenses = ["CC0-1.0"]
         @data_path = cache_dir_path + "#{species}.csv"
       end
@@ -46,7 +48,10 @@ module Datasets
       end
 
       private def open_data
-        download(data_path, metadata.url)
+        download(data_path,
+                 "https://portal.edirepository.org/nis/dataviewer",
+                 http_method: :post,
+                 http_parameters: http_parameters)
         CSV.open(data_path, headers: :first_row, converters: :all) do |csv|
           yield csv
         end
@@ -56,19 +61,37 @@ module Datasets
     # Adelie penguin data from: https://doi.org/10.6073/pasta/abc50eed9138b75f54eaada0841b9b86
     class Adelie < SpeciesBase
       DOI = "doi.org/10.6073/pasta/abc50eed9138b75f54eaada0841b9b86".freeze
-      URL = "https://portal.edirepository.org/nis/dataviewer?packageid=knb-lter-pal.219.3&entityid=002f3893385f710df69eeebe893144ff".freeze
+
+      private def http_parameters
+        {
+          "packageid" => "knb-lter-pal.219.3",
+          "entityid" => "002f3893385f710df69eeebe893144ff",
+        }
+      end
     end
 
     # Chinstrap penguin data from: https://doi.org/10.6073/pasta/409c808f8fc9899d02401bdb04580af7
     class Chinstrap < SpeciesBase
       DOI = "doi.org/10.6073/pasta/409c808f8fc9899d02401bdb04580af7".freeze
-      URL = "https://portal.edirepository.org/nis/dataviewer?packageid=knb-lter-pal.221.2&entityid=fe853aa8f7a59aa84cdd3197619ef462".freeze
+
+      private def http_parameters
+        {
+          "packageid" => "knb-lter-pal.221.2",
+          "entityid" => "fe853aa8f7a59aa84cdd3197619ef462",
+        }
+      end
     end
 
     # Gentoo penguin data from: https://doi.org/10.6073/pasta/2b1cff60f81640f182433d23e68541ce
     class Gentoo < SpeciesBase
       DOI = "doi.org/10.6073/pasta/2b1cff60f81640f182433d23e68541ce".freeze
-      URL = "https://portal.edirepository.org/nis/dataviewer?packageid=knb-lter-pal.220.3&entityid=e03b43c924f226486f2f0ab6709d2381".freeze
+
+      private def http_parameters
+        {
+          "packageid" => "knb-lter-pal.220.3",
+          "entityid" => "e03b43c924f226486f2f0ab6709d2381",
+        }
+      end
     end
   end
 

data/lib/datasets/version.rb

@@ -1,3 +1,3 @@
 module Datasets
-  VERSION = "0.1.8"
+  VERSION = "0.1.9"
 end

data/test/test-cldr-plurals.rb

@@ -14,7 +14,7 @@ class CLDRPluralsTest < Test::Unit::TestCase
   test("#each") do
     locales = @dataset.each.to_a
     assert_equal([
-                   222,
+                   220,
                    locale("bm",
                           [
                             rule("other",

data/test/test-house-of-councillor.rb

@@ -75,9 +75,9 @@ class HouseOfCouncillorTest < Test::Unit::TestCase
                      record(Date.parse("2024-01-26"),
                             "自由民主党",
                             "自民",
-                            Date.parse("2024-08-30"),
-                            114,
-                            23,
+                            Date.parse("2024-05-25"),
+                            115,
+                            24,
                             Date.parse("2025-07-28"),
                             19,
                             5,
@@ -88,14 +88,14 @@ class HouseOfCouncillorTest < Test::Unit::TestCase
                             Date.parse("2028-07-25"),
                             18,
                             5,
-                            44,
-                            7,
-                            62,
-                            12),
+                            45,
+                            8,
+                            63,
+                            13),
                      record(Date.parse("2024-01-26"),
                             "各派に属しない議員",
                             "無所属",
-                            Date.parse("2024-08-30"),
+                            Date.parse("2024-05-25"),
                             12,
                             4,
                             Date.parse("2025-07-28"),
@@ -133,7 +133,7 @@ class HouseOfCouncillorTest < Test::Unit::TestCase
     test("#each") do
       records = @dataset.each.to_a
       assert_equal([
-                     245,
+                     247,
                      record("足立　敏之",
                             nil,
                             "https://www.sangiin.go.jp/japanese/joho1/kousei/giin/profile/7016001.htm",
@@ -145,7 +145,7 @@ class HouseOfCouncillorTest < Test::Unit::TestCase
                             [2016, 2022],
                             2,
                             "財政金融委員会（長）",
-                            Date.parse("2024-08-30"),
+                            Date.parse("2024-05-25"),
                             "昭和29年5月20日兵庫県西宮市生まれ。（本籍地・京都府福知山市）昭和48年和歌山県立桐蔭高等学校卒業、昭和52年京都大学工学部土木工学科卒業、昭和54年京都大学大学院工学研究科修士課程修了、同年建設省入省後、兵庫県庁、東北及び関東地方整備局、河川局河川計画課河川事業調整官、内閣官房（安全保障・危機管理担当）等を経て、平成15年近畿地方整備局企画部長、平成18年河川局河川計画課長、平成21年四国地方整備局長、平成23年中部地方整備局長、平成24年水管理・国土保全局長、平成25年技監、平成26年国土交通省を退職。平成28年第24回参議院議員通常選挙で初当選○参議院予算委員会理事、災害対策特別委員会理事○著書「激甚化する水害」「いいね！建設産業本当の魅力」（日経BP社）",
                             Date.parse("2022-11-30")),
                      record("渡辺　猛之",
@@ -159,7 +159,7 @@ class HouseOfCouncillorTest < Test::Unit::TestCase
                             [2010, 2016, 2022],
                             3,
                             "経済産業委員会、議院運営委員会（理）",
-                            Date.parse("2024-08-30"),
+                            Date.parse("2024-05-25"),
                             "昭和43年4月18日生、岐阜県加茂郡八百津町出身。岐阜県立加茂高等学校、名古屋大学経済学部卒業。平成4年、財団法人松下政経塾入塾（第13期生）。平成7年、同塾卒業後、26歳で岐阜県議会議員に初当選。以後通算4期当選。在任中は、自民党岐阜県連副幹事長、岐阜県商工会青年部連合会会長、岐阜県商工政治連盟会長、県監査委員、県政自民クラブ幹事長を歴任。平成22年7月、参議院議員初当選○農林水産委員長、政治倫理の確立及び選挙制度に関する特別委員長、参議院自民党筆頭副幹事長、国土交通副大臣兼内閣府副大臣兼復興副大臣を歴任○現在議院運営委員会筆頭理事。環境委員",
                             Date.parse("2022-11-30")),
                    ],
@@ -183,7 +183,7 @@ class HouseOfCouncillorTest < Test::Unit::TestCase
     test("#each") do
       records = @dataset.each.to_a
       assert_equal([
-                     7833,
+                     7739,
                      record(1,
                             1,
                             "食生活安定に関する質問主意書",
@@ -199,19 +199,19 @@ class HouseOfCouncillorTest < Test::Unit::TestCase
                             Date.parse("1947-06-28"),
                             nil),
                      record(213,
-                            239,
-                            "地方自治法第二百五十二条の二十六の五に定める各大臣の「生命等の保護の措置に関する指示」と地方自治の本旨等との関係に関する質問主意書",
-                            "小西　洋之",
+                            145,
+                            "地方自治体職員の国籍に関する質問主意書",
+                            "神谷　宗幣",
                             1,
-                            "https://www.sangiin.go.jp/japanese/joho1/kousei/syuisyo/213/syuh/s213239.htm",
-                            "https://www.sangiin.go.jp/japanese/joho1/kousei/syuisyo/213/touh/t213239.htm",
-                            "https://www.sangiin.go.jp/japanese/joho1/kousei/syuisyo/213/syup/s213239.pdf",
-                            "https://www.sangiin.go.jp/japanese/joho1/kousei/syuisyo/213/toup/t213239.pdf",
-                            "https://www.sangiin.go.jp/japanese/joho1/kousei/syuisyo/213/meisai/m213239.htm",
-                            Date.parse("2024-06-21"),
-                            Date.parse("2024-06-21"),
-                            Date.parse("2024-07-02"),
-                            "6月25日内閣から通知書受領（7月2日まで答弁延期）"),
+                            nil,
+                            nil,
+                            nil,
+                            nil,
+                            "https://www.sangiin.go.jp/japanese/joho1/kousei/syuisyo/213/meisai/m213145.htm",
+                            Date.parse("2024-05-23"),
+                            nil,
+                            nil,
+                            nil),
                    ],
                    [
                      records.size,

data/test/test-rdataset.rb

@@ -48,7 +48,7 @@ class RdatasetTest < Test::Unit::TestCase
       test("without package_name") do
         records = @dataset.each.to_a
         assert_equal([
-                       2337,
+                       2293,
                        {
                          package: "AER",
                          dataset: "Affairs",

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: red-datasets
 version: !ruby/object:Gem::Version
-  version: 0.1.8
+  version: 0.1.9
 platform: ruby
 authors:
 - tomisuker
 - Kouhei Sutou
 bindir: bin
 cert_chain: []
-date: 2025-02-07 00:00:00.000000000 Z
+date: 2025-04-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: csv