gtfs_df 0.3.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 128899724d4613f0170fa601cb0c4cdc9f88a51b2fcd6c06108e7cc9acf9201c
-  data.tar.gz: 14c30a678bc9a623233c6631be1c7d497fff6bae4b30abd88406c297bc76190b
+  metadata.gz: 205ec058b41c5bd1d2b01ff3950d4cd2ebb20f304d02d5bcd3dd0c447b4e0a6e
+  data.tar.gz: 54cca637de421c26d2144df100f8430f5dddb639b3a37d1160dfbd9704630e33
 SHA512:
-  metadata.gz: b7e3d1ac85953b995b82d0abbb9f872cf5307564ba67949320d607190ef5a7ff7f8e34df241cc584d7b36fc8db9c4159a7bf74ef1ce6d6868e4a3155784a9ec9
-  data.tar.gz: a1067ff3a0912b3eb56e3348e707c777563b50f440084a5b8c83b2d3da6ebf8a8a7ed49093a756d8871f1d3d849906daa844a0a66c04e56f97b03bfe2a106d79
+  metadata.gz: ef853b504ee701e77911259352d0057f6e327db8022370a1fa0dbaa597bfadcac7ac1c8fbe746c04ec4669d86384279a259ac35d4126f1483286937812cb7cce
+  data.tar.gz: 728d684dd02b653cc779aa5e8152766f5dccafcb097e8fca5810b90f994c442645119f32efb62c84d8fbf1645109f579a3c74f6eb88c9e21dd03faa72193cdd0

data/CHANGELOG.md

@@ -1,4 +1,28 @@
-## [Unreleased]
+## [0.4.1] - 2025-12-05
+
+### Added
+
+- handle extra whitespace in csvs
+
+### Maintenance
+
+- remove unused initializer format
+## [0.4.0] - 2025-12-04
+
+### Added
+
+- allow setting maintain_trip_dependencies=false
+
+### Fixed
+
+- parse stop_lat as float
+- add missing agency -> fare_attributes edge
+- allow null for fare_rules
+
+### Maintenance
+
+- provide accessor for gtfs_files (utility)
+- add yard docs
 
 ## [0.3.0] - 2025-12-04
 

data/lib/gtfs_df/base_gtfs_table.rb

@@ -10,20 +10,21 @@ module GtfsDf
         if input.is_a?(Polars::DataFrame)
           input
         elsif input.is_a?(String)
-          # We need to account for extra columns due to: https://github.com/ankane/ruby-polars/issues/125
-          all_columns = Polars.scan_csv(input).columns
-          default_schema = all_columns.map { |c| [c, Polars::String] }.to_h
-          dtypes = default_schema.merge(self.class::SCHEMA)
-          Polars.read_csv(input, null_values: [""], dtypes:)
-        elsif input.is_a?(Array)
-          head, *body = input
-          df_input = body.each_with_object({}) do |row, acc|
-            head.each_with_index do |column, index|
-              acc[column] ||= []
-              acc[column] << row[index]
-            end
-          end
-          Polars::DataFrame.new(df_input, schema_overrides: self.class::SCHEMA, strict: false)
+          # TODO: use `infer_schema: false` instead of `infer_schema_length` after polars release:
+          # https://github.com/ankane/ruby-polars/blob/master/CHANGELOG.md#100-unreleased
+          df = Polars.read_csv(input, infer_schema_length: 0)
+          dtypes = self.class::SCHEMA.slice(*df.columns)
+
+          df
+            .with_columns(dtypes.keys.map do |col|
+              stripped = Polars.col(col).str.strip
+              Polars.when(stripped.str.len_chars.gt(0))
+                .then(stripped)
+                .otherwise(Polars.lit(nil))
+            end)
+            .with_columns(dtypes.map do |name, type|
+                            Polars.col(name).cast(type)
+                          end)
         else
           throw GtfsDf::Error, "Unrecognized input"
         end

data/lib/gtfs_df/feed.rb

@@ -36,7 +36,8 @@ module GtfsDf
       booking_rules
     ].freeze
 
-    attr_reader(*GTFS_FILES, :graph)
+    attr_accessor(*GTFS_FILES)
+    attr_reader(:graph)
 
     # Initialize with a hash of DataFrames
     REQUIRED_GTFS_FILES = %w[agency stops routes trips stop_times].freeze
@@ -71,8 +72,16 @@ module GtfsDf
     end
 
     # Filter the feed using a view hash
-    # Example view: { 'routes' => { 'route_id' => '123' }, 'trips' => { 'service_id' => 'A' } }
-    def filter(view)
+    #
+    # @param view [Hash] The view used to filter the feed, with format { file => filters }.
+    #   Example view: { 'routes' => { 'route_id' => '123' }, 'trips' => { 'service_id' => 'A' } }
+    # @param maintain_trip_dependencies [Boolean] Whether trip dependencies should be preserved.
+    #   By default, we treat trips as the atomic unit of GTFS. Therefore, if we filter to one stop
+    #   referenced by TripA, we will preserve _all stops_ referenced by TripA. However, it is
+    #   occasionally useful to prune bad data and _not_ maintain all trip dependencies.
+    #   For example, if StopA contains invalid coordinates, we may wish to filter it out but keep
+    #   the other stops for TripA. In this case, `maintain_trip_dependencies` should be set to false.
+    def filter(view, maintain_trip_dependencies = true)
       filtered = {}
 
       GTFS_FILES.each do |file|
@@ -82,22 +91,28 @@ module GtfsDf
         filtered[file] = df
       end
 
-      # Trips are the atomic unit of GTFS, we will generate a new view
-      # based on the set of trips that would be included for each invidual filter
-      # and cascade changes from this view in order to retain referential integrity
-      trip_ids = nil
-
-      view.each do |file, filters|
-        new_filtered = filter!(file, filters, filtered.dup)
-        trip_ids = if trip_ids.nil?
-          new_filtered["trips"]["trip_id"]
-        else
-          trip_ids & new_filtered["trips"]["trip_id"]
+      if maintain_trip_dependencies
+        # Trips are the atomic unit of GTFS, we will generate a new view
+        # based on the set of trips that would be included for each invidual filter
+        # and cascade changes from this view in order to retain referential integrity
+        trip_ids = nil
+
+        view.each do |file, filters|
+          new_filtered = filter!(file, filters, filtered.dup)
+          trip_ids = if trip_ids.nil?
+            new_filtered["trips"]["trip_id"]
+          else
+            trip_ids & new_filtered["trips"]["trip_id"]
+          end
         end
-      end
 
-      if trip_ids
-        filtered = filter!("trips", {"trip_id" => trip_ids.to_a}, filtered)
+        if trip_ids
+          filtered = filter!("trips", {"trip_id" => trip_ids.to_a}, filtered)
+        end
+      else
+        view.each do |file, filters|
+          filtered = filter!(file, filters, filtered.dup)
+        end
       end
 
       # Remove files that are empty, but keep required files even if empty
@@ -163,6 +178,7 @@ module GtfsDf
         attrs[:dependencies].each do |dep|
           parent_col = dep[parent_node_id]
           child_col = dep[child_node_id]
+          allow_null = !!dep[:allow_null]
 
           next unless parent_col && child_col &&
             parent_df.columns.include?(parent_col) && child_df.columns.include?(child_col)
@@ -172,9 +188,11 @@ module GtfsDf
 
           # Filter child to only include rows that reference valid parent values
           before = child_df.height
-          child_df = child_df.filter(
-            Polars.col(child_col).is_in(valid_values)
-          )
+          filter = Polars.col(child_col).is_in(valid_values)
+          if allow_null
+            filter = (filter | Polars.col(child_col).is_null)
+          end
+          child_df = child_df.filter(filter)
           changed = child_df.height < before
 
           # If we removed a part of the child_df earlier, concat it back on

data/lib/gtfs_df/graph.rb

@@ -50,12 +50,16 @@ module GtfsDf
         ["agency", "routes", {dependencies: [
           {"agency" => "agency_id", "routes" => "agency_id"}
         ]}],
+        ["fare_attributes", "agency", {dependencies: [
+          {"fare_attributes" => "agency_id",
+           "agency" => "agency_id"}
+        ]}],
         ["fare_attributes", "fare_rules", {dependencies: [
           {"fare_attributes" => "fare_id",
            "fare_rules" => "fare_id"}
         ]}],
         ["fare_rules", "routes", {dependencies: [
-          {"fare_rules" => "route_id", "routes" => "route_id"}
+          {"fare_rules" => "route_id", "routes" => "route_id", :allow_null => true}
         ]}],
         ["routes", "trips", {dependencies: [
           {"routes" => "route_id", "trips" => "route_id"}

data/lib/gtfs_df/schema/stops.rb

@@ -9,7 +9,7 @@ module GtfsDf
         "stop_name" => Polars::String,
         "tts_stop_name" => Polars::String,
         "stop_desc" => Polars::String,
-        "stop_lat" => Polars::String,
+        "stop_lat" => Polars::Float64,
         "stop_lon" => Polars::Float64,
         "zone_id" => Polars::String,
         "stop_url" => Polars::String,

data/lib/gtfs_df/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module GtfsDf
-  VERSION = "0.3.0"
+  VERSION = "0.4.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gtfs_df
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.1
 platform: ruby
 authors:
 - David Mejorado