gtfs_df 0.2.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5b90534f9b41229026b91e4632c12dac2235ff615ac8138c301702e51bef3dfa
-  data.tar.gz: 95ac49f7dcdecea5f08c2fe51f8f50b0466bcb75a1d8ae67720b464c2983ed88
+  metadata.gz: 127f50187dd26c4824abd94a6a44f64242715bf9092072276cdf6c811e0c28a6
+  data.tar.gz: d5a6bd9b25830d0574c6c315712d1ec4e0b14959d25f6a1cf12f2e13eeb738a2
 SHA512:
-  metadata.gz: 2fc195cb47d81dd4799d99d8182f8c5ff7b56cdfdee08281ea8a129568b87f38d719956fd98532604dc26bf97dee344bbdd65943cc1e292248a2375b76290eda
-  data.tar.gz: 0d4b2e22968e5ed37349c82ff84c9ba8a6f4d12c143a8709caa40998e16afe4a2178f820f68bb384dc4cf12de445a9f0d265d163d48071621e69f64501db76d5
+  metadata.gz: 9312f173960b069a31bb2bb37368bd65ad5345d527d5a29818a7485c731bcb6f6c5fb74639403a3bcbd5d5704ae2bbf629ed5cdfe3f5f629f36ea719663356da
+  data.tar.gz: 19563c256e1cbe52a34eef47c31f6b50806a0cb07b773bacb015ad40730c6d804941aef1c936aaeb6c17ea49719ab7db64d1144eeb05c7c4bd1d83ebcd87e12f

data/CHANGELOG.md

@@ -1,3 +1,35 @@
+## [0.4.0] - 2025-12-04
+
+### Added
+
+- allow setting maintain_trip_dependencies=false
+
+### Fixed
+
+- parse stop_lat as float
+- add missing agency -> fare_attributes edge
+- allow null for fare_rules
+
+### Maintenance
+
+- provide accessor for gtfs_files (utility)
+- add yard docs
+
+## [0.3.0] - 2025-12-04
+
+### Added
+
+- keep parent stations linked to used stops
+
+### Fixed
+
+- handle null values
+- update lock on version bump
+
+### Maintenance
+
+- reuse load_from_dir logic in reader
+- clean up unused method + better comments
 ## [0.1.0] - 2025-11-10
 
 - Initial release

data/README.md

@@ -86,11 +86,25 @@ After checking out the repo, run `bin/setup` to install dependencies. Then, run
 
 To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 
+## Release process
+
+1. `bin/bump-version`
+
+- Bump the version in `lib/gtfs_df/version.rb`
+- Update the `CHANGELOG.md` using the git log since the last version
+- Create and push a new release branch with those changes
+- Create a PR for that release
+
+2. `bin/create-tag`
+
+Creates and pushes the git tag for the release. That will trigger the GitHub action: `.github/workflows/publish.yml` to publish to RubyGems.
+
 ## TODO
 
 - [ ] Time parsing
-    Just like partridge, we should parse Time as seconds since midnight. There's a draft in `lib/gtfs_df/utils.rb` but it's not used anywhere.
-    I haven't figured out how to properly implement with Polars.
+
+  Just like partridge, we should parse Time as seconds since midnight. There's a draft in `lib/gtfs_df/utils.rb` but it's not used anywhere.
+  I haven't figured out how to properly implement that with Polars.
 
 ## Contributing
 

data/lib/gtfs_df/feed.rb

@@ -36,7 +36,8 @@ module GtfsDf
       booking_rules
     ].freeze
 
-    attr_reader(*GTFS_FILES, :graph)
+    attr_accessor(*GTFS_FILES)
+    attr_reader(:graph)
 
     # Initialize with a hash of DataFrames
     REQUIRED_GTFS_FILES = %w[agency stops routes trips stop_times].freeze
@@ -71,8 +72,16 @@ module GtfsDf
     end
 
     # Filter the feed using a view hash
-    # Example view: { 'routes' => { 'route_id' => '123' }, 'trips' => { 'service_id' => 'A' } }
-    def filter(view)
+    #
+    # @param view [Hash] The view used to filter the feed, with format { file => filters }.
+    #   Example view: { 'routes' => { 'route_id' => '123' }, 'trips' => { 'service_id' => 'A' } }
+    # @param maintain_trip_dependencies [Boolean] Whether trip dependencies should be preserved.
+    #   By default, we treat trips as the atomic unit of GTFS. Therefore, if we filter to one stop
+    #   referenced by TripA, we will preserve _all stops_ referenced by TripA. However, it is
+    #   occasionally useful to prune bad data and _not_ maintain all trip dependencies.
+    #   For example, if StopA contains invalid coordinates, we may wish to filter it out but keep
+    #   the other stops for TripA. In this case, `maintain_trip_dependencies` should be set to false.
+    def filter(view, maintain_trip_dependencies = true)
       filtered = {}
 
       GTFS_FILES.each do |file|
@@ -82,22 +91,28 @@ module GtfsDf
         filtered[file] = df
       end
 
-      # Trips are the atomic unit of GTFS, we will generate a new view
-      # based on the set of trips that would be included for each invidual filter
-      # and cascade changes from this view in order to retain referential integrity
-      trip_ids = nil
-
-      view.each do |file, filters|
-        new_filtered = filter!(file, filters, filtered.dup)
-        trip_ids = if trip_ids.nil?
-          new_filtered["trips"]["trip_id"]
-        else
-          trip_ids & new_filtered["trips"]["trip_id"]
+      if maintain_trip_dependencies
+        # Trips are the atomic unit of GTFS, we will generate a new view
+        # based on the set of trips that would be included for each invidual filter
+        # and cascade changes from this view in order to retain referential integrity
+        trip_ids = nil
+
+        view.each do |file, filters|
+          new_filtered = filter!(file, filters, filtered.dup)
+          trip_ids = if trip_ids.nil?
+            new_filtered["trips"]["trip_id"]
+          else
+            trip_ids & new_filtered["trips"]["trip_id"]
+          end
         end
-      end
 
-      if trip_ids
-        filtered = filter!("trips", {"trip_id" => trip_ids.to_a}, filtered)
+        if trip_ids
+          filtered = filter!("trips", {"trip_id" => trip_ids.to_a}, filtered)
+        end
+      else
+        view.each do |file, filters|
+          filtered = filter!(file, filters, filtered.dup)
+        end
       end
 
       # Remove files that are empty, but keep required files even if empty
@@ -135,19 +150,35 @@ module GtfsDf
       filtered
     end
 
+    # Traverses the grah to prune unreferenced entities from child dataframes
+    # based on parent relationships. See GtfsDf::Graph::STOP_NODES
     def prune!(root, filtered)
-      graph.each_bfs_edge(root) do |parent_file, child_file|
-        parent_df = filtered[parent_file]
+      graph.each_bfs_edge(root) do |parent_node_id, child_node_id|
+        parent_node = Graph::NODES[parent_node_id]
+        child_node = Graph::NODES[child_node_id]
+        parent_df = filtered[parent_node.fetch(:file)]
         next unless parent_df
 
-        child_df = filtered[child_file]
+        child_df = filtered[child_node.fetch(:file)]
+        # Certain nodes are pre-filtered because they reference only
+        # a piece of the dataframe
+        filter_attrs = child_node[:filter_attrs]
+        if filter_attrs && child_df.columns.include?(filter_attrs.fetch(:filter_col))
+          filter = filter_attrs.fetch(:filter)
+          # Temporarily remove rows that do not match node filter criteria to process them
+          # separately (e.g., when filtering stops, parent stations that should be preserved
+          # regardless of direct references)
+          saved_vals = child_df.filter(filter.is_not)
+          child_df = child_df.filter(filter)
+        end
         next unless child_df && child_df.height > 0
 
-        attrs = graph.get_edge_data(parent_file, child_file)
+        attrs = graph.get_edge_data(parent_node_id, child_node_id)
 
         attrs[:dependencies].each do |dep|
-          parent_col = dep[parent_file]
-          child_col = dep[child_file]
+          parent_col = dep[parent_node_id]
+          child_col = dep[child_node_id]
+          allow_null = !!dep[:allow_null]
 
           next unless parent_col && child_col &&
             parent_df.columns.include?(parent_col) && child_df.columns.include?(child_col)
@@ -157,10 +188,20 @@ module GtfsDf
 
           # Filter child to only include rows that reference valid parent values
           before = child_df.height
-          child_df = child_df.filter(Polars.col(child_col).is_in(valid_values))
+          filter = Polars.col(child_col).is_in(valid_values)
+          if allow_null
+            filter = (filter | Polars.col(child_col).is_null)
+          end
+          child_df = child_df.filter(filter)
+          changed = child_df.height < before
+
+          # If we removed a part of the child_df earlier, concat it back on
+          if saved_vals
+            child_df = Polars.concat([child_df, saved_vals], how: "vertical")
+          end
 
-          if child_df.height < before
-            filtered[child_file] = child_df
+          if changed
+            filtered[child_node.fetch(:file)] = child_df
           end
         end
       end

data/lib/gtfs_df/graph.rb

@@ -2,27 +2,64 @@
 
 module GtfsDf
   class Graph
+    FILES = %w[
+      agency routes trips stop_times calendar calendar_dates shapes transfers frequencies fare_attributes fare_rules
+      fare_leg_join_rules fare_transfer_rules areas networks route_networks location_groups location_group_stops booking_rules
+      stop_areas fare_leg_rules
+    ]
+
+    STANDARD_FILE_NODES = FILES.map do |file|
+      [file, {id: file, file: file, filter: nil}]
+    end.to_h.freeze
+
+    # Separate node definitions for stops and parent stations to handle the self-referential
+    # relationship in stops.txt where stops reference parent stations via parent_station column.
+    # This allows filtering to preserve parent stations when their child stops are referenced.
+    STOP_NODES = {
+      "stops" => {
+        id: "stops",
+        file: "stops",
+        filter_attrs: {
+          filter_col: "location_type",
+          filter: Polars.col("location_type").is_in(
+            Schema::EnumValues::STOP_LOCATION_TYPES.map(&:first)
+          ) | Polars.col("location_type").is_null
+        }
+      },
+      "parent_stations" => {
+        id: "parent_stations",
+        file: "stops",
+        filter_attrs: {
+          filter_col: "location_type",
+          filter: Polars.col("location_type").is_in(
+            Schema::EnumValues::STATION_LOCATION_TYPES.map(&:first)
+          ) & Polars.col("location_type").is_not_null
+        }
+      }
+    }.freeze
+
+    NODES = STANDARD_FILE_NODES.merge(STOP_NODES).freeze
+
     # Returns a directed graph of GTFS file dependencies
     def self.build
       g = NetworkX::Graph.new
-      # Nodes: GTFS files
-      files = %w[
-        agency routes trips stop_times stops calendar calendar_dates shapes transfers frequencies fare_attributes fare_rules
-        fare_leg_join_rules fare_transfer_rules areas networks route_networks location_groups location_group_stops booking_rules
-      ]
-      files.each { |f| g.add_node(f) }
+      NODES.keys.each { |node| g.add_node(node) }
 
       # TODO: Add fare_rules -> stops + test
       edges = [
         ["agency", "routes", {dependencies: [
           {"agency" => "agency_id", "routes" => "agency_id"}
         ]}],
+        ["fare_attributes", "agency", {dependencies: [
+          {"fare_attributes" => "agency_id",
+           "agency" => "agency_id"}
+        ]}],
         ["fare_attributes", "fare_rules", {dependencies: [
           {"fare_attributes" => "fare_id",
            "fare_rules" => "fare_id"}
         ]}],
         ["fare_rules", "routes", {dependencies: [
-          {"fare_rules" => "route_id", "routes" => "route_id"}
+          {"fare_rules" => "route_id", "routes" => "route_id", :allow_null => true}
         ]}],
         ["routes", "trips", {dependencies: [
           {"routes" => "route_id", "trips" => "route_id"}
@@ -33,6 +70,10 @@ module GtfsDf
         ["stop_times", "stops", {dependencies: [
           {"stop_times" => "stop_id", "stops" => "stop_id"}
         ]}],
+        # Self-referential edge: stops can reference parent stations (location_type=1)
+        ["stops", "parent_stations", {dependencies: [
+          {"stops" => "parent_station", "parent_stations" => "stop_id"}
+        ]}],
         ["stops", "transfers", {dependencies: [
           {"stops" => "stop_id", "transfers" => "from_stop_id"},
           {"stops" => "stop_id", "transfers" => "to_stop_id"}

data/lib/gtfs_df/reader.rb

@@ -4,24 +4,21 @@ module GtfsDf
   class Reader
     # Loads a GTFS zip file and returns a Feed
     def self.load_from_zip(zip_path)
-      data = {}
+      data = nil
+
       Dir.mktmpdir do |tmpdir|
         Zip::File.open(zip_path) do |zip_file|
           zip_file.each do |entry|
             next unless entry.file?
-
-            GtfsDf::Feed::GTFS_FILES.each do |gtfs_file|
-              next unless entry.name == "#{gtfs_file}.txt"
-
-              out_path = File.join(tmpdir, entry.name)
-              entry.extract(out_path)
-
-              data[gtfs_file] = data_frame(gtfs_file, out_path)
-            end
+            out_path = File.join(tmpdir, entry.name)
+            entry.extract(out_path)
           end
         end
+
+        data = load_from_dir(tmpdir)
       end
-      GtfsDf::Feed.new(data)
+
+      data
     end
 
     # Loads a GTFS dir and returns a Feed

data/lib/gtfs_df/schema/enum_values.rb

@@ -82,13 +82,16 @@ module GtfsDf
 
       # stops.txt
       # location_type: Type of location
-      LOCATION_TYPE = [
+      STOP_LOCATION_TYPES = [
         ["0", "Stop or platform"],
-        ["1", "Station"],
         ["2", "Entrance/Exit"],
         ["3", "Generic Node"],
         ["4", "Boarding Area"]
       ]
+      STATION_LOCATION_TYPES = [
+        ["1", "Station"]
+      ]
+      LOCATION_TYPE = STOP_LOCATION_TYPES + STATION_LOCATION_TYPES
 
       # wheelchair_boarding: Indicates wheelchair boarding possibility
       WHEELCHAIR_BOARDING = [

data/lib/gtfs_df/schema/stops.rb

@@ -9,7 +9,7 @@ module GtfsDf
         "stop_name" => Polars::String,
         "tts_stop_name" => Polars::String,
         "stop_desc" => Polars::String,
-        "stop_lat" => Polars::String,
+        "stop_lat" => Polars::Float64,
         "stop_lon" => Polars::Float64,
         "zone_id" => Polars::String,
         "stop_url" => Polars::String,

data/lib/gtfs_df/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module GtfsDf
-  VERSION = "0.2.0"
+  VERSION = "0.4.0"
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: gtfs_df
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.4.0
 platform: ruby
 authors:
 - David Mejorado
 bindir: exe
 cert_chain: []
-date: 1980-01-01 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: networkx