gtfs_df 0.4.1 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 205ec058b41c5bd1d2b01ff3950d4cd2ebb20f304d02d5bcd3dd0c447b4e0a6e
-  data.tar.gz: 54cca637de421c26d2144df100f8430f5dddb639b3a37d1160dfbd9704630e33
+  metadata.gz: c7c1d87e57bbb44ceb4ce8112da7172c5a8c76f8e88d3b1e8fbb610aad850cf3
+  data.tar.gz: 6a1b68dfc723d3c70b779687a70100b735c9a57e91603b302685318660473c66
 SHA512:
-  metadata.gz: ef853b504ee701e77911259352d0057f6e327db8022370a1fa0dbaa597bfadcac7ac1c8fbe746c04ec4669d86384279a259ac35d4126f1483286937812cb7cce
-  data.tar.gz: 728d684dd02b653cc779aa5e8152766f5dccafcb097e8fca5810b90f994c442645119f32efb62c84d8fbf1645109f579a3c74f6eb88c9e21dd03faa72193cdd0
+  metadata.gz: 85684d79eac1479bac56995cd00a4d22106e5247979e2b54b07dee0d2b1948c4e86bf005be674b57656c0a9c14acf731a11d17163edde72d0302561a2d489159
+  data.tar.gz: 7033b279730614870ff8710e51b29ff20f2b399aff9cd1b13a8a2a3202940a88221d9f0edc5dc82a5c3593c9f2314fdc6082fd8c89673f8ec956ba9eada11e2a

data/CHANGELOG.md

@@ -1,3 +1,18 @@
+## [0.6.0] - 2025-12-09
+### Fixed
+
+- visit nodes multiple times
+## [0.5.0] - 2025-12-08
+
+### Added
+
+- add Feed#filter filter_only_children param
+
+### Maintenance
+
+- arrange edges so parent is always first
+- build directed graph
+- allow ! in commit messages
 ## [0.4.1] - 2025-12-05
 
 ### Added

data/lib/gtfs_df/feed.rb

@@ -75,13 +75,17 @@ module GtfsDf
     #
     # @param view [Hash] The view used to filter the feed, with format { file => filters }.
     #   Example view: { 'routes' => { 'route_id' => '123' }, 'trips' => { 'service_id' => 'A' } }
-    # @param maintain_trip_dependencies [Boolean] Whether trip dependencies should be preserved.
-    #   By default, we treat trips as the atomic unit of GTFS. Therefore, if we filter to one stop
-    #   referenced by TripA, we will preserve _all stops_ referenced by TripA. However, it is
-    #   occasionally useful to prune bad data and _not_ maintain all trip dependencies.
-    #   For example, if StopA contains invalid coordinates, we may wish to filter it out but keep
-    #   the other stops for TripA. In this case, `maintain_trip_dependencies` should be set to false.
-    def filter(view, maintain_trip_dependencies = true)
+    # @param filter_only_children [Boolean] Whether only child dependencies should be pruned.
+    #   When false, we:
+    #   - Treat trips as the atomic unit of GTFS. Therefore, if we filter to one stop
+    #     referenced by TripA, we will preserve _all stops_ referenced by TripA.
+    #   - Prune unreferenced parent objects (e.g. route is a parent of trip. Unreferenced routes
+    #     will be pruned.)
+    #   When true we:
+    #   - Do not treat trips as atomic. I can filter stopA without maintaining other stops for
+    #     trips that reference it.
+    #   - Only filter child objects
+    def filter(view, filter_only_children: false)
       filtered = {}
 
       GTFS_FILES.each do |file|
@@ -91,7 +95,11 @@ module GtfsDf
         filtered[file] = df
       end
 
-      if maintain_trip_dependencies
+      if filter_only_children
+        view.each do |file, filters|
+          filtered = filter!(file, filters, filtered.dup, filter_only_children: true)
+        end
+      else
         # Trips are the atomic unit of GTFS, we will generate a new view
         # based on the set of trips that would be included for each invidual filter
         # and cascade changes from this view in order to retain referential integrity
@@ -107,11 +115,7 @@ module GtfsDf
         end
 
         if trip_ids
-          filtered = filter!("trips", {"trip_id" => trip_ids.to_a}, filtered)
-        end
-      else
-        view.each do |file, filters|
-          filtered = filter!(file, filters, filtered.dup)
+          filtered = filter!("trips", {"trip_id" => trip_ids.to_a}, filtered.dup)
         end
       end
 
@@ -128,7 +132,7 @@ module GtfsDf
 
     private
 
-    def filter!(file, filters, filtered)
+    def filter!(file, filters, filtered, filter_only_children: false)
       unless filters.empty?
         df = filtered[file]
 
@@ -144,7 +148,7 @@ module GtfsDf
 
         filtered[file] = df
 
-        prune!(file, filtered)
+        prune!(file, filtered, filter_only_children:)
       end
 
       filtered
@@ -152,59 +156,77 @@ module GtfsDf
 
     # Traverses the grah to prune unreferenced entities from child dataframes
     # based on parent relationships. See GtfsDf::Graph::STOP_NODES
-    def prune!(root, filtered)
-      graph.each_bfs_edge(root) do |parent_node_id, child_node_id|
-        parent_node = Graph::NODES[parent_node_id]
-        child_node = Graph::NODES[child_node_id]
-        parent_df = filtered[parent_node.fetch(:file)]
-        next unless parent_df
-
-        child_df = filtered[child_node.fetch(:file)]
-        # Certain nodes are pre-filtered because they reference only
-        # a piece of the dataframe
-        filter_attrs = child_node[:filter_attrs]
-        if filter_attrs && child_df.columns.include?(filter_attrs.fetch(:filter_col))
-          filter = filter_attrs.fetch(:filter)
-          # Temporarily remove rows that do not match node filter criteria to process them
-          # separately (e.g., when filtering stops, parent stations that should be preserved
-          # regardless of direct references)
-          saved_vals = child_df.filter(filter.is_not)
-          child_df = child_df.filter(filter)
-        end
-        next unless child_df && child_df.height > 0
-
-        attrs = graph.get_edge_data(parent_node_id, child_node_id)
-
-        attrs[:dependencies].each do |dep|
-          parent_col = dep[parent_node_id]
-          child_col = dep[child_node_id]
-          allow_null = !!dep[:allow_null]
-
-          next unless parent_col && child_col &&
-            parent_df.columns.include?(parent_col) && child_df.columns.include?(child_col)
-
-          # Get valid values from parent
-          valid_values = parent_df[parent_col].to_a.uniq.compact
-
-          # Filter child to only include rows that reference valid parent values
-          before = child_df.height
-          filter = Polars.col(child_col).is_in(valid_values)
-          if allow_null
-            filter = (filter | Polars.col(child_col).is_null)
-          end
-          child_df = child_df.filter(filter)
-          changed = child_df.height < before
-
-          # If we removed a part of the child_df earlier, concat it back on
-          if saved_vals
-            child_df = Polars.concat([child_df, saved_vals], how: "vertical")
+    def prune!(root, filtered, filter_only_children: false)
+      seen_edges = Set.new
+      maybe_digraph = filter_only_children ? graph : graph.to_undirected
+
+      queue = [root]
+
+      while queue.length > 0
+        parent_node_id = queue.shift
+        maybe_digraph.adj[parent_node_id].each do |child_node_id, attrs|
+          edge = edge_id(parent_node_id, child_node_id)
+
+          next if seen_edges.include?(edge)
+          seen_edges.add(edge)
+
+          parent_node = Graph::NODES[parent_node_id]
+          child_node = Graph::NODES[child_node_id]
+          parent_df = filtered[parent_node.fetch(:file)]
+          next unless parent_df
+
+          child_df = filtered[child_node.fetch(:file)]
+          # Certain nodes are pre-filtered because they reference only
+          # a piece of the dataframe
+          filter_attrs = child_node[:filter_attrs]
+          if filter_attrs && child_df.columns.include?(filter_attrs.fetch(:filter_col))
+            filter = filter_attrs.fetch(:filter)
+            # Temporarily remove rows that do not match node filter criteria to process them
+            # separately (e.g., when filtering stops, parent stations that should be preserved
+            # regardless of direct references)
+            saved_vals = child_df.filter(filter.is_not)
+            child_df = child_df.filter(filter)
           end
-
-          if changed
-            filtered[child_node.fetch(:file)] = child_df
+          next unless child_df && child_df.height > 0
+
+          queue << child_node_id
+
+          attrs[:dependencies].each do |dep|
+            parent_col = dep[parent_node_id]
+            child_col = dep[child_node_id]
+            allow_null = !!dep[:allow_null]
+
+            next unless parent_col && child_col &&
+              parent_df.columns.include?(parent_col) && child_df.columns.include?(child_col)
+
+            # Get valid values from parent
+            valid_values = parent_df[parent_col].to_a.uniq.compact
+
+            # Filter child to only include rows that reference valid parent values
+            before = child_df.height
+            filter = Polars.col(child_col).is_in(valid_values)
+            if allow_null
+              filter = (filter | Polars.col(child_col).is_null)
+            end
+            child_df = child_df.filter(filter)
+            changed = child_df.height < before
+
+            # If we removed a part of the child_df earlier, concat it back on
+            if saved_vals
+              child_df = Polars.concat([child_df, saved_vals], how: "vertical")
+            end
+
+            if changed
+              filtered[child_node.fetch(:file)] = child_df
+            end
           end
         end
       end
     end
+
+    def edge_id(parent, child)
+      # Alphabetize to make sure this works with undirected graph
+      [parent, child].sort.join("-")
+    end
   end
 end

data/lib/gtfs_df/graph.rb

@@ -42,15 +42,16 @@ module GtfsDf
 
     # Returns a directed graph of GTFS file dependencies
     def self.build
-      g = NetworkX::Graph.new
+      g = NetworkX::DiGraph.new
       NODES.keys.each { |node| g.add_node(node) }
 
+      # Edges should be parent, child
       # TODO: Add fare_rules -> stops + test
       edges = [
         ["agency", "routes", {dependencies: [
           {"agency" => "agency_id", "routes" => "agency_id"}
         ]}],
-        ["fare_attributes", "agency", {dependencies: [
+        ["agency", "fare_attributes", {dependencies: [
           {"fare_attributes" => "agency_id",
            "agency" => "agency_id"}
         ]}],
@@ -58,7 +59,7 @@ module GtfsDf
           {"fare_attributes" => "fare_id",
            "fare_rules" => "fare_id"}
         ]}],
-        ["fare_rules", "routes", {dependencies: [
+        ["routes", "fare_rules", {dependencies: [
           {"fare_rules" => "route_id", "routes" => "route_id", :allow_null => true}
         ]}],
         ["routes", "trips", {dependencies: [
@@ -67,24 +68,24 @@ module GtfsDf
         ["trips", "stop_times", {dependencies: [
           {"trips" => "trip_id", "stop_times" => "trip_id"}
         ]}],
-        ["stop_times", "stops", {dependencies: [
+        ["stops", "stop_times", {dependencies: [
           {"stop_times" => "stop_id", "stops" => "stop_id"}
         ]}],
         # Self-referential edge: stops can reference parent stations (location_type=1)
-        ["stops", "parent_stations", {dependencies: [
+        ["parent_stations", "stops", {dependencies: [
           {"stops" => "parent_station", "parent_stations" => "stop_id"}
         ]}],
         ["stops", "transfers", {dependencies: [
           {"stops" => "stop_id", "transfers" => "from_stop_id"},
           {"stops" => "stop_id", "transfers" => "to_stop_id"}
         ]}],
-        ["trips", "calendar", {dependencies: [
+        ["calendar", "trips", {dependencies: [
           {"trips" => "service_id", "calendar" => "service_id"}
         ]}],
-        ["trips", "calendar_dates", {dependencies: [
+        ["calendar_dates", "trips", {dependencies: [
           {"trips" => "service_id", "calendar_dates" => "service_id"}
         ]}],
-        ["trips", "shapes", {dependencies: [
+        ["shapes", "trips", {dependencies: [
           {"trips" => "shape_id", "shapes" => "shape_id"}
         ]}],
         ["trips", "frequencies", {dependencies: [
@@ -97,11 +98,11 @@ module GtfsDf
             {"stops" => "stop_id", "fare_leg_join_rules" => "from_stop_id"},
             {"stops" => "stop_id", "fare_leg_join_rules" => "to_stop_id"}
           ]}],
-        ["fare_leg_join_rules", "networks", {dependencies: [
+        ["networks", "fare_leg_join_rules", {dependencies: [
           {"fare_leg_join_rules" => "from_network_id", "networks" => "network_id"},
           {"fare_leg_join_rules" => "to_network_id", "networks" => "network_id"}
         ]}],
-        ["fare_leg_join_rules", "fare_leg_rules",
+        ["fare_leg_rules", "fare_leg_join_rules",
           {dependencies: [
             {"fare_leg_join_rules" => "fare_leg_rule_id", "fare_leg_rules" => "fare_leg_rule_id"}
           ]}],
@@ -110,14 +111,14 @@ module GtfsDf
             {"fare_transfer_rules" => "from_leg_group_id", "fare_leg_rules" => "leg_group_id"},
             {"fare_transfer_rules" => "to_leg_group_id", "fare_leg_rules" => "leg_group_id"}
           ]}],
-        ["fare_transfer_rules", "fare_products",
+        ["fare_products", "fare_transfer_rules",
           {dependencies: [
             {"fare_transfer_rules" => "fare_product_id", "fare_products" => "fare_product_id"}
           ]}],
         ["areas", "stop_areas", {dependencies: [
           {"areas" => "area_id", "stop_areas" => "area_id"}
         ]}],
-        ["stops", "areas", {dependencies: [
+        ["areas", "stops", {dependencies: [
           {"stops" => "area_id", "areas" => "area_id"}
         ]}],
         ["areas", "fare_leg_rules", {dependencies: [
@@ -133,10 +134,10 @@ module GtfsDf
         ["networks", "fare_leg_rules", {dependencies: [
           {"networks" => "network_id", "fare_leg_rules" => "network_id"}
         ]}],
-        ["route_networks", "routes", {dependencies: [
+        ["routes", "route_networks", {dependencies: [
           {"route_networks" => "route_id", "routes" => "route_id"}
         ]}],
-        ["route_networks", "networks", {dependencies: [
+        ["networks", "route_networks", {dependencies: [
           {"route_networks" => "network_id", "networks" => "network_id"}
         ]}],
         ["location_groups", "location_group_stops", {dependencies: [
@@ -145,13 +146,13 @@ module GtfsDf
         ["location_groups", "stops", {dependencies: [
           {"location_groups" => "location_group_id", "stops" => "location_group_id"}
         ]}],
-        ["location_group_stops", "stops", {dependencies: [
+        ["stops", "location_group_stops", {dependencies: [
           {"location_group_stops" => "stop_id", "stops" => "stop_id"}
         ]}],
         ["stops", "location_group_stops", {dependencies: [
           {"stops" => "stop_id", "location_group_stops" => "stop_id"}
         ]}],
-        ["location_group_stops", "location_groups", {dependencies: [
+        ["location_groups", "location_group_stops", {dependencies: [
           {"location_group_stops" => "location_group_id", "location_groups" => "location_group_id"}
         ]}],
         ["booking_rules", "stop_times", {dependencies: [

data/lib/gtfs_df/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module GtfsDf
-  VERSION = "0.4.1"
+  VERSION = "0.6.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gtfs_df
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.6.0
 platform: ruby
 authors:
 - David Mejorado