gtfs_df 0.7.0 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0b218f60e33e005576eb278a74ba89e6164fddccfe99e9f8e5665d105d9fa471
-  data.tar.gz: cb4023192418d5ba89f08b605b2ec8ac79073b28d29f6161f252d97df3291dff
+  metadata.gz: b04b659c98fd64e3a9ccbe37390dbd1904b37efba1e001ddb3bfab6ada41e912
+  data.tar.gz: 5c003f2cdc866a1232c8988604040dbf0523c9605a96d5bea2138253011cea92
 SHA512:
-  metadata.gz: cefa279582b0579a637e4ebc39fe4c3d90d56582aa1cd0d00ffda041b92f681831334d46aeae95dc1c291c1295758c500a0c7bfe32876f226605f5b150fc0a9c
-  data.tar.gz: e4b5bbe129674304121c61cb732a8261421158f206370289e67d3e6d514bbb089bc0c35f88fde6e46012a18afd3700b86eb4c2a2b457b9880270ef3a4f74cf79
+  metadata.gz: 7d7b6ef1ed68ffd05273a01c6e17a900a757b2c933c8d0c199378e1b94d79b1d7650d8094e22e2f9e2f56555bb7082418f8b2765675a48a4a0ec2d5ef7f0f6d9
+  data.tar.gz: 9d1c54c97ca7f1480693a736b44dcb656f664291e46f04af94bdd25027ea9d976772ce8ba2d30c28f470ecb4f570991784157e7492db20c8320f5cb7a2db480f

data/CHANGELOG.md

@@ -1,3 +1,36 @@
+## [0.9.0] - 2026-02-17
+
+### 🚀 Features
+
+- Add helper utilities
+
+### 🐛 Bug Fixes
+
+- [**breaking**] Bump rubyzip min version to 3.0
+## [0.8.0] - 2026-01-09
+
+### 🐛 Bug Fixes
+
+- Ignore extra newlines when parsing csv
+- Bump minimum ruby version to 3.2.0
+- Fix fare_attributes filtering
+- Fix exceptions edge case
+- Replace dynamic graph traversal with bidirectional graph option
+
+### 📚 Documentation
+
+- Document dev environment
+- Clarify the actions made by the bump-version script
+- Update example transitive dependencies
+
+### ⚙️ Miscellaneous Tasks
+
+- Reduce the test run frequency
+- Update dependabot schedule
+- Consolidate test fixtures
+- Add test for additional fares case
+- Update readme
+- Bump version to 0.8.0
 ## [0.7.0] - 2025-12-30
 
 ### 🚀 Features
@@ -22,6 +55,7 @@
 ### ⚙️ Miscellaneous Tasks
 
 - Include the util helpers in the console and the test spec
+- Bump version to 0.7.0
 ## [0.6.2] - 2025-12-15
 
 ### 🐛 Bug Fixes

data/README.md

@@ -35,6 +35,9 @@ feed = GtfsDf::Reader.load_from_zip('path/to/gtfs.zip')
 # Or, load from a directory
 feed = GtfsDf::Reader.load_from_dir('path/to/gtfs_dir')
 
+# Parse times as seconds since midnight instead of string
+feed = GtfsDf::Reader.load_from_dir('path/to/gtfs_dir', parse_times: true)
+
 # Access dataframes for each GTFS file
 puts feed.agency.head
 puts feed.routes.head
@@ -71,11 +74,25 @@ When you filter by a field, the library automatically:
 
 For example, filtering by `agency_id` will automatically filter routes, trips, stop_times, and stops to only include data for that agency.
 
+By default gtfs_df treats trips as the atomic unit of GTFS. Therefore, if we
+filter to one stop referenced by TripA, we will preserve _all stops_ referenced
+by TripA.
+
+To avoid this behavior, you can pass the `filter_only_children` param. In this case, only the children of the specified filter will be pruned and trip integrity will not be maintained. In the below example, stop 1 and related stop_times will be pruned.
+
+```ruby
+filtered_feed = feed.filter({ 'stop' => { 'stop_id' => ['1'] } }, filter_only_children: true)
+```
+
+
 ### Writing filtered feeds
 
 ```ruby
 # Write to a new zip file
 GtfsDf::Writer.write_to_zip(filtered_feed, 'output/filtered_gtfs.zip')
+
+# Write to a directory
+GtfsDf::Writer.write_to_dir(filtered_feed, 'output/filtered_gtfs')
 ```
 
 ### Example: Split feed by agency
@@ -84,30 +101,41 @@ See [examples/split-by-agency](examples/split-by-agency) for a complete example
 
 ## Development
 
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+### Environment
+
+This project manages its development environment with nix, specifically [devenv].
+
+After checking out the repo:
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+- Install devenv: https://devenv.sh/getting-started/
+
+- To enable the environment you can either:
+    - Use [direnv] to enable the environment as soon as you enter the project's path.
+    - Enable it only when you needed by running: `devenv shell`
+
+- Run `bin/setup` to install the gem dependencies.
+
+### Tests
+
+Run `rake spec` to run the tests.
+
+### REPL
+
+You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
 ## Release process
 
 1. `bin/bump-version`
 
-- Bump the version in `lib/gtfs_df/version.rb`
-- Update the `CHANGELOG.md` using the git log since the last version
-- Create and push a new release branch with those changes
-- Create a PR for that release
+- Bumps the version in `lib/gtfs_df/version.rb`
+- Updates the `CHANGELOG.md` using the git log since the last version
+- Creates and push a new release branch with those changes
+- Creates a PR for that release
 
 2. `bin/create-tag`
 
 Creates and pushes the git tag for the release. That will trigger the GitHub action: `.github/workflows/publish.yml` to publish to RubyGems.
 
-## TODO
-
-- [ ] Time parsing
-
-  Just like partridge, we should parse Time as seconds since midnight. There's a draft in `lib/gtfs_df/utils.rb` but it's not used anywhere.
-  I haven't figured out how to properly implement that with Polars.
-
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/davidmh/ruby-gtfs_df.
@@ -120,3 +148,5 @@ The gem is available as open source under the terms of the [MIT License](https:/
 [Polars]: https://pola.rs/
 [ruby-polars]: https://github.com/ankane/ruby-polars
 [partridge]: https://github.com/remix/partridge
+[devenv]: https://devenv.sh
+[direnv]: https://direnv.net

data/examples/split-by-agency/Gemfile.lock

@@ -1,20 +1,21 @@
 PATH
   remote: ../..
   specs:
-    gtfs_df (0.7.0)
+    gtfs_df (0.9.0)
       networkx (~> 0.4)
       polars-df (~> 0.22)
-      rubyzip (~> 2.3)
+      rubyzip (>= 3.0, < 4.0)
 
 GEM
   remote: https://gem.coop/
   specs:
-    bigdecimal (3.3.1)
+    bigdecimal (4.0.1)
+    json (2.18.0)
     matrix (0.4.3)
     networkx (0.4.0)
       matrix (~> 0.4)
       rb_heap (~> 1.0)
-    optparse (0.8.0)
+    optparse (0.8.1)
     polars-df (0.23.0-aarch64-linux)
       bigdecimal
     polars-df (0.23.0-aarch64-linux-musl)
@@ -28,11 +29,12 @@ GEM
     polars-df (0.23.0-x86_64-linux-musl)
       bigdecimal
     rb_heap (1.1.0)
-    rubyzip (2.4.1)
+    rubyzip (3.2.2)
     unicode-display_width (3.2.0)
       unicode-emoji (~> 4.1)
-    unicode-emoji (4.1.0)
-    whirly (0.3.0)
+    unicode-emoji (4.2.0)
+    whirly (0.4.0)
+      json
       unicode-display_width (>= 1.1)
 
 PLATFORMS

data/lib/gtfs_df/base_gtfs_table.rb

@@ -15,6 +15,10 @@ module GtfsDf
           df = Polars.read_csv(input, infer_schema_length: 0, encoding: "utf8-lossy")
             .rename(->(col) { col.strip })
 
+          # Strip out empty lines. Unfortunately read_csv does not support the drop_empty_rows
+          # option right now.
+          df = df.filter(Polars.all_horizontal(Polars.all.is_null).is_not)
+
           dtypes = self.class::SCHEMA.slice(*df.columns)
           df
             .with_columns(dtypes.keys.map do |col|

data/lib/gtfs_df/feed.rb

@@ -148,6 +148,32 @@ module GtfsDf
       self.class.new(filtered, parse_times: @parse_times)
     end
 
+    # Utility method that returns a hash of dataframes by file name
+    #
+    # @return [{file_name => dataframe}]
+    def by_dataframe_name
+      GTFS_FILES.filter_map do |file|
+        dataframe = send(file)
+        dataframe ? [file, dataframe] : nil
+      end.to_h
+    end
+
+    # Utility method for getting a dataframe, e.g. feed['agency']
+    #
+    # @param [string] file name
+    # @return [dataframe]
+    def [](file_name)
+      send(file_name)
+    end
+
+    # Utility method for setting a dataframe, e.g. feed['agency'] = new_dataframe
+    #
+    # @param [string] file name
+    # @value [dataframe] the new dataframe
+    def []=(file_name, value)
+      send("#{file_name}=", value)
+    end
+
     private
 
     def filter!(file, filters, filtered, filter_only_children: false)
@@ -172,17 +198,17 @@ module GtfsDf
       filtered
     end
 
-    # Traverses the grah to prune unreferenced entities from child dataframes
+    # Traverses the graph to prune unreferenced entities from child dataframes
     # based on parent relationships. See GtfsDf::Graph::STOP_NODES
     def prune!(root, filtered, filter_only_children: false)
       seen_edges = Set.new
-      maybe_digraph = filter_only_children ? graph : graph.to_undirected
+      rerooted_graph = Graph.build(bidirectional: !filter_only_children)
 
       queue = [root]
 
       while queue.length > 0
         parent_node_id = queue.shift
-        maybe_digraph.adj[parent_node_id].each do |child_node_id, attrs|
+        rerooted_graph.adj[parent_node_id].each do |child_node_id, attrs|
           edge = edge_id(parent_node_id, child_node_id)
 
           next if seen_edges.include?(edge)
@@ -209,6 +235,13 @@ module GtfsDf
 
           queue << child_node_id
 
+          # If the edge is weak (e.g. reverse edge of an optional relationship),
+          # we traverse to ensure connectivity but do NOT apply the filter.
+          if attrs[:type] == :weak
+            # puts "Skipping weak filter: #{edge}"
+            next
+          end
+
           attrs[:dependencies].each do |dep|
             parent_col = dep[parent_node_id]
             child_col = dep[child_node_id]
@@ -220,6 +253,13 @@ module GtfsDf
             # Get valid values from parent
             valid_values = parent_df[parent_col].to_a.uniq.compact
 
+            # Annoying special case to make sure that if we have a calendar with exceptions,
+            # the calendar_dates file doesn't end up pruning other files
+            if parent_node_id == "calendar_dates" && parent_col == "service_id" &&
+                filtered["calendar"]
+              valid_values = (valid_values + calendar["service_id"].to_a).uniq
+            end
+
             # Filter child to only include rows that reference valid parent values
             before = child_df.height
             filter = Polars.col(child_col).is_in(valid_values)
@@ -243,8 +283,7 @@ module GtfsDf
     end
 
     def edge_id(parent, child)
-      # Alphabetize to make sure this works with undirected graph
-      [parent, child].sort.join("-")
+      [parent, child].join("-")
     end
   end
 end

data/lib/gtfs_df/graph.rb

@@ -41,7 +41,7 @@ module GtfsDf
     NODES = STANDARD_FILE_NODES.merge(STOP_NODES).freeze
 
     # Returns a directed graph of GTFS file dependencies
-    def self.build
+    def self.build(bidirectional: false)
       g = NetworkX::DiGraph.new
       NODES.keys.each { |node| g.add_node(node) }
 
@@ -53,15 +53,15 @@ module GtfsDf
         ]}],
         ["agency", "fare_attributes", {dependencies: [
           {"fare_attributes" => "agency_id",
-           "agency" => "agency_id"}
-        ]}],
+           "agency" => "agency_id", :allow_null => true}
+        ], optional: true}],
         ["fare_attributes", "fare_rules", {dependencies: [
           {"fare_attributes" => "fare_id",
            "fare_rules" => "fare_id"}
         ]}],
         ["routes", "fare_rules", {dependencies: [
           {"fare_rules" => "route_id", "routes" => "route_id", :allow_null => true}
-        ]}],
+        ], optional: true}],
         ["routes", "trips", {dependencies: [
           {"routes" => "route_id", "trips" => "route_id"}
         ]}],
@@ -73,12 +73,12 @@ module GtfsDf
         ]}],
         # Self-referential edge: stops can reference parent stations (location_type=1)
         ["parent_stations", "stops", {dependencies: [
-          {"stops" => "parent_station", "parent_stations" => "stop_id"}
+          {"stops" => "parent_station", "parent_stations" => "stop_id", :allow_null => true}
         ]}],
         ["stops", "transfers", {dependencies: [
           {"stops" => "stop_id", "transfers" => "from_stop_id"},
           {"stops" => "stop_id", "transfers" => "to_stop_id"}
-        ]}],
+        ], optional: true}],
         ["calendar", "trips", {dependencies: [
           {"trips" => "service_id", "calendar" => "service_id"}
         ]}],
@@ -86,11 +86,11 @@ module GtfsDf
           {"trips" => "service_id", "calendar_dates" => "service_id"}
         ]}],
         ["shapes", "trips", {dependencies: [
-          {"trips" => "shape_id", "shapes" => "shape_id"}
+          {"trips" => "shape_id", "shapes" => "shape_id", :allow_null => true}
         ]}],
         ["trips", "frequencies", {dependencies: [
           {"trips" => "trip_id", "frequencies" => "trip_id"}
-        ]}],
+        ], optional: true}],
 
         # --- GTFS Extensions ---
         ["stops", "fare_leg_join_rules",
@@ -163,6 +163,16 @@ module GtfsDf
 
       edges.each do |from, to, attrs|
         g.add_edge(from, to, **attrs)
+        if bidirectional
+          # When adding the reverse edge, if the relationship is optional (child is not required),
+          # mark the reverse edge as weak. This prevents empty child tables (e.g. fare_rules)
+          # from filtering parent tables (e.g. routes) into emptiness.
+          reverse_attrs = attrs.dup
+          if attrs[:optional]
+            reverse_attrs[:type] = :weak
+          end
+          g.add_edge(to, from, **reverse_attrs)
+        end
       end
       g
     end

data/lib/gtfs_df/reader.rb

@@ -14,8 +14,7 @@ module GtfsDf
         Zip::File.open(zip_path) do |zip_file|
           zip_file.each do |entry|
             next unless entry.file?
-            out_path = File.join(tmpdir, entry.name)
-            entry.extract(out_path)
+            entry.extract(destination_directory: tmpdir)
           end
         end
 

data/lib/gtfs_df/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module GtfsDf
-  VERSION = "0.7.0"
+  VERSION = "0.9.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gtfs_df
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.9.0
 platform: ruby
 authors:
 - David Mejorado
@@ -41,16 +41,22 @@ dependencies:
   name: rubyzip
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '2.3'
+        version: '4.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '2.3'
+        version: '4.0'
 description: 'A Ruby gem to load, filter, and manipulate GTFS (General Transit Feed
   Specification) feeds using DataFrames powered by Polars. Supports cascading filters
   that maintain referential integrity across related tables. NOTE: This gem is not
@@ -137,7 +143,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.1.0
+      version: 3.2.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="