gtfs_df 0.11.1 → 0.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 70a7d05d5600baac0b36b21d0ebdd8fd0719cfe572844e44ae710c25fbf353f8
-  data.tar.gz: df2b5f47cd19bfe213644dcda42b22778acf98d8f104bf71c4e231eaa4860bba
+  metadata.gz: 35a4116956b65a7dfdfd539af912bf67f590a2b99b35b3ea0bb62eafb2e60690
+  data.tar.gz: 14e43ea9a946392ac3044306a166544ae0183868b7305811be0c168fc6b9b153
 SHA512:
-  metadata.gz: ee94819792b5c1389783a7844ce23af40de13264c5c069e608ae9c2383254c3fdcdc09915f5c33348d204675e5e373f8c26b039ff97144be37721aa7f4bc1b49
-  data.tar.gz: 28f28bb69a62738c817dafbc36c756a96f56f9f96d4d587aab2b9dffa8df5b6948b25d6b024e70856639cc2a97c75c49ff510bb7695c825f54405bf23701582d
+  metadata.gz: 7c26d8662f454706e3ab4d641cb2f7f4f076acbd80fa9ce3689c2746bbec30d5c4added43129a7f984f6137bc1eee3a3d4597c1ea0d68b08d35caf79e3abe2c9
+  data.tar.gz: 824f21c6e8cabec24b31222ce9cb45ae71e0455edf5346ca56294e9490b5223915a782f0501f5800fc7f3aacd13ae1f5c6120012ffa7a9828ed6e55b9de93485

data/CHANGELOG.md

@@ -1,3 +1,28 @@
+## [0.13.0] - 2026-10-07
+
+### 🚀 Features
+
+- Add custom file processing to reader
+- Add custom file processing to feed and graph
+- Add custom file processing to writer
+
+### 💼 Other
+
+- Devenv 2
+## [0.12.0] - 2026-06-12
+
+### 🚀 Features
+
+- Allow loading a subset of tables
+
+### 🐛 Bug Fixes
+
+- Truncate ragged lines
+- Consider BOM when looking for empty files
+
+### ⚙️ Miscellaneous Tasks
+
+- Bump version to 0.12.0
 ## [0.11.1] - 2026-04-28
 
 ### 🐛 Bug Fixes
@@ -8,6 +33,10 @@
 
 - Validate filtering from feed with empty calendar
 - Validate unreferenced calendar dates
+
+### ⚙️ Miscellaneous Tasks
+
+- Bump version to 0.11.1
 ## [0.11.0] - 2026-03-30
 
 ### ⚙️ Miscellaneous Tasks

data/README.md

@@ -35,6 +35,18 @@ feed = GtfsDf::Reader.load_from_zip('path/to/gtfs.zip')
 # Or, load from a directory
 feed = GtfsDf::Reader.load_from_dir('path/to/gtfs_dir')
 
+# Load methods accept an 'extra' parameter that specifies custom files and how to build them
+extra = {
+  files: %w[directions],
+  classes: {
+    directions: DirectionsClass,
+  },
+  graph_edges: [
+    ["routes", "directions", {dependencies: [{"directions" => "route_id", "routes" => "route_id"}], optional: true}]
+  ],
+}
+GtfsDf::Reader.load_from_zip('path/to/gtfs.zip', extra:)
+
 # Parse times as seconds since midnight instead of string
 feed = GtfsDf::Reader.load_from_dir('path/to/gtfs_dir', parse_times: true)
 

data/devenv.lock

@@ -3,10 +3,11 @@
     "devenv": {
       "locked": {
         "dir": "src/modules",
-        "lastModified": 1765397744,
+        "lastModified": 1783109842,
+        "narHash": "sha256-Fi7lW+sg1ChduOOQRBCK5JeA1cw4x5wSjIJcppRGGqQ=",
         "owner": "cachix",
         "repo": "devenv",
-        "rev": "fd121886248781fee8546e4b6edc6d27e1d4efd5",
+        "rev": "e30eb49258bee68353bd9c619823f635f4afa86c",
         "type": "github"
       },
       "original": {
@@ -19,14 +20,15 @@
     "flake-compat": {
       "flake": false,
       "locked": {
-        "lastModified": 1765121682,
-        "owner": "edolstra",
+        "lastModified": 1767039857,
+        "narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=",
+        "owner": "NixOS",
         "repo": "flake-compat",
-        "rev": "65f23138d8d09a92e30f1e5c87611b23ef451bf3",
+        "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab",
         "type": "github"
       },
       "original": {
-        "owner": "edolstra",
+        "owner": "NixOS",
         "repo": "flake-compat",
         "type": "github"
       }
@@ -34,10 +36,11 @@
     "flake-compat_2": {
       "flake": false,
       "locked": {
-        "lastModified": 1765121682,
+        "lastModified": 1767039857,
+        "narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=",
         "owner": "edolstra",
         "repo": "flake-compat",
-        "rev": "65f23138d8d09a92e30f1e5c87611b23ef451bf3",
+        "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab",
         "type": "github"
       },
       "original": {
@@ -52,6 +55,7 @@
       },
       "locked": {
         "lastModified": 1731533236,
+        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
         "owner": "numtide",
         "repo": "flake-utils",
         "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
@@ -66,16 +70,16 @@
     "git-hooks": {
       "inputs": {
         "flake-compat": "flake-compat",
-        "gitignore": "gitignore",
         "nixpkgs": [
           "nixpkgs"
         ]
       },
       "locked": {
-        "lastModified": 1765404074,
+        "lastModified": 1783008725,
+        "narHash": "sha256-jGiy6+sxjNWXSjp25uoJuNfyH9zBK1PEDY0lVoL4ibQ=",
         "owner": "cachix",
         "repo": "git-hooks.nix",
-        "rev": "2d6f58930fbcd82f6f9fd59fb6d13e37684ca529",
+        "rev": "bca82caa46d5ec0f5d422c61fb1e30bc51313cbe",
         "type": "github"
       },
       "original": {
@@ -84,32 +88,16 @@
         "type": "github"
       }
     },
-    "gitignore": {
+    "nixpkgs": {
       "inputs": {
-        "nixpkgs": [
-          "git-hooks",
-          "nixpkgs"
-        ]
+        "nixpkgs-src": "nixpkgs-src"
       },
       "locked": {
-        "lastModified": 1762808025,
-        "owner": "hercules-ci",
-        "repo": "gitignore.nix",
-        "rev": "cb5e3fdca1de58ccbc3ef53de65bd372b48f567c",
-        "type": "github"
-      },
-      "original": {
-        "owner": "hercules-ci",
-        "repo": "gitignore.nix",
-        "type": "github"
-      }
-    },
-    "nixpkgs": {
-      "locked": {
-        "lastModified": 1764580874,
+        "lastModified": 1782924808,
+        "narHash": "sha256-tn2ahNv3ZNXyVHdPx/uw+N0xa7YSMtXUr6OEvu+s8ng=",
         "owner": "cachix",
         "repo": "devenv-nixpkgs",
-        "rev": "dcf61356c3ab25f1362b4a4428a6d871e84f1d1d",
+        "rev": "e2c881cb8d5f1cac6cef9d71f0d450a55691b999",
         "type": "github"
       },
       "original": {
@@ -128,10 +116,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1765345660,
+        "lastModified": 1782898183,
+        "narHash": "sha256-2wRgS4D9TnY68y14I69FfYVLzcWXMnystgfGLkBuAMI=",
         "owner": "bobvanderlinden",
         "repo": "nixpkgs-ruby",
-        "rev": "35e7b0919db2859f4022d1aa00b9c01253be1bce",
+        "rev": "edd32327ce414b547db75ca35eece1bd5329073c",
         "type": "github"
       },
       "original": {
@@ -140,20 +129,35 @@
         "type": "github"
       }
     },
+    "nixpkgs-src": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1782636521,
+        "narHash": "sha256-OG8laCOGtkxlB1JH3XqOnXxnkGJme4mQdXo5hkhCQIY=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "e1c1b84752fb0897897380a3cae9dc7fcab91ca3",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixpkgs-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
     "root": {
       "inputs": {
         "devenv": "devenv",
         "git-hooks": "git-hooks",
         "nixpkgs": "nixpkgs",
-        "nixpkgs-ruby": "nixpkgs-ruby",
-        "pre-commit-hooks": [
-          "git-hooks"
-        ]
+        "nixpkgs-ruby": "nixpkgs-ruby"
       }
     },
     "systems": {
       "locked": {
         "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
         "owner": "nix-systems",
         "repo": "default",
         "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
@@ -168,4 +172,4 @@
   },
   "root": "root",
   "version": 7
-}
+}

data/devenv.nix

@@ -7,6 +7,7 @@
     enable = true;
     versionFile = ./.ruby-version;
     bundler.enable = false;
+    lsp.enable = false;
   };
 
   git-hooks.hooks = {

data/devenv.yaml

@@ -6,3 +6,8 @@ inputs:
     inputs:
       nixpkgs:
         follows: nixpkgs
+  git-hooks:
+    url: github:cachix/git-hooks.nix
+    inputs:
+      nixpkgs:
+        follows: nixpkgs

data/lib/gtfs_df/base_gtfs_table.rb

@@ -12,7 +12,13 @@ module GtfsDf
         elsif input.is_a?(String)
           # TODO: use `infer_schema: false` instead of `infer_schema_length` after polars release:
           # https://github.com/ankane/ruby-polars/blob/master/CHANGELOG.md#100-unreleased
-          df = Polars.read_csv(input, infer_schema_length: 0, encoding: "utf8-lossy")
+          df = Polars
+            .read_csv(
+              input,
+              infer_schema_length: 0,
+              encoding: "utf8-lossy",
+              truncate_ragged_lines: true
+            )
             .rename(->(col) { col.strip })
 
           # Strip out empty lines. Unfortunately read_csv does not support the drop_empty_rows

data/lib/gtfs_df/feed.rb

@@ -38,15 +38,17 @@ module GtfsDf
 
     attr_accessor(*GTFS_FILES)
     attr_accessor(:parse_times)
-    attr_reader(:graph)
+    attr_reader(:graph, :extra)
 
     # Initialize with a hash of DataFrames
     REQUIRED_GTFS_FILES = %w[agency stops routes trips stop_times].freeze
 
     # @param data [Hash] Hash of DataFrames for each GTFS file
     # @param parse_times [Boolean] Whether to parse time fields to seconds since midnight (default: false)
-    def initialize(data = {}, parse_times: false)
+    # @param extra [Hash] Extra file metadata to include in the feed; see README
+    def initialize(data = {}, parse_times: false, extra: {})
       @parse_times = parse_times
+      @extra = extra
 
       missing = REQUIRED_GTFS_FILES.reject { |file| data[file].is_a?(Polars::DataFrame) }
       # At least one of calendar or calendar_dates must be present
@@ -59,12 +61,16 @@ module GtfsDf
         end.join(", ")}"
       end
 
-      @graph = GtfsDf::Graph.build
+      extra_files = @extra[:files] || []
+      extra_edges = @extra[:graph_edges] || []
+      extra_classes = @extra[:classes] || {}
+      @graph = GtfsDf::Graph.build(extra_edges:, extra_files:)
+      singleton_class.attr_accessor(*extra_files)
 
-      GTFS_FILES.each do |file|
+      (GTFS_FILES + extra_files).each do |file|
         df = data[file]
-        schema_class_name = file.split("_").map(&:capitalize).join
-        schema_class = begin
+        schema_class = extra_classes[file.to_sym] || begin
+          schema_class_name = file.split("_").map(&:capitalize).join
           GtfsDf::Schema.const_get(schema_class_name)
         rescue
           nil
@@ -106,7 +112,7 @@ module GtfsDf
     def filter(view, filter_only_children: false)
       filtered = {}
 
-      GTFS_FILES.each do |file|
+      (GTFS_FILES + (@extra[:files] || [])).each do |file|
         df = send(file)
         next unless df
 
@@ -145,7 +151,7 @@ module GtfsDf
 
         (!df || df.height == 0) && !is_required_file
       end
-      self.class.new(filtered, parse_times: @parse_times)
+      self.class.new(filtered, parse_times: @parse_times, extra: @extra)
     end
 
     # Utility method that returns a hash of dataframes by file name
@@ -374,7 +380,10 @@ module GtfsDf
     # not dropped when another edge is processed first.
     def prune!(root, filtered, filter_only_children: false)
       seen_edges = Set.new
-      rerooted_graph = Graph.build(bidirectional: !filter_only_children)
+      extra_files = @extra[:files] || []
+      extra_edges = @extra[:graph_edges] || []
+      all_nodes = Graph.all_nodes(extra_files:)
+      rerooted_graph = Graph.build(bidirectional: !filter_only_children, extra_edges:, extra_files:)
       accumulated_service_ids = Polars::Series.new("service_id", dtype: Polars::String)
       trips_base_df = nil
 
@@ -388,8 +397,8 @@ module GtfsDf
           next if seen_edges.include?(edge)
           seen_edges.add(edge)
 
-          parent_node = Graph::NODES[parent_node_id]
-          child_node = Graph::NODES[child_node_id]
+          parent_node = all_nodes[parent_node_id]
+          child_node = all_nodes[child_node_id]
           parent_df = filtered[parent_node.fetch(:file)]
           next unless parent_df
 

data/lib/gtfs_df/graph.rb

@@ -40,10 +40,17 @@ module GtfsDf
 
     NODES = STANDARD_FILE_NODES.merge(STOP_NODES).freeze
 
+    def self.all_nodes(extra_files: [])
+      extra_nodes = extra_files.map do |file|
+        [file, {id: file, file: file, filter: nil}]
+      end.to_h
+      NODES.merge(extra_nodes)
+    end
+
     # Returns a directed graph of GTFS file dependencies
-    def self.build(bidirectional: false)
+    def self.build(bidirectional: false, extra_edges: [], extra_files: [])
       g = NetworkX::DiGraph.new
-      NODES.keys.each { |node| g.add_node(node) }
+      all_nodes(extra_files:).keys.each { |node| g.add_node(node) }
 
       # Edges should be parent, child
       # TODO: Add fare_rules -> stops + test
@@ -160,6 +167,7 @@ module GtfsDf
           {"booking_rules" => "booking_rule_id", "stop_times" => "drop_off_booking_rule_id"}
         ]}]
       ]
+      edges += extra_edges
 
       edges.each do |from, to, attrs|
         g.add_edge(from, to, **attrs)

data/lib/gtfs_df/reader.rb

@@ -6,13 +6,16 @@ module GtfsDf
     #
     # @param zip_path [String] Path to the GTFS zip file
     # @param parse_times [Boolean] Whether to parse time fields to seconds since midnight (default: false)
+    # @param relevant_files [Array<String>] A list of file names, useful to avoid loading tables you don't care about.
     # @return [Feed] The loaded GTFS feed
-    def self.load_from_zip(zip_path, parse_times: false)
+    def self.load_from_zip(zip_path, parse_times: false, relevant_files: nil, extra: nil)
       data = nil
 
-      relevant_files = GtfsDf::Feed::GTFS_FILES
-        .map { |name| "#{name}.txt" }
-        .to_set
+      relevant_files ||= GtfsDf::Feed::GTFS_FILES.map { |name| "#{name}.txt" }
+      relevant_files = relevant_files.to_set
+      extra ||= {}
+      extra_files = extra[:files] || []
+      extra_classes = extra[:classes] || {}
 
       seen = {}
 
@@ -27,9 +30,11 @@ module GtfsDf
             end
 
             # We're skipping:
-            # - unrelated files
+            # - files neither relevant nor supported by extra
             # - empty feed files
-            next unless relevant_files.include?(file_name) && has_header?(entry)
+            file_name_truncated = file_name.split(".").first # without the .txt extension
+            relevant_or_extra = relevant_files.include?(file_name) || (extra_files.include?(file_name_truncated) && extra_classes.key?(file_name_truncated.to_sym))
+            next unless relevant_or_extra && has_header?(entry)
 
             seen[file_name] = entry.name
 
@@ -37,7 +42,7 @@ module GtfsDf
           end
         end
 
-        data = load_from_dir(tmpdir, parse_times: parse_times)
+        data = load_from_dir(tmpdir, parse_times:, relevant_files:, extra:)
       end
 
       data
@@ -47,26 +52,42 @@ module GtfsDf
     #
     # @param dir_path [String] Path to the GTFS directory
     # @param parse_times [Boolean] Whether to parse time fields to seconds since midnight (default: false)
+    # @param relevant_files [Array<String>] A list of file names, useful to avoid loading tables you don't care about.
     # @return [Feed] The loaded GTFS feed
-    def self.load_from_dir(dir_path, parse_times: false)
+    def self.load_from_dir(dir_path, parse_times: false, relevant_files: nil, extra: nil)
+      relevant_files ||= GtfsDf::Feed::GTFS_FILES.map { |name| "#{name}.txt" }
+      relevant_files = relevant_files.to_set
+      extra ||= {}
+      extra_files = extra[:files] || []
+      extra_classes = extra[:classes] || {}
+
       data = {}
-      GtfsDf::Feed::GTFS_FILES.each do |gtfs_file|
-        path = File.join(dir_path, "#{gtfs_file}.txt")
-        next unless File.exist?(path)
+      (GtfsDf::Feed::GTFS_FILES + extra_files).each do |gtfs_file|
+        basename = "#{gtfs_file}.txt"
+        path = File.join(dir_path, basename)
+        relevant_or_extra = relevant_files.include?(basename) || (extra_files.include?(gtfs_file) && extra_classes.key?(gtfs_file.to_sym))
+        next unless relevant_or_extra && File.exist?(path)
 
-        data[gtfs_file] = data_frame(gtfs_file, path)
+        data[gtfs_file] = data_frame(gtfs_file, path, extra_classes)
       end
 
-      GtfsDf::Feed.new(data, parse_times: parse_times)
+      GtfsDf::Feed.new(data, parse_times: parse_times, extra: extra)
     end
 
-    private_class_method def self.data_frame(gtfs_file, path)
+    private_class_method def self.data_frame(gtfs_file, path, extra_classes)
+      custom_class = extra_classes[gtfs_file.to_sym]
+      return custom_class.new(path).df if custom_class
+
       schema_class_name = gtfs_file.split("_").map(&:capitalize).join
       GtfsDf::Schema.const_get(schema_class_name).new(path).df
     end
 
     private_class_method def self.has_header?(zip_entry)
-      zip_entry.get_input_stream.readline.strip != ""
+      zip_entry
+        .get_input_stream
+        .readline
+        .delete_prefix("\xEF\xBB\xBF".b) # BOM
+        .strip != ""
     rescue
       false
     end

data/lib/gtfs_df/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module GtfsDf
-  VERSION = "0.11.1"
+  VERSION = "0.13.0"
 end

data/lib/gtfs_df/writer.rb

@@ -11,13 +11,13 @@ module GtfsDf
       require "zlib"
       FileUtils.mkdir_p(File.dirname(zip_path))
       Zip::File.open(zip_path, create: true) do |zipfile|
-        GtfsDf::Feed::GTFS_FILES.each do |file|
+        output_files(feed).each do |file|
           df = feed.send(file)
           next unless df.is_a?(Polars::DataFrame)
 
           # Convert time fields back to strings if parse_times was enabled
           if feed.parse_times
-            df = format_time_fields(file, df)
+            df = format_time_fields(file, df, feed)
           end
 
           # Write CSV to StringIO
@@ -37,26 +37,34 @@ module GtfsDf
     # @param dir_path [String] The path where the directory will be created
     def self.write_to_dir(feed, dir_path)
       FileUtils.mkdir_p(dir_path)
-      GtfsDf::Feed::GTFS_FILES.each do |file|
+      output_files(feed).each do |file|
         df = feed.send(file)
         next unless df.is_a?(Polars::DataFrame)
 
         # Convert time fields back to strings if parse_times was enabled
-        df = format_time_fields(file, df) if feed.parse_times
+        df = format_time_fields(file, df, feed) if feed.parse_times
 
         # Write CSV directly to file
         df.write_csv(File.join(dir_path, "#{file}.txt"))
       end
     end
 
+    def self.output_files(feed)
+      extra_files = feed.extra&.[](:files) || []
+      GtfsDf::Feed::GTFS_FILES + extra_files
+    end
+    private_class_method :output_files
+
     # Formats time fields back to HH:MM:SS strings for a given GTFS file
     #
     # @param file [String] The GTFS file name (e.g., "stop_times")
     # @param df [Polars::DataFrame] The DataFrame to format
     # @return [Polars::DataFrame] DataFrame with time fields formatted as strings
-    def self.format_time_fields(file, df)
-      schema_class_name = file.split("_").map(&:capitalize).join
-      schema_class = begin
+    def self.format_time_fields(file, df, feed = nil)
+      extra_classes = feed&.extra&.[](:classes) || {}
+      schema_class = extra_classes[file.to_sym]
+      schema_class ||= begin
+        schema_class_name = file.split("_").map(&:capitalize).join
         GtfsDf::Schema.const_get(schema_class_name)
       rescue
         nil

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gtfs_df
 version: !ruby/object:Gem::Version
-  version: 0.11.1
+  version: 0.13.0
 platform: ruby
 authors:
 - David Mejorado
@@ -74,7 +74,6 @@ files:
 - ".rspec"
 - ".rubocop.yml"
 - ".ruby-version"
-- ".solargraph.yml"
 - ".standard.yml"
 - CHANGELOG.md
 - LICENSE.txt

data/.solargraph.yml

@@ -1,26 +0,0 @@
----
-include:
-- Rakefile
-- Gemfile
-- "*.gemspec"
-- "**/*.rb"
-exclude:
-- spec/**/*
-- test/**/*
-- vendor/**/*
-- ".bundle/**/*"
-require: []
-domains: []
-reporters:
-- rubocop
-- require_not_found
-formatter:
-  rubocop:
-    cops: safe
-    except: []
-    only: []
-    extra_args: []
-    lint: true
-require_paths: []
-plugins: []
-max_files: 5000