swiss-netex 1.0.0

data/lib/swiss_netex/cli.rb

@@ -0,0 +1,546 @@
+# frozen_string_literal: true
+
+require "optparse"
+
+module SwissNetex
+  class CLI
+    EXIT_OK = 0
+    EXIT_USAGE = 64
+    EXIT_FAILURE = 1
+
+    COMMANDS = {
+      "download" => "Download the latest national NeTEx package",
+      "extract" => "Download (if needed) and filter by operator/line",
+      "filter" => "Filter a local package by operator/line",
+      "help" => "Show help for the CLI or a command",
+      "lines" => "List lines in a package (table/TSV)",
+      "operators" => "List operators in a package (table/TSV)",
+      "version" => "Print the swiss-netex version"
+    }.freeze
+
+    def self.start(argv = ARGV, stdout: $stdout, stderr: $stderr, http: nil)
+      new(stdout: stdout, stderr: stderr, http: http).run(argv)
+    end
+
+    def initialize(stdout: $stdout, stderr: $stderr, http: nil)
+      @stdout = stdout
+      @stderr = stderr
+      @http = http
+      @active_help = nil
+    end
+
+    def run(argv)
+      command, *args = argv
+      command = "help" if command.nil? || %w[-h --help].include?(command)
+      command = "version" if %w[-v --version].include?(command)
+
+      dispatch(command, args)
+    rescue Error => e
+      fail_error(e.message)
+    rescue OptionParser::ParseError => e
+      fail_usage(e.message)
+    rescue Zip::Error => e
+      fail_error("Corrupt or unreadable zip: #{e.message}")
+    end
+
+    private
+
+    def dispatch(command, args)
+      case command
+      when "download" then download(args)
+      when "extract" then extract(args)
+      when "filter" then filter(args)
+      when "operators" then operators(args)
+      when "lines" then lines(args)
+      when "help" then help(args.first)
+      when "version" then version
+      else unknown_command(command)
+      end
+    end
+
+    def version
+      @stdout.puts "swiss-netex #{VERSION}"
+      EXIT_OK
+    end
+
+    def help(command = nil)
+      if command.nil? || command.empty?
+        @stdout.puts top_level_help
+        return EXIT_OK
+      end
+
+      return unknown_command(command) unless COMMANDS.key?(command)
+
+      @stdout.puts command_help(command)
+      EXIT_OK
+    end
+
+    def download(argv)
+      @active_help = download_help
+      options = blank_download_options
+      leftover = download_parser(options).parse(argv)
+      return show_command_help(download_help) if options[:show_help]
+      return unexpected_args(leftover, download_help) if leftover.any?
+
+      dataset = resolve_download_dataset(options[:dataset], options[:year])
+      result = Download.new(
+        dataset: dataset,
+        output: options[:output],
+        cache_dir: options[:cache_dir],
+        force: options[:force],
+        http: @http || Http.new,
+        progress_io: @stderr
+      ).call
+
+      @stdout.puts result.path
+      EXIT_OK
+    end
+
+    def filter(argv)
+      @active_help = filter_help
+      options = blank_filter_options
+      leftover = filter_parser(options).parse(argv)
+      return show_command_help(filter_help) if options[:show_help]
+      return unexpected_args(leftover, filter_help) if leftover.any?
+
+      result = Filter.new(
+        from: options[:from],
+        output: options[:output],
+        operators: options[:operators],
+        lines: options[:lines],
+        allow_empty: options[:allow_empty],
+        verbose: options[:verbose],
+        progress_io: @stderr
+      ).call
+
+      @stdout.puts result.path
+      EXIT_OK
+    end
+
+    def extract(argv)
+      @active_help = extract_help
+      options = blank_extract_options
+      leftover = extract_parser(options).parse(argv)
+      return show_command_help(extract_help) if options[:show_help]
+      return unexpected_args(leftover, extract_help) if leftover.any?
+
+      reject_unused_download_opts_with_from!(options)
+      dataset = resolve_download_dataset(options[:dataset], options[:year])
+      result = Extract.new(
+        operators: options[:operators],
+        lines: options[:lines],
+        from: options[:from],
+        output: options[:output],
+        dataset: dataset,
+        cache_dir: options[:cache_dir],
+        force: options[:force],
+        allow_empty: options[:allow_empty],
+        verbose: options[:verbose],
+        http: @http,
+        progress_io: @stderr
+      ).call
+
+      @stdout.puts result.path
+      EXIT_OK
+    end
+
+    def operators(argv)
+      @active_help = operators_help
+      options = blank_discovery_options
+      leftover = operators_parser(options).parse(argv)
+      return show_command_help(operators_help) if options[:show_help]
+      return unexpected_args(leftover, operators_help) if leftover.any?
+
+      path = resolve_package_path(options)
+      index = Operators.load(path)
+      raise Error, "No operators found in #{path}" if index.empty?
+
+      Tsv.write(@stdout, Operators::HEADERS, index.rows)
+      EXIT_OK
+    end
+
+    def lines(argv)
+      @active_help = lines_help
+      options = blank_discovery_options.merge(operators: [], lines: [])
+      leftover = lines_parser(options).parse(argv)
+      return show_command_help(lines_help) if options[:show_help]
+      return unexpected_args(leftover, lines_help) if leftover.any?
+
+      path = resolve_package_path(options)
+      entries = Lines.load(
+        path,
+        operators: options[:operators],
+        lines: options[:lines]
+      )
+      raise Error, "No lines matched" if entries.empty?
+
+      Tsv.write(@stdout, Lines::HEADERS, entries.map(&:to_row))
+      EXIT_OK
+    end
+
+    def blank_download_options
+      {
+        dataset: nil,
+        year: nil,
+        output: nil,
+        cache_dir: Download::DEFAULT_CACHE_DIR,
+        cache_dir_set: false,
+        force: false,
+        show_help: false
+      }
+    end
+
+    def blank_filter_options
+      {
+        from: nil,
+        output: nil,
+        operators: [],
+        lines: [],
+        allow_empty: false,
+        verbose: false,
+        show_help: false
+      }
+    end
+
+    def blank_extract_options
+      blank_filter_options.merge(blank_package_source_options)
+    end
+
+    def blank_discovery_options
+      blank_package_source_options.merge(from: nil, show_help: false)
+    end
+
+    def blank_package_source_options
+      {
+        dataset: nil,
+        year: nil,
+        cache_dir: Download::DEFAULT_CACHE_DIR,
+        cache_dir_set: false,
+        force: false
+      }
+    end
+
+    def download_parser(options)
+      OptionParser.new do |opts|
+        opts.banner = <<~BANNER.chomp
+          Usage: swiss-netex download [options]
+
+          Download the latest Swiss national NeTEx zip from opentransportdata.swiss.
+          Resource downloads are public; no API token is required on the happy path.
+        BANNER
+        opts.separator ""
+        opts.separator "Options:"
+        add_dataset_options(opts, options)
+        opts.on("-o", "--output PATH", "--out PATH",
+                "Output file or directory (default: under --cache-dir)") do |value|
+          options[:output] = value
+        end
+        add_cache_dir_option(opts, options)
+        add_force_option(opts, options)
+        add_help_option(opts, options)
+        opts.separator ""
+        opts.separator "Prints the downloaded (or cached) file path on stdout."
+        opts.separator "Progress and status messages go to stderr."
+      end
+    end
+
+    def filter_parser(options)
+      OptionParser.new do |opts|
+        opts.banner = <<~BANNER.chomp
+          Usage: swiss-netex filter [options]
+
+          Filter a local Swiss national NeTEx package (zip or directory) by operator
+          and optional line. Writes a smaller zip with transitive frame contents.
+        BANNER
+        opts.separator ""
+        opts.separator "Options:"
+        add_from_option(opts, options, required_note: true)
+        add_output_option(opts, options)
+        add_operator_option(opts, options)
+        add_line_option(opts, options)
+        add_allow_empty_option(opts, options)
+        add_verbose_option(opts, options)
+        add_help_option(opts, options)
+        opts.separator ""
+        opts.separator "Prints the output zip path on stdout."
+        opts.separator "Progress and status messages go to stderr."
+        opts.separator "--operator may be repeated (union of operators)."
+        opts.separator "National filter needs several GB of free RAM (TIMETABLE streaming)."
+      end
+    end
+
+    def extract_parser(options)
+      OptionParser.new do |opts|
+        opts.banner = <<~BANNER.chomp
+          Usage: swiss-netex extract [options]
+
+          Download the latest national package when --from is omitted, then filter by
+          operator/line. Equivalent to download + filter for the common workflow.
+        BANNER
+        opts.separator ""
+        opts.separator "Options:"
+        add_operator_option(opts, options)
+        add_line_option(opts, options)
+        add_from_option(opts, options, required_note: false)
+        add_dataset_options(opts, options)
+        add_output_option(opts, options)
+        add_cache_dir_option(opts, options)
+        add_force_option(opts, options)
+        add_allow_empty_option(opts, options)
+        add_verbose_option(opts, options)
+        add_help_option(opts, options)
+        opts.separator ""
+        opts.separator "Prints the output zip path on stdout."
+        opts.separator "Progress and status messages go to stderr."
+      end
+    end
+
+    def operators_parser(options)
+      OptionParser.new do |opts|
+        opts.banner = <<~BANNER.chomp
+          Usage: swiss-netex operators [options]
+
+          List operators from the RESOURCE frame on stdout (TSV when piped;
+          aligned columns on a TTY). Scans RESOURCE only (no TIMETABLE).
+          Omit --from to download (or reuse cache) first.
+        BANNER
+        opts.separator ""
+        opts.separator "Options:"
+        add_from_option(opts, options, required_note: false)
+        add_dataset_options(opts, options)
+        add_cache_dir_option(opts, options)
+        add_force_option(opts, options)
+        add_help_option(opts, options)
+        opts.separator ""
+        opts.separator "Columns: #{Operators::HEADERS.join(", ")}"
+      end
+    end
+
+    def lines_parser(options)
+      OptionParser.new do |opts|
+        opts.banner = <<~BANNER.chomp
+          Usage: swiss-netex lines [options]
+
+          List lines from the SERVICE frame on stdout (TSV when piped; aligned
+          columns on a TTY). Scans SERVICE only. Optional --operator / --line
+          narrow the list (same alias rules as filter). Omit --from to download
+          (or reuse cache) first.
+        BANNER
+        opts.separator ""
+        opts.separator "Options:"
+        add_from_option(opts, options, required_note: false)
+        add_dataset_options(opts, options)
+        add_cache_dir_option(opts, options)
+        add_force_option(opts, options)
+        add_operator_option(opts, options)
+        add_line_option(opts, options)
+        add_help_option(opts, options)
+        opts.separator ""
+        opts.separator "Columns: #{Lines::HEADERS.join(", ")}"
+      end
+    end
+
+    def add_from_option(opts, options, required_note:)
+      label =
+        if required_note
+          "Source package zip or extracted directory (required)"
+        else
+          "Local package zip/dir (skips download when set)"
+        end
+      opts.on("--from PATH", label) { |value| options[:from] = value }
+    end
+
+    def add_output_option(opts, options)
+      opts.on("-o", "--output PATH", "--out PATH",
+              "Output zip path or directory (bare names get .zip)") do |value|
+        options[:output] = value
+      end
+    end
+
+    def add_operator_option(opts, options)
+      opts.on("-O", "--operator ALIAS",
+              "Operator alias (GO, NeTEx id, SBOID, ShortName); repeatable") do |value|
+        options[:operators] << value
+      end
+    end
+
+    def add_line_option(opts, options)
+      opts.on("-l", "--line ALIAS", "--lines ALIAS",
+              "Optional line filter (PublicCode, id, SLNID, ShortName, Name);",
+              "repeatable") do |value|
+        options[:lines] << value
+      end
+    end
+
+    def add_allow_empty_option(opts, options)
+      opts.on("--allow-empty",
+              "Write output even when no lines or ServiceJourneys match") do
+        options[:allow_empty] = true
+      end
+    end
+
+    def add_verbose_option(opts, options)
+      opts.on("-v", "--verbose",
+              "Per-shard and per-frame keep/drop counters on stderr") do
+        options[:verbose] = true
+      end
+    end
+
+    def add_help_option(opts, options)
+      opts.on("-h", "--help", "Show this help") { options[:show_help] = true }
+    end
+
+    def add_dataset_options(opts, options)
+      opts.on("-d", "--dataset URL_OR_SLUG",
+              "Dataset page URL or slug",
+              "(default: latest timetablenetex_YYYY from catalog)") do |value|
+        options[:dataset] = value
+      end
+      opts.on("-y", "--year YYYY",
+              "Shortcut for --dataset timetablenetex_YYYY",
+              "(mutually exclusive with --dataset)") do |value|
+        options[:year] = value
+      end
+    end
+
+    def add_cache_dir_option(opts, options)
+      opts.on("-c", "--cache-dir PATH",
+              "Cache directory when downloading",
+              "(default: #{Download::DEFAULT_CACHE_DIR})") do |value|
+        options[:cache_dir] = value
+        options[:cache_dir_set] = true
+      end
+    end
+
+    def add_force_option(opts, options)
+      opts.on("-f", "--force", "Re-download even if a cache file already exists") do
+        options[:force] = true
+      end
+    end
+
+    def download_help = download_parser({}).help.rstrip
+    def filter_help = filter_parser(blank_filter_options).help.rstrip
+    def extract_help = extract_parser(blank_extract_options).help.rstrip
+    def operators_help = operators_parser(blank_discovery_options).help.rstrip
+
+    def lines_help
+      lines_parser(blank_discovery_options.merge(operators: [], lines: [])).help.rstrip
+    end
+
+    def resolve_package_path(options)
+      reject_unused_download_opts_with_from!(options)
+      dataset = resolve_download_dataset(options[:dataset], options[:year])
+      PackageSource.resolve(
+        from: options[:from],
+        dataset: dataset,
+        cache_dir: options[:cache_dir],
+        force: options[:force],
+        http: @http,
+        progress_io: @stderr
+      )
+    end
+
+    def resolve_download_dataset(dataset, year)
+      if dataset && year
+        raise OptionParser::ParseError, "--dataset and --year are mutually exclusive"
+      end
+
+      return dataset if dataset
+      return nil if year.nil?
+
+      unless year.match?(/\A\d{4}\z/)
+        raise OptionParser::ParseError, "--year must be a 4-digit Fahrplanjahr (e.g. 2026)"
+      end
+
+      "timetablenetex_#{year}"
+    end
+
+    # --from skips download; reject flags that would otherwise be silently ignored.
+    def reject_unused_download_opts_with_from!(options)
+      return if options[:from].to_s.strip.empty?
+
+      used = []
+      used << "--dataset" if options[:dataset]
+      used << "--year" if options[:year]
+      used << "--force" if options[:force]
+      used << "--cache-dir" if options[:cache_dir_set]
+      return if used.empty?
+
+      raise OptionParser::ParseError,
+            "--from cannot be combined with #{used.join(", ")}"
+    end
+
+    def show_command_help(text)
+      @stdout.puts text
+      EXIT_OK
+    end
+
+    def unexpected_args(leftover, help_text)
+      @stderr.puts "Error: unexpected argument(s): #{leftover.join(" ")}"
+      @stderr.puts
+      @stderr.puts help_text
+      EXIT_USAGE
+    end
+
+    def unknown_command(command)
+      @stderr.puts "Unknown command: #{command}"
+      @stderr.puts
+      @stderr.puts top_level_help
+      EXIT_USAGE
+    end
+
+    def fail_error(message)
+      @stderr.puts "Error: #{message}"
+      EXIT_FAILURE
+    end
+
+    def fail_usage(message)
+      @stderr.puts "Error: #{message}"
+      @stderr.puts
+      @stderr.puts(@active_help || top_level_help)
+      EXIT_USAGE
+    end
+
+    def top_level_help
+      lines = [
+        "Usage: swiss-netex <command> [options]",
+        "",
+        "CLI for Swiss national NeTEx timetable packages",
+        "(opentransportdata.swiss): download, inspect, and filter by operator/line.",
+        "",
+        "Commands:"
+      ]
+
+      COMMANDS.each do |name, summary|
+        lines << format("  %<name>-12s %<summary>s", name: name, summary: summary)
+      end
+
+      (lines + [
+        "",
+        "Run `swiss-netex help <command>` for command-specific usage."
+      ]).join("\n")
+    end
+
+    def command_help(command)
+      case command
+      when "download" then download_help
+      when "filter" then filter_help
+      when "extract" then extract_help
+      when "operators" then operators_help
+      when "lines" then lines_help
+      when "help"
+        <<~HELP.chomp
+          Usage: swiss-netex help [command]
+
+          Show top-level help, or detailed usage for a command.
+        HELP
+      when "version"
+        <<~HELP.chomp
+          Usage: swiss-netex version
+
+          Print the installed swiss-netex version.
+        HELP
+      end
+    end
+  end
+end

data/lib/swiss_netex/dataset_page.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require "uri"
+
+module SwissNetex
+  # Parses an opentransportdata.swiss CKAN dataset HTML page and picks the
+  # newest national NeTEx package by the export timestamp embedded in the
+  # filename (`…_YYYYMMDDHHMM.zip`).
+  class DatasetPage
+    HREF_RE = /href\s*=\s*["']([^"']+)["']/i
+    PACKAGE_NAME_RE = /\A(PROD_NETEX_TT_.+\.zip)\z/i
+    TIMESTAMP_RE = /_(\d{12})\.zip\z/i
+
+    Resource = Data.define(:url, :filename, :timestamp)
+
+    def self.latest(html, base_url:)
+      new(html, base_url: base_url).latest
+    end
+
+    def initialize(html, base_url:)
+      @html = html.to_s
+      @base_url = base_url.to_s.chomp("/")
+    end
+
+    def resources
+      seen = {}
+      @html.scan(HREF_RE).filter_map do |(href)|
+        resource = resource_from_href(unescape_attr(href))
+        next unless resource
+        next if seen[resource.url]
+
+        seen[resource.url] = true
+        resource
+      end
+    end
+
+    def latest
+      list = resources
+      raise Error, "No PROD_NETEX_TT_*.zip resource found on dataset page" if list.empty?
+
+      list.max_by(&:timestamp)
+    end
+
+    private
+
+    def resource_from_href(href)
+      uri = absolute_uri(href)
+      return unless uri
+
+      filename = File.basename(uri.path.to_s)
+      match = PACKAGE_NAME_RE.match(filename)
+      return unless match
+
+      timestamp = TIMESTAMP_RE.match(filename)&.captures&.first
+      return unless timestamp
+
+      Resource.new(
+        url: uri.to_s,
+        filename: canonicalize_filename(match[1]),
+        timestamp: timestamp
+      )
+    rescue URI::InvalidURIError
+      nil
+    end
+
+    def absolute_uri(href)
+      return if href.nil? || href.empty? || href.start_with?("#", "javascript:")
+
+      uri = URI.parse(href)
+      return uri if uri.absolute?
+
+      URI.join("#{@base_url}/", href)
+    end
+
+    # CKAN download paths are lowercased; keep the conventional display casing.
+    def canonicalize_filename(name)
+      stem = name.sub(/\.zip\z/i, "")
+      "#{stem.upcase}.zip"
+    end
+
+    def unescape_attr(value)
+      value
+        .gsub("&", "&")
+        .gsub(""", "\"")
+        .gsub("'", "'")
+        .gsub("&lt;", "<")
+        .gsub("&gt;", ">")
+    end
+  end
+end