swiss-netex 1.0.0

data/lib/swiss_netex/filter/xml_sieve.rb

@@ -0,0 +1,189 @@
+# frozen_string_literal: true
+
+require "nokogiri"
+require "stringio"
+require "tempfile"
+
+module SwissNetex
+  class Filter
+    # Stream an XML document and keep/drop selected entity elements by local name.
+    #
+    # Yields an {Entity} so callers can decide from attributes (or lazy outer_xml)
+    # without always materializing dropped subtrees. Non-filterable structure is
+    # copied through. Output goes to a Tempfile by default.
+    #
+    # Pure whitespace between siblings is held until the next keep or structural
+    # node. A drop clears that buffer so filtered packages do not retain blank
+    # "holes" where removed entities used to sit (no full DOM pretty-print).
+    class XmlSieve
+      TYPE_ELEMENT = Nokogiri::XML::Reader::TYPE_ELEMENT
+      TYPE_END = Nokogiri::XML::Reader::TYPE_END_ELEMENT
+      TYPE_TEXT = Nokogiri::XML::Reader::TYPE_TEXT
+      TYPE_CDATA = Nokogiri::XML::Reader::TYPE_CDATA
+      TYPE_WS = Nokogiri::XML::Reader::TYPE_WHITESPACE
+      TYPE_SWS = Nokogiri::XML::Reader::TYPE_SIGNIFICANT_WHITESPACE
+      TYPE_COMMENT = Nokogiri::XML::Reader::TYPE_COMMENT
+      PURE_WS = /\A\s*\z/
+
+      Result = Data.define(:io, :kept, :dropped)
+
+      # Read-only view of the current filterable element.
+      class Entity
+        attr_reader :local_name, :id
+
+        def initialize(reader)
+          @reader = reader
+          @local_name = reader.local_name
+          @id = reader.attribute("id")
+        end
+
+        def outer_xml
+          @outer_xml ||= @reader.outer_xml.to_s
+        end
+
+        def to_s = outer_xml
+      end
+
+      def initialize(filterable:)
+        @filterable = filterable.to_h { |name| [name.to_s, true] }
+      end
+
+      # Returns Filter::XmlSieve::Result. Caller owns `io` (rewinded Tempfile by default).
+      def call(source, out: nil, &)
+        reader = Nokogiri::XML::Reader(to_io(source))
+        out = new_tempfile if out.nil?
+        out << "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n"
+        @pending_ws = +""
+        kept = 0
+        dropped = 0
+
+        while reader.read
+          case handle_node(reader, out, &)
+          when :kept then kept += 1
+          when :dropped then dropped += 1
+          end
+        end
+
+        flush_pending_ws(out)
+        out.flush
+        out.rewind if out.respond_to?(:rewind)
+        Result.new(io: out, kept: kept, dropped: dropped)
+      end
+
+      private
+
+      def new_tempfile
+        file = Tempfile.new(["swiss-netex-sieve", ".xml"])
+        file.binmode
+        file
+      end
+
+      def handle_node(reader, out, &)
+        case reader.node_type
+        when TYPE_ELEMENT
+          handle_element(reader, out, &)
+        when TYPE_END
+          flush_pending_ws(out)
+          out << "</#{reader.name}>"
+          :structure
+        when TYPE_WS, TYPE_SWS
+          buffer_ws(reader.value.to_s)
+          :structure
+        when TYPE_TEXT
+          handle_text(reader, out)
+        when TYPE_CDATA
+          flush_pending_ws(out)
+          out << reader.value.to_s
+          :structure
+        when TYPE_COMMENT
+          flush_pending_ws(out)
+          out << "<!--#{reader.value}-->"
+          :structure
+        end
+      end
+
+      def handle_text(reader, out)
+        value = reader.value.to_s
+        if pure_ws?(value)
+          buffer_ws(value)
+        else
+          flush_pending_ws(out)
+          out << value
+        end
+        :structure
+      end
+
+      def handle_element(reader, out, &)
+        if @filterable[reader.local_name]
+          filter_entity(reader, out, &)
+        else
+          flush_pending_ws(out)
+          write_start(reader, out)
+          :structure
+        end
+      end
+
+      def filter_entity(reader, out)
+        entity = Entity.new(reader)
+        if yield(entity)
+          flush_pending_ws(out)
+          out << entity.outer_xml
+          skip_element(reader)
+          :kept
+        else
+          clear_pending_ws
+          skip_element(reader)
+          :dropped
+        end
+      end
+
+      def buffer_ws(value)
+        @pending_ws << value
+      end
+
+      def flush_pending_ws(out)
+        return if @pending_ws.empty?
+
+        out << @pending_ws
+        @pending_ws.clear
+      end
+
+      def clear_pending_ws
+        @pending_ws.clear
+      end
+
+      def pure_ws?(value)
+        value.match?(PURE_WS)
+      end
+
+      def write_start(reader, out)
+        out << "<#{reader.name}"
+        reader.attributes.each { |key, value| out << " #{key}=\"#{esc_attr(value)}\"" }
+        out << (reader.empty_element? ? "/>" : ">")
+      end
+
+      def skip_element(reader)
+        return if reader.empty_element?
+
+        depth = reader.depth
+        while reader.read
+          break if reader.node_type == TYPE_END && reader.depth == depth
+        end
+      end
+
+      def esc_attr(value)
+        value.to_s
+             .gsub("&", "&amp;")
+             .gsub('"', "&quot;")
+             .gsub("<", "&lt;")
+      end
+
+      def to_io(source)
+        return StringIO.new(source) if source.is_a?(String)
+        return source if source.respond_to?(:read)
+
+        raise Error, "XmlSieve expects a String or IO-like object"
+      end
+    end
+  end
+end

data/lib/swiss_netex/filter.rb

@@ -0,0 +1,346 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "tmpdir"
+require "zip"
+
+require_relative "filter/refs"
+require_relative "filter/xml_sieve"
+require_relative "filter/service_lines"
+require_relative "filter/timetable"
+require_relative "filter/timetable_batch"
+require_relative "filter/common_frame"
+require_relative "filter/frames"
+require_relative "filter/frame_worker"
+
+module SwissNetex
+  # Filter a national NeTEx package down to selected operator(s) / line(s).
+  #
+  # Pipeline: resolve operators → SERVICE lines → TIMETABLE journeys (+ refs) →
+  # resolve StopPlaces via assignments → rewrite support frames → zip + README.
+  class Filter
+    Result = Data.define(
+      :path,
+      :operator_ids,
+      :line_ids,
+      :journey_count,
+      :line_count,
+      :timetable_shards,
+      :counts
+    )
+
+    def initialize(
+      from:,
+      operators:,
+      output:,
+      lines: [],
+      progress_io: nil,
+      allow_empty: false,
+      verbose: false
+    )
+      @from = from.to_s
+      @operator_aliases = Array(operators)
+      @line_aliases = Array(lines)
+      @output = output.to_s
+      @progress_io = progress_io
+      @allow_empty = allow_empty
+      @verbose = verbose
+    end
+
+    def call
+      validate!
+      operator_ids = resolve_operators
+      refs = Refs.new
+      refs.add(:operators, operator_ids)
+
+      Dir.mktmpdir("swiss-netex-filter-") do |tmp|
+        line_ids = select_lines(operator_ids, refs)
+        ensure_lines!(line_ids)
+
+        journey_count, shard_names = filter_timetables(tmp, operator_ids, line_ids, refs)
+        ensure_journeys!(journey_count)
+
+        expand_stop_places(refs)
+        expand_day_types(refs)
+        write_support_frames(tmp, refs)
+        write_readme(tmp, operator_ids, line_ids, journey_count, shard_names, refs)
+        path = write_zip(tmp)
+        log_summary(path, operator_ids, line_ids, journey_count, shard_names, refs)
+
+        Result.new(
+          path: path,
+          operator_ids: operator_ids,
+          line_ids: line_ids,
+          journey_count: journey_count,
+          line_count: line_ids.size,
+          timetable_shards: shard_names.size,
+          counts: refs.counts
+        )
+      end
+    end
+
+    private
+
+    def validate!
+      raise Error, "Missing --from package path" if @from.empty?
+      raise Error, "Package path not found: #{@from}" unless File.exist?(@from)
+      raise Error, "Missing --output path" if @output.empty?
+      raise Error, "No --operator given" if @operator_aliases.empty?
+    end
+
+    def resolve_operators
+      log "Indexing operators…"
+      Operators.load(@from).resolve(@operator_aliases)
+    end
+
+    def select_lines(operator_ids, refs)
+      log "Selecting SERVICE lines…"
+      lines = {}
+      Package.open(@from) do |pkg|
+        pkg.open_one(:service) do |io|
+          lines = ServiceLines.new(
+            operator_ids: operator_ids,
+            line_aliases: @line_aliases,
+            refs: refs
+          ).call(io)
+        end
+      end
+
+      log "  kept #{lines.size} line(s)"
+      lines.keys
+    end
+
+    def ensure_lines!(line_ids)
+      return if line_ids.any?
+      return if @allow_empty
+
+      raise Error,
+            "No lines matched the operator/line filter " \
+            "(use --allow-empty to write an empty package)"
+    end
+
+    def filter_timetables(tmp, operator_ids, line_ids, refs)
+      names = Package.open(@from) { |pkg| pkg.names_for(:timetable) }
+      log "Filtering #{names.size} TIMETABLE shard(s)…"
+      TimetableBatch.new(
+        package_path: @from,
+        tmp: tmp,
+        operator_ids: operator_ids,
+        line_ids: line_ids,
+        line_filter: @line_aliases.any?,
+        progress_io: @progress_io,
+        verbose: @verbose
+      ).call(names, refs)
+    end
+
+    def ensure_journeys!(count)
+      return if count.positive?
+      return if @allow_empty
+
+      raise Error,
+            "No ServiceJourneys matched the filter (use --allow-empty to write anyway)"
+    end
+
+    def expand_stop_places(refs)
+      return if refs.scheduled_stop_points.empty?
+
+      log "Resolving StopPlace refs from SERVICE assignments…"
+      Package.open(@from) do |pkg|
+        pkg.open_one(:service) do |io|
+          collect_stop_place_refs(io, refs)
+        end
+      end
+      log "  stop places: #{refs.size(:stop_places)}"
+    end
+
+    def collect_stop_place_refs(io, refs)
+      reader = Nokogiri::XML::Reader(io)
+      while reader.read
+        next unless assignment_element?(reader)
+
+        xml = reader.outer_xml
+        ssp = xml[/<ScheduledStopPointRef\b[^>]*\bref="([^"]+)"/, 1]
+        next unless ssp && refs.include?(:scheduled_stop_points, ssp)
+
+        stop = xml[/<StopPlaceRef\b[^>]*\bref="([^"]+)"/, 1]
+        refs.add(:stop_places, stop) if stop
+      end
+    end
+
+    def assignment_element?(reader)
+      reader.node_type == Nokogiri::XML::Reader::TYPE_ELEMENT &&
+        reader.local_name == "PassengerStopAssignment"
+    end
+
+    # DayTypes often sit before AvailabilityConditions; scan kept ACs first so
+    # DayType/DayTypeAssignment keep decisions see nested DayTypeRef values.
+    def expand_day_types(refs)
+      return if refs.availability_conditions.empty?
+
+      log "Resolving DayType refs from SERVICECALENDAR…"
+      Package.open(@from) do |pkg|
+        pkg.open_one(:service_calendar) do |io|
+          collect_day_type_refs(io, refs)
+        end
+      end
+      log "  day types: #{refs.size(:day_types)}"
+    end
+
+    def collect_day_type_refs(io, refs)
+      reader = Nokogiri::XML::Reader(io)
+      while reader.read
+        next unless ac_element?(reader)
+
+        xml = reader.outer_xml
+        id = xml[/\bid="([^"]+)"/, 1]
+        next unless id && refs.include?(:availability_conditions, id)
+
+        refs.merge_from_xml(xml)
+      end
+    end
+
+    def ac_element?(reader)
+      reader.node_type == Nokogiri::XML::Reader::TYPE_ELEMENT &&
+        reader.local_name == "AvailabilityCondition"
+    end
+
+    def write_support_frames(tmp, refs)
+      log "Rewriting support frames…"
+      names = Package.open(@from) do |pkg|
+        {
+          resource: pkg.names_for(:resource).first,
+          service: pkg.names_for(:service).first,
+          site: pkg.names_for(:site).first,
+          service_calendar: pkg.names_for(:service_calendar).first,
+          common: pkg.names_for(:common).first
+        }
+      end
+
+      pairs = names.filter_map { |kind, name| [kind, name] if name }
+      worker = FrameWorker.new(package_path: @from, tmp: tmp, refs: refs)
+      results = worker.call_many(pairs)
+      return unless @verbose
+
+      pairs.each do |kind,|
+        result = results.fetch(kind)
+        log "  #{kind}: kept #{result.fetch("kept")}, dropped #{result.fetch("dropped")}"
+      end
+    end
+
+    def write_readme(tmp, operator_ids, line_ids, journey_count, shard_names, refs)
+      File.write(
+        File.join(tmp, "README.txt"),
+        readme_body(operator_ids, line_ids, journey_count, shard_names, refs)
+      )
+    end
+
+    def readme_body(operator_ids, line_ids, journey_count, shard_names, refs)
+      <<~TEXT
+        swiss-netex filtered package
+        ============================
+
+        Tool version: #{VERSION}
+        Source: #{@from}
+        Generated: #{Time.now.utc.strftime("%Y-%m-%dT%H:%M:%SZ")}
+
+        Filters
+        -------
+        Operators: #{operator_ids.join(", ")}
+        Operator aliases: #{@operator_aliases.join(", ")}
+        Lines filter: #{lines_filter_label}
+        Line ids kept: #{line_ids.size}
+
+        Counts
+        ------
+        ServiceJourneys: #{journey_count}
+        TIMETABLE shards: #{shard_names.size}
+        ScheduledStopPoints: #{refs.size(:scheduled_stop_points)}
+        StopPlaces: #{refs.size(:stop_places)}
+        DestinationDisplays: #{refs.size(:destination_displays)}
+        AvailabilityConditions: #{refs.size(:availability_conditions)}
+
+        Notes
+        -----
+        - Timetable data remains under OpenTransportData terms.
+        - This package is a subset of the Swiss national NeTEx export.
+        - Empty TIMETABLE shards were omitted; kept shards keep their original
+          national indices (gaps are normal).
+        - Operating days often use AvailabilityCondition ValidDayBits rather
+          than DayType / DayTypeRef. DayTypes: #{refs.size(:day_types)} collected.
+        - COMMON JourneyMeeting / InterchangeRule entities are kept only when
+          every tracked journey, stop, and line ref still resolves in this
+          package. Cross-operator rules disappear unless both sides are in the
+          filter; an empty connection section is valid.
+      TEXT
+    end
+
+    def lines_filter_label
+      @line_aliases.empty? ? "(all lines for operators)" : @line_aliases.join(", ")
+    end
+
+    def write_zip(tmp)
+      destination = resolve_output_path
+      FileUtils.mkdir_p(File.dirname(destination))
+      # Write beside the destination then rename so an interrupted run cannot
+      # leave a truncated zip where a previous good output was.
+      temp_path = "#{destination}.partial.#{Process.pid}"
+
+      begin
+        Zip::File.open(temp_path, create: true) do |zip|
+          Dir.children(tmp).sort.each do |name|
+            path = File.join(tmp, name)
+            next unless File.file?(path)
+
+            zip.add(name, path)
+          end
+        end
+        File.rename(temp_path, destination)
+      ensure
+        FileUtils.rm_f(temp_path)
+      end
+
+      log "Wrote #{destination}" unless @verbose
+      destination
+    end
+
+    def resolve_output_path
+      expanded = File.expand_path(@output)
+      return expanded if @output.match?(/\.zip\z/i)
+      return File.join(expanded, default_output_name) if directory_output?
+      return "#{expanded}.zip" if bare_output_stem?
+
+      expanded
+    end
+
+    def directory_output?
+      File.directory?(@output) || @output.end_with?("/")
+    end
+
+    # `-o tl` → tl.zip. Explicit extensions (`tl.tar`) and dirs stay unchanged.
+    def bare_output_stem?
+      File.extname(@output).empty?
+    end
+
+    def default_output_name
+      base = File.basename(@from, ".*")
+      slug = @operator_aliases.map { |alias_name| alias_name.to_s.gsub(/[^\w.-]+/, "-") }.join("-")
+      "#{base}_op-#{slug}.zip"
+    end
+
+    def log_summary(path, operator_ids, line_ids, journey_count, shard_names, refs)
+      log "Summary: operators=#{operator_ids.size} lines=#{line_ids.size} " \
+          "journeys=#{journey_count} timetable_shards=#{shard_names.size} " \
+          "stops=#{refs.size(:stop_places)}"
+      return unless @verbose
+
+      refs.counts.sort_by { |name, _| name.to_s }.each do |name, count|
+        log "  refs.#{name}: #{count}"
+      end
+      log "Wrote #{path}"
+    end
+
+    def log(message)
+      @progress_io&.puts(message)
+    end
+  end
+end

data/lib/swiss_netex/http.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "openssl"
+require "uri"
+
+module SwissNetex
+  # Minimal stdlib HTTP helper. Follows redirects and streams response bodies.
+  # Inject a stub in tests instead of hitting the network.
+  #
+  # Note: Cloudflare R2 signed URLs used by opentransportdata.swiss reject HEAD
+  # (403). Size comes from the streaming GET's Content-Length header instead.
+  class Http
+    DEFAULT_OPEN_TIMEOUT = 30
+    DEFAULT_READ_TIMEOUT = 60
+    DEFAULT_REDIRECT_LIMIT = 10
+
+    NETWORK_ERRORS = [
+      SocketError,
+      Timeout::Error,
+      Errno::ECONNREFUSED,
+      Errno::ECONNRESET,
+      Errno::EHOSTUNREACH,
+      OpenSSL::SSL::SSLError,
+      EOFError,
+      Net::OpenTimeout,
+      Net::ReadTimeout
+    ].freeze
+
+    def initialize(open_timeout: DEFAULT_OPEN_TIMEOUT, read_timeout: DEFAULT_READ_TIMEOUT)
+      @open_timeout = open_timeout
+      @read_timeout = read_timeout
+    end
+
+    def get_body(url)
+      request(url, &:body)
+    end
+
+    # Streams the response body.
+    #
+    # If a block is given, it is called as `block.call(chunk, content_length)`
+    # for each body chunk. `content_length` is the response Content-Length
+    # (Integer) or nil — the same value on every call.
+    #
+    # Returns the Content-Length Integer, or nil when absent.
+    def stream(url, &block)
+      content_length = nil
+
+      request(url) do |response|
+        content_length = parse_content_length(response)
+        response.read_body do |chunk|
+          block&.call(chunk, content_length)
+        end
+      end
+
+      content_length
+    end
+
+    private
+
+    def request(url, redirect_limit: DEFAULT_REDIRECT_LIMIT, &)
+      raise Error, "Too many HTTP redirects" if redirect_limit.negative?
+
+      uri = parse_http_uri(url)
+
+      Net::HTTP.start(
+        uri.host,
+        uri.port,
+        use_ssl: uri.scheme == "https",
+        open_timeout: @open_timeout,
+        read_timeout: @read_timeout
+      ) do |http|
+        # Identity encoding keeps Content-Length equal to yielded body bytes.
+        # Net::HTTP would otherwise decode gzip while CL stays wire size, which
+        # breaks Download size verification. Name is get (not request) so the
+        # recursive request(...) call below is not shadowed by a local.
+        get = Net::HTTP::Get.new(uri)
+        get["Accept-Encoding"] = "identity"
+
+        http.request(get) do |response|
+          case response
+          when Net::HTTPRedirection
+            location = response["location"]
+            raise Error, "Redirect without Location from #{uri}" if location.nil? || location.empty?
+
+            return request(
+              absolute_redirect(uri, location),
+              redirect_limit: redirect_limit - 1,
+              &
+            )
+          when Net::HTTPSuccess
+            return yield(response)
+          else
+            raise Error, "HTTP #{response.code} for #{uri}"
+          end
+        end
+      end
+    rescue Error
+      raise
+    rescue *NETWORK_ERRORS => e
+      raise Error, "HTTP request failed for #{url}: #{e.message}"
+    end
+
+    def parse_http_uri(url)
+      uri = URI.parse(url.to_s)
+      raise Error, "URL must be HTTP(S): #{url}" unless uri.is_a?(URI::HTTP)
+      raise Error, "URL must include a host: #{url}" if uri.host.to_s.empty?
+
+      uri
+    rescue URI::InvalidURIError
+      raise Error, "Invalid URL: #{url}"
+    end
+
+    def absolute_redirect(from_uri, location)
+      redirect = URI.parse(location)
+      return redirect.to_s if redirect.absolute?
+
+      URI.join(from_uri, location).to_s
+    end
+
+    def parse_content_length(response)
+      value = response["content-length"]
+      return if value.nil? || value.empty?
+
+      Integer(value)
+    rescue ArgumentError
+      nil
+    end
+  end
+end