swiss-netex 1.0.0

data/lib/swiss_netex/download.rb

@@ -0,0 +1,239 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "uri"
+require "zip"
+
+module SwissNetex
+  # Resolve the latest national NeTEx zip from a dataset page and stream it to
+  # disk. Shared by the `download` CLI command and later `extract`.
+  class Download
+    DEFAULT_CACHE_DIR = File.join(Dir.home, ".cache", "swiss-netex").freeze
+    DATA_HOST = "https://data.opentransportdata.swiss"
+    CATALOG_SEARCH_URL = "#{DATA_HOST}/en/dataset/?q=timetablenetex".freeze
+    PROGRESS_INTERVAL_BYTES = 1024 * 1024 # 1 MiB
+
+    Result = Data.define(:path, :url, :filename, :bytes, :cached, :dataset)
+
+    # dataset: nil → discover highest published timetablenetex_YYYY from catalog.
+    def initialize(
+      dataset: nil,
+      output: nil,
+      cache_dir: DEFAULT_CACHE_DIR,
+      force: false,
+      http: Http.new,
+      progress_io: nil
+    )
+      @dataset = dataset
+      @output = output
+      @cache_dir = cache_dir
+      @force = force
+      @http = http
+      @progress_io = progress_io
+    end
+
+    def call
+      page_url, dataset_slug = resolve_dataset_page
+      log "Resolving latest package from #{page_url}…"
+
+      html = @http.get_body(page_url)
+      resource = DatasetPage.latest(html, base_url: page_origin(page_url))
+      destination = resolve_destination(resource.filename)
+
+      # Package filenames embed the export timestamp, so a non-empty file with
+      # the same name is a cache hit. OTD's R2 signed URLs reject HEAD, so we
+      # cannot cheaply verify remote size without starting a full GET.
+      if !@force && cache_hit?(destination)
+        bytes = File.size(destination)
+        log "Using cached package (#{human_bytes(bytes)}): #{destination}"
+        return Result.new(
+          path: destination,
+          url: resource.url,
+          filename: resource.filename,
+          bytes: bytes,
+          cached: true,
+          dataset: dataset_slug
+        )
+      end
+
+      bytes = stream_to(destination, resource)
+      log "Downloaded #{human_bytes(bytes)} → #{destination}"
+
+      Result.new(
+        path: destination,
+        url: resource.url,
+        filename: resource.filename,
+        bytes: bytes,
+        cached: false,
+        dataset: dataset_slug
+      )
+    end
+
+    private
+
+    def resolve_dataset_page
+      value = @dataset
+      value = value.to_s.strip unless value.nil?
+
+      if value.nil? || value.empty?
+        slug = discover_latest_dataset_slug
+        return ["#{DATA_HOST}/en/dataset/#{slug}", slug]
+      end
+
+      return [value, slug_from_url(value)] if value.match?(%r{\Ahttps?://}i)
+
+      slug = value.delete_prefix("/").delete_prefix("dataset/")
+      ["#{DATA_HOST}/en/dataset/#{slug}", slug]
+    end
+
+    def discover_latest_dataset_slug
+      log "Discovering latest national dataset from catalog…"
+      html = @http.get_body(CATALOG_SEARCH_URL)
+      slug = Catalog.latest_slug(html)
+      log "Using dataset #{slug}"
+      slug
+    end
+
+    def slug_from_url(url)
+      path = URI.parse(url).path.to_s
+      match = path.match(%r{/dataset/([^/]+)}i)
+      match ? match[1] : path
+    rescue URI::InvalidURIError
+      url
+    end
+
+    def page_origin(page_url)
+      uri = URI.parse(page_url)
+      return DATA_HOST unless uri.is_a?(URI::HTTP) && uri.host
+
+      origin = "#{uri.scheme}://#{uri.host}"
+      origin += ":#{uri.port}" if uri.port && uri.port != uri.default_port
+      origin
+    rescue URI::InvalidURIError
+      DATA_HOST
+    end
+
+    def resolve_destination(filename)
+      if @output && !@output.empty?
+        path = File.expand_path(@output)
+        return File.join(path, filename) if directory_output?(path)
+
+        return path
+      end
+
+      File.join(File.expand_path(@cache_dir), filename)
+    end
+
+    def directory_output?(path)
+      return true if @output.end_with?("/") || @output.end_with?(File::SEPARATOR)
+      return true if File.directory?(path)
+
+      # Treat bare/non-zip paths that don't exist yet as directories when they
+      # look like dirs (no extension). Explicit .zip paths stay files.
+      !File.exist?(path) && File.extname(path).empty?
+    end
+
+    # Filename match is the cache key (export timestamp embedded). Still open the
+    # zip central directory so a truncated / non-zip file is not treated as a hit.
+    def cache_hit?(path)
+      return false unless File.file?(path) && File.size(path).positive?
+
+      Zip::File.open(path) { true }
+    rescue Zip::Error, Errno::ENOENT, Errno::EACCES
+      false
+    end
+
+    def stream_to(destination, resource)
+      FileUtils.mkdir_p(File.dirname(destination))
+      temp_path = "#{destination}.partial.#{Process.pid}"
+
+      log "Downloading #{resource.filename}…"
+
+      bytes = 0
+      last_report_at = 0
+      expected_bytes = nil
+
+      begin
+        File.open(temp_path, "wb") do |file|
+          @http.stream(resource.url) do |chunk, content_length|
+            expected_bytes ||= content_length
+            file.write(chunk)
+            bytes += chunk.bytesize
+
+            if should_report_progress?(bytes, last_report_at, expected_bytes)
+              report_progress(bytes, expected_bytes)
+              last_report_at = bytes
+            end
+          end
+        end
+        finish_progress_line
+
+        if expected_bytes && bytes != expected_bytes
+          raise Error,
+                "Download size mismatch for #{resource.filename}: " \
+                "expected #{expected_bytes} bytes, got #{bytes}"
+        end
+
+        raise Error, "Downloaded empty file: #{resource.filename}" if bytes.zero?
+
+        File.rename(temp_path, destination)
+        bytes
+      ensure
+        # Interrupt / signal and size errors leave junk otherwise.
+        FileUtils.rm_f(temp_path)
+      end
+    end
+
+    def should_report_progress?(bytes, last_report_at, expected_bytes)
+      return false unless @progress_io
+      return true if expected_bytes && bytes >= expected_bytes
+
+      (bytes - last_report_at) >= PROGRESS_INTERVAL_BYTES
+    end
+
+    def report_progress(bytes, expected_bytes)
+      return unless @progress_io
+
+      message =
+        if expected_bytes&.positive?
+          percent = [(bytes * 100.0 / expected_bytes).floor, 100].min
+          format(
+            "\r  %<done>s / %<total>s (%<percent>d%%)",
+            done: human_bytes(bytes),
+            total: human_bytes(expected_bytes),
+            percent: percent
+          )
+        else
+          format("\r  %<done>s", done: human_bytes(bytes))
+        end
+
+      @progress_io.print message
+      @progress_io.flush
+    end
+
+    def finish_progress_line
+      return unless @progress_io
+
+      @progress_io.print "\n"
+      @progress_io.flush
+    end
+
+    def log(message)
+      return unless @progress_io
+
+      @progress_io.puts message
+      @progress_io.flush
+    end
+
+    def human_bytes(bytes)
+      units = %w[B KB MB GB TB]
+      value = bytes.to_f
+      units.each_with_index do |unit, index|
+        last_unit = index == units.length - 1
+        return format("%<n>.1f %<u>s", n: value, u: unit) if value < 1024.0 || last_unit
+
+        value /= 1024.0
+      end
+    end
+  end
+end

data/lib/swiss_netex/error.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module SwissNetex
+  class Error < StandardError; end
+end

data/lib/swiss_netex/extract.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module SwissNetex
+  # Convenience pipeline: resolve a national package (download if needed) then
+  # filter by operator/line. Shared CLI surface for `extract`.
+  class Extract
+    def initialize(
+      operators:,
+      output:,
+      lines: [],
+      from: nil,
+      dataset: nil,
+      cache_dir: Download::DEFAULT_CACHE_DIR,
+      force: false,
+      allow_empty: false,
+      verbose: false,
+      http: nil,
+      progress_io: nil
+    )
+      @operators = Array(operators)
+      @lines = Array(lines)
+      @output = output
+      @from = from
+      @dataset = dataset
+      @cache_dir = cache_dir
+      @force = force
+      @allow_empty = allow_empty
+      @verbose = verbose
+      @http = http
+      @progress_io = progress_io
+    end
+
+    def call
+      raise Error, "No --operator given" if @operators.empty?
+      raise Error, "Missing --output path" if @output.to_s.empty?
+
+      source = PackageSource.resolve(
+        from: @from,
+        dataset: @dataset,
+        cache_dir: @cache_dir,
+        force: @force,
+        http: @http,
+        progress_io: @progress_io
+      )
+      Filter.new(
+        from: source,
+        output: @output,
+        operators: @operators,
+        lines: @lines,
+        allow_empty: @allow_empty,
+        verbose: @verbose,
+        progress_io: @progress_io
+      ).call
+    end
+  end
+end

data/lib/swiss_netex/filter/common_frame.rb

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+require "stringio"
+require "tempfile"
+
+module SwissNetex
+  class Filter
+    # COMMON holds JourneyMeeting, InterchangeRule, TypeOfService, ServiceFacilitySet.
+    # Chunked byte scanner (no Nokogiri Reader) keeps peak RSS near frame size.
+    # Regexes use /n (NOENCODING) so zip entry streams (ASCII-8BIT) match cleanly.
+    # Assumes Swiss default-namespace elements (unprefixed local names). Prefixed
+    # entity tags would pass through as structure.
+    #
+    # Pure-whitespace structure before a dropped entity is discarded so filtered
+    # output does not keep blank holes between kept siblings (XmlSieve parity).
+    class CommonFrame
+      CHUNK = 1 << 20 # 1 MiB
+      TAIL = 64
+      OPEN_RE = %r{<(JourneyMeeting|InterchangeRule|ServiceFacilitySet|TypeOfService)([\s>/])}n
+      JOURNEY_REF_RE = /<(?:FromJourneyRef|ToJourneyRef|ServiceJourneyRef)\b[^>]*\bref="([^"]+)"/n
+      STOP_REF_RE = /<(?:StopPlaceRef|AdjacentStopPlaceRef)\b[^>]*\bref="([^"]+)"/n
+      LINE_REF_RE = /<LineRef\b[^>]*\bref="([^"]+)"/n
+      ID_RE = /\bid="([^"]+)"/n
+      XML_DECL_RE = /\A(?:\xEF\xBB\xBF)?\s*<\?xml\b.*?\?>\s*/nm
+      NON_WS_RE = /\S/n
+
+      def initialize(refs)
+        @refs = refs
+      end
+
+      def call(source)
+        out = new_tempfile
+        out << "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n"
+        kept = 0
+        dropped = 0
+        buffer = +"".b
+        io = to_io(source)
+        pending_decl = true
+
+        while (chunk = io.read(CHUNK))
+          buffer << chunk.b
+          if pending_decl
+            stripped = strip_xml_decl(buffer)
+            pending_decl = false if stripped.bytesize != buffer.bytesize || buffer.match?(NON_WS_RE)
+            buffer = stripped
+          end
+
+          k, d, buffer = drain(buffer, out, final: false)
+          kept += k
+          dropped += d
+        end
+
+        k, d, remainder = drain(buffer, out, final: true)
+        kept += k
+        dropped += d
+        out << remainder
+
+        out.flush
+        out.rewind
+        XmlSieve::Result.new(io: out, kept: kept, dropped: dropped)
+      end
+
+      private
+
+      def drain(buffer, out, final:)
+        kept = 0
+        dropped = 0
+
+        loop do
+          match = OPEN_RE.match(buffer)
+          return [kept, dropped, flush_structure(buffer, out, final: final)] unless match
+
+          prefix = +"".b
+          if match.begin(0).positive?
+            prefix = buffer.byteslice(0, match.begin(0))
+            buffer = buffer.byteslice(match.begin(0), buffer.bytesize - match.begin(0))
+          end
+
+          name = match[1]
+          block, rest = extract_element(buffer, name)
+          unless block
+            # Entity not complete yet — put prefix back so the next chunk can retry.
+            return [kept, dropped, prefix + buffer]
+          end
+
+          buffer = rest
+          if keep_block?(name, block)
+            out << prefix unless prefix.empty?
+            out << block
+            kept += 1
+          else
+            # Drop: only keep structural prefix that is not pure whitespace.
+            out << prefix if prefix.match?(NON_WS_RE)
+            dropped += 1
+          end
+        end
+      end
+
+      def flush_structure(buffer, out, final:)
+        return buffer if final || buffer.bytesize <= TAIL
+
+        flush_len = buffer.bytesize - TAIL
+        out << buffer.byteslice(0, flush_len)
+        buffer.byteslice(flush_len, TAIL)
+      end
+
+      def extract_element(buffer, name)
+        gt = buffer.index(">".b)
+        return [nil, buffer] unless gt
+
+        start = buffer.byteslice(0, gt + 1)
+        if start.end_with?("/>".b)
+          return [start, buffer.byteslice(gt + 1, buffer.bytesize - gt - 1)]
+        end
+
+        close = "</#{name}>".b
+        idx = buffer.index(close)
+        return [nil, buffer] unless idx
+
+        stop = idx + close.bytesize
+        [buffer.byteslice(0, stop), buffer.byteslice(stop, buffer.bytesize - stop)]
+      end
+
+      def keep_block?(name, block)
+        case name
+        when "JourneyMeeting".b
+          journey_ref_kept?(block)
+        when "InterchangeRule".b
+          interchange_kept?(block)
+        when "ServiceFacilitySet".b
+          id_kept?(block, :service_facility_sets)
+        when "TypeOfService".b
+          id_kept?(block, :type_of_services)
+        else
+          false
+        end
+      end
+
+      # Require every tracked journey side to resolve — one-sided meetings leave
+      # dangling refs in the filtered package. No journey refs → drop.
+      def journey_ref_kept?(block)
+        refs = scan_refs(block, JOURNEY_REF_RE)
+        return false if refs.empty?
+
+        all_refs_kept?(refs, :journeys)
+      end
+
+      # Same closure rule as SERVICE SiteConnection: keep only when every tracked
+      # journey / stop / line ref is present. Entities with no tracked refs drop.
+      def interchange_kept?(block)
+        journey_refs = scan_refs(block, JOURNEY_REF_RE)
+        stop_refs = scan_refs(block, STOP_REF_RE)
+        line_refs = scan_refs(block, LINE_REF_RE)
+        return false if journey_refs.empty? && stop_refs.empty? && line_refs.empty?
+
+        all_refs_kept?(journey_refs, :journeys) &&
+          all_refs_kept?(stop_refs, :stop_places) &&
+          all_refs_kept?(line_refs, :lines)
+      end
+
+      def all_refs_kept?(refs, bucket)
+        return true if refs.empty?
+
+        refs.all? { |ref| @refs.include?(bucket, ref) }
+      end
+
+      def id_kept?(block, bucket)
+        id = block[ID_RE, 1]&.force_encoding(Encoding::UTF_8)
+        id && @refs.include?(bucket, id)
+      end
+
+      def scan_refs(block, regex)
+        block.scan(regex).flatten.map { |ref| ref.force_encoding(Encoding::UTF_8) }
+      end
+
+      def strip_xml_decl(text)
+        text.sub(XML_DECL_RE, "".b)
+      end
+
+      def new_tempfile
+        file = Tempfile.new(["swiss-netex-common", ".xml"])
+        file.binmode
+        file
+      end
+
+      def to_io(source)
+        return StringIO.new(source.b) if source.is_a?(String)
+        return source if source.respond_to?(:read)
+
+        raise Error, "CommonFrame expects a String or IO-like object"
+      end
+    end
+  end
+end

data/lib/swiss_netex/filter/frame_worker.rb

@@ -0,0 +1,188 @@
+# frozen_string_literal: true
+
+require "etc"
+require "json"
+
+module SwissNetex
+  class Filter
+    # Rewrite support frames in short-lived children so Nokogiri Reader heap is
+    # freed at process exit. Falls back to in-process when fork is unavailable.
+    # Multiple kinds run concurrently when fork is available (SWISS_NETEX_TT_WORKERS).
+    class FrameWorker
+      def initialize(package_path:, tmp:, refs:)
+        @package_path = package_path.to_s
+        @tmp = tmp.to_s
+        @refs = refs
+        @workers = resolve_workers
+      end
+
+      def call(kind, name)
+        call_many([[kind, name]]).fetch(kind)
+      end
+
+      # Parallel rewrite of independent support frames. +pairs+ is [[kind, name], …].
+      # Returns { kind => { "kept" => …, "dropped" => … } }.
+      def call_many(pairs)
+        pairs = pairs.select { |_kind, name| name }
+        return {} if pairs.empty?
+        return run_pool(pairs) if fork_supported? && pairs.size > 1 && @workers > 1
+
+        pairs.each_with_object({}) do |(kind, name), out|
+          out[kind] = run_one(kind, name)
+        end
+      end
+
+      private
+
+      def run_one(kind, name)
+        return process_frame(kind, name, @refs) unless fork_supported?
+
+        read_io, write_io = IO.pipe
+        refs_export = @refs.export
+        pid = fork do
+          read_io.close
+          local_refs = Refs.new.merge_export(refs_export)
+          result = process_frame(kind, name, local_refs)
+          write_io.write(JSON.generate(result))
+          write_io.close
+          exit! 0
+        end
+        write_io.close
+
+        raw = nil
+        status = nil
+        begin
+          raw = read_io.read
+        ensure
+          read_io.close
+          _pid, status = Process.wait2(pid)
+        end
+        raise Error, "#{kind} worker failed (exit #{status.exitstatus})" unless status.success?
+
+        JSON.parse(raw)
+      rescue JSON::ParserError => e
+        raise Error, "#{kind} worker returned invalid payload: #{e.message}"
+      end
+
+      def run_pool(pairs)
+        queue = pairs.dup
+        inflight = {}
+        results = {}
+        refs_export = @refs.export
+
+        while queue.any? || inflight.any?
+          while inflight.size < @workers && queue.any?
+            kind, name = queue.shift
+            spawn_job(kind, name, refs_export, inflight)
+          end
+
+          pid, status = Process.wait2(-1)
+          meta = inflight.delete(pid)
+          next unless meta
+
+          raw = meta[:reader].value
+          meta[:read_io].close
+          unless status.success?
+            reap_inflight(inflight)
+            raise Error, "#{meta[:kind]} worker failed (exit #{status.exitstatus})"
+          end
+
+          results[meta[:kind]] = JSON.parse(raw)
+        end
+
+        results
+      rescue JSON::ParserError => e
+        reap_inflight(inflight || {})
+        raise Error, "support-frame worker returned invalid payload: #{e.message}"
+      rescue StandardError
+        reap_inflight(inflight || {})
+        raise
+      end
+
+      def spawn_job(kind, name, refs_export, inflight)
+        read_io, write_io = IO.pipe
+        pid = fork do
+          read_io.close
+          local_refs = Refs.new.merge_export(refs_export)
+          result = process_frame(kind, name, local_refs)
+          write_io.write(JSON.generate(result))
+          write_io.close
+          exit! 0
+        end
+        write_io.close
+        reader = Thread.new { read_io.read }
+        inflight[pid] = { kind: kind, read_io: read_io, reader: reader }
+      end
+
+      def reap_inflight(inflight)
+        inflight.each_key do |pid|
+          Process.kill("TERM", pid)
+        rescue Errno::ESRCH
+          # already gone
+        end
+        inflight.each do |pid, meta|
+          Process.wait2(pid)
+        rescue Errno::ECHILD
+          # already reaped
+        ensure
+          meta[:reader]&.kill
+          meta[:read_io]&.close unless meta[:read_io]&.closed?
+        end
+        inflight.clear
+      end
+
+      def process_frame(kind, name, refs)
+        destination = File.join(@tmp, File.basename(name))
+        kept = 0
+        dropped = 0
+
+        Package.open(@package_path) do |pkg|
+          pkg.open_member(name) do |io|
+            result = Frames.public_send(kind, io, refs)
+            begin
+              kept = result.kept
+              dropped = result.dropped
+              copy_io_to_file(result.io, destination)
+            ensure
+              close_temp(result.io)
+            end
+          end
+        end
+
+        { "kept" => kept, "dropped" => dropped }
+      end
+
+      def fork_supported?
+        return false if ENV["SWISS_NETEX_NO_FORK"]
+
+        Process.respond_to?(:fork)
+      end
+
+      def resolve_workers
+        raw = ENV["SWISS_NETEX_TT_WORKERS"].to_s.strip
+        if raw.empty?
+          n = Etc.nprocessors
+          return n.positive? ? n : 1
+        end
+
+        size = Integer(raw, 10)
+        size.positive? ? size : 1
+      rescue ArgumentError
+        1
+      end
+
+      def copy_io_to_file(io, destination)
+        io.rewind if io.respond_to?(:rewind)
+        File.open(destination, "wb") { |out| IO.copy_stream(io, out) }
+      end
+
+      def close_temp(io)
+        return unless io
+
+        io.close!
+      rescue StandardError
+        io.close if io.respond_to?(:close) && !io.closed?
+      end
+    end
+  end
+end