git-fit 0.17.2 → 0.17.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 472d41138e7c8607cbc63dbff14db13079b6b824c342ec03661696c77f08f8f9
-  data.tar.gz: 291394e38307334eaedd49106b5c6c057331d972ccf4fa2619ce68904ca89a86
+  metadata.gz: 4c9c026f3d44b75a14eae1f36cc7dc6a56434cd0039996bfb38163fc3bb617a5
+  data.tar.gz: b11245bfa2b0fef405e5f0f369fd02582d1a2ba509b7fb91972f38249ae8ec01
 SHA512:
-  metadata.gz: 2f7e4971cc16db4f139ae122a65fe19842bfd1e999362bfc57294e4fa2b41a3adbd228824e9126b13a13fc2499643dec07ad195e1d6050618137a54f12b34a48
-  data.tar.gz: 713ff0444661494cb22469f5f3535de7b20f89229ec82b5db07398654b6cd2622267db18a0d1841517f796a351bbc1d29800d79b53469009454532dba855b61e
+  metadata.gz: 1baed90c0c86bbb46edefd951646ba0f2fffd8cd08b07eb900479a74f372de43d50c7a0f39c56f728f8fb476cfe264075eaafcc3721b2bcd5dddc692b9469592
+  data.tar.gz: e98542328ef0210b5934a1c85dfb388b35563df9a4998bb52de7dd6847c05f7ada8a57402d9c359ed08bb6f9420d3a5205d91454c61b40801faa6bfd6fad769f

data/lib/git-fit.rb

@@ -93,6 +93,8 @@ require_relative 'git_fit/cli/export'
 require_relative 'git_fit/cli/import_cli'
 require_relative 'git_fit/cli/geo_cli'
 require_relative 'git_fit/strava_web/file_check'
+require_relative 'git_fit/cli/elevation'
+require_relative 'git_fit/elevation/backfill'
 require_relative 'git_fit/cli/strava'
 require_relative 'git_fit/auth/garmin_token'
 require_relative 'git_fit/auth/strava'

data/lib/git_fit/cli/elevation.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require 'thor'
+
+module GitFit
+  class ElevationCLI < Thor
+    no_commands do
+      def git_fit_config
+        @git_fit_config ||= GitFit::Config.new(options[:config])
+      end
+    end
+
+    desc 'backfill [SOURCE]', 'Backfill elevation data (gain/loss/min/max) from raw FIT files or L2 data'
+    option :force, type: :boolean, default: false, desc: 'Overwrite existing values (default: only fill NULLs)'
+    def backfill(source = nil)
+      config = git_fit_config
+      conn = GitFit::DB::Connection.new(config.db_path)
+      conn.migrate!
+      db = conn.db
+
+      result = GitFit::Elevation::Backfill.new(db, force: options[:force]).call(source)
+
+      msg = "Elevation backfill: #{result[:updated]} updated, " \
+            "#{result[:skipped]} skipped, #{result[:failed]} failed"
+      say_status :done, msg, :green
+    rescue StandardError => e
+      say_status :error, "Elevation backfill failed: #{e.message}", :red
+    end
+  end
+end

data/lib/git_fit/cli.rb

@@ -117,6 +117,9 @@ module GitFit
       say_status :error, "Dedup failed: #{e.message}", :red
     end
 
+    desc 'elevation SUBCOMMAND', 'Elevation data operations'
+    subcommand 'elevation', ElevationCLI
+
     desc 'strava SUBCOMMAND', 'Strava web operations'
     subcommand 'strava', StravaCLI
 

data/lib/git_fit/elevation/backfill.rb

@@ -0,0 +1,182 @@
+# frozen_string_literal: true
+
+module GitFit
+  module Elevation
+    class Backfill
+      SOURCES = %w[igpsport xoss strava garmin garmin_cn xingzhe].freeze
+      BATCH_SIZE = 100
+
+      def initialize(db, force: false)
+        @db = db
+        @force = force
+        @updated = 0
+        @skipped = 0
+        @failed = 0
+      end
+
+      def call(source = nil)
+        sources = source ? [source] : SOURCES
+        sources.each do |src|
+          backfill_source(src)
+        end
+        { updated: @updated, skipped: @skipped, failed: @failed }
+      end
+
+      private
+
+      def backfill_source(source)
+        offset = 0
+        loop do
+          rows = fetch_rows(source, offset)
+          break if rows.empty?
+
+          @db.transaction do
+            rows.each { |row| process_row(row, source) }
+          end
+
+          offset += BATCH_SIZE
+        end
+      end
+
+      def fetch_rows(source, offset)
+        scope = @db[:activities].where(source: source)
+        unless @force
+          scope = scope.where(Sequel.|(
+                                elevation_gain: nil,
+                                elevation_loss: nil,
+                                elevation_min: nil,
+                                elevation_max: nil,
+                              ))
+        end
+        scope.order(:id).limit(BATCH_SIZE, offset).all
+      end
+
+      def process_row(row, source)
+        run_id = row[:run_id]
+        natural_id = run_id.sub("#{source}_", '')
+
+        elev = case source
+               when 'igpsport' then compute_from_fit(source, natural_id)
+               when 'xoss' then compute_from_fit(source, natural_id)
+               when 'garmin' then compute_from_fit(source, natural_id)
+               when 'garmin_cn' then compute_from_fit(source, natural_id)
+               when 'strava' then compute_from_l2(source, natural_id)
+               when 'xingzhe' then compute_from_xingzhe(natural_id)
+               else return
+               end
+
+        return skip_row(run_id) unless elev
+
+        update_row(row, elev)
+      end
+
+      def compute_from_fit(source, natural_id)
+        fit_path = fit_path_for(source, natural_id)
+        return nil unless fit_path && File.exist?(fit_path)
+
+        records = GitFit::FIT::Decoder.decode(fit_path)
+        return nil if records.empty?
+
+        alts = records.filter_map { |r| r['enhancedAltitude'] || r['altitude'] }
+        return nil if alts.size < 2
+
+        session = begin
+          GitFit::FIT::Decoder.session(fit_path)
+        rescue StandardError
+          nil
+        end
+
+        gain = session&.dig('totalAscent').is_a?(Numeric) ? session['totalAscent'] : elevation_gain_from_alts(alts)
+        loss = session&.dig('totalDescent').is_a?(Numeric) ? session['totalDescent'] : elevation_loss_from_alts(alts)
+
+        {
+          elevation_gain: gain&.round(1),
+          elevation_loss: loss&.round(1),
+          elevation_min: alts.min&.round(1),
+          elevation_max: alts.max&.round(1),
+        }
+      rescue StandardError => e
+        warn "Elevation backfill [#{source}][#{natural_id}]: #{e.message}"
+        nil
+      end
+
+      def compute_from_l2(source, natural_id)
+        l2_path = File.join('data', 'std', source, "#{natural_id}.json")
+        return nil unless File.exist?(l2_path)
+
+        l2 = JSON.parse(File.read(l2_path))
+        alts = l2.filter_map { |pt| pt['altitude'] || pt[:altitude] }
+        return nil if alts.size < 2
+
+        {
+          elevation_gain: elevation_gain_from_alts(alts)&.round(1),
+          elevation_loss: elevation_loss_from_alts(alts)&.round(1),
+          elevation_min: alts.min&.round(1),
+          elevation_max: alts.max&.round(1),
+        }
+      rescue StandardError => e
+        warn "Elevation backfill [#{source}][#{natural_id}]: #{e.message}"
+        nil
+      end
+
+      def compute_from_xingzhe(natural_id)
+        raw_path = File.join('data', 'raw', 'xingzhe', "#{natural_id}.json")
+        return nil unless File.exist?(raw_path)
+
+        raw = JSON.parse(File.read(raw_path))
+        alts = raw.dig('stream', 'altitude') || []
+        return nil if alts.size < 2
+
+        {
+          elevation_gain: elevation_gain_from_alts(alts)&.round(1),
+          elevation_loss: elevation_loss_from_alts(alts)&.round(1),
+          elevation_min: alts.min&.round(1),
+          elevation_max: alts.max&.round(1),
+        }
+      rescue StandardError => e
+        warn "Elevation backfill [xingzhe][#{natural_id}]: #{e.message}"
+        nil
+      end
+
+      def fit_path_for(source, natural_id)
+        case source
+        when 'garmin', 'garmin_cn'
+          File.join('data', 'raw', source, natural_id, 'activity.fit')
+        else
+          File.join('data', 'raw', source, "#{natural_id}.fit")
+        end
+      end
+
+      def elevation_gain_from_alts(alts)
+        return 0.0 if alts.size < 2
+        (1...alts.size).sum do |i|
+          d = alts[i] - alts[i - 1]
+          d > 0 ? d : 0.0
+        end
+      end
+
+      def elevation_loss_from_alts(alts)
+        return 0.0 if alts.size < 2
+        (1...alts.size).sum do |i|
+          d = alts[i - 1] - alts[i]
+          d > 0 ? d : 0.0
+        end
+      end
+
+      def update_row(row, elev)
+        @db[:activities].where(id: row[:id]).update(
+          elevation_gain: elev[:elevation_gain],
+          elevation_loss: elev[:elevation_loss],
+          elevation_min: elev[:elevation_min],
+          elevation_max: elev[:elevation_max],
+          updated_at: Time.now.utc,
+        )
+        @updated += 1
+      end
+
+      def skip_row(_run_id)
+        @skipped += 1
+      end
+    end
+  end
+end

data/lib/git_fit/sync/xingzhe.rb

@@ -14,6 +14,7 @@ module GitFit
       WORKOUTS_PATH = '/api/v1/pgworkout/'
       STREAM_PATH = '/api/v1/pgworkout/'
       DETAIL_PATH = '/api/v1/pgworkout/'
+      EXPORT_BASE = '/api/v1/workout/'
       RSA_PUBKEY = 'MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDmuQkBbijudDAJgfffDeeIButqWHZvUwcRuvWdg89393FSdz3IJUHc0rgI/S3WuU8N0VePJLmVAZtCOK4qe4FY/eKmWpJmn7JfXB4HTMWjPVoyRZmSYjW4L8GrWmh51Qj7DwpTADadF3aq04o+s1b8LXJa8r6+TIqqL5WUHtRqmQIDAQAB'
 
       SPORT_MAP = {
@@ -21,8 +22,6 @@ module GitFit
         5 => 'swim', 6 => 'walk', 11 => 'ski', 13 => 'workout'
       }.freeze
 
-      RAW_EXT = 'json'
-
       SESSION_TTL = 30 * 24 * 60 * 60
 
       register_adapter
@@ -85,6 +84,48 @@ module GitFit
         @last_http_code
       end
 
+      # ── Raw directory layout (Phase 1, mirrors garmin Phase 1) ──────────
+      #   data/raw/xingzhe/{id}/stream.json   — stream envelope
+      #   data/raw/xingzhe/{id}/detail.json   — full detail response (workout+user+...)
+      #   data/raw/xingzhe/{id}/{id}.gpx      — GPX export (when available)
+      #   data/raw/xingzhe/{id}/{id}.fit      — FIT export (when is_fit)
+      def raw_path(platform_id)
+        File.join(raw_dir, platform_id, 'stream.json')
+      end
+
+      def raw_detail_path(platform_id)
+        File.join(raw_dir, platform_id, 'detail.json')
+      end
+
+      def raw_gpx_path(platform_id)
+        File.join(raw_dir, platform_id, "#{platform_id}.gpx")
+      end
+
+      def raw_fit_path(platform_id)
+        File.join(raw_dir, platform_id, "#{platform_id}.fit")
+      end
+
+      def raw_file_exists?(platform_id)
+        File.exist?(raw_path(platform_id))
+      end
+
+      def write_source_archive(data, platform_id, _ext = 'json')
+        # Back-compat entry point — now writes stream.json into per-id dir
+        write_raw_file(data, 'stream.json', platform_id)
+      end
+
+      def write_raw_file(data, filename, platform_id)
+        dir = File.join(raw_dir, platform_id)
+        FileUtils.mkdir_p(dir)
+        tmp = File.join(dir, ".#{filename}.tmp")
+        dest = File.join(dir, filename)
+        case data
+        when String then File.write(tmp, data)
+        when Hash, Array then File.write(tmp, JSON.generate(data))
+        end
+        File.rename(tmp, dest)
+      end
+
       private
 
       def http_request(method, path, body: nil, params: nil, headers: {})
@@ -252,9 +293,28 @@ module GitFit
           average_speed: (act['avg_speed'].to_f / 3.6).round(2).nonzero?,
           elevation_gain: act['elevation_gain'].to_f.nonzero?,
           elevation_loss: act['elevation_loss'].to_f.nonzero?,
+          elevation_max: act['max_altitude']&.to_f&.nonzero?,
           source: 'xingzhe',
         }
 
+        # Phase 1: fetch and persist detail + stream; Phase 2: exports
+        detail_workout = nil
+        full_detail = nil
+
+        # Fetch full detail (public, with segments/slopes/pois/laps) for raw archive
+        fetched_full, fetched_workout = fetch_full_detail(aid)
+        if fetched_full
+          full_detail = fetched_full
+          detail_workout = fetched_workout
+          # Persist full detail response (code/data/msg) as detail.json
+          write_raw_file(full_detail, 'detail.json', aid)
+          # Prefer workout from full_detail for attrs that benefit from richer data
+          if detail_workout
+            attrs[:elevation_max] ||= detail_workout['max_altitude']&.to_f&.nonzero?
+            # equipment_info / power / grade etc. are archived in detail.json for downstream use
+          end
+        end
+
         pts = nil
         stream = fetch_stream(aid)
         if stream
@@ -270,19 +330,36 @@ module GitFit
           end
         end
 
-        if attrs[:summary_polyline].nil? && (detail = fetch_activity_detail(aid)) && (segments = detail['segments_km'])
-          pts = segments.map { |s| [s['latitude'], s['longitude']] }
-          attrs[:summary_polyline] = Geo::Polyline.encode(pts) if pts.size >= 2
+        if attrs[:summary_polyline].nil?
+          fallback_workout = detail_workout || fetch_activity_detail(aid)
+          if fallback_workout && (segments = fallback_workout['segments_km'])
+            pts = segments.map { |s| [s['latitude'], s['longitude']] }
+            attrs[:summary_polyline] = Geo::Polyline.encode(pts) if pts.size >= 2
+          end
         end
 
+        # Elevation min: prefer stream altitude min, fallback to segments_km altitude min (see wiki)
+        attrs[:elevation_min] = derive_elevation_min(stream, detail_workout || act)
+
+        # Exports (best-effort, per-record format probing — see issue #86)
+        try_download_exports(aid, detail_workout || act) if stream || full_detail
+
         attrs[:start_date_local] = resolve_local_time(start_t, pts)
 
         upsert_activity(attrs)
         attrs
       end
 
-      def raw_path(platform_id)
-        File.join(raw_dir, "#{platform_id}.#{RAW_EXT}")
+      def derive_elevation_min(stream, detail_workout)
+        if stream && (alt = stream['altitude']) && alt.is_a?(Array) && alt.any?
+          v = alt.compact.min
+          return v.to_f if v
+        end
+        if detail_workout && (segs = detail_workout['segments_km']) && segs.is_a?(Array) && segs.any?
+          v = segs.filter_map { |s| s['altitude'] }.min
+          return v.to_f if v
+        end
+        nil
       end
 
       def fetch_stream(id)
@@ -338,6 +415,114 @@ module GitFit
         data.dig('data', 'workout')
       end
 
+      # Full detail with segments/slopes/pois/laps — returns [full_json, workout_hash]
+      def fetch_full_detail(id)
+        resp = http_request(:get, "#{DETAIL_PATH}#{id}/",
+          params: { segments: 'true', slopes: 'true', pois: 'true', laps: 'true' })
+        return [nil, nil] unless resp.code.to_i == 200
+
+        data = JSON.parse(resp.body)
+        return [nil, nil] unless data['code'] == 200 || data.dig('data', 'workout')
+
+        workout = data.dig('data', 'workout')
+        [data, workout]
+      rescue JSON::ParserError
+        [nil, nil]
+      end
+
+      def try_download_exports(activity_id, detail_workout)
+        # GPX: generally available — probe via authed GET
+        download_and_save_gpx(activity_id)
+        # FIT: only when is_fit truthy (per detail), else probe still but expect 404/302 empty
+        is_fit = detail_workout && detail_workout['is_fit']
+        # Always attempt FIT when summary suggests ride with device; but gate on is_fit to avoid noise
+        download_and_save_fit(activity_id) if is_fit
+      rescue StandardError => e
+        warn "XingZhe: export download failed for #{activity_id}: #{e.message}"
+      end
+
+      def download_and_save_gpx(activity_id)
+        resp = authed_get("#{EXPORT_BASE}#{activity_id}/gpx/")
+        return false unless resp && resp.code.to_i == 200
+        body = resp.body
+        return false if body.nil? || body.empty?
+        # GPX endpoint returns XML but Content-Type may be application/json — validate by prefix
+        stripped = body.lstrip
+        unless stripped.start_with?('<?xml', '<gpx')
+          # Some error responses are JSON with code/msg
+          begin
+            j = JSON.parse(body)
+            return false if j['code'] && j['code'] != 200
+          rescue JSON::ParserError
+            return false
+          end
+          return false unless stripped.include?('<gpx') || stripped.include?('<trk')
+        end
+        write_raw_file(body, "#{activity_id}.gpx", activity_id)
+        true
+      rescue StandardError => e
+        warn "XingZhe: failed to download gpx for #{activity_id}: #{e.message}"
+        false
+      end
+
+      def download_and_save_fit(activity_id)
+        # FIT export: 302 → presigned OSS URL (see issue #86 verification)
+        resp = http_request(:get, "#{EXPORT_BASE}#{activity_id}/fit/", headers: @headers || {})
+        code = resp.code.to_i
+        # 302/301 redirect to OSS is the expected success path — handle before auth check
+        if [301, 302, 303, 307, 308].include?(code)
+          location = resp['location'] || resp['Location']
+          if location && !location.empty?
+            uri = URI(location)
+            http = Net::HTTP.new(uri.host, uri.port)
+            http.use_ssl = uri.scheme == 'https'
+            http.open_timeout = 30
+            http.read_timeout = 60
+            req = Net::HTTP::Get.new(uri)
+            oss_resp = http.start { |h| h.request(req) }
+            return false unless oss_resp.code.to_i == 200
+
+            # Binary FIT — write as String (Ruby handles binary)
+            write_raw_file(oss_resp.body, "#{activity_id}.fit", activity_id)
+            return true
+          end
+        end
+        if auth_failure?(resp)
+          remove_stale_session
+          return false unless login
+
+          resp = http_request(:get, "#{EXPORT_BASE}#{activity_id}/fit/", headers: @headers)
+          code = resp.code.to_i
+          if [301, 302, 303, 307, 308].include?(code)
+            location = resp['location'] || resp['Location']
+            return false unless location && !location.empty?
+
+            uri = URI(location)
+            http = Net::HTTP.new(uri.host, uri.port)
+            http.use_ssl = uri.scheme == 'https'
+            http.open_timeout = 30
+            http.read_timeout = 60
+            req = Net::HTTP::Get.new(uri)
+            oss_resp = http.start { |h| h.request(req) }
+            return false unless oss_resp.code.to_i == 200
+
+            write_raw_file(oss_resp.body, "#{activity_id}.fit", activity_id)
+            return true
+          end
+        end
+        if code == 200
+          # Direct FIT (rare)
+          return false if resp.body.nil? || resp.body.empty?
+          write_raw_file(resp.body, "#{activity_id}.fit", activity_id)
+          true
+        else
+          false
+        end
+      rescue StandardError => e
+        warn "XingZhe: failed to download fit for #{activity_id}: #{e.message}"
+        false
+      end
+
       def map_sport(sport)
         SPORT_MAP[sport] || 'other'
       end

data/lib/git_fit/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module GitFit
-  VERSION = '0.17.2'
+  VERSION = '0.17.4'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: git-fit
 version: !ruby/object:Gem::Version
-  version: 0.17.2
+  version: 0.17.4
 platform: ruby
 authors:
 - Lax
@@ -257,6 +257,7 @@ files:
 - lib/git_fit/auth/strava.rb
 - lib/git_fit/cli.rb
 - lib/git_fit/cli/checkpointable.rb
+- lib/git_fit/cli/elevation.rb
 - lib/git_fit/cli/export.rb
 - lib/git_fit/cli/geo_cli.rb
 - lib/git_fit/cli/gh_cli.rb
@@ -277,6 +278,7 @@ files:
 - lib/git_fit/dedup/matcher.rb
 - lib/git_fit/dedup/service.rb
 - lib/git_fit/dedup/trajectory.rb
+- lib/git_fit/elevation/backfill.rb
 - lib/git_fit/export.rb
 - lib/git_fit/export/calculator.rb
 - lib/git_fit/export/csv.rb