git-fit 0.20.0 → 0.21.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3c268d464a4e366bef89848504d3a6b9f3c4dd47139f1ab9d8051b869ae38f00
-  data.tar.gz: 25060c54bb7135b5400ac03b3f7b310760e1a2ccba681e32b2780be59f976ae7
+  metadata.gz: 30f139ea0a7c21e9a99189ac7541c8b80c35377f40742823843dbcd5b2ce604a
+  data.tar.gz: b554587d97956af6e3a532ab67b6eca755377005f82eec38c500414d7bd1c4c4
 SHA512:
-  metadata.gz: 41445db3b930293a97320e977e3b5e3338ab402722ab50ddba938af79a1bf9b1504f42547d5e5ca90811bd86c76e9a32289470b39c302e25546bc7edf0dd45f3
-  data.tar.gz: 7fa3934d598cb97e7061157c812404858033d0f96460b9022944a9e6f604c3188c2ffb22b1b9933c7653785c9d47a8d1daf04c5c497e579fb24cc31652b8f069
+  metadata.gz: 5b23770d805a5c42fdbe3b4322e3557bd4938033e2e7e3094e934354d16fbc8c6bb7d7ed028ae5ffe9684b785d7e5fdca560e17480fd020713c1b96302795bb2
+  data.tar.gz: 32dde756e5dc46d3f859a992779187ccf2e478e9de05b681f79184ba2c35838822f29586f9d4e9c79d26e1d8d767c1771d4ed80172fa6fd21ec9ff7851345720

data/db/migrations/004_add_elevation_terrain_fields.rb

@@ -0,0 +1,19 @@
+Sequel.migration do
+  up do
+    alter_table(:activities) do
+      add_column :elevation_gain_terrain, Float
+      add_column :elevation_loss_terrain, Float
+      add_column :elevation_min_terrain,  Float
+      add_column :elevation_max_terrain,  Float
+    end
+  end
+
+  down do
+    alter_table(:activities) do
+      drop_column :elevation_max_terrain
+      drop_column :elevation_min_terrain
+      drop_column :elevation_loss_terrain
+      drop_column :elevation_gain_terrain
+    end
+  end
+end

data/lib/git-fit.rb

@@ -95,6 +95,7 @@ require_relative 'git_fit/cli/geo_cli'
 require_relative 'git_fit/strava_web/file_check'
 require_relative 'git_fit/cli/elevation'
 require_relative 'git_fit/elevation/backfill'
+require_relative 'git_fit/elevation/correct'
 require_relative 'git_fit/elevation/terrain_coeff'
 require_relative 'git_fit/cli/strava'
 require_relative 'git_fit/auth/garmin_token'

data/lib/git_fit/cli/elevation.rb

@@ -11,7 +11,7 @@ module GitFit
       end
     end
 
-    desc 'backfill [SOURCE]', 'Backfill elevation data (gain/loss/min/max) from raw FIT files or L2 data'
+    desc 'backfill [SOURCE]', 'Backfill raw elevation data (device/API values) from FIT files or L2 data'
     option :force, type: :boolean, default: false, desc: 'Overwrite existing values (default: only fill NULLs)'
     def backfill(source = nil)
       config = git_fit_config
@@ -28,7 +28,33 @@ module GitFit
       say_status :error, "Elevation backfill failed: #{e.message}", :red
     end
 
-    desc 'train', 'Train DEM terrain coefficient model from activity data'
+    desc 'correct [SOURCE]', 'Apply elevation correction into _terrain columns (adaptive hysteresis; --dem adds DEM)'
+    option :force, type: :boolean, default: false, desc: 'Recompute even when _terrain columns are already populated'
+    option :'dry-run', type: :boolean, default: false, desc: 'Count records without writing to database'
+    option :dem, type: :boolean, default: true,
+                 desc: 'Enable DEM correction (default: true; --no-dem for hysteresis only)'
+    def correct(source = nil)
+      config = git_fit_config
+      conn = GitFit::DB::Connection.new(config.db_path)
+      conn.migrate!
+      db = conn.db
+
+      result = GitFit::Elevation::Correct.new(
+        db,
+        force: options[:force],
+        dry_run: options[:'dry-run'],
+        dem: options[:dem],
+      ).call(source)
+
+      msg = "Elevation correct: #{result[:updated]} updated, " \
+            "#{result[:skipped]} skipped, #{result[:failed]} failed, " \
+            "#{result[:no_model]} no_model"
+      say_status :done, msg, :green
+    rescue StandardError => e
+      say_status :error, "Elevation correct failed: #{e.message}", :red
+    end
+
+    desc 'train', 'Train DEM terrain coefficient model from activity data', hide: true
     option :sources, type: :string, default: nil, desc: 'Comma-separated source adapters (default: all)'
     option :output, type: :string, default: nil, desc: 'Output path for model file'
     def train
@@ -51,7 +77,7 @@ module GitFit
       exit 1
     end
 
-    desc 'infer GAIN_PER_KM RANGE_PER_KM SEG_DENSITY', 'Infer optimal h threshold for terrain'
+    desc 'infer GAIN_PER_KM RANGE_PER_KM SEG_DENSITY', 'Infer optimal h threshold for terrain', hide: true
     def infer(gain_per_km, range_per_km, seg_density)
       g = gain_per_km.to_f
       r = range_per_km.to_f
@@ -77,7 +103,7 @@ module GitFit
       exit 1
     end
 
-    desc 'model-info', 'Show information about the loaded DEM model'
+    desc 'model-info', 'Show information about the loaded DEM model', hide: true
     def model_info
       model = GitFit::Elevation::TerrainCoeff.model
       path = GitFit::Elevation::TerrainCoeff.model_path

data/lib/git_fit/elevation/backfill.rb

@@ -5,8 +5,6 @@ module GitFit
     class Backfill
       SOURCES = %w[igpsport xoss strava garmin garmin_cn xingzhe].freeze
       BATCH_SIZE = 100
-      # Fallback threshold when detail.json elevation is absent (issue #83: baro 2m / GPS 3m).
-      ELEV_THRESHOLD = 2.0
 
       def initialize(db, force: false)
         @db = db
@@ -88,8 +86,8 @@ module GitFit
           nil
         end
 
-        gain = session&.dig('totalAscent').is_a?(Numeric) ? session['totalAscent'] : elevation_gain_from_alts(alts)
-        loss = session&.dig('totalDescent').is_a?(Numeric) ? session['totalDescent'] : elevation_loss_from_alts(alts)
+        gain = session&.dig('totalAscent').is_a?(Numeric) ? session['totalAscent'] : nil
+        loss = session&.dig('totalDescent').is_a?(Numeric) ? session['totalDescent'] : nil
 
         {
           elevation_gain: gain&.round(1),
@@ -102,22 +100,7 @@ module GitFit
         nil
       end
 
-      def compute_from_l2(source, natural_id)
-        l2_path = File.join('data', 'std', source, "#{natural_id}.json")
-        return nil unless File.exist?(l2_path)
-
-        l2 = JSON.parse(File.read(l2_path))
-        alts = l2.filter_map { |pt| pt['altitude'] || pt[:altitude] }
-        return nil if alts.size < 2
-
-        {
-          elevation_gain: elevation_gain_from_alts(alts)&.round(1),
-          elevation_loss: elevation_loss_from_alts(alts)&.round(1),
-          elevation_min: alts.min&.round(1),
-          elevation_max: alts.max&.round(1),
-        }
-      rescue StandardError => e
-        warn "Elevation backfill [#{source}][#{natural_id}]: #{e.message}"
+      def compute_from_l2(_source, _natural_id)
         nil
       end
 
@@ -141,8 +124,8 @@ module GitFit
         dl = detail && detail['elevation_loss']
         dm = detail && detail['max_altitude']
 
-        gain = detail_gain?(dg) ? dg : elevation_gain_from_alts(alts, threshold: ELEV_THRESHOLD)
-        loss = detail_gain?(dl) ? dl : elevation_loss_from_alts(alts, threshold: ELEV_THRESHOLD)
+        gain = detail_gain?(dg) ? dg : nil
+        loss = detail_gain?(dl) ? dl : nil
 
         # max: detail max_altitude is reliable (matches stream max in samples); else raw max.
         # min: stream is per-track baro-offset, so only drop low spikes, keep the bulk.

data/lib/git_fit/elevation/correct.rb

@@ -0,0 +1,189 @@
+# frozen_string_literal: true
+
+module GitFit
+  module Elevation
+    # Strategy-corrected elevation: L2 std → (optional DEM) → adaptive hysteresis
+    # threshold → activities.elevation_*_terrain. Raw device/API values stay in
+    # elevation_* (written by Backfill); this class only fills the _terrain columns.
+    class Correct
+      SOURCES = %w[garmin garmin_cn strava keep igpsport xoss xingzhe 2bulu apple_health].freeze
+      BARO_SOURCES = %w[xingzhe 2bulu apple_health].freeze
+      BATCH_SIZE = 100
+      STATIC_HYST_BARO = 2.0
+      STATIC_HYST_GPS = 3.0
+
+      def initialize(db, force: false, dry_run: false, dem: true)
+        @db = db
+        @force = force
+        @dry_run = dry_run
+        @dem = dem
+        @updated = 0
+        @skipped = 0
+        @failed = 0
+        @no_model = 0
+      end
+
+      def call(source = nil)
+        sources = source ? [source] : SOURCES
+        sources.each do |src|
+          correct_source(src)
+        end
+        { updated: @updated, skipped: @skipped, failed: @failed, no_model: @no_model }
+      end
+
+      private
+
+      def correct_source(source)
+        offset = 0
+        loop do
+          rows = fetch_rows(source, offset)
+          break if rows.empty?
+
+          @db.transaction do
+            rows.each { |row| process_row(row, source) }
+          end
+
+          offset += BATCH_SIZE
+        end
+      end
+
+      def fetch_rows(source, offset)
+        scope = @db[:activities].where(source: source)
+        unless @force
+          scope = scope.where(Sequel.|(
+                                { elevation_gain_terrain: nil },
+                                { elevation_loss_terrain: nil },
+                                { elevation_min_terrain: nil },
+                                { elevation_max_terrain: nil },
+                              ))
+        end
+        scope.order(:id).limit(BATCH_SIZE, offset).all
+      end
+
+      def process_row(row, source)
+        run_id = row[:run_id]
+        natural_id = run_id.sub("#{source}_", '')
+
+        elev = compute_from_l2(source, natural_id)
+        return skip_row(run_id) unless elev
+
+        update_row(row, elev) unless @dry_run
+        @updated += 1
+      rescue StandardError => e
+        warn "Elevation correct [#{source}][#{natural_id}]: #{e.message}"
+        @failed += 1
+      end
+
+      def compute_from_l2(source, natural_id)
+        l2_path = File.join('data', 'std', source, "#{natural_id}.json")
+        return nil unless File.exist?(l2_path)
+
+        l2 = JSON.parse(File.read(l2_path))
+        return nil unless l2.is_a?(Array)
+
+        pts = l2.filter_map do |pt|
+          next unless pt.is_a?(Hash)
+          next unless pt['latitude'] && pt['longitude'] && pt['altitude']
+
+          pt
+        end
+        return nil if pts.size < 2
+
+        locations = pts.map { |pt| [pt['latitude'].to_f, pt['longitude'].to_f] }
+        alts_raw = pts.map { |pt| pt['altitude'].to_f }
+
+        dem_alts = dem_corrected_altitudes(locations, alts_raw)
+        h = hysteresis_threshold(source, locations, alts_raw)
+
+        gain = elevation_gain_from_alts(dem_alts, threshold: h)
+        loss = elevation_loss_from_alts(dem_alts, threshold: h)
+        filtered = Backfill.filter_low_outliers(dem_alts)
+
+        {
+          elevation_gain_terrain: gain.round(1),
+          elevation_loss_terrain: loss.round(1),
+          elevation_min_terrain: (filtered.min || dem_alts.min)&.round(1),
+          elevation_max_terrain: dem_alts.max&.round(1),
+        }
+      rescue StandardError => e
+        warn "Elevation correct [#{source}][#{natural_id}]: #{e.message}"
+        nil
+      end
+
+      def dem_corrected_altitudes(locations, alts_raw)
+        return alts_raw unless @dem
+
+        elevs = TerrainCoeff.elevations_for(locations)
+        # Partial DEM batch failures leave nil holes; mixing them with raw
+        # altitudes would corrupt gain/loss — fall back wholesale instead.
+        return alts_raw if elevs.nil? || elevs.any?(&:nil?)
+
+        elevs
+      rescue StandardError => e
+        warn "DEM correction failed: #{e.message}; using raw altitudes"
+        @no_model += 1
+        alts_raw
+      end
+
+      def hysteresis_threshold(source, locations, alts_raw)
+        gain_per_km, range_per_km, seg_density = terrain_features(locations, alts_raw)
+        m = TerrainCoeff.terrain_m(
+          gain_per_km: gain_per_km,
+          range_per_km: range_per_km,
+          seg_density: seg_density,
+        )
+        TerrainCoeff.hysteresis_threshold(m)
+      rescue StandardError => e
+        warn "adaptive threshold failed: #{e.message}; using static threshold"
+        static_threshold(source)
+      end
+
+      def terrain_features(locations, alts)
+        distances = Sync::Base.cumulative_distances(locations)
+        total_dist_km = (distances.last || 0) / 1000.0
+        if total_dist_km.positive?
+          gain_per_km = elevation_gain_from_alts(alts, threshold: 0.0) / total_dist_km
+          range_per_km = (alts.max - alts.min) / total_dist_km
+          seg_density = alts.size / total_dist_km
+        end
+        [gain_per_km || 0.0, range_per_km || 0.0, seg_density || 0.0]
+      end
+
+      def static_threshold(source)
+        BARO_SOURCES.include?(source) ? STATIC_HYST_BARO : STATIC_HYST_GPS
+      end
+
+      def elevation_gain_from_alts(alts, threshold: 0.0)
+        return 0.0 if alts.size < 2
+
+        (1...alts.size).sum do |i|
+          d = alts[i] - alts[i - 1]
+          d > threshold ? d : 0.0
+        end
+      end
+
+      def elevation_loss_from_alts(alts, threshold: 0.0)
+        return 0.0 if alts.size < 2
+
+        (1...alts.size).sum do |i|
+          d = alts[i - 1] - alts[i]
+          d > threshold ? d : 0.0
+        end
+      end
+
+      def update_row(row, elev)
+        @db[:activities].where(id: row[:id]).update(
+          elevation_gain_terrain: elev[:elevation_gain_terrain],
+          elevation_loss_terrain: elev[:elevation_loss_terrain],
+          elevation_min_terrain: elev[:elevation_min_terrain],
+          elevation_max_terrain: elev[:elevation_max_terrain],
+          updated_at: Time.now.utc,
+        )
+      end
+
+      def skip_row(_run_id)
+        @skipped += 1
+      end
+    end
+  end
+end

data/lib/git_fit/elevation/terrain_coeff.rb

@@ -16,6 +16,10 @@ module GitFit
       GRID = 0.0003
 
       class << self
+        # OpenTopography free tier allows 1 request/second — pacing below that
+        # guarantees a 429 on every cold-cache slice (observed in workouts CI).
+        DEM_SLICE_INTERVAL = 1.05
+
         def model_path
           @model_path ||= find_model
         end
@@ -62,18 +66,26 @@ module GitFit
         def elevations_for(locations)
           uniq_vals = uniquify_locations(locations)
           elevs = {}
+          last_api = nil
           uniq_vals.each_slice(100).with_index do |slice, bi|
             cached = DemCache.load_batch(slice)
             elevs.merge!(cached)
             missing = DemCache.missing_keys(slice, cached)
-            if missing.any?
-              missing_locations = missing.map { |lat, lng| [lat, lng] }
-              batch = {}
-              DemFetch.fetch_batch(missing_locations, bi, batch, method(:grid_key))
-              elevs.merge!(batch)
-              DemCache.save_batch(batch)
+            next if missing.empty?
+
+            # Pace API→API independently of interleaved cache-hit slices:
+            # cache processing is free, only consecutive fetches must respect
+            # the provider's 1 req/s limit.
+            if last_api
+              elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - last_api
+              sleep(DEM_SLICE_INTERVAL - elapsed) if elapsed < DEM_SLICE_INTERVAL
             end
-            sleep 0.15
+            missing_locations = missing.map { |lat, lng| [lat, lng] }
+            batch = {}
+            DemFetch.fetch_batch(missing_locations, bi, batch, method(:grid_key))
+            elevs.merge!(batch)
+            DemCache.save_batch(batch)
+            last_api = Process.clock_gettime(Process::CLOCK_MONOTONIC)
           end
           locations.map { |lat, lng| elevs[grid_key(lat, lng)] }
         end

data/lib/git_fit/export/calculator.rb

@@ -44,8 +44,8 @@ module GitFit
           cat = r[:sport_category] || 'other'
           d = r[:distance] || 0
           t = r[:moving_time] || 0
-          e = r[:elevation_gain] || 0
-          l = r[:elevation_loss] || 0
+          e = r[:elevation_gain_terrain] || r[:elevation_gain] || 0
+          l = r[:elevation_loss_terrain] || r[:elevation_loss] || 0
           s = r[:steps] || 0
 
           total_dist += d
@@ -113,8 +113,8 @@ module GitFit
         acc[:count] += 1
         acc[:distance_km] += (row[:distance] || 0) / 1000.0
         acc[:moving_time_h] += (row[:moving_time] || 0) / 3600.0
-        acc[:total_elev] += row[:elevation_gain] || 0
-        acc[:total_loss] += row[:elevation_loss] || 0
+        acc[:total_elev] += row[:elevation_gain_terrain] || row[:elevation_gain] || 0
+        acc[:total_loss] += row[:elevation_loss_terrain] || row[:elevation_loss] || 0
         acc[:total_steps] += row[:steps] || 0
         add_if_positive(acc, :total_hr, :hr_count, row[:average_heartrate])
         add_if_positive(acc, :total_cadence, :cadence_count, row[:average_cadence])
@@ -167,8 +167,8 @@ module GitFit
         by_year.sort.map do |year, acts|
           total_dist = acts.sum { |r| r[:distance] || 0 } / 1000.0
           total_time = acts.sum { |r| r[:moving_time] || 0 }
-          total_elev = acts.sum { |r| r[:elevation_gain] || 0 }
-          total_loss = acts.sum { |r| r[:elevation_loss] || 0 }
+          total_elev = acts.sum { |r| r[:elevation_gain_terrain] || r[:elevation_gain] || 0 }
+          total_loss = acts.sum { |r| r[:elevation_loss_terrain] || r[:elevation_loss] || 0 }
           total_steps = acts.sum { |r| r[:steps] || 0 }
           longest = acts.map { |r| r[:distance] || 0 }.max / 1000.0
 

data/lib/git_fit/export/csv.rb

@@ -12,6 +12,7 @@ module GitFit
         average_heartrate max_heartrate average_cadence max_cadence
         average_power max_power calories average_temperature
         average_speed elevation_gain elevation_loss elevation_min elevation_max
+        elevation_gain_terrain elevation_loss_terrain elevation_min_terrain elevation_max_terrain
         steps source dedup_group_id
       ].freeze
 

data/lib/git_fit/export/json.rb

@@ -79,10 +79,10 @@ module GitFit
           calories: a[:calories],
           average_temperature: a[:average_temperature],
           average_speed: a[:average_speed],
-          elevation_gain: a[:elevation_gain],
-          elevation_loss: a[:elevation_loss],
-          elevation_min: a[:elevation_min],
-          elevation_max: a[:elevation_max],
+          elevation_gain: a[:elevation_gain_terrain] || a[:elevation_gain],
+          elevation_loss: a[:elevation_loss_terrain] || a[:elevation_loss],
+          elevation_min: a[:elevation_min_terrain] || a[:elevation_min],
+          elevation_max: a[:elevation_max_terrain] || a[:elevation_max],
           steps: a[:steps],
           source: a[:source],
         }
@@ -109,7 +109,9 @@ module GitFit
           composite = members.first&.slice(:run_id, :distance, :moving_time, :elapsed_time,
             :average_heartrate, :max_heartrate, :elevation_gain, :elevation_loss,
             :elevation_min, :elevation_max, :steps,
-            :average_speed, :average_cadence, :max_cadence) || {}
+            :average_speed, :average_cadence, :max_cadence,
+            :elevation_gain_terrain, :elevation_loss_terrain,
+            :elevation_min_terrain, :elevation_max_terrain) || {}
           composite[:summary_polyline] = pick_best_polyline(group_activities)
           composite
         end

data/lib/git_fit/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module GitFit
-  VERSION = '0.20.0'
+  VERSION = '0.21.1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: git-fit
 version: !ruby/object:Gem::Version
-  version: 0.20.0
+  version: 0.21.1
 platform: ruby
 authors:
 - Lax
@@ -259,6 +259,7 @@ files:
 - db/migrations/001_full_schema.rb
 - db/migrations/002_iso8601_time_format.rb
 - db/migrations/003_add_elevation_fields.rb
+- db/migrations/004_add_elevation_terrain_fields.rb
 - exe/git-fit
 - lib/git-fit.rb
 - lib/git_fit/auth/garmin.rb
@@ -293,6 +294,7 @@ files:
 - lib/git_fit/dedup/service.rb
 - lib/git_fit/dedup/trajectory.rb
 - lib/git_fit/elevation/backfill.rb
+- lib/git_fit/elevation/correct.rb
 - lib/git_fit/elevation/data/dem_coeff.json
 - lib/git_fit/elevation/data/dem_lookup.json
 - lib/git_fit/elevation/dem_cache.rb