git-fit 0.19.0 → 0.21.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 977f82c280b5d5f2c0787f5b455d8b181036cddb8010fd7fd6d2c8947ad98c07
-  data.tar.gz: 74367d8b3b9d783fecc92e9c98ec0ba32d9d8383cc2fa0cb06aca4fc3397e0bd
+  metadata.gz: e9ac100a25388c8d2e9a82f541b14b369a08f4307cfa70949032c25a8afc7c74
+  data.tar.gz: 60eb1e4c683317df12d66260517b21db9841823b9b20de6defa2819bbecd4b65
 SHA512:
-  metadata.gz: e65db09935d0182ddadce280b108a241ef272d37617f01544f8062c1f3e1c2c7ff8fcb0b956b37889da7c38d7fe0d291b1d01f73eb50eddddb0950767faa9238
-  data.tar.gz: e1480e0a9920618456681c4f0421b9520054d8943e1fc6aa0e81d41b19eadf712351581da86252837d6fb29d94ac3c02d40f022beba88f970e1eae104eb07a6b
+  metadata.gz: a50e41061abe840e12c2a2f5741cef761dd97f5d87867d830bcb04e6ce7da61e8da48ff54e2500252c8c78c3f5a6a033dde69d695d49257a8d08f97c24799a02
+  data.tar.gz: 6db8e171e162cb31c9aca678c0db13e8271f1d61d0d1bf2097f12c1189d8130861349a69991907a7f0cc36c69e36b2892db7dd6f4950b984ec3473e15e2fd5b8

data/db/migrations/004_add_elevation_terrain_fields.rb

@@ -0,0 +1,19 @@
+Sequel.migration do
+  up do
+    alter_table(:activities) do
+      add_column :elevation_gain_terrain, Float
+      add_column :elevation_loss_terrain, Float
+      add_column :elevation_min_terrain,  Float
+      add_column :elevation_max_terrain,  Float
+    end
+  end
+
+  down do
+    alter_table(:activities) do
+      drop_column :elevation_max_terrain
+      drop_column :elevation_min_terrain
+      drop_column :elevation_loss_terrain
+      drop_column :elevation_gain_terrain
+    end
+  end
+end

data/lib/git-fit.rb

@@ -95,6 +95,7 @@ require_relative 'git_fit/cli/geo_cli'
 require_relative 'git_fit/strava_web/file_check'
 require_relative 'git_fit/cli/elevation'
 require_relative 'git_fit/elevation/backfill'
+require_relative 'git_fit/elevation/correct'
 require_relative 'git_fit/elevation/terrain_coeff'
 require_relative 'git_fit/cli/strava'
 require_relative 'git_fit/auth/garmin_token'

data/lib/git_fit/cli/elevation.rb

@@ -11,7 +11,7 @@ module GitFit
       end
     end
 
-    desc 'backfill [SOURCE]', 'Backfill elevation data (gain/loss/min/max) from raw FIT files or L2 data'
+    desc 'backfill [SOURCE]', 'Backfill raw elevation data (device/API values) from FIT files or L2 data'
     option :force, type: :boolean, default: false, desc: 'Overwrite existing values (default: only fill NULLs)'
     def backfill(source = nil)
       config = git_fit_config
@@ -28,7 +28,33 @@ module GitFit
       say_status :error, "Elevation backfill failed: #{e.message}", :red
     end
 
-    desc 'train', 'Train DEM terrain coefficient model from activity data'
+    desc 'correct [SOURCE]', 'Apply elevation correction into _terrain columns (adaptive hysteresis; --dem adds DEM)'
+    option :force, type: :boolean, default: false, desc: 'Recompute even when _terrain columns are already populated'
+    option :'dry-run', type: :boolean, default: false, desc: 'Count records without writing to database'
+    option :dem, type: :boolean, default: true,
+                 desc: 'Enable DEM correction (default: true; --no-dem for hysteresis only)'
+    def correct(source = nil)
+      config = git_fit_config
+      conn = GitFit::DB::Connection.new(config.db_path)
+      conn.migrate!
+      db = conn.db
+
+      result = GitFit::Elevation::Correct.new(
+        db,
+        force: options[:force],
+        dry_run: options[:'dry-run'],
+        dem: options[:dem],
+      ).call(source)
+
+      msg = "Elevation correct: #{result[:updated]} updated, " \
+            "#{result[:skipped]} skipped, #{result[:failed]} failed, " \
+            "#{result[:no_model]} no_model"
+      say_status :done, msg, :green
+    rescue StandardError => e
+      say_status :error, "Elevation correct failed: #{e.message}", :red
+    end
+
+    desc 'train', 'Train DEM terrain coefficient model from activity data', hide: true
     option :sources, type: :string, default: nil, desc: 'Comma-separated source adapters (default: all)'
     option :output, type: :string, default: nil, desc: 'Output path for model file'
     def train
@@ -51,7 +77,7 @@ module GitFit
       exit 1
     end
 
-    desc 'infer GAIN_PER_KM RANGE_PER_KM SEG_DENSITY', 'Infer optimal h threshold for terrain'
+    desc 'infer GAIN_PER_KM RANGE_PER_KM SEG_DENSITY', 'Infer optimal h threshold for terrain', hide: true
     def infer(gain_per_km, range_per_km, seg_density)
       g = gain_per_km.to_f
       r = range_per_km.to_f
@@ -77,7 +103,7 @@ module GitFit
       exit 1
     end
 
-    desc 'model-info', 'Show information about the loaded DEM model'
+    desc 'model-info', 'Show information about the loaded DEM model', hide: true
     def model_info
       model = GitFit::Elevation::TerrainCoeff.model
       path = GitFit::Elevation::TerrainCoeff.model_path

data/lib/git_fit/elevation/backfill.rb

@@ -5,8 +5,6 @@ module GitFit
     class Backfill
       SOURCES = %w[igpsport xoss strava garmin garmin_cn xingzhe].freeze
       BATCH_SIZE = 100
-      # Fallback threshold when detail.json elevation is absent (issue #83: baro 2m / GPS 3m).
-      ELEV_THRESHOLD = 2.0
 
       def initialize(db, force: false)
         @db = db
@@ -88,8 +86,8 @@ module GitFit
           nil
         end
 
-        gain = session&.dig('totalAscent').is_a?(Numeric) ? session['totalAscent'] : elevation_gain_from_alts(alts)
-        loss = session&.dig('totalDescent').is_a?(Numeric) ? session['totalDescent'] : elevation_loss_from_alts(alts)
+        gain = session&.dig('totalAscent').is_a?(Numeric) ? session['totalAscent'] : nil
+        loss = session&.dig('totalDescent').is_a?(Numeric) ? session['totalDescent'] : nil
 
         {
           elevation_gain: gain&.round(1),
@@ -102,22 +100,7 @@ module GitFit
         nil
       end
 
-      def compute_from_l2(source, natural_id)
-        l2_path = File.join('data', 'std', source, "#{natural_id}.json")
-        return nil unless File.exist?(l2_path)
-
-        l2 = JSON.parse(File.read(l2_path))
-        alts = l2.filter_map { |pt| pt['altitude'] || pt[:altitude] }
-        return nil if alts.size < 2
-
-        {
-          elevation_gain: elevation_gain_from_alts(alts)&.round(1),
-          elevation_loss: elevation_loss_from_alts(alts)&.round(1),
-          elevation_min: alts.min&.round(1),
-          elevation_max: alts.max&.round(1),
-        }
-      rescue StandardError => e
-        warn "Elevation backfill [#{source}][#{natural_id}]: #{e.message}"
+      def compute_from_l2(_source, _natural_id)
         nil
       end
 
@@ -141,8 +124,8 @@ module GitFit
         dl = detail && detail['elevation_loss']
         dm = detail && detail['max_altitude']
 
-        gain = detail_gain?(dg) ? dg : elevation_gain_from_alts(alts, threshold: ELEV_THRESHOLD)
-        loss = detail_gain?(dl) ? dl : elevation_loss_from_alts(alts, threshold: ELEV_THRESHOLD)
+        gain = detail_gain?(dg) ? dg : nil
+        loss = detail_gain?(dl) ? dl : nil
 
         # max: detail max_altitude is reliable (matches stream max in samples); else raw max.
         # min: stream is per-track baro-offset, so only drop low spikes, keep the bulk.

data/lib/git_fit/elevation/correct.rb

@@ -0,0 +1,189 @@
+# frozen_string_literal: true
+
+module GitFit
+  module Elevation
+    # Strategy-corrected elevation: L2 std → (optional DEM) → adaptive hysteresis
+    # threshold → activities.elevation_*_terrain. Raw device/API values stay in
+    # elevation_* (written by Backfill); this class only fills the _terrain columns.
+    class Correct
+      SOURCES = %w[garmin garmin_cn strava keep igpsport xoss xingzhe 2bulu apple_health].freeze
+      BARO_SOURCES = %w[xingzhe 2bulu apple_health].freeze
+      BATCH_SIZE = 100
+      STATIC_HYST_BARO = 2.0
+      STATIC_HYST_GPS = 3.0
+
+      def initialize(db, force: false, dry_run: false, dem: true)
+        @db = db
+        @force = force
+        @dry_run = dry_run
+        @dem = dem
+        @updated = 0
+        @skipped = 0
+        @failed = 0
+        @no_model = 0
+      end
+
+      def call(source = nil)
+        sources = source ? [source] : SOURCES
+        sources.each do |src|
+          correct_source(src)
+        end
+        { updated: @updated, skipped: @skipped, failed: @failed, no_model: @no_model }
+      end
+
+      private
+
+      def correct_source(source)
+        offset = 0
+        loop do
+          rows = fetch_rows(source, offset)
+          break if rows.empty?
+
+          @db.transaction do
+            rows.each { |row| process_row(row, source) }
+          end
+
+          offset += BATCH_SIZE
+        end
+      end
+
+      def fetch_rows(source, offset)
+        scope = @db[:activities].where(source: source)
+        unless @force
+          scope = scope.where(Sequel.|(
+                                { elevation_gain_terrain: nil },
+                                { elevation_loss_terrain: nil },
+                                { elevation_min_terrain: nil },
+                                { elevation_max_terrain: nil },
+                              ))
+        end
+        scope.order(:id).limit(BATCH_SIZE, offset).all
+      end
+
+      def process_row(row, source)
+        run_id = row[:run_id]
+        natural_id = run_id.sub("#{source}_", '')
+
+        elev = compute_from_l2(source, natural_id)
+        return skip_row(run_id) unless elev
+
+        update_row(row, elev) unless @dry_run
+        @updated += 1
+      rescue StandardError => e
+        warn "Elevation correct [#{source}][#{natural_id}]: #{e.message}"
+        @failed += 1
+      end
+
+      def compute_from_l2(source, natural_id)
+        l2_path = File.join('data', 'std', source, "#{natural_id}.json")
+        return nil unless File.exist?(l2_path)
+
+        l2 = JSON.parse(File.read(l2_path))
+        return nil unless l2.is_a?(Array)
+
+        pts = l2.filter_map do |pt|
+          next unless pt.is_a?(Hash)
+          next unless pt['latitude'] && pt['longitude'] && pt['altitude']
+
+          pt
+        end
+        return nil if pts.size < 2
+
+        locations = pts.map { |pt| [pt['latitude'].to_f, pt['longitude'].to_f] }
+        alts_raw = pts.map { |pt| pt['altitude'].to_f }
+
+        dem_alts = dem_corrected_altitudes(locations, alts_raw)
+        h = hysteresis_threshold(source, locations, alts_raw)
+
+        gain = elevation_gain_from_alts(dem_alts, threshold: h)
+        loss = elevation_loss_from_alts(dem_alts, threshold: h)
+        filtered = Backfill.filter_low_outliers(dem_alts)
+
+        {
+          elevation_gain_terrain: gain.round(1),
+          elevation_loss_terrain: loss.round(1),
+          elevation_min_terrain: (filtered.min || dem_alts.min)&.round(1),
+          elevation_max_terrain: dem_alts.max&.round(1),
+        }
+      rescue StandardError => e
+        warn "Elevation correct [#{source}][#{natural_id}]: #{e.message}"
+        nil
+      end
+
+      def dem_corrected_altitudes(locations, alts_raw)
+        return alts_raw unless @dem
+
+        elevs = TerrainCoeff.elevations_for(locations)
+        # Partial DEM batch failures leave nil holes; mixing them with raw
+        # altitudes would corrupt gain/loss — fall back wholesale instead.
+        return alts_raw if elevs.nil? || elevs.any?(&:nil?)
+
+        elevs
+      rescue StandardError => e
+        warn "DEM correction failed: #{e.message}; using raw altitudes"
+        @no_model += 1
+        alts_raw
+      end
+
+      def hysteresis_threshold(source, locations, alts_raw)
+        gain_per_km, range_per_km, seg_density = terrain_features(locations, alts_raw)
+        m = TerrainCoeff.terrain_m(
+          gain_per_km: gain_per_km,
+          range_per_km: range_per_km,
+          seg_density: seg_density,
+        )
+        TerrainCoeff.hysteresis_threshold(m)
+      rescue StandardError => e
+        warn "adaptive threshold failed: #{e.message}; using static threshold"
+        static_threshold(source)
+      end
+
+      def terrain_features(locations, alts)
+        distances = Sync::Base.cumulative_distances(locations)
+        total_dist_km = (distances.last || 0) / 1000.0
+        if total_dist_km.positive?
+          gain_per_km = elevation_gain_from_alts(alts, threshold: 0.0) / total_dist_km
+          range_per_km = (alts.max - alts.min) / total_dist_km
+          seg_density = alts.size / total_dist_km
+        end
+        [gain_per_km || 0.0, range_per_km || 0.0, seg_density || 0.0]
+      end
+
+      def static_threshold(source)
+        BARO_SOURCES.include?(source) ? STATIC_HYST_BARO : STATIC_HYST_GPS
+      end
+
+      def elevation_gain_from_alts(alts, threshold: 0.0)
+        return 0.0 if alts.size < 2
+
+        (1...alts.size).sum do |i|
+          d = alts[i] - alts[i - 1]
+          d > threshold ? d : 0.0
+        end
+      end
+
+      def elevation_loss_from_alts(alts, threshold: 0.0)
+        return 0.0 if alts.size < 2
+
+        (1...alts.size).sum do |i|
+          d = alts[i - 1] - alts[i]
+          d > threshold ? d : 0.0
+        end
+      end
+
+      def update_row(row, elev)
+        @db[:activities].where(id: row[:id]).update(
+          elevation_gain_terrain: elev[:elevation_gain_terrain],
+          elevation_loss_terrain: elev[:elevation_loss_terrain],
+          elevation_min_terrain: elev[:elevation_min_terrain],
+          elevation_max_terrain: elev[:elevation_max_terrain],
+          updated_at: Time.now.utc,
+        )
+      end
+
+      def skip_row(_run_id)
+        @skipped += 1
+      end
+    end
+  end
+end

data/lib/git_fit/export/calculator.rb

@@ -44,8 +44,8 @@ module GitFit
           cat = r[:sport_category] || 'other'
           d = r[:distance] || 0
           t = r[:moving_time] || 0
-          e = r[:elevation_gain] || 0
-          l = r[:elevation_loss] || 0
+          e = r[:elevation_gain_terrain] || r[:elevation_gain] || 0
+          l = r[:elevation_loss_terrain] || r[:elevation_loss] || 0
           s = r[:steps] || 0
 
           total_dist += d
@@ -113,8 +113,8 @@ module GitFit
         acc[:count] += 1
         acc[:distance_km] += (row[:distance] || 0) / 1000.0
         acc[:moving_time_h] += (row[:moving_time] || 0) / 3600.0
-        acc[:total_elev] += row[:elevation_gain] || 0
-        acc[:total_loss] += row[:elevation_loss] || 0
+        acc[:total_elev] += row[:elevation_gain_terrain] || row[:elevation_gain] || 0
+        acc[:total_loss] += row[:elevation_loss_terrain] || row[:elevation_loss] || 0
         acc[:total_steps] += row[:steps] || 0
         add_if_positive(acc, :total_hr, :hr_count, row[:average_heartrate])
         add_if_positive(acc, :total_cadence, :cadence_count, row[:average_cadence])
@@ -167,8 +167,8 @@ module GitFit
         by_year.sort.map do |year, acts|
           total_dist = acts.sum { |r| r[:distance] || 0 } / 1000.0
           total_time = acts.sum { |r| r[:moving_time] || 0 }
-          total_elev = acts.sum { |r| r[:elevation_gain] || 0 }
-          total_loss = acts.sum { |r| r[:elevation_loss] || 0 }
+          total_elev = acts.sum { |r| r[:elevation_gain_terrain] || r[:elevation_gain] || 0 }
+          total_loss = acts.sum { |r| r[:elevation_loss_terrain] || r[:elevation_loss] || 0 }
           total_steps = acts.sum { |r| r[:steps] || 0 }
           longest = acts.map { |r| r[:distance] || 0 }.max / 1000.0
 

data/lib/git_fit/export/csv.rb

@@ -12,6 +12,7 @@ module GitFit
         average_heartrate max_heartrate average_cadence max_cadence
         average_power max_power calories average_temperature
         average_speed elevation_gain elevation_loss elevation_min elevation_max
+        elevation_gain_terrain elevation_loss_terrain elevation_min_terrain elevation_max_terrain
         steps source dedup_group_id
       ].freeze
 

data/lib/git_fit/export/json.rb

@@ -79,10 +79,10 @@ module GitFit
           calories: a[:calories],
           average_temperature: a[:average_temperature],
           average_speed: a[:average_speed],
-          elevation_gain: a[:elevation_gain],
-          elevation_loss: a[:elevation_loss],
-          elevation_min: a[:elevation_min],
-          elevation_max: a[:elevation_max],
+          elevation_gain: a[:elevation_gain_terrain] || a[:elevation_gain],
+          elevation_loss: a[:elevation_loss_terrain] || a[:elevation_loss],
+          elevation_min: a[:elevation_min_terrain] || a[:elevation_min],
+          elevation_max: a[:elevation_max_terrain] || a[:elevation_max],
           steps: a[:steps],
           source: a[:source],
         }
@@ -109,7 +109,9 @@ module GitFit
           composite = members.first&.slice(:run_id, :distance, :moving_time, :elapsed_time,
             :average_heartrate, :max_heartrate, :elevation_gain, :elevation_loss,
             :elevation_min, :elevation_max, :steps,
-            :average_speed, :average_cadence, :max_cadence) || {}
+            :average_speed, :average_cadence, :max_cadence,
+            :elevation_gain_terrain, :elevation_loss_terrain,
+            :elevation_min_terrain, :elevation_max_terrain) || {}
           composite[:summary_polyline] = pick_best_polyline(group_activities)
           composite
         end

data/lib/git_fit/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module GitFit
-  VERSION = '0.19.0'
+  VERSION = '0.21.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: git-fit
 version: !ruby/object:Gem::Version
-  version: 0.19.0
+  version: 0.21.0
 platform: ruby
 authors:
 - Lax
@@ -259,6 +259,7 @@ files:
 - db/migrations/001_full_schema.rb
 - db/migrations/002_iso8601_time_format.rb
 - db/migrations/003_add_elevation_fields.rb
+- db/migrations/004_add_elevation_terrain_fields.rb
 - exe/git-fit
 - lib/git-fit.rb
 - lib/git_fit/auth/garmin.rb
@@ -293,6 +294,7 @@ files:
 - lib/git_fit/dedup/service.rb
 - lib/git_fit/dedup/trajectory.rb
 - lib/git_fit/elevation/backfill.rb
+- lib/git_fit/elevation/correct.rb
 - lib/git_fit/elevation/data/dem_coeff.json
 - lib/git_fit/elevation/data/dem_lookup.json
 - lib/git_fit/elevation/dem_cache.rb