git-fit 0.17.1 → 0.17.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1091af70dde8e2b640208f49633b3572ddf64e41c5063ec4541349a873829988
-  data.tar.gz: aa450db53cfe1eacc076d32f35c78da9bb0bfe86ab014d73554127d9467db876
+  metadata.gz: 8e88e21cd60e366ac83ce9f0dac135a5532d6c8953dc9557873a39b55de07174
+  data.tar.gz: 38d23b0f68e6e15ecbe9413d3ac0992fdab84faaa7b2977c6b4284a1f1a223ca
 SHA512:
-  metadata.gz: 122b1ecaac183df2215020f86fdd9f534aed6f356e9bee3e0e53c281f88a6d0c519007aa9809f89c99cfbaeb2e563c712b597065373280160f655188c633749a
-  data.tar.gz: 5c4e2a6f3832390138c1d6ea4d96ede15e8119fb85d67829da6801b81ff0c810d1e42c072bfcd647586b67c5482ec7209168015d0c97bc870326a5b6302916e2
+  metadata.gz: 77e28b8840ad990c41c82585de950af40761f52cfddba38f9b7d6d64386fdc5a9e87191e33d4be9d5a945da950a13e0e3f5182a92a1eaeb0f38ecc481dcb6917
+  data.tar.gz: f6fcee3ebc58afdcbaeb849cb2eca576de1039ed13783424abda28da6f6fccc80cc9fd5690efdf492e533039b22adcfb16b8a98b28cbc5e5494165dd05b88328

data/lib/git-fit.rb

@@ -93,6 +93,8 @@ require_relative 'git_fit/cli/export'
 require_relative 'git_fit/cli/import_cli'
 require_relative 'git_fit/cli/geo_cli'
 require_relative 'git_fit/strava_web/file_check'
+require_relative 'git_fit/cli/elevation'
+require_relative 'git_fit/elevation/backfill'
 require_relative 'git_fit/cli/strava'
 require_relative 'git_fit/auth/garmin_token'
 require_relative 'git_fit/auth/strava'

data/lib/git_fit/cli/elevation.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require 'thor'
+
+module GitFit
+  class ElevationCLI < Thor
+    no_commands do
+      def git_fit_config
+        @git_fit_config ||= GitFit::Config.new(options[:config])
+      end
+    end
+
+    desc 'backfill [SOURCE]', 'Backfill elevation data (gain/loss/min/max) from raw FIT files or L2 data'
+    option :force, type: :boolean, default: false, desc: 'Overwrite existing values (default: only fill NULLs)'
+    def backfill(source = nil)
+      config = git_fit_config
+      conn = GitFit::DB::Connection.new(config.db_path)
+      conn.migrate!
+      db = conn.db
+
+      result = GitFit::Elevation::Backfill.new(db, force: options[:force]).call(source)
+
+      msg = "Elevation backfill: #{result[:updated]} updated, " \
+            "#{result[:skipped]} skipped, #{result[:failed]} failed"
+      say_status :done, msg, :green
+    rescue StandardError => e
+      say_status :error, "Elevation backfill failed: #{e.message}", :red
+    end
+  end
+end

data/lib/git_fit/cli.rb

@@ -117,6 +117,9 @@ module GitFit
       say_status :error, "Dedup failed: #{e.message}", :red
     end
 
+    desc 'elevation SUBCOMMAND', 'Elevation data operations'
+    subcommand 'elevation', ElevationCLI
+
     desc 'strava SUBCOMMAND', 'Strava web operations'
     subcommand 'strava', StravaCLI
 

data/lib/git_fit/elevation/backfill.rb

@@ -0,0 +1,162 @@
+# frozen_string_literal: true
+
+module GitFit
+  module Elevation
+    class Backfill
+      SOURCES = %w[igpsport xoss strava garmin garmin_cn].freeze
+      BATCH_SIZE = 100
+
+      def initialize(db, force: false)
+        @db = db
+        @force = force
+        @updated = 0
+        @skipped = 0
+        @failed = 0
+      end
+
+      def call(source = nil)
+        sources = source ? [source] : SOURCES
+        sources.each do |src|
+          backfill_source(src)
+        end
+        { updated: @updated, skipped: @skipped, failed: @failed }
+      end
+
+      private
+
+      def backfill_source(source)
+        offset = 0
+        loop do
+          rows = fetch_rows(source, offset)
+          break if rows.empty?
+
+          @db.transaction do
+            rows.each { |row| process_row(row, source) }
+          end
+
+          offset += BATCH_SIZE
+        end
+      end
+
+      def fetch_rows(source, offset)
+        scope = @db[:activities].where(source: source)
+        unless @force
+          scope = scope.where(Sequel.|(
+                                elevation_gain: nil,
+                                elevation_loss: nil,
+                                elevation_min: nil,
+                                elevation_max: nil,
+                              ))
+        end
+        scope.order(:id).limit(BATCH_SIZE, offset).all
+      end
+
+      def process_row(row, source)
+        run_id = row[:run_id]
+        natural_id = run_id.sub("#{source}_", '')
+
+        elev = case source
+               when 'igpsport' then compute_from_fit(source, natural_id)
+               when 'xoss' then compute_from_fit(source, natural_id)
+               when 'garmin' then compute_from_fit(source, natural_id)
+               when 'garmin_cn' then compute_from_fit(source, natural_id)
+               when 'strava' then compute_from_l2(source, natural_id)
+               else return
+               end
+
+        return skip_row(run_id) unless elev
+
+        update_row(row, elev)
+      end
+
+      def compute_from_fit(source, natural_id)
+        fit_path = fit_path_for(source, natural_id)
+        return nil unless fit_path && File.exist?(fit_path)
+
+        records = GitFit::FIT::Decoder.decode(fit_path)
+        return nil if records.empty?
+
+        alts = records.filter_map { |r| r['enhancedAltitude'] || r['altitude'] }
+        return nil if alts.size < 2
+
+        session = begin
+          GitFit::FIT::Decoder.session(fit_path)
+        rescue StandardError
+          nil
+        end
+
+        gain = session&.dig('totalAscent').is_a?(Numeric) ? session['totalAscent'] : elevation_gain_from_alts(alts)
+        loss = session&.dig('totalDescent').is_a?(Numeric) ? session['totalDescent'] : elevation_loss_from_alts(alts)
+
+        {
+          elevation_gain: gain&.round(1),
+          elevation_loss: loss&.round(1),
+          elevation_min: alts.min&.round(1),
+          elevation_max: alts.max&.round(1),
+        }
+      rescue StandardError => e
+        warn "Elevation backfill [#{source}][#{natural_id}]: #{e.message}"
+        nil
+      end
+
+      def compute_from_l2(source, natural_id)
+        l2_path = File.join('data', 'std', source, "#{natural_id}.json")
+        return nil unless File.exist?(l2_path)
+
+        l2 = JSON.parse(File.read(l2_path))
+        alts = l2.filter_map { |pt| pt['altitude'] || pt[:altitude] }
+        return nil if alts.size < 2
+
+        {
+          elevation_gain: elevation_gain_from_alts(alts)&.round(1),
+          elevation_loss: elevation_loss_from_alts(alts)&.round(1),
+          elevation_min: alts.min&.round(1),
+          elevation_max: alts.max&.round(1),
+        }
+      rescue StandardError => e
+        warn "Elevation backfill [#{source}][#{natural_id}]: #{e.message}"
+        nil
+      end
+
+      def fit_path_for(source, natural_id)
+        case source
+        when 'garmin', 'garmin_cn'
+          File.join('data', 'raw', source, natural_id, 'activity.fit')
+        else
+          File.join('data', 'raw', source, "#{natural_id}.fit")
+        end
+      end
+
+      def elevation_gain_from_alts(alts)
+        return 0.0 if alts.size < 2
+        (1...alts.size).sum do |i|
+          d = alts[i] - alts[i - 1]
+          d > 0 ? d : 0.0
+        end
+      end
+
+      def elevation_loss_from_alts(alts)
+        return 0.0 if alts.size < 2
+        (1...alts.size).sum do |i|
+          d = alts[i - 1] - alts[i]
+          d > 0 ? d : 0.0
+        end
+      end
+
+      def update_row(row, elev)
+        @db[:activities].where(id: row[:id]).update(
+          elevation_gain: elev[:elevation_gain],
+          elevation_loss: elev[:elevation_loss],
+          elevation_min: elev[:elevation_min],
+          elevation_max: elev[:elevation_max],
+          updated_at: Time.now.utc,
+        )
+        @updated += 1
+      end
+
+      def skip_row(_run_id)
+        @skipped += 1
+      end
+    end
+  end
+end

data/lib/git_fit/source/keep.rb

@@ -39,6 +39,15 @@ module GitFit
         nil
       end
 
+      def decode_step_data(step_data)
+        return nil unless step_data
+        raw = Base64.decode64(step_data)
+        decompressed = zlib_inflate(raw)
+        JSON.parse(decompressed)
+      rescue StandardError
+        nil
+      end
+
       def aes_decrypt(data)
         cipher = OpenSSL::Cipher.new('AES-128-CBC')
         cipher.decrypt

data/lib/git_fit/sync/keep.rb

@@ -14,6 +14,8 @@ module GitFit
       KEEP_TYPES = %w[running hiking cycling].freeze
       KEEP_RAW_EXT = 'json'
 
+      STEP_ENCODED_FIELDS = %w[stepPoints stepFrequencies fullStepPoints].freeze
+
       TYPE_MAP = {
         'outdoorWalking' => 'walk',
         'outdoorRunning' => 'run',
@@ -208,9 +210,16 @@ module GitFit
         raw = run_data.dup
         if raw['heartRate'].is_a?(Hash)
           raw['heartRate'] = raw['heartRate'].dup
-          raw['heartRate'].delete('heartRates')
+          raw['heartRate']['heartRates'] = nil if raw['heartRate'].key?('heartRates')
+        end
+        raw['geoPoints'] = nil if raw.key?('geoPoints')
+
+        decoded_steps = {}
+        STEP_ENCODED_FIELDS.each do |field|
+          next unless raw.key?(field)
+          decoded_steps[field] = Source::Keep.decode_step_data(raw[field])
+          raw[field] = nil
         end
-        raw.delete('geoPoints')
 
         write_source_archive({
           archiveVersion: 0,
@@ -219,7 +228,7 @@ module GitFit
           raw: raw,
           geoPoints: payload[:geo_points],
           heartRates: payload[:hr_data],
-        }, keep_id)
+        }.merge(decoded_steps), keep_id)
 
         wgs_points = Source::Keep.compute_wgs_points(payload, run_data)
         write_standardized_json(wgs_points, payload, run_data, keep_id)

data/lib/git_fit/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module GitFit
-  VERSION = '0.17.1'
+  VERSION = '0.17.3'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: git-fit
 version: !ruby/object:Gem::Version
-  version: 0.17.1
+  version: 0.17.3
 platform: ruby
 authors:
 - Lax
@@ -257,6 +257,7 @@ files:
 - lib/git_fit/auth/strava.rb
 - lib/git_fit/cli.rb
 - lib/git_fit/cli/checkpointable.rb
+- lib/git_fit/cli/elevation.rb
 - lib/git_fit/cli/export.rb
 - lib/git_fit/cli/geo_cli.rb
 - lib/git_fit/cli/gh_cli.rb
@@ -277,6 +278,7 @@ files:
 - lib/git_fit/dedup/matcher.rb
 - lib/git_fit/dedup/service.rb
 - lib/git_fit/dedup/trajectory.rb
+- lib/git_fit/elevation/backfill.rb
 - lib/git_fit/export.rb
 - lib/git_fit/export/calculator.rb
 - lib/git_fit/export/csv.rb