git-fit 0.18.2 → 0.18.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 41afe213d0cfd11467217bc1fca559a90d791d08131f4ad2a8b1be69655cdfbe
-  data.tar.gz: bd97306411a2b55af3a0253646cbd43aff7907d4253175ea88372cea457f7e0c
+  metadata.gz: b68548ae0905b7374d85979f27053683d604ab26c8ae0e66011cd6cf776bf7ad
+  data.tar.gz: 863da16427ca424985a8f6ef62237b9a53e2c782fc390950323fd04ad2cb6929
 SHA512:
-  metadata.gz: eee9145cb899365d6cc669dcc065e6df9d9785ce97522c3e0f561f40225193400f664158b7335395d655b4c5a4d927e835098a39a1a4cb686ff38f4deac455e0
-  data.tar.gz: 748ed231a6b7ac6fe293e6c5481da6207cf87472645214019c468ed9f95628f2943bd955980df2e4d56456af6f7dae7891c53b0d67159d9ebb799c20b6e2dbeb
+  metadata.gz: fa7450b9a823e2b8dca5d5a81d6663caba96fdc9f43444cb53619b5f0cf3b4afc2016b3e69c74e03a4a53250cd4f6a7f79b38862a87a98dc1fb4a538f014117f
+  data.tar.gz: 1d7a706ea4b63ccf6f2607a236d73bbbcdeee9a49fb809846030667f8dec6b2b6f43eec3019891d00fbcb9d8b71198c3eec72243bc4a0d9e650bb012dfaff481

data/lib/git_fit/elevation/dem_cache.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'json'
+
+module GitFit
+  module Elevation
+    module DemCache
+      DEM_CACHE_DIR = ENV.fetch('GIT_FIT_DEM_CACHE_DIR', 'data/cache/dem')
+      CACHE_FILE = File.join(DEM_CACHE_DIR, 'dem.json')
+      GRID = 0.0003
+
+      class << self
+        def grid_key(lat, lng)
+          "#{(lat / GRID).round}/#{(lng / GRID).round}"
+        end
+
+        def load_all
+          return {} unless File.exist?(CACHE_FILE)
+          JSON.parse(File.read(CACHE_FILE))
+        rescue StandardError
+          {}
+        end
+
+        def load_batch(slice)
+          all = load_all
+          results = {}
+          slice.each do |lat, lng|
+            gk = grid_key(lat, lng)
+            results[gk] = all[gk] if all.key?(gk)
+          end
+          results
+        end
+
+        def missing_keys(slice, cached)
+          slice.reject { |lat, lng| cached.key?(grid_key(lat, lng)) }
+        end
+
+        def save_batch(elevs_by_gk)
+          return if elevs_by_gk.empty?
+          FileUtils.mkdir_p(DEM_CACHE_DIR)
+          existing = load_all
+          elevs_by_gk.each { |gk, elev| existing[gk] = elev }
+          File.write(CACHE_FILE, JSON.generate(existing))
+        rescue StandardError => e
+          warn "DEM cache write error: #{e.message}"
+        end
+      end
+    end
+  end
+end

data/lib/git_fit/elevation/dem_fetch.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require 'net/http'
+require 'uri'
+
+module GitFit
+  module Elevation
+    module DemFetch
+      DEM_URL = 'https://api.opentopodata.org/v1/srtm30m'
+
+      class << self
+        def fetch_batch(slice, batch_idx, elevs, grid_key_fn)
+          qs = slice.map { |la, ln| "#{la},#{ln}" }.join('|')
+          uri = URI("#{DEM_URL}?locations=#{qs.gsub(',', '%2C').gsub('|', '%7C')}")
+          attempt = 0
+          loop do
+            res = Net::HTTP.get_response(uri)
+            if res.code == '200'
+              results = JSON.parse(res.body)['results']
+              slice.each_with_index do |(lat, lng), i|
+                elevs[grid_key_fn.call(lat, lng)] = results[i]['elevation'] if results[i]
+              end
+              break
+            elsif res.code == '429'
+              attempt += 1
+              if attempt < 5
+                delay = 2**attempt
+                warn "DEM batch #{batch_idx}: rate limited, retrying in #{delay}s (attempt #{attempt})"
+                sleep(delay)
+              else
+                warn "DEM batch #{batch_idx}: rate limited, giving up after #{attempt} attempts"
+                break
+              end
+            else
+              warn "DEM batch #{batch_idx} HTTP #{res.code}"
+              break
+            end
+          rescue StandardError => e
+            warn "DEM batch #{batch_idx}: #{e.message}"
+            break
+          end
+        end
+      end
+    end
+  end
+end

data/lib/git_fit/elevation/terrain_coeff.rb

@@ -1,18 +1,18 @@
 # frozen_string_literal: true
 
 require 'msgpack'
-require 'net/http'
-require 'uri'
+
+require_relative 'dem_cache'
+require_relative 'dem_fetch'
+require_relative 'terrain_lookup'
 
 module GitFit
   module Elevation
     module TerrainCoeff
-      CACHE_DIR = ENV.fetch('GIT_FIT_DEM_CACHE_DIR', 'data/cache/dem_fit')
-      DEM_CACHE_DIR = File.join(CACHE_DIR, 'cache')
+      MODEL_CACHE_DIR = ENV.fetch('GIT_FIT_DEM_MODEL_DIR', 'data/cache/dem_fit')
       BUNDLED_MODEL_DIR = File.expand_path('model', __dir__)
       BUNDLED_MODEL = File.join(BUNDLED_MODEL_DIR, 'dem_model.msgpack')
       MODEL_FILE = 'dem_model.msgpack'
-      DEM_URL = 'https://api.opentopodata.org/v1/srtm30m'
       GRID = 0.0003
 
       class << self
@@ -25,10 +25,10 @@ module GitFit
         end
 
         def find_model
-          cache_path = File.join(CACHE_DIR, MODEL_FILE)
+          cache_path = File.join(MODEL_CACHE_DIR, MODEL_FILE)
           return cache_path if File.exist?(cache_path)
           return BUNDLED_MODEL if File.exist?(BUNDLED_MODEL)
-          raise "No #{MODEL_FILE} found in cache (#{CACHE_DIR}) or bundled (#{BUNDLED_MODEL})"
+          raise "No #{MODEL_FILE} found in cache (#{MODEL_CACHE_DIR}) or bundled (#{BUNDLED_MODEL})"
         end
 
         def reset!
@@ -49,52 +49,41 @@ module GitFit
         end
 
         def hysteresis_threshold(m_val)
-          return 2.0 if m_val <= 0
-          pav = model['pav_h']
-          return 2.0 if m_val <= pav[0][0]
-          return pav[-1][1] if m_val >= pav[-1][0]
-          pav.each_with_index do |(_xk, yk), i|
-            next unless i + 1 < pav.size
-            return yk if m_val <= pav[i + 1][0]
-          end
-          pav[-1][1]
+          Lookup.hysteresis_threshold(model['pav_h'], m_val)
         end
 
         def dem_spacing(gain_per_km:, range_per_km:, seg_density:)
-          nearest_neighbor_spacing(gain_per_km, range_per_km, seg_density)
+          Lookup.nearest_neighbor_spacing(lookup, gain_per_km, range_per_km, seg_density)
         rescue StandardError => e
           warn "TerrainCoeff: dem_spacing fallback to 181m (#{e.message})"
           181
         end
 
         def elevations_for(locations)
-          uniq = {}
-          locations.each { |lat, lng| uniq[grid_key(lat, lng)] ||= [lat, lng] }
-          uniq_vals = uniq.values
-
+          uniq_vals = uniquify_locations(locations)
           elevs = {}
           uniq_vals.each_slice(100).with_index do |slice, bi|
-            qs = slice.map { |la, ln| "#{la},#{ln}" }.join('|')
-            uri = URI("#{DEM_URL}?locations=#{qs.gsub(',', '%2C').gsub('|', '%7C')}")
-            begin
-              res = Net::HTTP.get_response(uri)
-              if res.code == '200'
-                results = JSON.parse(res.body)['results']
-                slice.each_with_index do |(lat, lng), i|
-                  elevs[grid_key(lat, lng)] = results[i]['elevation'] if results[i]
-                end
-              else
-                warn "DEM batch #{bi} HTTP #{res.code}"
-              end
-            rescue StandardError => e
-              warn "DEM batch #{bi}: #{e.message}"
+            cached = DemCache.load_batch(slice)
+            elevs.merge!(cached)
+            missing = DemCache.missing_keys(slice, cached)
+            if missing.any?
+              missing_locations = missing.map { |lat, lng| [lat, lng] }
+              batch = {}
+              DemFetch.fetch_batch(missing_locations, bi, batch, method(:grid_key))
+              elevs.merge!(batch)
+              DemCache.save_batch(batch)
             end
             sleep 0.15
           end
-
           locations.map { |lat, lng| elevs[grid_key(lat, lng)] }
         end
 
+        def uniquify_locations(locations)
+          uniq = {}
+          locations.each { |lat, lng| uniq[grid_key(lat, lng)] ||= [lat, lng] }
+          uniq.values
+        end
+
         def grid_key(lat, lng)
           "#{(lat / GRID).round}/#{(lng / GRID).round}"
         end
@@ -102,24 +91,6 @@ module GitFit
         def lookup
           model['lookup']
         end
-
-        private
-
-        def nearest_neighbor_spacing(gain_per_km, range_per_km, seg_density)
-          best = nil
-          best_dist = Float::INFINITY
-          lookup.each do |ref|
-            d = Math.sqrt(
-              (gain_per_km.to_f - ref['gk'])**2 +
-              (range_per_km.to_f - ref['rk'])**2 +
-              (seg_density.to_f - ref['sd'])**2,
-            )
-            next unless d < best_dist
-            best_dist = d
-            best = ref
-          end
-          best ? best['sp'] : 181
-        end
       end
     end
   end

data/lib/git_fit/elevation/terrain_lookup.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module GitFit
+  module Elevation
+    module TerrainCoeff
+      module Lookup
+        class << self
+          def nearest_neighbor_spacing(model_lookup, gain_per_km, range_per_km, seg_density)
+            best = nil
+            best_dist = Float::INFINITY
+            model_lookup.each do |ref|
+              d = Math.sqrt(
+                (gain_per_km.to_f - ref['gk'])**2 +
+                (range_per_km.to_f - ref['rk'])**2 +
+                (seg_density.to_f - ref['sd'])**2,
+              )
+              next unless d < best_dist
+              best_dist = d
+              best = ref
+            end
+            best ? best['sp'] : 181
+          end
+
+          def hysteresis_threshold(model_pav_h, m_val)
+            return 2.0 if m_val <= 0
+            return 2.0 if m_val <= model_pav_h[0][0]
+            return model_pav_h[-1][1] if m_val >= model_pav_h[-1][0]
+            model_pav_h.each_with_index do |(_xk, yk), i|
+              next unless i + 1 < model_pav_h.size
+              return yk if m_val <= model_pav_h[i + 1][0]
+            end
+            model_pav_h[-1][1]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/git_fit/elevation/train/collector.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative '../dem_cache'
 require_relative '../terrain_coeff'
 
 module GitFit
@@ -11,15 +12,25 @@ module GitFit
 
         def initialize(sources: nil)
           @sources = sources
+          @samples = []
         end
 
-        def collect
-          samples = []
+        attr_reader :samples
+
+        def count_samples
+          total = 0
+          each_source_sample do |_sample|
+            total += 1
+          end
+          total
+        end
+
+        def each_sample
           each_source_sample do |sample|
             result = evaluate_sample(sample)
-            samples << result if result
+            @samples << result if result
+            yield result
           end
-          samples
         end
 
         def each_source_sample(&block)
@@ -37,7 +48,7 @@ module GitFit
 
         def resolve_sources(names)
           names.map do |name|
-            Sync::Base.adapters.find { |a| a.name.demodulize.downcase == name.to_s.downcase }
+            Sync::Base.adapters.find { |a| a.config_key == name.to_s }
           end.compact
         end
 
@@ -50,11 +61,13 @@ module GitFit
 
           return nil if altitudes.size < 2
           return nil if distances.empty? || distances.last < 1000
-          return nil if sample[:altitudes].size < 2
 
+          cache_before = DemCache.load_batch(locations)
           dem = TerrainCoeff.elevations_for(locations)
           return nil if dem.empty? || dem.compact.size < 2
 
+          cache_hit = !cache_before.empty? && cache_before.values.all? { |v| !v.nil? }
+
           truth_min = dem.compact.min
           truth_max = dem.compact.max
           sm = smooth(altitudes)
@@ -97,6 +110,7 @@ module GitFit
             gain_per_km: f[:gain_per_km],
             range_per_km: f[:range_per_km],
             seg_density: f[:seg_density],
+            cache_hit: cache_hit,
           }
         end
 

data/lib/git_fit/elevation/train/runner.rb

@@ -17,19 +17,39 @@ module GitFit
         end
 
         def run
-          puts 'Collecting samples...'
           collector = Collector.new(sources: @sources)
-          samples = collector.collect
+          total = collector.count_samples
+          puts "Collecting samples (#{total} total)..."
 
-          if samples.empty?
-            warn 'No samples collected'
-            return false
+          collected = 0
+          cached = 0
+          missed = 0
+          skipped = 0
+
+          collector.each_sample do |result|
+            if result
+              collected += 1
+              if result[:cache_hit]
+                cached += 1
+              else
+                missed += 1
+              end
+              print_progress(collected, cached, missed, total)
+            else
+              skipped += 1
+            end
           end
 
-          puts "Collected #{samples.size} samples"
+          puts
+          puts "Collected: #{collected} samples (#{cached} hit / #{missed} miss / #{skipped} skipped)"
+
+          if collected.zero?
+            warn 'No valid samples collected'
+            return false
+          end
 
           puts 'Fitting model...'
-          fitter = Fitter.new(samples)
+          fitter = Fitter.new(collector.samples)
           model = fitter.fit
 
           unless model
@@ -43,15 +63,32 @@ module GitFit
             return false
           end
 
-          puts 'Model trained successfully!'
-          puts "  n_samples: #{model[:meta]['n_samples']}"
-          puts "  sources: #{model[:meta]['sources'].join(', ')}"
-          puts "  ridge_h: #{model[:ridge_h].map { |w| format('%.2f', w) }.join(', ')}"
-          puts "  output: #{@output}"
+          print_summary(model)
 
           true
         end
 
+        def print_progress(collected, cached, missed, total)
+          pct = (collected.to_f / total * 100).round(1)
+          bar_len = 30
+          filled = (collected.to_f / total * bar_len).round
+          bar = '=' * filled + '-' * (bar_len - filled)
+          $stdout.write "\r  [#{bar}] #{pct}% #{collected}/#{total} (h:#{cached} m:#{missed})"
+          $stdout.flush
+        end
+
+        def print_summary(model)
+          meta = model[:meta]
+          puts
+          puts '=== Model Trained ==='
+          puts "  samples:     #{meta['n_samples']}"
+          puts "  sources:     #{meta['sources'].join(', ')}"
+          puts "  trained_at:  #{meta['trained_at']}"
+          puts "  ridge_h:     #{model[:ridge_h].map { |w| format('%.3f', w) }.join(', ')}"
+          puts "  norms:       #{model[:norms].map { |n| format('%.2f', n) }.join(', ')}"
+          puts "  output:      #{@output}"
+        end
+
         def default_output_path
           File.join(CACHE_DIR, MODEL_FILE)
         end

data/lib/git_fit/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module GitFit
-  VERSION = '0.18.2'
+  VERSION = '0.18.4'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: git-fit
 version: !ruby/object:Gem::Version
-  version: 0.18.2
+  version: 0.18.4
 platform: ruby
 authors:
 - Lax
@@ -295,7 +295,10 @@ files:
 - lib/git_fit/elevation/backfill.rb
 - lib/git_fit/elevation/data/dem_coeff.json
 - lib/git_fit/elevation/data/dem_lookup.json
+- lib/git_fit/elevation/dem_cache.rb
+- lib/git_fit/elevation/dem_fetch.rb
 - lib/git_fit/elevation/terrain_coeff.rb
+- lib/git_fit/elevation/terrain_lookup.rb
 - lib/git_fit/elevation/train/collector.rb
 - lib/git_fit/elevation/train/fitter.rb
 - lib/git_fit/elevation/train/runner.rb