git-fit 0.18.3 → 0.18.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4885ddbb01fd4680fac6ae16c6fd3c1b90092910abcef29b317cacd351ee3c1a
-  data.tar.gz: e5257d8d6ddcbd46e8e8e07fda87316fcea76ebc1f47ed0da784ba744e6f4e35
+  metadata.gz: b68548ae0905b7374d85979f27053683d604ab26c8ae0e66011cd6cf776bf7ad
+  data.tar.gz: 863da16427ca424985a8f6ef62237b9a53e2c782fc390950323fd04ad2cb6929
 SHA512:
-  metadata.gz: c398fbe3e1740edf11ee63e02dd7f22816d6c0304144844f66e6a41e42415f79e263a6a8d6b8d26b4959a64c6d38a4e7e971d2bc97465daacc3475a611ebd380
-  data.tar.gz: 3cf2651b1d8754e28f23f682970757b4e4ba3f1cecda3126c127ccb71ebe09f0c9ce2244ea6f48a1119b46d97a110ed15fcc59c0a8f403c8960fc67e6c745b50
+  metadata.gz: fa7450b9a823e2b8dca5d5a81d6663caba96fdc9f43444cb53619b5f0cf3b4afc2016b3e69c74e03a4a53250cd4f6a7f79b38862a87a98dc1fb4a538f014117f
+  data.tar.gz: 1d7a706ea4b63ccf6f2607a236d73bbbcdeee9a49fb809846030667f8dec6b2b6f43eec3019891d00fbcb9d8b71198c3eec72243bc4a0d9e650bb012dfaff481

data/lib/git_fit/elevation/dem_cache.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'json'
+
+module GitFit
+  module Elevation
+    module DemCache
+      DEM_CACHE_DIR = ENV.fetch('GIT_FIT_DEM_CACHE_DIR', 'data/cache/dem')
+      CACHE_FILE = File.join(DEM_CACHE_DIR, 'dem.json')
+      GRID = 0.0003
+
+      class << self
+        def grid_key(lat, lng)
+          "#{(lat / GRID).round}/#{(lng / GRID).round}"
+        end
+
+        def load_all
+          return {} unless File.exist?(CACHE_FILE)
+          JSON.parse(File.read(CACHE_FILE))
+        rescue StandardError
+          {}
+        end
+
+        def load_batch(slice)
+          all = load_all
+          results = {}
+          slice.each do |lat, lng|
+            gk = grid_key(lat, lng)
+            results[gk] = all[gk] if all.key?(gk)
+          end
+          results
+        end
+
+        def missing_keys(slice, cached)
+          slice.reject { |lat, lng| cached.key?(grid_key(lat, lng)) }
+        end
+
+        def save_batch(elevs_by_gk)
+          return if elevs_by_gk.empty?
+          FileUtils.mkdir_p(DEM_CACHE_DIR)
+          existing = load_all
+          elevs_by_gk.each { |gk, elev| existing[gk] = elev }
+          File.write(CACHE_FILE, JSON.generate(existing))
+        rescue StandardError => e
+          warn "DEM cache write error: #{e.message}"
+        end
+      end
+    end
+  end
+end

data/lib/git_fit/elevation/dem_fetch.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require 'net/http'
+require 'uri'
+
+module GitFit
+  module Elevation
+    module DemFetch
+      DEM_URL = 'https://api.opentopodata.org/v1/srtm30m'
+
+      class << self
+        def fetch_batch(slice, batch_idx, elevs, grid_key_fn)
+          qs = slice.map { |la, ln| "#{la},#{ln}" }.join('|')
+          uri = URI("#{DEM_URL}?locations=#{qs.gsub(',', '%2C').gsub('|', '%7C')}")
+          attempt = 0
+          loop do
+            res = Net::HTTP.get_response(uri)
+            if res.code == '200'
+              results = JSON.parse(res.body)['results']
+              slice.each_with_index do |(lat, lng), i|
+                elevs[grid_key_fn.call(lat, lng)] = results[i]['elevation'] if results[i]
+              end
+              break
+            elsif res.code == '429'
+              attempt += 1
+              if attempt < 5
+                delay = 2**attempt
+                warn "DEM batch #{batch_idx}: rate limited, retrying in #{delay}s (attempt #{attempt})"
+                sleep(delay)
+              else
+                warn "DEM batch #{batch_idx}: rate limited, giving up after #{attempt} attempts"
+                break
+              end
+            else
+              warn "DEM batch #{batch_idx} HTTP #{res.code}"
+              break
+            end
+          rescue StandardError => e
+            warn "DEM batch #{batch_idx}: #{e.message}"
+            break
+          end
+        end
+      end
+    end
+  end
+end

data/lib/git_fit/elevation/terrain_coeff.rb

@@ -1,20 +1,18 @@
 # frozen_string_literal: true
 
 require 'msgpack'
-require 'net/http'
-require 'uri'
 
+require_relative 'dem_cache'
+require_relative 'dem_fetch'
 require_relative 'terrain_lookup'
 
 module GitFit
   module Elevation
     module TerrainCoeff
-      CACHE_DIR = ENV.fetch('GIT_FIT_DEM_CACHE_DIR', 'data/cache/dem_fit')
-      DEM_CACHE_DIR = File.join(CACHE_DIR, 'cache')
+      MODEL_CACHE_DIR = ENV.fetch('GIT_FIT_DEM_MODEL_DIR', 'data/cache/dem_fit')
       BUNDLED_MODEL_DIR = File.expand_path('model', __dir__)
       BUNDLED_MODEL = File.join(BUNDLED_MODEL_DIR, 'dem_model.msgpack')
       MODEL_FILE = 'dem_model.msgpack'
-      DEM_URL = 'https://api.opentopodata.org/v1/srtm30m'
       GRID = 0.0003
 
       class << self
@@ -27,10 +25,10 @@ module GitFit
         end
 
         def find_model
-          cache_path = File.join(CACHE_DIR, MODEL_FILE)
+          cache_path = File.join(MODEL_CACHE_DIR, MODEL_FILE)
           return cache_path if File.exist?(cache_path)
           return BUNDLED_MODEL if File.exist?(BUNDLED_MODEL)
-          raise "No #{MODEL_FILE} found in cache (#{CACHE_DIR}) or bundled (#{BUNDLED_MODEL})"
+          raise "No #{MODEL_FILE} found in cache (#{MODEL_CACHE_DIR}) or bundled (#{BUNDLED_MODEL})"
         end
 
         def reset!
@@ -65,7 +63,16 @@ module GitFit
           uniq_vals = uniquify_locations(locations)
           elevs = {}
           uniq_vals.each_slice(100).with_index do |slice, bi|
-            elevs.merge!(fetch_dem_batch(slice, bi))
+            cached = DemCache.load_batch(slice)
+            elevs.merge!(cached)
+            missing = DemCache.missing_keys(slice, cached)
+            if missing.any?
+              missing_locations = missing.map { |lat, lng| [lat, lng] }
+              batch = {}
+              DemFetch.fetch_batch(missing_locations, bi, batch, method(:grid_key))
+              elevs.merge!(batch)
+              DemCache.save_batch(batch)
+            end
             sleep 0.15
           end
           locations.map { |lat, lng| elevs[grid_key(lat, lng)] }
@@ -77,44 +84,6 @@ module GitFit
           uniq.values
         end
 
-        def fetch_dem_batch(slice, batch_idx)
-          elevs = {}
-          qs = slice.map { |la, ln| "#{la},#{ln}" }.join('|')
-          uri = URI("#{DEM_URL}?locations=#{qs.gsub(',', '%2C').gsub('|', '%7C')}")
-          attempt = 0
-          loop do
-            # rubocop:disable Style/RedundantBegin
-            begin
-              res = Net::HTTP.get_response(uri)
-              if res.code == '200'
-                results = JSON.parse(res.body)['results']
-                slice.each_with_index do |(lat, lng), i|
-                  elevs[grid_key(lat, lng)] = results[i]['elevation'] if results[i]
-                end
-                break
-              elsif res.code == '429'
-                attempt += 1
-                if attempt < 5
-                  delay = 2**attempt
-                  warn "DEM batch #{batch_idx}: rate limited, retrying in #{delay}s (attempt #{attempt})"
-                  sleep(delay)
-                else
-                  warn "DEM batch #{batch_idx}: rate limited, giving up after #{attempt} attempts"
-                  break
-                end
-              else
-                warn "DEM batch #{batch_idx} HTTP #{res.code}"
-                break
-              end
-            rescue StandardError => e
-              warn "DEM batch #{batch_idx}: #{e.message}"
-              break
-            end
-            # rubocop:enable Style/RedundantBegin
-          end
-          elevs
-        end
-
         def grid_key(lat, lng)
           "#{(lat / GRID).round}/#{(lng / GRID).round}"
         end

data/lib/git_fit/elevation/train/collector.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative '../dem_cache'
 require_relative '../terrain_coeff'
 
 module GitFit
@@ -11,15 +12,25 @@ module GitFit
 
         def initialize(sources: nil)
           @sources = sources
+          @samples = []
         end
 
-        def collect
-          samples = []
+        attr_reader :samples
+
+        def count_samples
+          total = 0
+          each_source_sample do |_sample|
+            total += 1
+          end
+          total
+        end
+
+        def each_sample
           each_source_sample do |sample|
             result = evaluate_sample(sample)
-            samples << result if result
+            @samples << result if result
+            yield result
           end
-          samples
         end
 
         def each_source_sample(&block)
@@ -50,11 +61,13 @@ module GitFit
 
           return nil if altitudes.size < 2
           return nil if distances.empty? || distances.last < 1000
-          return nil if sample[:altitudes].size < 2
 
+          cache_before = DemCache.load_batch(locations)
           dem = TerrainCoeff.elevations_for(locations)
           return nil if dem.empty? || dem.compact.size < 2
 
+          cache_hit = !cache_before.empty? && cache_before.values.all? { |v| !v.nil? }
+
           truth_min = dem.compact.min
           truth_max = dem.compact.max
           sm = smooth(altitudes)
@@ -97,6 +110,7 @@ module GitFit
             gain_per_km: f[:gain_per_km],
             range_per_km: f[:range_per_km],
             seg_density: f[:seg_density],
+            cache_hit: cache_hit,
           }
         end
 

data/lib/git_fit/elevation/train/runner.rb

@@ -17,19 +17,39 @@ module GitFit
         end
 
         def run
-          puts 'Collecting samples...'
           collector = Collector.new(sources: @sources)
-          samples = collector.collect
+          total = collector.count_samples
+          puts "Collecting samples (#{total} total)..."
 
-          if samples.empty?
-            warn 'No samples collected'
-            return false
+          collected = 0
+          cached = 0
+          missed = 0
+          skipped = 0
+
+          collector.each_sample do |result|
+            if result
+              collected += 1
+              if result[:cache_hit]
+                cached += 1
+              else
+                missed += 1
+              end
+              print_progress(collected, cached, missed, total)
+            else
+              skipped += 1
+            end
           end
 
-          puts "Collected #{samples.size} samples"
+          puts
+          puts "Collected: #{collected} samples (#{cached} hit / #{missed} miss / #{skipped} skipped)"
+
+          if collected.zero?
+            warn 'No valid samples collected'
+            return false
+          end
 
           puts 'Fitting model...'
-          fitter = Fitter.new(samples)
+          fitter = Fitter.new(collector.samples)
           model = fitter.fit
 
           unless model
@@ -43,15 +63,32 @@ module GitFit
             return false
           end
 
-          puts 'Model trained successfully!'
-          puts "  n_samples: #{model[:meta]['n_samples']}"
-          puts "  sources: #{model[:meta]['sources'].join(', ')}"
-          puts "  ridge_h: #{model[:ridge_h].map { |w| format('%.2f', w) }.join(', ')}"
-          puts "  output: #{@output}"
+          print_summary(model)
 
           true
         end
 
+        def print_progress(collected, cached, missed, total)
+          pct = (collected.to_f / total * 100).round(1)
+          bar_len = 30
+          filled = (collected.to_f / total * bar_len).round
+          bar = '=' * filled + '-' * (bar_len - filled)
+          $stdout.write "\r  [#{bar}] #{pct}% #{collected}/#{total} (h:#{cached} m:#{missed})"
+          $stdout.flush
+        end
+
+        def print_summary(model)
+          meta = model[:meta]
+          puts
+          puts '=== Model Trained ==='
+          puts "  samples:     #{meta['n_samples']}"
+          puts "  sources:     #{meta['sources'].join(', ')}"
+          puts "  trained_at:  #{meta['trained_at']}"
+          puts "  ridge_h:     #{model[:ridge_h].map { |w| format('%.3f', w) }.join(', ')}"
+          puts "  norms:       #{model[:norms].map { |n| format('%.2f', n) }.join(', ')}"
+          puts "  output:      #{@output}"
+        end
+
         def default_output_path
           File.join(CACHE_DIR, MODEL_FILE)
         end

data/lib/git_fit/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module GitFit
-  VERSION = '0.18.3'
+  VERSION = '0.18.4'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: git-fit
 version: !ruby/object:Gem::Version
-  version: 0.18.3
+  version: 0.18.4
 platform: ruby
 authors:
 - Lax
@@ -295,6 +295,8 @@ files:
 - lib/git_fit/elevation/backfill.rb
 - lib/git_fit/elevation/data/dem_coeff.json
 - lib/git_fit/elevation/data/dem_lookup.json
+- lib/git_fit/elevation/dem_cache.rb
+- lib/git_fit/elevation/dem_fetch.rb
 - lib/git_fit/elevation/terrain_coeff.rb
 - lib/git_fit/elevation/terrain_lookup.rb
 - lib/git_fit/elevation/train/collector.rb