git-fit 0.9.5 → 0.9.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 28eab6fd810e6ed7b29e2b04ae7c08dc37ff619aa37b87e21e73f139fc3b147c
-  data.tar.gz: 6e4554743b020986de0803ba39f86044f3f607f8efbf81e57dd51ff174eb1080
+  metadata.gz: b08e4aac65236128444a75e0890cdaf1e8b25f08a1db325833c7e6c240019b98
+  data.tar.gz: b028cc241a518348434988519e5602cbe624d73e80f0b3e5bfbfb0bbe13660ab
 SHA512:
-  metadata.gz: 432140f8a5aeff892215acacde7ffa4184c9bcc156754234f676a407b9466aef84b8df70987b380339ca2e2ad33c96882f9a5508f7929e1ccab184de30bef9de
-  data.tar.gz: 1bf2814bc40fd83b253aa4424d371c3fcb0318dcc655ae34974bb284e45ad737d9f8eb00e130f14203ea3713b6869859e287a2535f5100f03d9a7603d68fa82e
+  metadata.gz: 62d0a2b6b23b44c8d56254fa695fcd5735052319261cb6b146143b88910f704c2eb2df77544bcb23b490409ed271e495d7bade5fff25d6c32e8a9dbb9d717e63
+  data.tar.gz: 6ab10823f7a8d558de35ef6eb51d11d7aa88d03a9d61d175623355d5a1f159cfd6a23c7985836b8edf26ce0284e61d99f1506792c1f3017a62852dfbdc493d1c

data/lib/git_fit/config_template.rb

@@ -10,7 +10,7 @@ module GitFit
 
         sync:
           # workers: 3                                 # env: GIT_FIT_SYNC_WORKERS
-          # time_budget: 300                           # env: GIT_FIT_SYNC_TIME_BUDGET (seconds)
+          # time_budget: 90                            # env: GIT_FIT_SYNC_TIME_BUDGET (seconds)
           # total_timeout: 600                         # env: GIT_FIT_SYNC_TOTAL_TIMEOUT (seconds)
 
           # strava:                                    # env: GIT_FIT_STRAVA_CLIENT_ID

data/lib/git_fit/sync/base.rb

@@ -42,6 +42,14 @@ module GitFit
         true
       end
 
+      def auth_error?
+        false
+      end
+
+      def last_auth_code
+        nil
+      end
+
       def pending_ids
         raise NotImplementedError, "#{self.class} must implement #pending_ids"
       end

data/lib/git_fit/sync/keep.rb

@@ -53,9 +53,14 @@ module GitFit
           'Content-Type' => 'application/x-www-form-urlencoded;charset=utf-8',
         }
         @session = nil
+        @cached_pending_ids = nil
+        @last_http_code = nil
+        @token_issued_at = nil
+        @requests_since_login = 0
       end
 
       def before_call
+        @cached_pending_ids = nil
         return false if @phone.nil? || @password.nil?
         super
       end
@@ -67,6 +72,7 @@ module GitFit
       end
 
       def pending_ids
+        return @cached_pending_ids if @cached_pending_ids
         existing = existing_run_ids
         ids = []
         KEEP_TYPES.each do |sport|
@@ -80,6 +86,7 @@ module GitFit
             ids << { id: cid, keep_id: keep_id, sport: sport, has_raw: has_raw }
           end
         end
+        @cached_pending_ids = ids
         ids
       end
 
@@ -91,6 +98,14 @@ module GitFit
         end
       end
 
+      def auth_error?
+        [401, 403].include?(@last_http_code)
+      end
+
+      def last_auth_code
+        @last_http_code
+      end
+
       private
 
       def http_post(url, body, headers)
@@ -102,7 +117,9 @@ module GitFit
         http.use_ssl = uri.scheme == 'https'
         http.open_timeout = 30
         http.read_timeout = 60
-        http.start { |h| h.request(req) }
+        resp = http.start { |h| h.request(req) }
+        @last_http_code = resp.code.to_i
+        resp
       end
 
       def http_get(url, headers)
@@ -113,7 +130,10 @@ module GitFit
         http.use_ssl = uri.scheme == 'https'
         http.open_timeout = 30
         http.read_timeout = 60
-        http.start { |h| h.request(req) }
+        resp = http.start { |h| h.request(req) }
+        @last_http_code = resp.code.to_i
+        @requests_since_login += 1
+        resp
       end
 
       def login
@@ -126,6 +146,8 @@ module GitFit
 
         @headers['Authorization'] = "Bearer #{token}"
         @session = true
+        @token_issued_at = Time.now
+        @requests_since_login = 0
       rescue JSON::ParserError
         nil
       end
@@ -147,7 +169,12 @@ module GitFit
         loop do
           url = STATS_URL % { sport: sport, last_date: last_date }
           resp = @session ? http_get(url, @headers) : nil
-          break unless resp&.code&.to_i == 200
+          break unless resp
+          if [401, 403].include?(resp.code.to_i)
+            report_token_expiry
+            break
+          end
+          break unless resp.code.to_i == 200
 
           data = JSON.parse(resp.body)['data']
           break unless data
@@ -175,6 +202,10 @@ module GitFit
       def sync_single_run_with_raw(compound_id, sport)
         url = LOG_URL % { sport: sport, run_id: compound_id }
         resp = http_get(url, @headers)
+        if [401, 403].include?(resp.code.to_i)
+          report_token_expiry
+          return false
+        end
         return false unless resp.code.to_i == 200
 
         data = JSON.parse(resp.body)
@@ -213,6 +244,7 @@ module GitFit
       end
 
       def upsert_from_raw(compound_id)
+        @last_http_code = nil
         keep_id = extract_id(compound_id) || compound_id
         raw = JSON.parse(File.read(raw_path(keep_id)))
         run_data = raw['meta'].merge('id' => compound_id)
@@ -497,6 +529,11 @@ module GitFit
       def spider_sleep
         sleep 0.1
       end
+
+      def report_token_expiry
+        lifetime = @token_issued_at ? (Time.now - @token_issued_at).round(1) : nil
+        warn "Keep: token expired after #{@requests_since_login} requests#{lifetime ? " (#{lifetime}s)" : ''}"
+      end
     end
   end
 end

data/lib/git_fit/sync/runner.rb

@@ -6,7 +6,7 @@ module GitFit
   module Sync
     class Runner
       POISON = Object.new.freeze
-      BATCH_SIZE = 5
+      BATCH_SIZE = 10
 
       def initialize(sources:, db:, config:, activity_filter: nil,
                      privacy: nil, num_workers: 3, time_budget: nil, total_timeout: nil)
@@ -15,7 +15,7 @@ module GitFit
         @config = config
         @activity_filter = activity_filter
         @privacy = privacy
-        @num_workers = [@num_workers.to_i, 1].max
+        @num_workers = [num_workers.to_i, 1].max
         @time_budget = time_budget.is_a?(Numeric) ? time_budget.to_f : nil
         @total_timeout = total_timeout.is_a?(Numeric) ? total_timeout.to_f : nil
         @sync_start = Time.now
@@ -32,6 +32,8 @@ module GitFit
         @shutdown = false
         @source_start_times = {}
         @source_activated = {}
+        @run_totals = {}
+        @run_processed = {}
       end
 
       def run
@@ -83,36 +85,43 @@ module GitFit
           return false
         end
 
-        output_queue << { type: :banner, source:, text: 'authenticating' }
-        adapter = klass.new(config: cfg, db: @db, activity_filter: @activity_filter, privacy: @privacy,
-                            time_budget: @time_budget)
-        unless adapter.before_call
-          output_queue << { type: :banner, source:, text: 'before_call failed' }
-          clear_source(source)
-          return false
-        end
+        unless @adapters[source]
+          output_queue << { type: :banner, source:, text: 'authenticating' }
+          adapter = klass.new(config: cfg, db: @db, activity_filter: @activity_filter, privacy: @privacy,
+                              time_budget: @time_budget)
+          unless adapter.before_call
+            output_queue << { type: :banner, source:, text: 'before_call failed' }
+            clear_source(source)
+            return false
+          end
 
-        unless adapter.authenticate
-          output_queue << { type: :banner, source:, text: 'auth failed' }
-          clear_source(source)
-          return false
-        end
+          unless adapter.authenticate
+            output_queue << { type: :banner, source:, text: 'auth failed' }
+            clear_source(source)
+            return false
+          end
 
-        method = adapter.auth_method || 'unknown'
-        output_queue << { type: :banner, source:, text: "authenticated (#{method})" }
-        output_queue << { type: :banner, source:, text: 'fetching activity list' }
-        ids = adapter.pending_ids
-        if ids.nil? || ids.empty?
-          output_queue << { type: :banner, source:, text: 'no new activities' }
-          clear_source(source)
-          return false
+          method = adapter.auth_method || 'unknown'
+          output_queue << { type: :banner, source:, text: "authenticated (#{method})" }
+          @adapters[source] = adapter
+
+          output_queue << { type: :banner, source:, text: 'fetching activity list' }
+          ids = adapter.pending_ids
+          if adapter.auth_error?
+            output_queue << { type: :banner, source:, text: "auth failed (#{adapter.last_auth_code})" }
+            clear_source(source)
+            return false
+          end
+          if ids.nil? || ids.empty?
+            output_queue << { type: :banner, source:, text: 'no new activities' }
+            clear_source(source)
+            return false
+          end
+          @pending_ids[source] = ids
+          output_queue << { type: :banner, source:, text: "#{ids.size} pending" }
         end
 
-        @adapters[source] = adapter
-        @pending_ids[source] = ids
-        @source_start_times[source] = Time.now
         @source_activated[source] = true
-        output_queue << { type: :banner, source:, text: "#{ids.size} pending" }
         true
       end
 
@@ -121,6 +130,8 @@ module GitFit
         @adapters.delete(source)
         @source_start_times.delete(source)
         @source_activated.delete(source)
+        @run_totals.delete(source)
+        @run_processed.delete(source)
       end
 
       def worker_loop
@@ -153,10 +164,12 @@ module GitFit
           finish_activity(source)
           return
         end
+        @stats_lock.synchronize { @source_start_times[source] ||= Time.now }
 
-        total = @pending_ids[source].size
+        total = @run_totals[source] ||= @pending_ids[source].size
         processed = 0
         timed_out_local = false
+        auth_failed_local = false
 
         loop do
           activity = nil
@@ -187,33 +200,76 @@ module GitFit
           begin
             result = @adapters[source].process_one(activity)
             processed += 1
-            if result
-              info = @adapters[source].build_progress_info(result)
-              @output_queue << {
-                type: :progress, source:, id: activity[:id],
-                info: info, done: processed, total: total
-              }
-            else
-              @output_queue << {
-                type: :progress, source:, id: activity[:id],
-                info: nil, done: processed, total: total
-              }
-            end
+            outcome = handle_result(source, activity, result, processed, total)
           rescue StandardError => e
             @output_queue << { type: :error, source:, message: e.message }
+            outcome = :ok
           end
 
+          if outcome == :auth_failed
+            auth_failed_local = true
+            break
+          end
           break if processed >= BATCH_SIZE
         end
 
-        finish_activity(source, timed_out_local)
+        @run_processed[source] = (@run_processed[source] || 0) + processed
+
+        if auth_failed_local
+          finish_activity(source, true, 're-auth failed')
+        else
+          finish_activity(source, timed_out_local)
+        end
+      end
+
+      def handle_result(source, activity, result, processed, total)
+        if result
+          report_activity_progress(source, activity, result, processed, total)
+          :ok
+        elsif @adapters[source].auth_error?
+          code = @adapters[source].last_auth_code
+          @output_queue << { type: :banner, source:, text: "auth error (#{code}), re-authenticating" }
+          if @adapters[source].authenticate
+            result = @adapters[source].process_one(activity)
+            if result
+              report_activity_progress(source, activity, result, processed, total)
+            elsif @adapters[source].auth_error?
+              still_code = @adapters[source].last_auth_code
+              msg = "#{activity[:id]} still #{still_code} after re-auth"
+              @output_queue << { type: :error, source:, message: msg }
+              report_activity_progress(source, activity, nil, processed, total)
+            else
+              report_activity_progress(source, activity, nil, processed, total)
+            end
+            :ok
+          else
+            :auth_failed
+          end
+        else
+          report_activity_progress(source, activity, nil, processed, total)
+          :ok
+        end
+      end
+
+      def report_activity_progress(source, activity, result, processed, total)
+        acc = (@run_processed[source] || 0) + processed
+        info = result ? @adapters[source].build_progress_info(result) : nil
+        @output_queue << {
+          type: :progress, source:, id: activity[:id],
+          info:, done: acc, total:
+        }
       end
 
-      def finish_activity(source, timed_out = false)
+      def finish_activity(source, timed_out = false, timeout_note = nil)
         @stats_lock.synchronize do
           if timed_out
             remaining = @pending_ids[source]&.size.to_i
-            @output_queue << { type: :timeout, source:, remaining: }
+            msg = if timeout_note
+                    { type: :banner, source:, text: "#{timeout_note} — #{remaining} remaining, ending source" }
+                  else
+                    { type: :timeout, source:, remaining: }
+                  end
+            @output_queue << msg
             @pending_ids[source]&.clear
           end
 
@@ -277,11 +333,12 @@ module GitFit
             ps[:done] = msg[:done].nil? ? ps.fetch(:done, 0) + 1 : msg[:done]
             total = msg[:total] || ps[:total]
             next unless show_progress?(msg[:done], total)
-            line = if msg[:info]
-                     " [#{msg[:done]}/#{total}] #{msg[:info]}"
-                   else
-                     " [#{msg[:done]}/#{total}] (skipped)"
-                   end
+            line = " [#{msg[:done]}/#{total}] #{msg[:id]}"
+            line += if msg[:info]
+                      " | #{msg[:info]}"
+                    else
+                      ' | (skipped)'
+                    end
             $stdout.puts "  #{s.rjust(10)} |#{line}"
           when :error
             ps[:errors] = (ps[:errors] || 0) + 1
@@ -297,13 +354,15 @@ module GitFit
         @sources.each do |s|
           info = per_source[s] || {}
           err = info[:errors] || 0
-          if info[:auth_failed]
-            $stdout.puts "    done: #{s}: 0 synced (auth failed)"
-          elsif info[:up_to_date]
+          if info[:up_to_date]
             $stdout.puts "    done: #{s}: up to date"
           elsif (cnt = info[:done]) && cnt > 0
-            label = err > 0 ? "#{cnt} synced, #{err} errors" : "#{cnt} synced"
+            label = "#{cnt} synced"
+            label += ", #{err} errors" if err > 0
+            label += ' (auth failed)' if info[:auth_failed]
             $stdout.puts "    done: #{s}: #{label}"
+          elsif info[:auth_failed]
+            $stdout.puts "    done: #{s}: 0 synced (auth failed)"
           elsif info.key?(:done)
             $stdout.puts "    done: #{s}: 0 synced"
           else
@@ -317,10 +376,9 @@ module GitFit
       end
 
       def show_progress?(done, total)
-        return true if done.nil? || done <= 3
+        return true if done.nil?
         return true if total && (total - done) <= 5
-        step = total && total > 0 ? [1, (total / 15.0).ceil].max : 1
-        step <= 1 || done % step == 0
+        done % BATCH_SIZE == 1
       end
     end
   end

data/lib/git_fit/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module GitFit
-  VERSION = '0.9.5'
+  VERSION = '0.9.7'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: git-fit
 version: !ruby/object:Gem::Version
-  version: 0.9.5
+  version: 0.9.7
 platform: ruby
 authors:
 - Lax