git-fit 0.6.1 → 0.7.1

data/lib/git_fit/sync/runner.rb

@@ -0,0 +1,322 @@
+require "thread"
+
+module GitFit
+  module Sync
+    class Runner
+      POISON = Object.new.freeze
+      BATCH_SIZE = 5
+
+      def initialize(sources:, db:, config:, activity_filter: nil,
+                     privacy: nil, num_workers: 3, time_budget: nil, total_timeout: nil)
+        @sources = sources
+        @db = db
+        @config = config
+        @activity_filter = activity_filter
+        @privacy = privacy
+        @num_workers = [@num_workers.to_i, 1].max
+        @time_budget = time_budget.is_a?(Numeric) ? time_budget.to_f : nil
+        @total_timeout = total_timeout.is_a?(Numeric) ? total_timeout.to_f : nil
+        @sync_start = Time.now
+        @watchdog = nil
+
+        @pending_ids = {}
+        @adapters = {}
+        @source_queue = Queue.new
+        @output_queue = Queue.new
+        @stats_lock = Mutex.new
+        @unfinished = 0
+        @completed = {}
+        @timed_out = false
+        @shutdown = false
+        @source_start_times = {}
+        @source_activated = {}
+      end
+
+      def run
+        output_thread = Thread.new { output_loop }
+        start_watchdog if @total_timeout
+        init_pending
+        unless @source_queue.empty?
+          worker_threads = @num_workers.times.map { Thread.new { worker_loop } }
+          worker_threads.each(&:join)
+        end
+        output_queue << POISON
+        output_thread.join(60) || output_thread.kill
+
+        # TODO: Dedup 需要独立设计三种语义后再实现
+        #   A. 同源防重: upsert_activity 已通过 run_id 唯一约束天然保证
+        #   B. 跨设备同活动: 需轨迹相似度匹配 + 时间窗口聚类
+        #   C. 跨平台迁移复制: 需 external_id 映射表 + 数据指纹比对
+        #   触发时机、匹配算法、合并策略见 docs/dedup-design.md
+        # Dedup::Service.new(@db).run if defined?(Dedup::Service) && ...
+      ensure
+        @watchdog&.kill
+      end
+
+      private
+
+      attr_reader :output_queue
+
+      def init_pending
+        @sources.each do |source|
+          cfg = @config.sync_config(source)
+          next if cfg.nil? || cfg.empty?
+          @source_queue << source
+        end
+        @unfinished = @source_queue.size
+      end
+
+      def adapter_class(source, _cfg)
+        (GitFit::Sync::Base.adapters + GitFit::Sync::Base.test_adapters)
+          .find { |a| a.name.split("::").last.downcase == source }
+      end
+
+      def init_source(source)
+        klass = adapter_class(source, nil)
+        actual_key = klass&.config_key || source
+        cfg = @config.sync_config(actual_key)
+        unless klass
+          output_queue << { type: :banner, source:, text: "no adapter" }
+          clear_source(source)
+          return false
+        end
+
+        output_queue << { type: :banner, source:, text: "authenticating" }
+        adapter = klass.new(config: cfg, db: @db, activity_filter: @activity_filter, privacy: @privacy, time_budget: @time_budget)
+        unless adapter.before_call
+          output_queue << { type: :banner, source:, text: "before_call failed" }
+          clear_source(source)
+          return false
+        end
+
+        unless adapter.authenticate
+          output_queue << { type: :banner, source:, text: "auth failed" }
+          clear_source(source)
+          return false
+        end
+
+        output_queue << { type: :banner, source:, text: "fetching activity list" }
+        ids = adapter.pending_ids
+        if ids.nil? || ids.empty?
+          output_queue << { type: :banner, source:, text: "no new activities" }
+          clear_source(source)
+          return false
+        end
+
+        @adapters[source] = adapter
+        @pending_ids[source] = ids
+        @source_start_times[source] = Time.now
+        @source_activated[source] = true
+        output_queue << { type: :banner, source:, text: "#{ids.size} pending" }
+        true
+      end
+
+      def clear_source(source)
+        @pending_ids.delete(source)
+        @adapters.delete(source)
+        @source_start_times.delete(source)
+        @source_activated.delete(source)
+      end
+
+      def worker_loop
+        loop do
+          source = @source_queue.pop
+          break if source == POISON
+
+          begin
+            process_source(source)
+          rescue => e
+            @output_queue << { type: :error, source:, message: "Worker crashed: #{e.message}" }
+            @stats_lock.synchronize do
+              @pending_ids.delete(source)
+              @adapters.delete(source)
+              return if @completed[source]
+              @completed[source] = true
+              @unfinished -= 1
+              @unfinished = 0 if @unfinished < 0
+              if @unfinished <= 0
+                @shutdown = true
+                @num_workers.times { @source_queue << POISON }
+              end
+            end
+          end
+        end
+      end
+
+      def process_source(source)
+        unless init_source(source)
+          finish_activity(source)
+          return
+        end
+
+        total = @pending_ids[source].size
+        processed = 0
+        timed_out_local = false
+
+        loop do
+          activity = nil
+          @stats_lock.synchronize { activity = @pending_ids[source]&.shift }
+          break unless activity
+
+          if @stats_lock.synchronize { @shutdown }
+            @stats_lock.synchronize do
+              @pending_ids[source]&.unshift(activity) if @pending_ids[source]
+            end
+            timed_out_local = true
+            break
+          end
+
+          if source_budget_expired?(source)
+            @stats_lock.synchronize do
+              @timed_out = true
+              @pending_ids[source]&.unshift(activity) if @pending_ids[source]
+            end
+            timed_out_local = true
+            break
+          end
+
+          if @adapters[source].respond_to?(:skip_type?, true) && @adapters[source].send(:skip_type?, activity[:type])
+            next
+          end
+
+          begin
+            result = @adapters[source].process_one(activity)
+            processed += 1
+            if result
+              info = @adapters[source].build_progress_info(result)
+              @output_queue << {
+                type: :progress, source:, id: activity[:id],
+                info: info, done: processed, total: total
+              }
+            else
+              @output_queue << {
+                type: :progress, source:, id: activity[:id],
+                info: nil, done: processed, total: total
+              }
+            end
+          rescue => e
+            @output_queue << { type: :error, source:, message: e.message }
+          end
+
+          break if processed >= BATCH_SIZE
+        end
+
+        finish_activity(source, timed_out_local)
+      end
+
+      def finish_activity(source, timed_out = false)
+        @stats_lock.synchronize do
+          if timed_out
+            remaining = @pending_ids[source]&.size.to_i
+            @output_queue << { type: :timeout, source:, remaining: }
+            @pending_ids[source]&.clear
+          end
+
+          more = @pending_ids[source]&.any?
+          if more && !@shutdown
+            @source_queue << source
+          else
+            return if @completed[source]
+            @completed[source] = true
+            @unfinished -= 1
+            @unfinished = 0 if @unfinished < 0
+            if @unfinished <= 0
+              @shutdown = true
+              @num_workers.times { @source_queue << POISON }
+            end
+          end
+        end
+      end
+
+      def source_budget_expired?(source)
+        return false unless @time_budget&.positive?
+        start_time = @stats_lock.synchronize { @source_start_times[source] }
+        return false unless start_time
+        (Time.now - start_time) > @time_budget
+      end
+
+      def start_watchdog
+        @watchdog = Thread.new do
+          sleep @total_timeout
+          @stats_lock.synchronize { @shutdown = true }
+          @output_queue << { type: :total_timeout, seconds: @total_timeout }
+          @num_workers.times { @source_queue << POISON }
+        end
+      end
+
+      def output_loop
+        $stdout.sync = true
+        $stdout.puts "Runner started with #{@num_workers} workers, #{@sources.size} sources"
+        $stdout.flush
+        per_source = Hash.new { |h, k| h[k] = {} }
+
+        loop do
+          msg = @output_queue.pop
+          break if msg == POISON
+
+          s = msg[:source]
+          ps = per_source[s]
+
+          case msg[:type]
+          when :banner
+            text = msg[:text]
+            if text =~ /auth failed/
+              ps[:auth_failed] = true
+            elsif text =~ /no new activities/
+              ps[:up_to_date] = true
+            elsif text =~ /(\d+) pending/
+              ps[:total] = $1.to_i
+            end
+            $stdout.puts "  #{s.rjust(10)} | #{text}"
+          when :progress
+            ps[:done] = msg[:done].nil? ? ps.fetch(:done, 0) + 1 : msg[:done]
+            total = msg[:total] || ps[:total]
+            next unless show_progress?(msg[:done], total)
+            line = if msg[:info]
+                     " [#{msg[:done]}/#{total}] #{msg[:info]}"
+                   else
+                     " [#{msg[:done]}/#{total}] (skipped)"
+                   end
+            $stdout.puts "  #{s.rjust(10)} |#{line}"
+          when :error
+            ps[:errors] = (ps[:errors] || 0) + 1
+            $stdout.puts "  #{s.rjust(10)} | ERROR: #{msg[:message]}"
+          when :timeout
+            $stdout.puts "  #{s.rjust(10)} | TIME BUDGET — #{msg[:remaining]} remaining"
+          when :total_timeout
+            $stdout.puts "  *** TOTAL TIMEOUT #{msg[:seconds]}s — forcing shutdown"
+          end
+          $stdout.flush
+        end
+
+        @sources.each do |s|
+          info = per_source[s] || {}
+          err = info[:errors] || 0
+          if info[:auth_failed]
+            $stdout.puts "    done: #{s}: 0 synced (auth failed)"
+          elsif info[:up_to_date]
+            $stdout.puts "    done: #{s}: up to date"
+          elsif (cnt = info[:done]) && cnt > 0
+            label = err > 0 ? "#{cnt} synced, #{err} errors" : "#{cnt} synced"
+            $stdout.puts "    done: #{s}: #{label}"
+          elsif info.key?(:done)
+            $stdout.puts "    done: #{s}: 0 synced"
+          else
+            $stdout.puts "    done: #{s}: 0 synced"
+          end
+        end
+        $stdout.flush
+      rescue => e
+        $stdout.puts "    Runner output error: #{e.message}"
+        $stdout.flush
+      end
+
+      def show_progress?(done, total)
+        return true if done.nil? || done <= 3
+        return true if total && (total - done) <= 5
+        step = total && total > 0 ? [1, (total / 15.0).ceil].max : 1
+        step <= 1 || done % step == 0
+      end
+    end
+  end
+end

data/lib/git_fit/sync/strava.rb

@@ -1,7 +1,194 @@
+require_relative "base"
+require "faraday"
+
 module GitFit
   module Sync
+    class SyncError < RuntimeError; end unless const_defined?(:SyncError)
+
     class Strava < Base
+      BASE_URL = "https://www.strava.com/api/v3"
+
+      register_adapter
       register_config :client_id, :client_secret, :refresh_token
+
+      def authenticate
+        return false unless @config["client_id"].to_s != "" &&
+                            @config["client_secret"].to_s != "" &&
+                            @config["refresh_token"].to_s != ""
+        @access_token = refresh_access_token
+        @access_token ? true : false
+      end
+
+      def pending_ids
+        existing = existing_run_ids
+        ids = []
+        page = 1
+        loop do
+          activities = fetch_activities(@access_token, page)
+          break if activities.empty?
+          activities.each do |act|
+            act_id = act["id"].to_s
+            next if existing.include?(act_id) && raw_file_exists?(act_id)
+            type = map_sport_type(act["type"], act["sport_type"])
+            next if skip_type?(type)
+            ids << { id: act_id, type: type, summary: act }
+          end
+          page += 1
+        end
+        ids
+      end
+
+      def process_one(pending)
+        result = sync_activity(pending[:summary], pending[:id], @access_token)
+        result
+      end
+
+      private
+
+      def refresh_access_token
+        resp = Faraday.post("https://www.strava.com/oauth/token") do |req|
+          req.body = {
+            client_id: @config["client_id"],
+            client_secret: @config["client_secret"],
+            refresh_token: @config["refresh_token"],
+            grant_type: "refresh_token"
+          }
+        end
+        raise SyncError, "Strava token refresh failed: #{resp.status}" unless resp.success?
+        JSON.parse(resp.body)["access_token"]
+      end
+
+      def sync_activity(act, act_id, token)
+        type = map_sport_type(act["type"], act["sport_type"])
+        return false if skip_type?(type)
+
+        detail = fetch_detail(act_id, token)
+        return false unless detail
+
+        external_id = detail["external_id"]
+
+        streams = fetch_streams(act_id, token)
+
+        write_source_archive({ source: "strava", detail: detail, streams: streams }, act_id)
+
+        l2 = streams_to_l2(detail, streams)
+        write_standardized_json(l2, act_id) if l2
+
+        polyline = l2 ? Geo::Polyline.encode(l2.map { |pt| [pt[:positionLat], pt[:positionLong]] }) : act.dig("map", "summary_polyline")
+
+        attrs = build_attrs(act, type, polyline, external_id: external_id)
+        upsert_activity(attrs)
+        attrs
+      end
+
+      def fetch_activities(token, page)
+        resp = Faraday.get("#{BASE_URL}/athlete/activities") do |req|
+          req.headers["Authorization"] = "Bearer #{token}"
+          req.params = { page: page, per_page: 100 }
+        end
+        return [] unless resp.success?
+        JSON.parse(resp.body)
+      end
+
+      def fetch_detail(act_id, token)
+        resp = Faraday.get("#{BASE_URL}/activities/#{act_id}") do |req|
+          req.headers["Authorization"] = "Bearer #{token}"
+        end
+        return nil unless resp.success?
+        JSON.parse(resp.body)
+      end
+
+      def fetch_streams(act_id, token)
+        resp = Faraday.get("#{BASE_URL}/activities/#{act_id}/streams") do |req|
+          req.headers["Authorization"] = "Bearer #{token}"
+          req.params = { keys: "lat_lng,altitude,time,heartrate,cadence,watts,temp,velocity_smooth,distance", key_by_type: true }
+        end
+        return {} unless resp.success?
+        JSON.parse(resp.body)
+      end
+
+      def write_standardized_json(l2, act_id)
+        dir = std_dir
+        FileUtils.mkdir_p(dir)
+        tmp = File.join(dir, ".#{act_id}.json.tmp")
+        final = File.join(dir, "#{act_id}.json")
+        File.write(tmp, JSON.generate(l2))
+        File.rename(tmp, final)
+      end
+
+      def streams_to_l2(detail, streams)
+        polyline = detail.dig("map", "polyline")
+        return nil unless polyline
+
+        coords = Geo::Polyline.decode(polyline)
+        return nil if coords.empty?
+
+        alt_data = streams["altitude"]&.dig("data")
+        time_data = streams["time"]&.dig("data")
+        hr_data = streams["heartrate"]&.dig("data")
+        cad_data = streams["cadence"]&.dig("data")
+        watts_data = streams["watts"]&.dig("data")
+        temp_data = streams["temp"]&.dig("data")
+        speed_data = streams["velocity_smooth"]&.dig("data")
+        dist_data = streams["distance"]&.dig("data")
+
+        coords.each_with_index.map do |latlng, i|
+          pt = { positionLat: latlng[0], positionLong: latlng[1] }
+          pt[:altitude] = alt_data[i] if alt_data && i < alt_data.size
+          pt[:timestamp] = time_data[i] if time_data && i < time_data.size
+          pt[:heartRate] = hr_data[i] if hr_data && i < hr_data.size
+          pt[:cadence] = cad_data[i] if cad_data && i < cad_data.size
+          pt[:power] = watts_data[i] if watts_data && i < watts_data.size
+          pt[:temperature] = temp_data[i] if temp_data && i < temp_data.size
+          pt[:speed] = speed_data[i] if speed_data && i < speed_data.size
+          pt[:distance] = dist_data[i] if dist_data && i < dist_data.size
+          pt
+        end
+      end
+
+      def build_attrs(act, category, polyline = nil, external_id: nil)
+        start_t = Time.parse(act["start_date"]) rescue nil
+        local_t = Time.parse(act["start_date_local"]) rescue nil
+
+        {
+          run_id: "strava_#{act["id"]}",
+          name: act["name"],
+          distance: act["distance"],
+          moving_time: act["moving_time"],
+          elapsed_time: act["elapsed_time"],
+          sport_category: category,
+          sport_type: (act["sport_type"] || act["type"])&.downcase,
+          start_date: start_t&.iso8601,
+          start_date_local: local_t&.iso8601,
+          location_country: act["location_country"],
+          summary_polyline: polyline,
+          average_heartrate: act["average_heartrate"],
+          max_heartrate: act["max_heartrate"],
+          average_cadence: act["average_cadence"],
+          max_cadence: nil,
+          average_power: act["average_watts"],
+          max_power: act["max_watts"],
+          calories: act["calories"],
+          average_temperature: act["average_temp"],
+          average_speed: act["average_speed"],
+          elevation_gain: act["total_elevation_gain"],
+          source: "strava",
+          external_id: external_id
+        }
+      end
+
+      def map_sport_type(type, sport_type)
+        SportMapper.canonicalize(sport_type || type)
+      end
+
+      def raw_path(platform_id)
+        File.join(raw_dir, "#{platform_id}.json")
+      end
+
+      def existing_run_ids
+        @db[:activities].where(source: "strava").select_map(:run_id)
+          .map { |id| id.sub("strava_", "") }
+      end
     end
   end
 end