git-fit 0.10.3 → 0.10.5

data/lib/git_fit/export/json.rb

@@ -7,10 +7,11 @@ module GitFit
     class JSON
       PLACEHOLDER_POLYLINE = 'gqqrFurkeU??'
 
-      def initialize(db:, output:, activity_filter: nil)
+      def initialize(db:, output:, activity_filter: nil, tracks_output: nil)
         @db = db
         @output = output
         @activity_filter = activity_filter
+        @tracks_output = tracks_output
       end
 
       def call
@@ -24,23 +25,40 @@ module GitFit
 
         group_members_cache = {}
 
+        tracks = @tracks_output ? {} : nil
+
         formatted = activities.map do |a|
           gid = run_id_to_group[a[:run_id]]
           if gid
             group_members_cache[gid] ||= @db[:dedup_group_members].where(group_id: gid).all
-            composite_activity(a, gid, group_members_cache[gid])
+            composite_activity(a, gid, group_members_cache[gid], tracks)
           else
-            format_activity(a)
+            format_activity(a, tracks)
           end
         end
 
         File.write(@output, ::JSON.pretty_generate(formatted))
+        File.write(@tracks_output, ::JSON.generate(tracks)) if @tracks_output
         formatted.size
       end
 
       private
 
-      def format_activity(a)
+      def extract_polyline(a)
+        return nil if a[:source] == 'keep' && a[:summary_polyline] == PLACEHOLDER_POLYLINE
+
+        a[:summary_polyline]
+      end
+
+      def pick_best_polyline(group_activities)
+        candidates = group_activities.filter_map { |a| extract_polyline(a) }
+        candidates.max_by { |p| p.to_s.length }
+      end
+
+      def format_activity(a, tracks = nil)
+        polyline = extract_polyline(a)
+        tracks[a[:run_id]] = polyline if tracks && polyline
+
         base = {
           run_id: a[:run_id],
           name: a[:name],
@@ -51,8 +69,7 @@ module GitFit
           start_date: a[:start_date],
           start_date_local: a[:start_date_local],
           location_country: a[:location_country],
-          summary_polyline: a[:source] == 'keep' && a[:summary_polyline] == PLACEHOLDER_POLYLINE ?
-            nil : a[:summary_polyline],
+          summary_polyline: tracks ? nil : polyline,
           average_heartrate: a[:average_heartrate] && a[:average_heartrate] != 0 ? a[:average_heartrate] : nil,
           max_heartrate: a[:max_heartrate],
           average_cadence: a[:average_cadence],
@@ -71,7 +88,7 @@ module GitFit
         base
       end
 
-      def composite_activity(activity, group_id, members)
+      def composite_activity(activity, group_id, members, tracks = nil)
         run_ids = members.map { |m| m[:run_id] }
         group_activities = @db[:activities].where(run_id: run_ids).all
 
@@ -85,11 +102,17 @@ module GitFit
           end
           result
         else
-          members.first&.slice(:run_id, :distance, :moving_time, :elapsed_time,
+          composite = members.first&.slice(:run_id, :distance, :moving_time, :elapsed_time,
             :average_heartrate, :max_heartrate, :elevation_gain,
             :average_speed, :average_cadence, :max_cadence) || {}
+          composite[:summary_polyline] = pick_best_polyline(group_activities)
+          composite
         end
 
+        polyline = composite[:summary_polyline]
+        tracks[activity[:run_id]] = polyline if tracks && polyline
+        composite[:summary_polyline] = tracks ? nil : polyline
+
         composite[:run_id] = activity[:run_id]
         composite[:source] = activity[:source]
         composite[:sport_type] = activity[:sport_type]

data/lib/git_fit/export.rb

@@ -8,5 +8,6 @@ end
 require_relative 'export/defaults'
 require_relative 'export/json'
 require_relative 'export/csv'
+require_relative 'export/gpx'
 require_relative 'export/calculator'
 require_relative 'export/stats'

data/lib/git_fit/fit.rb

@@ -1,3 +1,9 @@
 # frozen_string_literal: true
 
+module GitFit
+  module FIT
+    FIT_EPOCH = 631_065_600
+  end
+end
+
 require_relative 'fit/decoder'

data/lib/git_fit/source/keep.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+require 'base64'
+require 'zlib'
+require 'openssl'
+require 'json'
+require 'time'
+require_relative '../geo/coord_transform'
+
+module GitFit
+  module Source
+    module Keep
+      PLACEHOLDER_POLYLINE = 'gqqrFurkeU??'
+
+      AES_KEY = Base64.decode64('NTZmZTU5OzgyZzpkODczYw==').freeze
+      AES_IV = Base64.decode64('MjM0Njg5MjQzMjkyMDMwMA==').freeze
+
+      TIMESTAMP_THRESHOLD = 3_600_000
+      HR_THRESHOLD = 100
+
+      module_function
+
+      def decode_geo_points(geo_data)
+        return nil unless geo_data
+        raw = Base64.decode64(geo_data)
+        decrypted = aes_decrypt(raw)
+        decompressed = zlib_inflate(decrypted)
+        JSON.parse(decompressed)
+      rescue StandardError
+        nil
+      end
+
+      def decode_hr_data(hr_data)
+        return nil unless hr_data
+        raw = Base64.decode64(hr_data)
+        decompressed = zlib_inflate(raw)
+        JSON.parse(decompressed)
+      rescue StandardError
+        nil
+      end
+
+      def aes_decrypt(data)
+        cipher = OpenSSL::Cipher.new('AES-128-CBC')
+        cipher.decrypt
+        cipher.key = AES_KEY
+        cipher.iv = AES_IV
+        cipher.update(data) + cipher.final
+      end
+
+      def zlib_inflate(data)
+        [-Zlib::MAX_WBITS, Zlib::MAX_WBITS, Zlib::MAX_WBITS + 16].each do |wbits|
+          inflater = Zlib::Inflate.new(wbits)
+          result = inflater.inflate(data)
+          begin
+            inflater.finish
+          rescue StandardError
+            nil
+          end
+          inflater.close
+          return result
+        rescue Zlib::DataError
+          next
+        end
+        raise Zlib::DataError, 'cannot inflate data'
+      end
+
+      def compute_wgs_points(payload, run_data)
+        geo_points = payload[:geo_points]
+        hr_data = payload[:hr_data]
+        return [] unless geo_points&.any?
+
+        start_time = run_data['startTime']
+        use_absolute_ts = geo_points.first['timestamp'].to_i > TIMESTAMP_THRESHOLD
+        effective_start = use_absolute_ts ? 0 : start_time
+
+        geo_points.map do |p|
+          lat, lng = Geo::CoordTransform.gcj02_to_wgs84_exact(p['latitude'], p['longitude'])
+          ts = p['timestamp'] || p['unixTimestamp']
+          pt_time = ts ? Time.at(effective_start / 1000.0 + ts.to_i / 10.0).utc : nil
+          hr = find_nearest_hr(hr_data, ts, start_time)
+          [lat, lng, p['altitude'], pt_time, hr]
+        end
+      end
+
+      def find_nearest_hr(hr_data, geo_ts, start_time, threshold: HR_THRESHOLD)
+        return nil unless hr_data&.any? && geo_ts
+
+        normalize = lambda { |ts|
+          if ts > TIMESTAMP_THRESHOLD
+            ts - start_time / 100
+          else
+            ts
+          end
+        }
+
+        target = normalize.call(geo_ts)
+
+        nearest = nil
+        min_diff = Float::INFINITY
+
+        hr_data.each do |item|
+          ts = item['timestamp']
+          next unless ts
+
+          diff = (normalize.call(ts) - target).abs
+          if diff <= threshold && diff < min_diff
+            nearest = item
+            min_diff = diff
+          end
+        end
+
+        hr = nearest&.dig('beatsPerMinute')
+        hr&.positive? ? hr : nil
+      end
+    end
+  end
+end

data/lib/git_fit/source.rb

@@ -7,5 +7,6 @@ end
 
 require_relative 'util/tally'
 require_relative 'source/base'
+require_relative 'source/keep'
 
 GitFit::Source::Tally = GitFit::Util::Tally

data/lib/git_fit/sync/base.rb

@@ -235,6 +235,17 @@ module GitFit
           .select_map(:run_id)
           .map { |rid| rid.sub(/\A#{Regexp.escape(prefix)}/, '') }
       end
+
+      def normalize_fit_timestamps(points)
+        points.map do |pt|
+          ts = pt['timestamp']
+          next pt unless ts.is_a?(Numeric)
+
+          pt = pt.dup
+          pt['timestamp'] = Time.at(FIT::FIT_EPOCH + ts).utc.iso8601
+          pt
+        end
+      end
     end
   end
 end

data/lib/git_fit/sync/igpsport.rb

@@ -140,7 +140,7 @@ module GitFit
         FileUtils.mkdir_p(dir)
         tmp = File.join(dir, ".#{ride_id}.json.tmp")
         final = File.join(dir, "#{ride_id}.json")
-        File.write(tmp, JSON.generate(wgs_points))
+        File.write(tmp, JSON.generate(normalize_fit_timestamps(wgs_points)))
         File.rename(tmp, final)
       end
 

data/lib/git_fit/sync/keep.rb

@@ -1,9 +1,6 @@
 # frozen_string_literal: true
 
 require_relative 'base'
-require 'base64'
-require 'zlib'
-require 'openssl'
 require 'time'
 require 'tzinfo'
 
@@ -14,11 +11,6 @@ module GitFit
       STATS_URL = 'https://api.gotokeep.com/pd/v3/stats/detail?dateUnit=all&type=%{sport}&lastDate=%{last_date}'
       LOG_URL = 'https://api.gotokeep.com/pd/v3/%{sport}log/%{run_id}'
 
-      AES_KEY = Base64.decode64('NTZmZTU5OzgyZzpkODczYw==')
-      AES_IV = Base64.decode64('MjM0Njg5MjQzMjkyMDMwMA==')
-
-      PLACEHOLDER_POLYLINE = 'gqqrFurkeU??'
-
       KEEP_TYPES = %w[running hiking cycling].freeze
       KEEP_RAW_EXT = 'json'
 
@@ -38,9 +30,6 @@ module GitFit
         'mountaineering' => 'Hiking',
       }.freeze
 
-      TIMESTAMP_THRESHOLD = 3_600_000
-      HR_THRESHOLD = 100
-
       register_adapter
       register_config :phone, :password
 
@@ -231,7 +220,7 @@ module GitFit
           heartRates: payload[:hr_data],
         }, keep_id)
 
-        wgs_points = compute_wgs_points(payload, run_data)
+        wgs_points = Source::Keep.compute_wgs_points(payload, run_data)
         write_standardized_json(wgs_points, payload, run_data, keep_id)
 
         attrs = build_activity_from_wgs(run_data, wgs_points)
@@ -273,35 +262,17 @@ module GitFit
         else
           payload = { geo_points: raw['geoPoints'], hr_data: raw['heartRates'] }
           run_data = raw['meta']
-          compute_wgs_points(payload, run_data)
+          Source::Keep.compute_wgs_points(payload, run_data)
         end
       end
 
       def decrypt_raw_payload(run_data)
         {
-          geo_points: decode_geo_points(run_data['geoPoints']),
-          hr_data: decode_hr_data(run_data.dig('heartRate', 'heartRates')),
+          geo_points: Source::Keep.decode_geo_points(run_data['geoPoints']),
+          hr_data: Source::Keep.decode_hr_data(run_data.dig('heartRate', 'heartRates')),
         }
       end
 
-      def compute_wgs_points(payload, run_data)
-        geo_points = payload[:geo_points]
-        hr_data = payload[:hr_data]
-        return [] unless geo_points&.any?
-
-        start_time = run_data['startTime']
-        use_absolute_ts = geo_points.first['timestamp'].to_i > TIMESTAMP_THRESHOLD
-        effective_start = use_absolute_ts ? 0 : start_time
-
-        geo_points.map do |p|
-          lat, lng = Geo::CoordTransform.gcj02_to_wgs84_exact(p['latitude'], p['longitude'])
-          ts = p['timestamp'] || p['unixTimestamp']
-          pt_time = ts ? Time.at(effective_start / 1000.0 + ts.to_i / 10.0).utc : nil
-          hr = find_nearest_hr(hr_data, ts, start_time)
-          [lat, lng, p['altitude'], pt_time, hr]
-        end
-      end
-
       def tz_resolver
         @tz_resolver ||= GitFit::Timezone::Resolver.new({})
       end
@@ -414,79 +385,6 @@ module GitFit
         id_str.split('_')[1]
       end
 
-      def decode_geo_points(geo_data)
-        return nil unless geo_data
-        raw = Base64.decode64(geo_data)
-        decrypted = aes_decrypt(raw)
-        decompressed = zlib_inflate(decrypted)
-        JSON.parse(decompressed)
-      rescue StandardError
-        nil
-      end
-
-      def decode_hr_data(hr_data)
-        return nil unless hr_data
-        raw = Base64.decode64(hr_data)
-        decompressed = zlib_inflate(raw)
-        JSON.parse(decompressed)
-      rescue StandardError
-        nil
-      end
-
-      def aes_decrypt(data)
-        cipher = OpenSSL::Cipher.new('AES-128-CBC')
-        cipher.decrypt
-        cipher.key = AES_KEY
-        cipher.iv = AES_IV
-        cipher.update(data) + cipher.final
-      end
-
-      def zlib_inflate(data)
-        [-Zlib::MAX_WBITS, Zlib::MAX_WBITS, Zlib::MAX_WBITS + 16].each do |wbits|
-          begin
-            inflater = Zlib::Inflate.new(wbits)
-            result = inflater.inflate(data)
-            inflater.finish rescue nil
-            inflater.close
-            return result
-          rescue Zlib::DataError
-            next
-          end
-        end
-        raise Zlib::DataError, 'cannot inflate data'
-      end
-
-      def find_nearest_hr(hr_data, geo_ts, start_time, threshold: HR_THRESHOLD)
-        return nil unless hr_data&.any? && geo_ts
-
-        normalize = ->(ts) {
-          if ts > TIMESTAMP_THRESHOLD
-            ts - start_time / 100
-          else
-            ts
-          end
-        }
-
-        target = normalize.call(geo_ts)
-
-        nearest = nil
-        min_diff = Float::INFINITY
-
-        hr_data.each do |item|
-          ts = item['timestamp']
-          next unless ts
-
-          diff = (normalize.call(ts) - target).abs
-          if diff <= threshold && diff < min_diff
-            nearest = item
-            min_diff = diff
-          end
-        end
-
-        hr = nearest&.dig('beatsPerMinute')
-        hr&.positive? ? hr : nil
-      end
-
       def compute_avg_hr(points)
         hrs = points.map { |p| p[4] }.compact
         return nil if hrs.empty?

data/lib/git_fit/sync/strava.rb

@@ -134,27 +134,40 @@ module GitFit
         coords = Geo::Polyline.decode(polyline)
         return nil if coords.empty?
 
-        alt_data = streams['altitude']&.dig('data')
-        time_data = streams['time']&.dig('data')
-        hr_data = streams['heartrate']&.dig('data')
-        cad_data = streams['cadence']&.dig('data')
-        watts_data = streams['watts']&.dig('data')
-        temp_data = streams['temp']&.dig('data')
-        speed_data = streams['velocity_smooth']&.dig('data')
-        dist_data = streams['distance']&.dig('data')
+        streams_data = %w[altitude time heartrate cadence watts temp velocity_smooth distance].to_h do |k|
+          [k, streams[k]&.dig('data')]
+        end
+
+        start_time = begin
+          Time.parse(detail['start_date'])
+        rescue StandardError
+          nil
+        end
 
         coords.each_with_index.map do |latlng, i|
-          pt = { positionLat: latlng[0], positionLong: latlng[1] }
-          pt[:altitude] = alt_data[i] if alt_data && i < alt_data.size
-          pt[:timestamp] = time_data[i] if time_data && i < time_data.size
-          pt[:heartRate] = hr_data[i] if hr_data && i < hr_data.size
-          pt[:cadence] = cad_data[i] if cad_data && i < cad_data.size
-          pt[:power] = watts_data[i] if watts_data && i < watts_data.size
-          pt[:temperature] = temp_data[i] if temp_data && i < temp_data.size
-          pt[:speed] = speed_data[i] if speed_data && i < speed_data.size
-          pt[:distance] = dist_data[i] if dist_data && i < dist_data.size
-          pt
+          build_l2_point(latlng, i, streams_data, start_time)
+        end
+      end
+
+      def build_l2_point(latlng, i, streams_data, start_time)
+        pt = { positionLat: latlng[0], positionLong: latlng[1] }
+        pt[:altitude] = stream_at(streams_data, 'altitude', i)
+        ts = stream_at(streams_data, 'time', i)
+        if ts
+          pt[:timestamp] = start_time ? (start_time + ts).utc.iso8601 : ts
         end
+        pt[:heartRate] = stream_at(streams_data, 'heartrate', i)
+        pt[:cadence] = stream_at(streams_data, 'cadence', i)
+        pt[:power] = stream_at(streams_data, 'watts', i)
+        pt[:temperature] = stream_at(streams_data, 'temp', i)
+        pt[:speed] = stream_at(streams_data, 'velocity_smooth', i)
+        pt[:distance] = stream_at(streams_data, 'distance', i)
+        pt
+      end
+
+      def stream_at(streams_data, key, i)
+        data = streams_data[key]
+        data && i < data.size ? data[i] : nil
       end
 
       def build_attrs(act, category, polyline = nil, external_id: nil)

data/lib/git_fit/sync/xoss.rb

@@ -348,7 +348,7 @@ module GitFit
         FileUtils.mkdir_p(dir)
         tmp = File.join(dir, ".#{act_id}.json.tmp")
         final = File.join(dir, "#{act_id}.json")
-        File.write(tmp, JSON.generate(points))
+        File.write(tmp, JSON.generate(normalize_fit_timestamps(points)))
         File.rename(tmp, final)
       end
 

data/lib/git_fit/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module GitFit
-  VERSION = '0.10.3'
+  VERSION = '0.10.5'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: git-fit
 version: !ruby/object:Gem::Version
-  version: 0.10.3
+  version: 0.10.5
 platform: ruby
 authors:
 - Lax
@@ -177,6 +177,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: webrick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
 - !ruby/object:Gem::Dependency
   name: ostruct
   requirement: !ruby/object:Gem::Requirement
@@ -218,6 +232,8 @@ files:
 - db/migrations/002_iso8601_time_format.rb
 - exe/git-fit
 - lib/git-fit.rb
+- lib/git_fit/auth/garmin_token.rb
+- lib/git_fit/auth/strava.rb
 - lib/git_fit/cli.rb
 - lib/git_fit/cli/export.rb
 - lib/git_fit/cli/geo_cli.rb
@@ -232,10 +248,17 @@ files:
 - lib/git_fit/db/cli.rb
 - lib/git_fit/db/connection.rb
 - lib/git_fit/db/rebuild.rb
+- lib/git_fit/dedup/field_elector.rb
+- lib/git_fit/dedup/group_consolidator.rb
+- lib/git_fit/dedup/log_writer.rb
+- lib/git_fit/dedup/matcher.rb
+- lib/git_fit/dedup/service.rb
+- lib/git_fit/dedup/trajectory.rb
 - lib/git_fit/export.rb
 - lib/git_fit/export/calculator.rb
 - lib/git_fit/export/csv.rb
 - lib/git_fit/export/defaults.rb
+- lib/git_fit/export/gpx.rb
 - lib/git_fit/export/json.rb
 - lib/git_fit/export/stats.rb
 - lib/git_fit/fit.rb
@@ -262,6 +285,7 @@ files:
 - lib/git_fit/privacy/polyline_filter.rb
 - lib/git_fit/source.rb
 - lib/git_fit/source/base.rb
+- lib/git_fit/source/keep.rb
 - lib/git_fit/source/tally.rb
 - lib/git_fit/sport_mapper.rb
 - lib/git_fit/strava_web/file_check.rb