emerge 0.2.2 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4a44399f9df18a4ce4d32984e12eaadb2fa921c34409d8e4daa0dd1b4acddd2c
-  data.tar.gz: 855d965c1478cbc23eee27f5f99281d1a144b7c40197bdb693fac7d93f0be72d
+  metadata.gz: e4ada02f9680b03cfcf2d4350d0aa719ae8c1a0399e7783b468b341e2eac5143
+  data.tar.gz: 7dfc873c8ba7cb9dd145b7ae181cdcadbe494741ac6f241fcdd0cb48f3c88526
 SHA512:
-  metadata.gz: 9d005834cf05f3d463df75af935e5e5ad075fe98cac3a05169df0e9f29c50366731f99b606af924ec33afce6fe332cd0e0ebccdd4768d52fa2dff59628e28b68
-  data.tar.gz: 184aad1844eacdf8aa3693ab5ff45acb36f11a16d039f258137f807f355f822b5f31565450927f5b6db453300785df287dcb5dc7c68227d635ccc7dc498a83ba
+  metadata.gz: 4ae1796a1d262846e12bf5b1f64301166209b0ad31f4981effb118a4c091d0f36c5396e169c881b8baea1f5fc93e18f8aa452eb1e05775eeab79b615e6b38370
+  data.tar.gz: ff14aac430f27b95be0d6267839d967e9869b31090c669d68c0e8fefb34dd6e3468bc627080f36ce27aaa1288d73e2698c5800614035f5218d7d71a2a5182aef

data/README.md

@@ -108,8 +108,33 @@ emerge upload snapshots \
   --project-root /my/awesomeapp/android/repo
 ```
 
-## Building
+## Reaper
 
-This depends on [Tree Sitter](https://tree-sitter.github.io/tree-sitter/) for part of its functionality.
+Experimental support has been added to interactively examine [Reaper](https://docs.emergetools.com/docs/reaper) results and also **delete them from your codebase**.
 
-In order to parse language grammars for Swift and Kotlin, both of which are third-party language grammars, we also depend on [tsdl](https://github.com/stackmystack/tsdl). This downloads and compiles the language grammars into dylibs for us to use.
+Use the `reaper` subcommand to get started, e.g.:
+
+```shell
+emerge reaper --upload-id 40f1dfe7-6c57-47c3-bc52-b621aec0ba8d \
+  --project-root /path/to/your/repo
+```
+
+After which it will prompt you to select classes to delete.
+
+### How it works
+
+Under the hood we are using [Tree Sitter](https://tree-sitter.github.io/tree-sitter/) to parse your source files into an AST which is then used for deletions. There are some obvious limitations to this approach, namely that Tree Sitter is designed for source code editors and only looks at a single file at a time. We are exploring some better long-term approaches but this works well enough for now!
+
+### Supported languages
+
+We currently support the following languages:
+
+- Swift
+- Kotlin
+- Java
+
+Please open an issue if you need an additional language grammar.
+
+### Building
+
+Because many of the language grammars we use are third-party, we have to package them with our CLI tool as shared libraries for distribution. We depend on [tsdl](https://github.com/stackmystack/tsdl) to build the grammars from our `parsers.toml` file.

data/lib/commands/global_options.rb

@@ -9,6 +9,8 @@ module EmergeCLI
       def before(args)
         log_level = args[:debug] ? ::Logger::DEBUG : ::Logger::INFO
         EmergeCLI::Logger.configure(log_level)
+
+        EmergeCLI::Utils::VersionCheck.new.check_version
       end
     end
   end

data/lib/commands/order_files/download_order_files.rb

@@ -0,0 +1,77 @@
+require 'dry/cli'
+
+module EmergeCLI
+  module Commands
+    class DownloadOrderFiles < EmergeCLI::Commands::GlobalOptions
+      desc 'Download order files from Emerge'
+
+      option :bundle_id, type: :string, required: true, desc: 'Bundle identifier to download order files for'
+
+      option :api_token, type: :string, required: false,
+                         desc: 'API token for authentication, defaults to ENV[EMERGE_API_TOKEN]'
+
+      option :app_version, type: :string, required: true,
+                           desc: 'App version to download order files for'
+
+      option :unzip, type: :boolean, required: false,
+                     desc: 'Unzip the order file after downloading'
+
+      option :output, type: :string, required: false,
+                      desc: 'Output name for the order file, defaults to bundle_id-app_version.gz'
+
+      EMERGE_ORDER_FILE_URL = 'order-files-prod.emergetools.com'.freeze
+
+      def initialize(network: nil)
+        @network = network
+      end
+
+      def call(**options)
+        @options = options
+        before(options)
+
+        begin
+          api_token = @options[:api_token] || ENV.fetch('EMERGE_API_TOKEN', nil)
+          raise 'API token is required' unless api_token
+
+          raise 'Bundle ID is required' unless @options[:bundle_id]
+          raise 'App version is required' unless @options[:app_version]
+
+          @network ||= EmergeCLI::Network.new(api_token:, base_url: EMERGE_ORDER_FILE_URL)
+          output_name = @options[:output] || "#{@options[:bundle_id]}-#{@options[:app_version]}.gz"
+          output_name = "#{output_name}.gz" unless output_name.end_with?('.gz')
+
+          Sync do
+            request = get_order_file(options[:bundle_id], options[:app_version])
+            response = request.read
+
+            File.write(output_name, response)
+
+            if @options[:unzip]
+              Logger.info 'Unzipping order file...'
+              Zlib::GzipReader.open(output_name) do |gz|
+                File.write(output_name.gsub('.gz', ''), gz.read)
+              end
+            end
+
+            Logger.info 'Order file downloaded successfully'
+          end
+        rescue StandardError => e
+          Logger.error "Failed to download order file: #{e.message}"
+          Logger.error 'Check your parameters and try again'
+          raise e
+        ensure
+          @network&.close
+        end
+      end
+
+      private
+
+      def get_order_file(bundle_id, app_version)
+        @network.get(
+          path: "/#{bundle_id}/#{app_version}",
+          max_retries: 0
+        )
+      end
+    end
+  end
+end

data/lib/commands/order_files/validate_linkmaps.rb

@@ -0,0 +1,55 @@
+require 'dry/cli'
+require 'cfpropertylist'
+
+module EmergeCLI
+  module Commands
+    class ValidateLinkmaps < EmergeCLI::Commands::GlobalOptions
+      desc 'Validate linkmaps in xcarchive'
+
+      option :path, type: :string, required: true, desc: 'Path to the xcarchive to validate'
+
+      def initialize(network: nil)
+        @network = network
+      end
+
+      def call(**options)
+        @options = options
+        before(options)
+
+        Sync do
+          executable_name = get_executable_name
+          raise 'Executable not found' if executable_name.nil?
+
+          Logger.info "Using executable: #{executable_name}"
+
+          linkmaps_path = File.join(@options[:path], 'Linkmaps')
+          raise 'Linkmaps folder not found' unless File.directory?(linkmaps_path)
+
+          linkmaps = Dir.glob("#{linkmaps_path}/*.txt")
+          raise 'No linkmaps found' if linkmaps.empty?
+
+          executable_linkmaps = linkmaps.select do |linkmap|
+            File.basename(linkmap).start_with?(executable_name)
+          end
+          raise 'No linkmaps found for executable' if executable_linkmaps.empty?
+
+          Logger.info "✅ Found linkmaps for #{executable_name}"
+        end
+      end
+
+      private
+
+      def get_executable_name
+        raise 'Path must be an xcarchive' unless @options[:path].end_with?('.xcarchive')
+
+        app_path = Dir.glob("#{@options[:path]}/Products/Applications/*.app").first
+        info_path = File.join(app_path, 'Info.plist')
+        plist_data = File.read(info_path)
+        plist = CFPropertyList::List.new(data: plist_data)
+        parsed_data = CFPropertyList.native_types(plist.value)
+
+        parsed_data['CFBundleExecutable']
+      end
+    end
+  end
+end

data/lib/commands/reaper/reaper.rb

@@ -0,0 +1,201 @@
+require 'dry/cli'
+require 'json'
+require 'tty-prompt'
+
+module EmergeCLI
+  module Commands
+    class Reaper < EmergeCLI::Commands::GlobalOptions
+      desc 'Analyze dead code from an Emerge upload'
+
+      option :upload_id, type: :string, required: true, desc: 'Upload ID to analyze'
+      option :project_root, type: :string, required: true,
+                            desc: 'Root directory of the project, defaults to current directory'
+
+      option :api_token, type: :string, required: false,
+                         desc: 'API token for authentication, defaults to ENV[EMERGE_API_TOKEN]'
+
+      option :profile, type: :boolean, default: false, desc: 'Enable performance profiling metrics'
+
+      option :skip_delete_usages, type: :boolean, default: false,
+                                  desc: 'Skip deleting usages of the type (experimental feature)'
+
+      def initialize(network: nil)
+        @network = network
+      end
+
+      def call(**options)
+        @options = options
+        @profiler = EmergeCLI::Profiler.new(enabled: options[:profile])
+        @prompt = TTY::Prompt.new
+        before(options)
+        success = false
+
+        begin
+          api_token = @options[:api_token] || ENV.fetch('EMERGE_API_TOKEN', nil)
+          raise 'API token is required' unless api_token
+
+          @network ||= EmergeCLI::Network.new(api_token:)
+          project_root = @options[:project_root] || Dir.pwd
+
+          Sync do
+            all_data = @profiler.measure('fetch_dead_code') { fetch_all_dead_code(@options[:upload_id]) }
+            result = @profiler.measure('parse_dead_code') { DeadCodeResult.new(all_data) }
+
+            Logger.info result.to_s
+
+            selected_types = prompt_class_selection(result.filtered_unseen_classes, result.metadata['platform'])
+            Logger.info 'Selected classes:'
+            selected_types.each do |selected_class|
+              Logger.info " - #{selected_class['class_name']}"
+            end
+
+            confirmed = confirm_deletion(selected_types.length)
+            if !confirmed
+              Logger.info 'Operation cancelled'
+              return false
+            end
+
+            Logger.info 'Proceeding with deletion...'
+            platform = result.metadata['platform']
+            deleter = EmergeCLI::Reaper::CodeDeleter.new(
+              project_root: project_root,
+              platform: platform,
+              profiler: @profiler,
+              skip_delete_usages: options[:skip_delete_usages]
+            )
+            @profiler.measure('delete_types') { deleter.delete_types(selected_types) }
+          end
+
+          @profiler.report if @options[:profile]
+          success = true
+        rescue StandardError => e
+          Logger.error "Failed to analyze dead code: #{e.message}"
+          raise e
+        ensure
+          @network&.close
+        end
+
+        success
+      end
+
+      private
+
+      def fetch_all_dead_code(upload_id)
+        Logger.info 'Fetching dead code analysis (this may take a while for large codebases)...'
+
+        page = 1
+        combined_data = nil
+
+        loop do
+          response = fetch_dead_code_page(upload_id, page)
+          data = JSON.parse(response.read)
+
+          if combined_data.nil?
+            combined_data = data
+          else
+            combined_data['dead_code'].concat(data.fetch('dead_code', []))
+          end
+
+          current_page = data.dig('pagination', 'current_page')
+          total_pages = data.dig('pagination', 'total_pages')
+
+          break unless current_page && total_pages && current_page < total_pages
+
+          page += 1
+          Logger.info "Fetching page #{page} of #{total_pages}..."
+        end
+
+        combined_data
+      end
+
+      def fetch_dead_code_page(upload_id, page)
+        @network.post(
+          path: '/deadCode/export',
+          query: {
+            uploadId: upload_id,
+            page: page
+          },
+          headers: { 'Accept' => 'application/json' },
+          body: nil
+        )
+      end
+
+      def prompt_class_selection(unseen_classes, platform)
+        return nil if unseen_classes.empty?
+
+        choices = unseen_classes.map do |item|
+          display_name = if item['paths']&.first && platform == 'ios'
+                           "#{item['class_name']} (#{item['paths'].first})"
+                         else
+                           item['class_name']
+                         end
+          {
+            name: display_name,
+            value: item
+          }
+        end
+
+        @prompt.multi_select(
+          'Select classes to delete:'.blue,
+          choices,
+          per_page: 15,
+          echo: false,
+          filter: true,
+          min: 1
+        )
+      end
+
+      def confirm_deletion(count)
+        @prompt.yes?("Are you sure you want to delete #{count} type#{count > 1 ? 's' : ''}?")
+      end
+
+      class DeadCodeResult
+        attr_reader :metadata, :dead_code, :counts, :pagination
+
+        def initialize(data)
+          @metadata = data['metadata']
+          @dead_code = data['dead_code']
+          @counts = data['counts']
+          @pagination = data['pagination']
+        end
+
+        def filtered_unseen_classes
+          @filtered_unseen_classes ||= dead_code
+                                       .reject { |item| item['seen'] }
+                                       .reject do |item|
+            paths = item['paths']
+            next false if paths.nil? || paths.empty?
+
+            next true if paths.any? do |path|
+              path.include?('/SourcePackages/checkouts/') ||
+              path.include?('/Pods/') ||
+              path.include?('/Carthage/') ||
+              path.include?('/Vendor/') ||
+              path.include?('/Sources/') ||
+              path.include?('/DerivedSources/')
+            end
+
+            next false if paths.none? do |path|
+              path.end_with?('.swift', '.java', '.kt')
+            end
+          end
+        end
+
+        def to_s
+          <<~SUMMARY.yellow
+
+            Dead Code Analysis Results:
+            App ID: #{@metadata['app_id']}
+            App Version: #{@metadata['version']}
+            Platform: #{@metadata['platform']}
+
+            Statistics:
+            - Total User Sessions: #{@counts['user_sessions']}
+            - Seen Classes: #{@counts['seen_classes']}
+            - Unseen Classes: #{@counts['unseen_classes']}
+          SUMMARY
+        end
+      end
+    end
+  end
+end

data/lib/commands/snapshots/validate_app.rb

@@ -0,0 +1,64 @@
+require 'dry/cli'
+require 'json'
+require 'uri'
+require 'yaml'
+require 'cfpropertylist'
+
+module EmergeCLI
+  module Commands
+    module Snapshots
+      class ValidateApp < EmergeCLI::Commands::GlobalOptions
+        desc 'Validate app for snapshot testing [iOS, macOS]'
+
+        # Optional options
+        option :path, type: :string, required: true, desc: 'Path to the app binary or xcarchive'
+
+        # Mangled names are deterministic, no need to demangle them
+        SWIFT_PREVIEWS_MANGLED_NAMES = [
+          '_$s21DeveloperToolsSupport15PreviewRegistryMp',
+          '_$s7SwiftUI15PreviewProviderMp'
+        ].freeze
+
+        def call(**options)
+          @options = options
+          before(options)
+
+          Sync do
+            binary_path = get_binary_path
+            Logger.info "Found binary: #{binary_path}"
+
+            Logger.info "Loading binary: #{binary_path}"
+            macho_parser = MachOParser.new
+            macho_parser.load_binary(binary_path)
+
+            use_chained_fixups, imported_symbols = macho_parser.read_linkedit_data_command
+            bound_symbols = macho_parser.read_dyld_info_only_command
+
+            found = macho_parser.find_protocols_in_swift_proto(use_chained_fixups, imported_symbols, bound_symbols,
+                                                               SWIFT_PREVIEWS_MANGLED_NAMES)
+
+            if found
+              Logger.info '✅ Found SwiftUI previews'
+            else
+              Logger.error '❌ No SwiftUI previews found'
+            end
+            found
+          end
+        end
+
+        private
+
+        def get_binary_path
+          return @options[:path] unless @options[:path].end_with?('.xcarchive')
+          app_path = Dir.glob("#{@options[:path]}/Products/Applications/*.app").first
+          info_path = File.join(app_path, 'Info.plist')
+          plist_data = File.read(info_path)
+          plist = CFPropertyList::List.new(data: plist_data)
+          parsed_data = CFPropertyList.native_types(plist.value)
+
+          File.join(app_path, parsed_data['CFBundleExecutable'])
+        end
+      end
+    end
+  end
+end

data/lib/emerge_cli.rb

@@ -9,8 +9,13 @@ require_relative 'commands/upload/snapshots/client_libraries/default'
 require_relative 'commands/integrate/fastlane'
 require_relative 'commands/config/snapshots/snapshots_ios'
 require_relative 'commands/config/orderfiles/orderfiles_ios'
+require_relative 'commands/reaper/reaper'
+require_relative 'commands/snapshots/validate_app'
+require_relative 'commands/order_files/download_order_files'
+require_relative 'commands/order_files/validate_linkmaps'
 
 require_relative 'reaper/ast_parser'
+require_relative 'reaper/code_deleter'
 
 require_relative 'utils/git_info_provider'
 require_relative 'utils/git_result'
@@ -20,10 +25,10 @@ require_relative 'utils/logger'
 require_relative 'utils/network'
 require_relative 'utils/profiler'
 require_relative 'utils/project_detector'
+require_relative 'utils/macho_parser'
+require_relative 'utils/version_check'
 
 require 'dry/cli'
-require 'pry'
-require 'pry-byebug'
 
 module EmergeCLI
   extend Dry::CLI::Registry
@@ -40,6 +45,17 @@ module EmergeCLI
     prefix.register 'snapshots-ios', Commands::Config::SnapshotsIOS
     prefix.register 'order-files-ios', Commands::Config::OrderFilesIOS
   end
+
+  register 'reaper', Commands::Reaper
+
+  register 'snapshots' do |prefix|
+    prefix.register 'validate-app-ios', Commands::Snapshots::ValidateApp
+  end
+
+  register 'order-files' do |prefix|
+    prefix.register 'download', Commands::DownloadOrderFiles
+    prefix.register 'validate-linkmaps', Commands::ValidateLinkmaps
+  end
 end
 
 # By default the log level is INFO, but can be overridden by the --debug flag