git-pkgs 0.3.0 → 0.5.0

data/lib/git/pkgs/commands/init.rb

@@ -6,8 +6,16 @@ module Git
       class Init
         include Output
 
-        BATCH_SIZE = 100
-        SNAPSHOT_INTERVAL = 20 # Store snapshot every N dependency-changing commits
+        DEFAULT_BATCH_SIZE = 500
+        DEFAULT_SNAPSHOT_INTERVAL = 50
+
+        def batch_size
+          Git::Pkgs.batch_size || DEFAULT_BATCH_SIZE
+        end
+
+        def snapshot_interval
+          Git::Pkgs.snapshot_interval || DEFAULT_SNAPSHOT_INTERVAL
+        end
 
         def initialize(args)
           @args = args
@@ -17,8 +25,11 @@ module Git
         def run
           repo = Repository.new
 
+          branch_name = @options[:branch] || repo.default_branch
+          error "Branch '#{branch_name}' not found. Check 'git branch -a' for available branches." unless repo.branch_exists?(branch_name)
+
           if Database.exists?(repo.git_dir) && !@options[:force]
-            puts "Database already exists. Use --force to rebuild."
+            info "Database already exists. Use --force to rebuild."
             return
           end
 
@@ -27,35 +38,36 @@ module Git
           Database.create_schema(with_indexes: false)
           Database.optimize_for_bulk_writes
 
-          branch_name = @options[:branch] || repo.default_branch
-          error "Branch '#{branch_name}' not found" unless repo.branch_exists?(branch_name)
-
           branch = Models::Branch.find_or_create(branch_name)
           analyzer = Analyzer.new(repo)
 
-          puts "Analyzing branch: #{branch_name}"
+          info "Analyzing branch: #{branch_name}"
 
+          print "Loading commits..." unless Git::Pkgs.quiet
           walker = repo.walk(branch_name, @options[:since])
           commits = walker.to_a
           total = commits.size
+          print "\rPrefetching diffs..." unless Git::Pkgs.quiet
+          repo.prefetch_blob_paths(commits)
+          print "\r#{' ' * 20}\r" unless Git::Pkgs.quiet
 
           stats = bulk_process_commits(commits, branch, analyzer, total)
 
           branch.update(last_analyzed_sha: repo.branch_target(branch_name))
 
-          print "\rCreating indexes..."
+          print "\rCreating indexes...#{' ' * 20}" unless Git::Pkgs.quiet
           Database.create_bulk_indexes
           Database.optimize_for_reads
 
           cache_stats = analyzer.cache_stats
 
-          puts "\rDone!#{' ' * 20}"
-          puts "Analyzed #{total} commits"
-          puts "Found #{stats[:dependency_commits]} commits with dependency changes"
-          puts "Stored #{stats[:snapshots_stored]} snapshots (every #{SNAPSHOT_INTERVAL} changes)"
-          puts "Blob cache: #{cache_stats[:cached_blobs]} unique blobs, #{cache_stats[:blobs_with_hits]} had cache hits"
+          info "\rDone!#{' ' * 20}"
+          info "Analyzed #{total} commits"
+          info "Found #{stats[:dependency_commits]} commits with dependency changes"
+          info "Stored #{stats[:snapshots_stored]} snapshots (every #{snapshot_interval} changes)"
+          info "Blob cache: #{cache_stats[:cached_blobs]} unique blobs, #{cache_stats[:blobs_with_hits]} had cache hits"
 
-          if @options[:hooks]
+          unless @options[:no_hooks]
             Commands::Hooks.new(["--install"]).run
           end
         end
@@ -73,6 +85,7 @@ module Git
           dependency_commit_count = 0
           snapshots_stored = 0
           processed = 0
+          last_processed_sha = nil
 
           flush = lambda do
             return if pending_commits.empty?
@@ -134,9 +147,11 @@ module Git
             pending_snapshots.clear
           end
 
+          progress_interval = [total / 100, 10].max
+
           commits.each do |rugged_commit|
             processed += 1
-            print "\rProcessing commit #{processed}/#{total}..." if processed % 50 == 0 || processed == total
+            print "\rProcessing commit #{processed}/#{total}..." if !Git::Pkgs.quiet && (processed % progress_interval == 0 || processed == total)
 
             next if rugged_commit.parents.length > 1 # skip merge commits
 
@@ -160,6 +175,8 @@ module Git
               position: processed
             }
 
+            last_processed_sha = rugged_commit.oid
+
             if has_changes
               dependency_commit_count += 1
 
@@ -188,7 +205,7 @@ module Git
               snapshot = result[:snapshot]
 
               # Store snapshot at intervals
-              if dependency_commit_count % SNAPSHOT_INTERVAL == 0
+              if dependency_commit_count % snapshot_interval == 0
                 snapshot.each do |(manifest_path, name), dep_info|
                   pending_snapshots << {
                     sha: rugged_commit.oid,
@@ -203,16 +220,15 @@ module Git
               end
             end
 
-            flush.call if pending_commits.size >= BATCH_SIZE
+            flush.call if pending_commits.size >= batch_size
           end
 
-          # Always store final snapshot for HEAD
-          if snapshot.any?
-            last_sha = commits.last&.oid
-            if last_sha && !pending_snapshots.any? { |s| s[:sha] == last_sha }
+          # Always store final snapshot for the last processed commit
+          if snapshot.any? && last_processed_sha
+            unless pending_snapshots.any? { |s| s[:sha] == last_processed_sha }
               snapshot.each do |(manifest_path, name), dep_info|
                 pending_snapshots << {
-                  sha: last_sha,
+                  sha: last_processed_sha,
                   manifest_path: manifest_path,
                   name: name,
                   ecosystem: dep_info[:ecosystem],
@@ -247,8 +263,8 @@ module Git
               options[:force] = true
             end
 
-            opts.on("--hooks", "Install git hooks for auto-updating") do
-              options[:hooks] = true
+            opts.on("--no-hooks", "Skip installing git hooks") do
+              options[:no_hooks] = true
             end
 
             opts.on("-h", "--help", "Show this help") do

data/lib/git/pkgs/commands/list.rb

@@ -20,7 +20,7 @@ module Git
           commit_sha = @options[:commit] || repo.head_sha
           target_commit = Models::Commit.find_by(sha: commit_sha)
 
-          error "Commit #{commit_sha[0, 7]} not found in database" unless target_commit
+          error "Commit #{commit_sha[0, 7]} not in database. Run 'git pkgs update' to index new commits." unless target_commit
 
           deps = compute_dependencies_at_commit(target_commit, repo)
 
@@ -90,7 +90,7 @@ module Git
           # Replay changes from snapshot to target
           if snapshot_commit && snapshot_commit.id != target_commit.id
             changes = Models::DependencyChange
-              .joins(:commit, :manifest)
+              .joins(:commit)
               .where(commits: { id: branch.commit_ids })
               .where("commits.committed_at > ? AND commits.committed_at <= ?",
                      snapshot_commit.committed_at, target_commit.committed_at)

data/lib/git/pkgs/commands/log.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "time"
-
 module Git
   module Pkgs
     module Commands
@@ -20,6 +18,7 @@ module Git
           Database.connect(repo.git_dir)
 
           commits = Models::Commit
+            .includes(:dependency_changes)
             .where(has_dependency_changes: true)
             .order(committed_at: :desc)
 
@@ -44,8 +43,8 @@ module Git
 
         def output_text(commits)
           commits.each do |commit|
-            changes = commit.dependency_changes
-            changes = changes.where(ecosystem: @options[:ecosystem]) if @options[:ecosystem]
+            changes = commit.dependency_changes.to_a
+            changes = changes.select { |c| c.ecosystem == @options[:ecosystem] } if @options[:ecosystem]
             next if changes.empty?
 
             puts "#{commit.short_sha} #{commit.message&.lines&.first&.strip}"
@@ -77,8 +76,8 @@ module Git
           require "json"
 
           data = commits.map do |commit|
-            changes = commit.dependency_changes
-            changes = changes.where(ecosystem: @options[:ecosystem]) if @options[:ecosystem]
+            changes = commit.dependency_changes.to_a
+            changes = changes.select { |c| c.ecosystem == @options[:ecosystem] } if @options[:ecosystem]
 
             {
               sha: commit.sha,
@@ -102,12 +101,6 @@ module Git
           puts JSON.pretty_generate(data)
         end
 
-        def parse_time(str)
-          Time.parse(str)
-        rescue ArgumentError
-          error "Invalid date format: #{str}"
-        end
-
         def parse_options
           options = {}
 

data/lib/git/pkgs/commands/show.rb

@@ -20,10 +20,10 @@ module Git
           Database.connect(repo.git_dir)
 
           sha = repo.rev_parse(ref)
-          error "Could not resolve '#{ref}'" unless sha
+          error "Could not resolve '#{ref}'. Check that the ref exists with 'git rev-parse #{ref}'." unless sha
 
-          commit = find_or_create_commit(repo, sha)
-          error "Commit '#{sha[0..7]}' not found" unless commit
+          commit = Models::Commit.find_or_create_from_repo(repo, sha)
+          error "Commit '#{sha[0..7]}' not in database. Run 'git pkgs update' to index new commits." unless commit
 
           changes = Models::DependencyChange
             .includes(:commit, :manifest)
@@ -107,26 +107,6 @@ module Git
           puts JSON.pretty_generate(data)
         end
 
-        def find_or_create_commit(repo, sha)
-          commit = Models::Commit.find_by(sha: sha) ||
-                   Models::Commit.where("sha LIKE ?", "#{sha}%").first
-          return commit if commit
-
-          rugged_commit = repo.lookup(sha)
-          return nil unless rugged_commit
-
-          Models::Commit.create!(
-            sha: rugged_commit.oid,
-            message: rugged_commit.message,
-            author_name: rugged_commit.author[:name],
-            author_email: rugged_commit.author[:email],
-            committed_at: rugged_commit.time,
-            has_dependency_changes: false
-          )
-        rescue Rugged::OdbError
-          nil
-        end
-
         def parse_options
           options = {}
 

data/lib/git/pkgs/commands/stale.rb

@@ -20,7 +20,7 @@ module Git
           branch_name = @options[:branch] || repo.default_branch
           branch = Models::Branch.find_by(name: branch_name)
 
-          error "No analysis found for branch '#{branch_name}'" unless branch&.last_analyzed_sha
+          error "No analysis found for branch '#{branch_name}'. Run 'git pkgs init' first." unless branch&.last_analyzed_sha
 
           current_commit = Models::Commit.find_by(sha: branch.last_analyzed_sha)
           snapshots = current_commit&.dependency_snapshots&.includes(:manifest) || []
@@ -34,19 +34,38 @@ module Git
             return
           end
 
+          # Batch fetch all changes for current dependencies
+          snapshot_keys = snapshots.map { |s| [s.name, s.manifest_id] }.to_set
+          manifest_ids = snapshots.map(&:manifest_id).uniq
+          names = snapshots.map(&:name).uniq
+
+          all_changes = Models::DependencyChange
+            .includes(:commit)
+            .where(manifest_id: manifest_ids, name: names)
+            .to_a
+
+          # Group by (name, manifest_id) and find latest by committed_at
+          latest_by_key = {}
+          all_changes.each do |change|
+            key = [change.name, change.manifest_id]
+            next unless snapshot_keys.include?(key)
+
+            existing = latest_by_key[key]
+            if existing.nil? || change.commit.committed_at > existing.commit.committed_at
+              latest_by_key[key] = change
+            end
+          end
+
           # Find last update for each dependency
           outdated_data = []
+          now = Time.now
 
           snapshots.each do |snapshot|
-            last_change = Models::DependencyChange
-              .includes(:commit)
-              .where(name: snapshot.name, manifest: snapshot.manifest)
-              .order("commits.committed_at DESC")
-              .first
+            last_change = latest_by_key[[snapshot.name, snapshot.manifest_id]]
 
             next unless last_change
 
-            days_since_update = ((Time.now - last_change.commit.committed_at) / 86400).to_i
+            days_since_update = ((now - last_change.commit.committed_at) / 86400).to_i
 
             outdated_data << {
               name: snapshot.name,

data/lib/git/pkgs/commands/stats.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "time"
-
 module Git
   module Pkgs
     module Commands
@@ -93,11 +91,16 @@ module Git
           manifests = Models::Manifest.all
           manifests = manifests.where(ecosystem: ecosystem) if ecosystem
 
+          manifest_ids = manifests.pluck(:id)
+          change_counts_query = Models::DependencyChange
+            .joins(:commit)
+            .where(manifest_id: manifest_ids)
+          change_counts_query = change_counts_query.where("commits.committed_at >= ?", since_time) if since_time
+          change_counts_query = change_counts_query.where("commits.committed_at <= ?", until_time) if until_time
+          change_counts = change_counts_query.group(:manifest_id).count
+
           data[:manifests] = manifests.map do |manifest|
-            manifest_changes = manifest.dependency_changes.joins(:commit)
-            manifest_changes = manifest_changes.where("commits.committed_at >= ?", since_time) if since_time
-            manifest_changes = manifest_changes.where("commits.committed_at <= ?", until_time) if until_time
-            { path: manifest.path, ecosystem: manifest.ecosystem, changes: manifest_changes.count }
+            { path: manifest.path, ecosystem: manifest.ecosystem, changes: change_counts[manifest.id] || 0 }
           end
 
           data
@@ -199,12 +202,6 @@ module Git
           end
         end
 
-        def parse_time(str)
-          Time.parse(str)
-        rescue ArgumentError
-          error "Invalid date format: #{str}"
-        end
-
         def parse_options
           options = {}
 

data/lib/git/pkgs/commands/tree.rb

@@ -21,7 +21,7 @@ module Git
           commit_sha = @options[:commit] || repo.head_sha
           commit = find_commit_with_snapshot(commit_sha, repo)
 
-          error "No dependency data found for commit #{commit_sha[0, 7]}" unless commit
+          error "No dependency data for commit #{commit_sha[0, 7]}. Run 'git pkgs update' to index new commits." unless commit
 
           # Get current snapshots
           snapshots = commit.dependency_snapshots.includes(:manifest)

data/lib/git/pkgs/commands/update.rb

@@ -26,7 +26,7 @@ module Git
           current_sha = repo.branch_target(branch_name)
 
           if since_sha == current_sha
-            puts "Already up to date."
+            info "Already up to date."
             return
           end
 
@@ -50,17 +50,19 @@ module Git
           walker = repo.walk(branch_name, since_sha)
           commits = walker.to_a
           total = commits.size
+          repo.prefetch_blob_paths(commits)
+
           processed = 0
           dependency_commits = 0
           last_position = Models::BranchCommit.where(branch: branch).maximum(:position) || 0
 
-          puts "Updating branch: #{branch_name}"
-          puts "Found #{total} new commits"
+          info "Updating branch: #{branch_name}"
+          info "Found #{total} new commits"
 
           ActiveRecord::Base.transaction do
             commits.each do |rugged_commit|
               processed += 1
-              print "\rProcessing commit #{processed}/#{total}..."
+              print "\rProcessing commit #{processed}/#{total}..." unless Git::Pkgs.quiet
 
               result = analyzer.analyze_commit(rugged_commit, snapshot)
 
@@ -114,9 +116,9 @@ module Git
             branch.update(last_analyzed_sha: current_sha)
           end
 
-          puts "\nDone!"
-          puts "Processed #{total} new commits"
-          puts "Found #{dependency_commits} commits with dependency changes"
+          info "\nDone!"
+          info "Processed #{total} new commits"
+          info "Found #{dependency_commits} commits with dependency changes"
         end
 
         def parse_options

data/lib/git/pkgs/commands/upgrade.rb

@@ -21,13 +21,13 @@ module Git
           current = Database::SCHEMA_VERSION
 
           if stored >= current
-            puts "Database is up to date (version #{current})"
+            info "Database is up to date (version #{current})"
             return
           end
 
-          puts "Upgrading database from version #{stored} to #{current}..."
-          puts "This requires re-indexing the repository."
-          puts
+          info "Upgrading database from version #{stored} to #{current}..."
+          info "This requires re-indexing the repository."
+          info ""
 
           # Run init --force
           Init.new(["--force"]).run

data/lib/git/pkgs/commands/where.rb

@@ -0,0 +1,166 @@
+# frozen_string_literal: true
+
+module Git
+  module Pkgs
+    module Commands
+      class Where
+        include Output
+
+        def initialize(args)
+          @args = args
+          @options = parse_options
+        end
+
+        def run
+          name = @args.first
+
+          error "Usage: git pkgs where <package-name>" unless name
+
+          repo = Repository.new
+          require_database(repo)
+
+          Database.connect(repo.git_dir)
+
+          workdir = File.dirname(repo.git_dir)
+          branch = Models::Branch.find_by(name: @options[:branch] || repo.default_branch)
+
+          unless branch
+            error "Branch not found. Run 'git pkgs init' first."
+          end
+
+          snapshots = Models::DependencySnapshot.current_for_branch(branch)
+          snapshots = snapshots.where(ecosystem: @options[:ecosystem]) if @options[:ecosystem]
+
+          manifest_paths = snapshots.for_package(name).joins(:manifest).pluck("manifests.path").uniq
+
+          if manifest_paths.empty?
+            empty_result "Package '#{name}' not found in current dependencies"
+            return
+          end
+
+          results = manifest_paths.flat_map do |path|
+            find_in_manifest(name, File.join(workdir, path), path)
+          end
+
+          if results.empty?
+            empty_result "Package '#{name}' tracked but not found in current files"
+            return
+          end
+
+          if @options[:format] == "json"
+            output_json(results)
+          else
+            paginate { output_text(results, name) }
+          end
+        end
+
+        def find_in_manifest(name, full_path, display_path)
+          return [] unless File.exist?(full_path)
+
+          lines = File.readlines(full_path)
+          matches = []
+
+          lines.each_with_index do |line, idx|
+            next unless line.include?(name)
+
+            match = { path: display_path, line: idx + 1, content: line.rstrip }
+
+            if context_lines > 0
+              match[:before] = context_before(lines, idx)
+              match[:after] = context_after(lines, idx)
+            end
+
+            matches << match
+          end
+
+          matches
+        end
+
+        def context_lines
+          @options[:context] || 0
+        end
+
+        def context_before(lines, idx)
+          start_idx = [0, idx - context_lines].max
+          (start_idx...idx).map { |i| { line: i + 1, content: lines[i].rstrip } }
+        end
+
+        def context_after(lines, idx)
+          end_idx = [lines.length - 1, idx + context_lines].min
+          ((idx + 1)..end_idx).map { |i| { line: i + 1, content: lines[i].rstrip } }
+        end
+
+        def output_text(results, name)
+          results.each_with_index do |result, i|
+            puts "--" if i > 0 && context_lines > 0
+
+            result[:before]&.each do |ctx|
+              puts format_context_line(result[:path], ctx[:line], ctx[:content])
+            end
+
+            puts format_match_line(result[:path], result[:line], result[:content], name)
+
+            result[:after]&.each do |ctx|
+              puts format_context_line(result[:path], ctx[:line], ctx[:content])
+            end
+          end
+        end
+
+        def format_match_line(path, line_num, content, name)
+          path_str = Color.magenta(path)
+          line_str = Color.green(line_num.to_s)
+          highlighted = content.gsub(name, Color.red(name))
+          "#{path_str}:#{line_str}:#{highlighted}"
+        end
+
+        def format_context_line(path, line_num, content)
+          path_str = Color.magenta(path)
+          line_str = Color.green(line_num.to_s)
+          content_str = Color.dim(content)
+          "#{path_str}-#{line_str}-#{content_str}"
+        end
+
+        def output_json(results)
+          require "json"
+          puts JSON.pretty_generate(results)
+        end
+
+        def parse_options
+          options = {}
+
+          parser = OptionParser.new do |opts|
+            opts.banner = "Usage: git pkgs where <package-name> [options]"
+
+            opts.on("-b", "--branch=NAME", "Branch to search (default: current)") do |v|
+              options[:branch] = v
+            end
+
+            opts.on("-e", "--ecosystem=NAME", "Filter by ecosystem") do |v|
+              options[:ecosystem] = v
+            end
+
+            opts.on("-C", "--context=NUM", Integer, "Show NUM lines of context") do |v|
+              options[:context] = v
+            end
+
+            opts.on("-f", "--format=FORMAT", "Output format (text, json)") do |v|
+              options[:format] = v
+            end
+
+            opts.on("--no-pager", "Do not pipe output into a pager") do
+              options[:no_pager] = true
+            end
+
+            opts.on("-h", "--help", "Show this help") do
+              puts opts
+              exit
+            end
+          end
+
+          parser.parse!(@args)
+          options
+        end
+      end
+    end
+  end
+end

data/lib/git/pkgs/config.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require "bibliothecary"
+
+module Git
+  module Pkgs
+    module Config
+      # Ecosystems that require remote parsing services - disabled by default
+      REMOTE_ECOSYSTEMS = %w[carthage clojars hackage hex swiftpm].freeze
+
+      # File patterns ignored by default (SBOM formats not supported)
+      DEFAULT_IGNORED_FILES = %w[
+        cyclonedx.xml
+        cyclonedx.json
+        *.cdx.xml
+        *.cdx.json
+        *.spdx
+        *.spdx.json
+      ].freeze
+
+      def self.ignored_dirs
+        @ignored_dirs ||= read_config_list("pkgs.ignoredDirs")
+      end
+
+      def self.ignored_files
+        @ignored_files ||= read_config_list("pkgs.ignoredFiles")
+      end
+
+      def self.ecosystems
+        @ecosystems ||= read_config_list("pkgs.ecosystems")
+      end
+
+      def self.configure_bibliothecary
+        dirs = ignored_dirs
+        files = DEFAULT_IGNORED_FILES + ignored_files
+
+        Bibliothecary.configure do |config|
+          config.ignored_dirs += dirs unless dirs.empty?
+          config.ignored_files += files
+        end
+      end
+
+      def self.filter_ecosystem?(platform)
+        platform_lower = platform.to_s.downcase
+
+        # Remote ecosystems are disabled unless explicitly enabled
+        if REMOTE_ECOSYSTEMS.include?(platform_lower)
+          return !ecosystems.map(&:downcase).include?(platform_lower)
+        end
+
+        # If no filter configured, allow all non-remote ecosystems
+        return false if ecosystems.empty?
+
+        # Otherwise, only allow explicitly listed ecosystems
+        !ecosystems.map(&:downcase).include?(platform_lower)
+      end
+
+      def self.remote_ecosystem?(platform)
+        REMOTE_ECOSYSTEMS.include?(platform.to_s.downcase)
+      end
+
+      def self.reset!
+        @ignored_dirs = nil
+        @ignored_files = nil
+        @ecosystems = nil
+      end
+
+      def self.read_config_list(key)
+        `git config --get-all #{key} 2>/dev/null`.split("\n").map(&:strip).reject(&:empty?)
+      end
+    end
+  end
+end