git-pkgs 0.4.0 → 0.5.0

data/lib/git/pkgs/commands/log.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "time"
-
 module Git
   module Pkgs
     module Commands
@@ -20,6 +18,7 @@ module Git
           Database.connect(repo.git_dir)
 
           commits = Models::Commit
+            .includes(:dependency_changes)
             .where(has_dependency_changes: true)
             .order(committed_at: :desc)
 
@@ -44,8 +43,8 @@ module Git
 
         def output_text(commits)
           commits.each do |commit|
-            changes = commit.dependency_changes
-            changes = changes.where(ecosystem: @options[:ecosystem]) if @options[:ecosystem]
+            changes = commit.dependency_changes.to_a
+            changes = changes.select { |c| c.ecosystem == @options[:ecosystem] } if @options[:ecosystem]
             next if changes.empty?
 
             puts "#{commit.short_sha} #{commit.message&.lines&.first&.strip}"
@@ -77,8 +76,8 @@ module Git
           require "json"
 
           data = commits.map do |commit|
-            changes = commit.dependency_changes
-            changes = changes.where(ecosystem: @options[:ecosystem]) if @options[:ecosystem]
+            changes = commit.dependency_changes.to_a
+            changes = changes.select { |c| c.ecosystem == @options[:ecosystem] } if @options[:ecosystem]
 
             {
               sha: commit.sha,
@@ -102,12 +101,6 @@ module Git
           puts JSON.pretty_generate(data)
         end
 
-        def parse_time(str)
-          Time.parse(str)
-        rescue ArgumentError
-          error "Invalid date format: #{str}"
-        end
-
         def parse_options
           options = {}
 

data/lib/git/pkgs/commands/show.rb

@@ -20,10 +20,10 @@ module Git
           Database.connect(repo.git_dir)
 
           sha = repo.rev_parse(ref)
-          error "Could not resolve '#{ref}'" unless sha
+          error "Could not resolve '#{ref}'. Check that the ref exists with 'git rev-parse #{ref}'." unless sha
 
-          commit = find_or_create_commit(repo, sha)
-          error "Commit '#{sha[0..7]}' not found" unless commit
+          commit = Models::Commit.find_or_create_from_repo(repo, sha)
+          error "Commit '#{sha[0..7]}' not in database. Run 'git pkgs update' to index new commits." unless commit
 
           changes = Models::DependencyChange
             .includes(:commit, :manifest)
@@ -107,26 +107,6 @@ module Git
           puts JSON.pretty_generate(data)
         end
 
-        def find_or_create_commit(repo, sha)
-          commit = Models::Commit.find_by(sha: sha) ||
-                   Models::Commit.where("sha LIKE ?", "#{sha}%").first
-          return commit if commit
-
-          rugged_commit = repo.lookup(sha)
-          return nil unless rugged_commit
-
-          Models::Commit.create!(
-            sha: rugged_commit.oid,
-            message: rugged_commit.message,
-            author_name: rugged_commit.author[:name],
-            author_email: rugged_commit.author[:email],
-            committed_at: rugged_commit.time,
-            has_dependency_changes: false
-          )
-        rescue Rugged::OdbError
-          nil
-        end
-
         def parse_options
           options = {}
 

data/lib/git/pkgs/commands/stale.rb

@@ -20,7 +20,7 @@ module Git
           branch_name = @options[:branch] || repo.default_branch
           branch = Models::Branch.find_by(name: branch_name)
 
-          error "No analysis found for branch '#{branch_name}'" unless branch&.last_analyzed_sha
+          error "No analysis found for branch '#{branch_name}'. Run 'git pkgs init' first." unless branch&.last_analyzed_sha
 
           current_commit = Models::Commit.find_by(sha: branch.last_analyzed_sha)
           snapshots = current_commit&.dependency_snapshots&.includes(:manifest) || []
@@ -34,19 +34,38 @@ module Git
             return
           end
 
+          # Batch fetch all changes for current dependencies
+          snapshot_keys = snapshots.map { |s| [s.name, s.manifest_id] }.to_set
+          manifest_ids = snapshots.map(&:manifest_id).uniq
+          names = snapshots.map(&:name).uniq
+
+          all_changes = Models::DependencyChange
+            .includes(:commit)
+            .where(manifest_id: manifest_ids, name: names)
+            .to_a
+
+          # Group by (name, manifest_id) and find latest by committed_at
+          latest_by_key = {}
+          all_changes.each do |change|
+            key = [change.name, change.manifest_id]
+            next unless snapshot_keys.include?(key)
+
+            existing = latest_by_key[key]
+            if existing.nil? || change.commit.committed_at > existing.commit.committed_at
+              latest_by_key[key] = change
+            end
+          end
+
           # Find last update for each dependency
           outdated_data = []
+          now = Time.now
 
           snapshots.each do |snapshot|
-            last_change = Models::DependencyChange
-              .includes(:commit)
-              .where(name: snapshot.name, manifest: snapshot.manifest)
-              .order("commits.committed_at DESC")
-              .first
+            last_change = latest_by_key[[snapshot.name, snapshot.manifest_id]]
 
             next unless last_change
 
-            days_since_update = ((Time.now - last_change.commit.committed_at) / 86400).to_i
+            days_since_update = ((now - last_change.commit.committed_at) / 86400).to_i
 
             outdated_data << {
               name: snapshot.name,

data/lib/git/pkgs/commands/stats.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "time"
-
 module Git
   module Pkgs
     module Commands
@@ -93,11 +91,16 @@ module Git
           manifests = Models::Manifest.all
           manifests = manifests.where(ecosystem: ecosystem) if ecosystem
 
+          manifest_ids = manifests.pluck(:id)
+          change_counts_query = Models::DependencyChange
+            .joins(:commit)
+            .where(manifest_id: manifest_ids)
+          change_counts_query = change_counts_query.where("commits.committed_at >= ?", since_time) if since_time
+          change_counts_query = change_counts_query.where("commits.committed_at <= ?", until_time) if until_time
+          change_counts = change_counts_query.group(:manifest_id).count
+
           data[:manifests] = manifests.map do |manifest|
-            manifest_changes = manifest.dependency_changes.joins(:commit)
-            manifest_changes = manifest_changes.where("commits.committed_at >= ?", since_time) if since_time
-            manifest_changes = manifest_changes.where("commits.committed_at <= ?", until_time) if until_time
-            { path: manifest.path, ecosystem: manifest.ecosystem, changes: manifest_changes.count }
+            { path: manifest.path, ecosystem: manifest.ecosystem, changes: change_counts[manifest.id] || 0 }
           end
 
           data
@@ -199,12 +202,6 @@ module Git
           end
         end
 
-        def parse_time(str)
-          Time.parse(str)
-        rescue ArgumentError
-          error "Invalid date format: #{str}"
-        end
-
         def parse_options
           options = {}
 

data/lib/git/pkgs/commands/tree.rb

@@ -21,7 +21,7 @@ module Git
           commit_sha = @options[:commit] || repo.head_sha
           commit = find_commit_with_snapshot(commit_sha, repo)
 
-          error "No dependency data found for commit #{commit_sha[0, 7]}" unless commit
+          error "No dependency data for commit #{commit_sha[0, 7]}. Run 'git pkgs update' to index new commits." unless commit
 
           # Get current snapshots
           snapshots = commit.dependency_snapshots.includes(:manifest)

data/lib/git/pkgs/commands/update.rb

@@ -26,7 +26,7 @@ module Git
           current_sha = repo.branch_target(branch_name)
 
           if since_sha == current_sha
-            puts "Already up to date."
+            info "Already up to date."
             return
           end
 
@@ -50,17 +50,19 @@ module Git
           walker = repo.walk(branch_name, since_sha)
           commits = walker.to_a
           total = commits.size
+          repo.prefetch_blob_paths(commits)
+
           processed = 0
           dependency_commits = 0
           last_position = Models::BranchCommit.where(branch: branch).maximum(:position) || 0
 
-          puts "Updating branch: #{branch_name}"
-          puts "Found #{total} new commits"
+          info "Updating branch: #{branch_name}"
+          info "Found #{total} new commits"
 
           ActiveRecord::Base.transaction do
             commits.each do |rugged_commit|
               processed += 1
-              print "\rProcessing commit #{processed}/#{total}..."
+              print "\rProcessing commit #{processed}/#{total}..." unless Git::Pkgs.quiet
 
               result = analyzer.analyze_commit(rugged_commit, snapshot)
 
@@ -114,9 +116,9 @@ module Git
             branch.update(last_analyzed_sha: current_sha)
           end
 
-          puts "\nDone!"
-          puts "Processed #{total} new commits"
-          puts "Found #{dependency_commits} commits with dependency changes"
+          info "\nDone!"
+          info "Processed #{total} new commits"
+          info "Found #{dependency_commits} commits with dependency changes"
         end
 
         def parse_options

data/lib/git/pkgs/commands/upgrade.rb

@@ -21,13 +21,13 @@ module Git
           current = Database::SCHEMA_VERSION
 
           if stored >= current
-            puts "Database is up to date (version #{current})"
+            info "Database is up to date (version #{current})"
             return
           end
 
-          puts "Upgrading database from version #{stored} to #{current}..."
-          puts "This requires re-indexing the repository."
-          puts
+          info "Upgrading database from version #{stored} to #{current}..."
+          info "This requires re-indexing the repository."
+          info ""
 
           # Run init --force
           Init.new(["--force"]).run

data/lib/git/pkgs/config.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require "bibliothecary"
+
+module Git
+  module Pkgs
+    module Config
+      # Ecosystems that require remote parsing services - disabled by default
+      REMOTE_ECOSYSTEMS = %w[carthage clojars hackage hex swiftpm].freeze
+
+      # File patterns ignored by default (SBOM formats not supported)
+      DEFAULT_IGNORED_FILES = %w[
+        cyclonedx.xml
+        cyclonedx.json
+        *.cdx.xml
+        *.cdx.json
+        *.spdx
+        *.spdx.json
+      ].freeze
+
+      def self.ignored_dirs
+        @ignored_dirs ||= read_config_list("pkgs.ignoredDirs")
+      end
+
+      def self.ignored_files
+        @ignored_files ||= read_config_list("pkgs.ignoredFiles")
+      end
+
+      def self.ecosystems
+        @ecosystems ||= read_config_list("pkgs.ecosystems")
+      end
+
+      def self.configure_bibliothecary
+        dirs = ignored_dirs
+        files = DEFAULT_IGNORED_FILES + ignored_files
+
+        Bibliothecary.configure do |config|
+          config.ignored_dirs += dirs unless dirs.empty?
+          config.ignored_files += files
+        end
+      end
+
+      def self.filter_ecosystem?(platform)
+        platform_lower = platform.to_s.downcase
+
+        # Remote ecosystems are disabled unless explicitly enabled
+        if REMOTE_ECOSYSTEMS.include?(platform_lower)
+          return !ecosystems.map(&:downcase).include?(platform_lower)
+        end
+
+        # If no filter configured, allow all non-remote ecosystems
+        return false if ecosystems.empty?
+
+        # Otherwise, only allow explicitly listed ecosystems
+        !ecosystems.map(&:downcase).include?(platform_lower)
+      end
+
+      def self.remote_ecosystem?(platform)
+        REMOTE_ECOSYSTEMS.include?(platform.to_s.downcase)
+      end
+
+      def self.reset!
+        @ignored_dirs = nil
+        @ignored_files = nil
+        @ecosystems = nil
+      end
+
+      def self.read_config_list(key)
+        `git config --get-all #{key} 2>/dev/null`.split("\n").map(&:strip).reject(&:empty?)
+      end
+    end
+  end
+end

data/lib/git/pkgs/database.rb

@@ -10,22 +10,21 @@ module Git
       SCHEMA_VERSION = 1
 
       def self.path(git_dir = nil)
-        return ENV["GIT_PKGS_DB"] if ENV["GIT_PKGS_DB"] && !ENV["GIT_PKGS_DB"].empty?
+        return Git::Pkgs.db_path if Git::Pkgs.db_path
 
         git_dir ||= find_git_dir
         File.join(git_dir, DB_FILE)
       end
 
       def self.find_git_dir
-        # Respect GIT_DIR environment variable
-        if ENV["GIT_DIR"] && !ENV["GIT_DIR"].empty?
-          git_dir = ENV["GIT_DIR"]
-          return git_dir if File.directory?(git_dir)
+        if Git::Pkgs.git_dir
+          return Git::Pkgs.git_dir if File.directory?(Git::Pkgs.git_dir)
 
-          raise NotInGitRepoError, "GIT_DIR '#{git_dir}' does not exist"
+          raise NotInGitRepoError, "GIT_DIR '#{Git::Pkgs.git_dir}' does not exist"
         end
 
-        dir = Dir.pwd
+        # Start from work_tree if set, otherwise current directory
+        dir = Git::Pkgs.work_tree || Dir.pwd
         loop do
           git_dir = File.join(dir, ".git")
           return git_dir if File.directory?(git_dir)
@@ -59,6 +58,7 @@ module Git
       end
 
       def self.create_schema(with_indexes: true)
+        ActiveRecord::Schema.verbose = false
         ActiveRecord::Schema.define do
           create_table :schema_info, if_not_exists: true do |t|
             t.integer :version, null: false

data/lib/git/pkgs/models/commit.rb

@@ -20,6 +20,25 @@ module Git
           end
         end
 
+        def self.find_or_create_from_repo(repo, sha)
+          commit = find_by(sha: sha) || where("sha LIKE ?", "#{sha}%").first
+          return commit if commit
+
+          rugged_commit = repo.lookup(sha)
+          return nil unless rugged_commit
+
+          create!(
+            sha: rugged_commit.oid,
+            message: rugged_commit.message,
+            author_name: rugged_commit.author[:name],
+            author_email: rugged_commit.author[:email],
+            committed_at: rugged_commit.time,
+            has_dependency_changes: false
+          )
+        rescue Rugged::OdbError
+          nil
+        end
+
         def short_sha
           sha[0, 7]
         end

data/lib/git/pkgs/output.rb

@@ -1,12 +1,25 @@
 # frozen_string_literal: true
 
 require "json"
+require "time"
 require_relative "pager"
 
 module Git
   module Pkgs
     module Output
       include Pager
+
+      def parse_time(str)
+        Time.parse(str)
+      rescue ArgumentError
+        error "Invalid date format: #{str}. Use YYYY-MM-DD format."
+      end
+
+      # Print informational/status message. Suppressed in quiet mode.
+      def info(msg)
+        puts msg unless Git::Pkgs.quiet
+      end
+
       # Print error message and exit with code 1.
       # Use for user errors (bad input, invalid refs) and system errors (db missing).
       # When format is :json, outputs JSON to stdout; otherwise outputs text to stderr.

data/lib/git/pkgs/repository.rb

@@ -8,8 +8,9 @@ module Git
       attr_reader :path
 
       def initialize(path = nil)
-        @path = path || ENV["GIT_DIR"] || Dir.pwd
+        @path = path || Git::Pkgs.git_dir || Git::Pkgs.work_tree || Dir.pwd
         @rugged = Rugged::Repository.new(@path)
+        @blob_paths_cache = {}
       end
 
       def git_dir
@@ -62,7 +63,40 @@ module Git
         @rugged.lookup(sha)
       end
 
+      DEFAULT_THREADS = 4
+
+      def prefetch_blob_paths(commits)
+        thread_count = Git::Pkgs.threads || DEFAULT_THREADS
+
+        # Serial is faster for small repos due to thread overhead
+        if commits.size < 1500 || thread_count <= 1
+          commits.each { |c| @blob_paths_cache[c.oid] = compute_blob_paths(c) }
+          return
+        end
+
+        queue = Queue.new
+        mutex = Mutex.new
+
+        commits.each { |c| queue << c }
+        thread_count.times { queue << nil }
+
+        thread_pool = thread_count.times.map do
+          Thread.new do
+            while (commit = queue.pop)
+              paths = compute_blob_paths(commit)
+              mutex.synchronize { @blob_paths_cache[commit.oid] = paths }
+            end
+          end
+        end
+
+        thread_pool.each(&:join)
+      end
+
       def blob_paths(rugged_commit)
+        @blob_paths_cache[rugged_commit.oid] || compute_blob_paths(rugged_commit)
+      end
+
+      def compute_blob_paths(rugged_commit)
         paths = []
 
         if rugged_commit.parents.empty?

data/lib/git/pkgs/version.rb

@@ -2,6 +2,6 @@
 
 module Git
   module Pkgs
-    VERSION = "0.4.0"
+    VERSION = "0.5.0"
   end
 end

data/lib/git/pkgs.rb

@@ -3,6 +3,7 @@
 require_relative "pkgs/version"
 require_relative "pkgs/output"
 require_relative "pkgs/color"
+require_relative "pkgs/config"
 require_relative "pkgs/cli"
 require_relative "pkgs/database"
 require_relative "pkgs/repository"
@@ -35,11 +36,44 @@ require_relative "pkgs/commands/log"
 require_relative "pkgs/commands/upgrade"
 require_relative "pkgs/commands/schema"
 require_relative "pkgs/commands/diff_driver"
+require_relative "pkgs/commands/completions"
 
 module Git
   module Pkgs
     class Error < StandardError; end
     class NotInitializedError < Error; end
     class NotInGitRepoError < Error; end
+
+    class << self
+      attr_accessor :quiet, :git_dir, :work_tree, :db_path, :batch_size, :snapshot_interval, :threads
+
+      def configure_from_env
+        @git_dir ||= presence(ENV["GIT_DIR"])
+        @work_tree ||= presence(ENV["GIT_WORK_TREE"])
+        @db_path ||= presence(ENV["GIT_PKGS_DB"])
+        @batch_size ||= int_presence(ENV["GIT_PKGS_BATCH_SIZE"])
+        @snapshot_interval ||= int_presence(ENV["GIT_PKGS_SNAPSHOT_INTERVAL"])
+        @threads ||= int_presence(ENV["GIT_PKGS_THREADS"])
+      end
+
+      def reset_config!
+        @quiet = false
+        @git_dir = nil
+        @work_tree = nil
+        @db_path = nil
+        @batch_size = nil
+        @snapshot_interval = nil
+        @threads = nil
+      end
+
+      def int_presence(value)
+        value && !value.empty? ? value.to_i : nil
+      end
+
+      def presence(value)
+        value && !value.empty? ? value : nil
+      end
+    end
+    self.quiet = false
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: git-pkgs
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Andrew Nesbitt
@@ -75,17 +75,8 @@ extensions: []
 extra_rdoc_files: []
 files:
 - CHANGELOG.md
-- CODE_OF_CONDUCT.md
-- CONTRIBUTING.md
 - LICENSE
 - README.md
-- Rakefile
-- SECURITY.md
-- benchmark_bulk.rb
-- benchmark_db.rb
-- benchmark_detailed.rb
-- benchmark_full.rb
-- docs/schema.md
 - exe/git-pkgs
 - lib/git/pkgs.rb
 - lib/git/pkgs/analyzer.rb
@@ -93,6 +84,7 @@ files:
 - lib/git/pkgs/color.rb
 - lib/git/pkgs/commands/blame.rb
 - lib/git/pkgs/commands/branch.rb
+- lib/git/pkgs/commands/completions.rb
 - lib/git/pkgs/commands/diff.rb
 - lib/git/pkgs/commands/diff_driver.rb
 - lib/git/pkgs/commands/history.rb
@@ -111,6 +103,7 @@ files:
 - lib/git/pkgs/commands/upgrade.rb
 - lib/git/pkgs/commands/where.rb
 - lib/git/pkgs/commands/why.rb
+- lib/git/pkgs/config.rb
 - lib/git/pkgs/database.rb
 - lib/git/pkgs/models/branch.rb
 - lib/git/pkgs/models/branch_commit.rb

data/CODE_OF_CONDUCT.md

@@ -1,10 +0,0 @@
-# Code of Conduct
-
-"git-pkgs" follows [The Ruby Community Conduct Guideline](https://www.ruby-lang.org/en/conduct) in all "collaborative space", which is defined as community communications channels (such as mailing lists, submitted patches, commit comments, etc.):
-
-* Participants will be tolerant of opposing views.
-* Participants must ensure that their language and actions are free of personal attacks and disparaging personal remarks.
-* When interpreting the words and actions of others, participants should always assume good intentions.
-* Behaviour which can be reasonably considered harassment will not be tolerated.
-
-If you have any concerns about behaviour within this project, please contact us at ["andrewnez@gmail.com"](mailto:"andrewnez@gmail.com").

data/CONTRIBUTING.md

@@ -1,27 +0,0 @@
-# Contributing
-
-## Setup
-
-```bash
-git clone https://github.com/andrew/git-pkgs
-cd git-pkgs
-bin/setup
-```
-
-## Running tests
-
-```bash
-bundle exec rake test
-```
-
-## Pull requests
-
-1. Fork the repo
-2. Create a branch
-3. Make your changes
-4. Run tests
-5. Open a PR
-
-## Reporting bugs
-
-Open an issue with steps to reproduce.

data/Rakefile

@@ -1,8 +0,0 @@
-# frozen_string_literal: true
-
-require "bundler/gem_tasks"
-require "minitest/test_task"
-
-Minitest::TestTask.create
-
-task default: :test

data/SECURITY.md

@@ -1,7 +0,0 @@
-# Security Policy
-
-## Reporting a Vulnerability
-
-To report a security vulnerability, please email andrewnez@gmail.com.
-
-Do not open a public issue for security vulnerabilities.