git-pkgs 0.1.0

data/benchmark_detailed.rb

@@ -0,0 +1,151 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "git/pkgs"
+require "benchmark"
+
+repo_path = ARGV[0] || "/Users/andrew/code/octobox"
+sample_size = (ARGV[1] || 500).to_i
+
+repo = Git::Pkgs::Repository.new(repo_path)
+analyzer = Git::Pkgs::Analyzer.new(repo)
+
+walker = repo.walk(repo.default_branch)
+commits = walker.take(sample_size)
+
+puts "Benchmarking #{commits.size} commits from #{repo_path}"
+puts "=" * 60
+
+timings = {
+  walk_iteration: 0.0,
+  blob_paths: 0.0,
+  regex_check: 0.0,
+  identify_manifests: 0.0,
+  parse_manifests: 0.0,
+  db_operations: 0.0
+}
+
+counts = {
+  total: 0,
+  merge_commits: 0,
+  regex_passed: 0,
+  identify_passed: 0,
+  has_changes: 0,
+  paths_by_commit: []
+}
+
+platform_times = Hash.new(0.0)
+platform_counts = Hash.new(0)
+
+commits.each do |rugged_commit|
+  counts[:total] += 1
+
+  if repo.merge_commit?(rugged_commit)
+    counts[:merge_commits] += 1
+    next
+  end
+
+  # Phase 1: Extract diff/file paths
+  blob_paths = nil
+  timings[:blob_paths] += Benchmark.realtime do
+    blob_paths = repo.blob_paths(rugged_commit)
+  end
+
+  all_paths = blob_paths.map { |p| p[:path] }
+  counts[:paths_by_commit] << all_paths.size
+
+  # Phase 2: Quick regex check
+  regex_match = nil
+  timings[:regex_check] += Benchmark.realtime do
+    regex_match = analyzer.might_have_manifests?(all_paths)
+  end
+
+  next unless regex_match
+  counts[:regex_passed] += 1
+
+  # Phase 3: Bibliothecary identify_manifests
+  added_paths = blob_paths.select { |p| p[:status] == :added }.map { |p| p[:path] }
+  modified_paths = blob_paths.select { |p| p[:status] == :modified }.map { |p| p[:path] }
+  removed_paths = blob_paths.select { |p| p[:status] == :deleted }.map { |p| p[:path] }
+
+  added_manifests = modified_manifests = removed_manifests = nil
+  timings[:identify_manifests] += Benchmark.realtime do
+    added_manifests = Bibliothecary.identify_manifests(added_paths)
+    modified_manifests = Bibliothecary.identify_manifests(modified_paths)
+    removed_manifests = Bibliothecary.identify_manifests(removed_paths)
+  end
+
+  all_manifests = added_manifests + modified_manifests + removed_manifests
+  next if all_manifests.empty?
+  counts[:identify_passed] += 1
+
+  # Phase 4: Parse manifests (with platform tracking)
+  timings[:parse_manifests] += Benchmark.realtime do
+    all_manifests.each do |manifest_path|
+      start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+      blob_oid = repo.blob_oid_at_commit(rugged_commit, manifest_path)
+      if blob_oid
+        content = repo.blob_content(blob_oid)
+        if content
+          result = Bibliothecary.analyse_file(manifest_path, content).first
+          if result
+            platform_counts[result[:platform]] += 1
+            platform_times[result[:platform]] += Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
+          end
+        end
+      end
+    end
+  end
+
+  counts[:has_changes] += 1
+end
+
+total_time = timings.values.sum
+
+puts "\nTiming breakdown:"
+puts "-" * 60
+timings.each do |phase, time|
+  pct = (time / total_time * 100).round(1)
+  puts "  #{phase.to_s.ljust(20)} #{time.round(3).to_s.rjust(8)}s  (#{pct}%)"
+end
+puts "-" * 60
+puts "  #{'Total'.ljust(20)} #{total_time.round(3).to_s.rjust(8)}s"
+
+puts "\nCommit counts:"
+puts "-" * 60
+puts "  Total commits:       #{counts[:total]}"
+puts "  Merge commits:       #{counts[:merge_commits]} (skipped)"
+puts "  Regex passed:        #{counts[:regex_passed]} (#{(counts[:regex_passed].to_f / (counts[:total] - counts[:merge_commits]) * 100).round(1)}%)"
+puts "  Identify passed:     #{counts[:identify_passed]}"
+puts "  Has actual changes:  #{counts[:has_changes]}"
+
+if counts[:paths_by_commit].any?
+  avg_paths = counts[:paths_by_commit].sum.to_f / counts[:paths_by_commit].size
+  max_paths = counts[:paths_by_commit].max
+  puts "\nPaths per commit:"
+  puts "  Average: #{avg_paths.round(1)}"
+  puts "  Max: #{max_paths}"
+end
+
+if platform_times.any?
+  puts "\nTime by platform:"
+  puts "-" * 60
+  platform_times.sort_by { |_, v| -v }.each do |platform, time|
+    count = platform_counts[platform]
+    avg = (time / count * 1000).round(2)
+    puts "  #{platform.ljust(20)} #{time.round(3).to_s.rjust(8)}s  (#{count} files, #{avg}ms avg)"
+  end
+end
+
+puts "\nPer-commit averages:"
+non_merge = counts[:total] - counts[:merge_commits]
+puts "  blob_paths:          #{(timings[:blob_paths] / non_merge * 1000).round(3)}ms"
+puts "  regex_check:         #{(timings[:regex_check] / non_merge * 1000).round(3)}ms"
+if counts[:regex_passed] > 0
+  puts "  identify_manifests:  #{(timings[:identify_manifests] / counts[:regex_passed] * 1000).round(3)}ms (when regex passes)"
+end
+
+commits_per_sec = counts[:total] / total_time
+puts "\nThroughput: #{commits_per_sec.round(1)} commits/sec"

data/benchmark_full.rb

@@ -0,0 +1,131 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "git/pkgs"
+require "benchmark"
+
+repo_path = ARGV[0] || "/Users/andrew/code/octobox"
+sample_size = (ARGV[1] || 500).to_i
+
+# Setup in-memory database for fair comparison
+Git::Pkgs::Database.connect_memory
+
+repo = Git::Pkgs::Repository.new(repo_path)
+analyzer = Git::Pkgs::Analyzer.new(repo)
+
+walker = repo.walk(repo.default_branch)
+commits = walker.take(sample_size)
+
+puts "Full pipeline benchmark: #{commits.size} commits"
+puts "=" * 60
+
+timings = {
+  git_diff: 0.0,
+  filtering: 0.0,
+  parsing: 0.0,
+  db_writes: 0.0
+}
+
+snapshot = {}
+branch = Git::Pkgs::Models::Branch.find_or_create("main")
+position = 0
+
+commits.each do |rugged_commit|
+  next if repo.merge_commit?(rugged_commit)
+  position += 1
+
+  # Git diff extraction
+  blob_paths = nil
+  timings[:git_diff] += Benchmark.realtime do
+    blob_paths = repo.blob_paths(rugged_commit)
+  end
+
+  all_paths = blob_paths.map { |p| p[:path] }
+
+  # Filtering (regex + identify_manifests)
+  result = nil
+  timings[:filtering] += Benchmark.realtime do
+    next unless analyzer.might_have_manifests?(all_paths)
+
+    added_paths = blob_paths.select { |p| p[:status] == :added }.map { |p| p[:path] }
+    modified_paths = blob_paths.select { |p| p[:status] == :modified }.map { |p| p[:path] }
+    removed_paths = blob_paths.select { |p| p[:status] == :deleted }.map { |p| p[:path] }
+
+    added_manifests = Bibliothecary.identify_manifests(added_paths)
+    modified_manifests = Bibliothecary.identify_manifests(modified_paths)
+    removed_manifests = Bibliothecary.identify_manifests(removed_paths)
+
+    result = (added_manifests + modified_manifests + removed_manifests).any?
+  end
+
+  # Full analysis with parsing
+  analysis_result = nil
+  if result
+    timings[:parsing] += Benchmark.realtime do
+      analysis_result = analyzer.analyze_commit(rugged_commit, snapshot)
+    end
+  end
+
+  # Database writes
+  timings[:db_writes] += Benchmark.realtime do
+    commit = Git::Pkgs::Models::Commit.find_or_create_from_rugged(rugged_commit)
+    Git::Pkgs::Models::BranchCommit.find_or_create_by(
+      branch: branch,
+      commit: commit,
+      position: position
+    )
+
+    if analysis_result && analysis_result[:changes].any?
+      commit.update(has_dependency_changes: true)
+
+      analysis_result[:changes].each do |change|
+        manifest = Git::Pkgs::Models::Manifest.find_or_create(
+          path: change[:manifest_path],
+          platform: change[:platform],
+          kind: change[:kind]
+        )
+
+        Git::Pkgs::Models::DependencyChange.create!(
+          commit: commit,
+          manifest: manifest,
+          name: change[:name],
+          platform: change[:platform],
+          change_type: change[:change_type],
+          requirement: change[:requirement],
+          previous_requirement: change[:previous_requirement],
+          dependency_type: change[:dependency_type]
+        )
+      end
+
+      snapshot = analysis_result[:snapshot]
+
+      snapshot.each do |(manifest_path, name), dep_info|
+        manifest = Git::Pkgs::Models::Manifest.find_by(path: manifest_path)
+        Git::Pkgs::Models::DependencySnapshot.find_or_create_by(
+          commit: commit,
+          manifest: manifest,
+          name: name
+        ) do |s|
+          s.platform = dep_info[:platform]
+          s.requirement = dep_info[:requirement]
+          s.dependency_type = dep_info[:dependency_type]
+        end
+      end
+    end
+  end
+end
+
+total = timings.values.sum
+
+puts "\nFull pipeline breakdown:"
+puts "-" * 60
+timings.each do |phase, time|
+  pct = total > 0 ? (time / total * 100).round(1) : 0
+  puts "  #{phase.to_s.ljust(15)} #{time.round(3).to_s.rjust(8)}s  (#{pct}%)"
+end
+puts "-" * 60
+puts "  #{'Total'.ljust(15)} #{total.round(3).to_s.rjust(8)}s"
+
+puts "\nThroughput: #{(position / total).round(1)} commits/sec"
+puts "Cache stats: #{analyzer.cache_stats}"

data/docs/schema.md

@@ -0,0 +1,129 @@
+# Database Schema
+
+git-pkgs stores dependency history in a SQLite database at `.git/pkgs.sqlite3`.
+
+## Tables
+
+### branches
+
+Tracks which branches have been analyzed.
+
+| Column | Type | Description |
+|--------|------|-------------|
+| id | integer | Primary key |
+| name | string | Branch name (e.g., "main", "develop") |
+| last_analyzed_sha | string | SHA of last commit analyzed for incremental updates |
+| created_at | datetime | |
+| updated_at | datetime | |
+
+Indexes: `name` (unique)
+
+### commits
+
+Stores commit metadata for commits that have been analyzed.
+
+| Column | Type | Description |
+|--------|------|-------------|
+| id | integer | Primary key |
+| sha | string | Full commit SHA |
+| message | text | Commit message |
+| author_name | string | Author name |
+| author_email | string | Author email |
+| committed_at | datetime | Commit timestamp |
+| has_dependency_changes | boolean | True if this commit modified dependencies |
+| created_at | datetime | |
+| updated_at | datetime | |
+
+Indexes: `sha` (unique)
+
+### branch_commits
+
+Join table linking commits to branches. A commit can belong to multiple branches.
+
+| Column | Type | Description |
+|--------|------|-------------|
+| id | integer | Primary key |
+| branch_id | integer | Foreign key to branches |
+| commit_id | integer | Foreign key to commits |
+| position | integer | Order of commit in branch history |
+
+Indexes: `(branch_id, commit_id)` (unique)
+
+### manifests
+
+Stores manifest file metadata.
+
+| Column | Type | Description |
+|--------|------|-------------|
+| id | integer | Primary key |
+| path | string | File path (e.g., "Gemfile", "package.json") |
+| platform | string | Package manager (e.g., "rubygems", "npm") |
+| kind | string | Manifest type (e.g., "manifest", "lockfile") |
+| created_at | datetime | |
+| updated_at | datetime | |
+
+Indexes: `path`
+
+### dependency_changes
+
+Records each dependency addition, modification, or removal.
+
+| Column | Type | Description |
+|--------|------|-------------|
+| id | integer | Primary key |
+| commit_id | integer | Foreign key to commits |
+| manifest_id | integer | Foreign key to manifests |
+| name | string | Package name |
+| platform | string | Package manager |
+| change_type | string | "added", "modified", or "removed" |
+| requirement | string | Version constraint after change |
+| previous_requirement | string | Version constraint before change (for modifications) |
+| dependency_type | string | "runtime", "development", etc. |
+| created_at | datetime | |
+| updated_at | datetime | |
+
+Indexes: `name`, `platform`, `(commit_id, name)`
+
+### dependency_snapshots
+
+Stores the complete dependency state at each commit that has changes. Enables O(1) queries for "what dependencies existed at commit X" without replaying history.
+
+| Column | Type | Description |
+|--------|------|-------------|
+| id | integer | Primary key |
+| commit_id | integer | Foreign key to commits |
+| manifest_id | integer | Foreign key to manifests |
+| name | string | Package name |
+| platform | string | Package manager |
+| requirement | string | Version constraint |
+| dependency_type | string | "runtime", "development", etc. |
+| created_at | datetime | |
+| updated_at | datetime | |
+
+Indexes: `(commit_id, manifest_id, name)` (unique), `name`, `platform`
+
+## Relationships
+
+```
+branches ──┬── branch_commits ──┬── commits
+           │                    │
+           │                    ├── dependency_changes ──── manifests
+           │                    │
+           │                    └── dependency_snapshots ── manifests
+           │
+           └── last_analyzed_sha (references commits.sha)
+```
+
+## Design Notes
+
+**Why snapshots?**
+
+Without snapshots, answering "what dependencies existed at commit X" requires replaying all changes from the beginning. With snapshots, it's a single query. The tradeoff is storage space, but SQLite handles this well.
+
+**Why branch_commits?**
+
+Git commits are branch-agnostic. The same commit can appear on multiple branches. This join table tracks which commits belong to which branches and their order, enabling branch-specific queries.
+
+**Platform field duplication**
+
+The platform appears in both `manifests` and `dependency_changes`/`dependency_snapshots`. This denormalization speeds up queries that filter by platform without requiring joins.

data/exe/git-pkgs

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "git/pkgs"
+
+Git::Pkgs::CLI.run(ARGV)

data/lib/git/pkgs/analyzer.rb

@@ -0,0 +1,270 @@
+# frozen_string_literal: true
+
+require "bibliothecary"
+
+module Git
+  module Pkgs
+    class Analyzer
+      attr_reader :repository
+
+      # Common manifest file patterns for quick pre-filtering
+      # This avoids calling Bibliothecary.identify_manifests for commits that clearly don't touch manifests
+      QUICK_MANIFEST_PATTERNS = %w[
+        Gemfile Gemfile.lock gems.rb gems.locked *.gemspec
+        package.json package-lock.json yarn.lock npm-shrinkwrap.json pnpm-lock.yaml bun.lock npm-ls.json
+        setup.py req*.txt req*.pip requirements/*.txt requirements/*.pip requirements.frozen
+        Pipfile Pipfile.lock pyproject.toml poetry.lock uv.lock pylock.toml
+        pip-resolved-dependencies.txt pip-dependency-graph.json
+        pom.xml ivy.xml build.gradle build.gradle.kts gradle-dependencies-q.txt
+        maven-resolved-dependencies.txt sbt-update-full.txt maven-dependency-tree.txt maven-dependency-tree.dot
+        Cargo.toml Cargo.lock
+        go.mod go.sum glide.yaml glide.lock Godeps Godeps/Godeps.json
+        vendor/manifest vendor/vendor.json Gopkg.toml Gopkg.lock go-resolved-dependencies.json
+        composer.json composer.lock
+        Podfile Podfile.lock *.podspec *.podspec.json
+        packages.config packages.lock.json Project.json Project.lock.json
+        *.nuspec paket.lock *.csproj project.assets.json
+        cyclonedx.xml cyclonedx.json *.cdx.xml *.cdx.json
+        *.spdx *.spdx.json
+        bower.json bentofile.yaml
+        META.json META.yml
+        environment.yml environment.yaml
+        cog.yaml versions.json MLmodel DESCRIPTION
+        pubspec.yaml pubspec.lock
+        dub.json dub.sdl
+        REQUIRE
+        shard.yml shard.lock
+        elm-package.json elm_dependencies.json elm-stuff/exact-dependencies.json
+        haxelib.json
+        action.yml action.yaml .github/workflows/*.yml .github/workflows/*.yaml
+        Dockerfile docker-compose*.yml docker-compose*.yaml
+        dvc.yaml vcpkg.json
+        Brewfile Brewfile.lock.json
+        Modelfile
+      ].freeze
+
+      QUICK_MANIFEST_REGEX = Regexp.union(
+        QUICK_MANIFEST_PATTERNS.map do |pattern|
+          if pattern.include?('*')
+            Regexp.new(pattern.gsub('.', '\\.').gsub('*', '.*'))
+          else
+            /(?:^|\/)#{Regexp.escape(pattern)}$/
+          end
+        end
+      ).freeze
+
+      def initialize(repository)
+        @repository = repository
+        @blob_cache = {}
+      end
+
+      # Quick check if any paths might be manifests (fast regex check)
+      def might_have_manifests?(paths)
+        paths.any? { |p| p.match?(QUICK_MANIFEST_REGEX) }
+      end
+
+      # Quick check if a commit touches any manifest files
+      def has_manifest_changes?(rugged_commit)
+        return false if repository.merge_commit?(rugged_commit)
+
+        blob_paths = repository.blob_paths(rugged_commit)
+        all_paths = blob_paths.map { |p| p[:path] }
+
+        return false unless might_have_manifests?(all_paths)
+
+        Bibliothecary.identify_manifests(all_paths).any?
+      end
+
+      def analyze_commit(rugged_commit, previous_snapshot = {})
+        return nil if repository.merge_commit?(rugged_commit)
+
+        blob_paths = repository.blob_paths(rugged_commit)
+
+        added_paths = blob_paths.select { |p| p[:status] == :added }.map { |p| p[:path] }
+        modified_paths = blob_paths.select { |p| p[:status] == :modified }.map { |p| p[:path] }
+        removed_paths = blob_paths.select { |p| p[:status] == :deleted }.map { |p| p[:path] }
+
+        all_paths = added_paths + modified_paths + removed_paths
+        return nil unless might_have_manifests?(all_paths)
+
+        added_manifests = Bibliothecary.identify_manifests(added_paths)
+        modified_manifests = Bibliothecary.identify_manifests(modified_paths)
+        removed_manifests = Bibliothecary.identify_manifests(removed_paths)
+
+        return nil if added_manifests.empty? && modified_manifests.empty? && removed_manifests.empty?
+
+        changes = []
+        new_snapshot = previous_snapshot.dup
+
+        # Process added manifest files
+        added_manifests.each do |manifest_path|
+          result = parse_manifest_at_commit(rugged_commit, manifest_path)
+          next unless result
+
+          result[:dependencies].each do |dep|
+            changes << {
+              manifest_path: manifest_path,
+              ecosystem: result[:platform],
+              kind: result[:kind],
+              name: dep[:name],
+              change_type: "added",
+              requirement: dep[:requirement],
+              dependency_type: dep[:type]
+            }
+
+            key = [manifest_path, dep[:name]]
+            new_snapshot[key] = {
+              ecosystem: result[:platform],
+              kind: result[:kind],
+              requirement: dep[:requirement],
+              dependency_type: dep[:type]
+            }
+          end
+        end
+
+        # Process modified manifest files
+        modified_manifests.each do |manifest_path|
+          before_result = parse_manifest_before_commit(rugged_commit, manifest_path)
+          after_result = parse_manifest_at_commit(rugged_commit, manifest_path)
+
+          next unless after_result
+
+          before_deps = (before_result&.dig(:dependencies) || []).map { |d| [d[:name], d] }.to_h
+          after_deps = (after_result[:dependencies] || []).map { |d| [d[:name], d] }.to_h
+
+          added_names = after_deps.keys - before_deps.keys
+          removed_names = before_deps.keys - after_deps.keys
+          common_names = after_deps.keys & before_deps.keys
+
+          added_names.each do |name|
+            dep = after_deps[name]
+            changes << {
+              manifest_path: manifest_path,
+              ecosystem: after_result[:platform],
+              kind: after_result[:kind],
+              name: name,
+              change_type: "added",
+              requirement: dep[:requirement],
+              dependency_type: dep[:type]
+            }
+
+            key = [manifest_path, name]
+            new_snapshot[key] = {
+              ecosystem: after_result[:platform],
+              kind: after_result[:kind],
+              requirement: dep[:requirement],
+              dependency_type: dep[:type]
+            }
+          end
+
+          removed_names.each do |name|
+            dep = before_deps[name]
+            changes << {
+              manifest_path: manifest_path,
+              ecosystem: before_result[:platform],
+              kind: before_result[:kind],
+              name: name,
+              change_type: "removed",
+              requirement: dep[:requirement],
+              dependency_type: dep[:type]
+            }
+
+            key = [manifest_path, name]
+            new_snapshot.delete(key)
+          end
+
+          common_names.each do |name|
+            before_dep = before_deps[name]
+            after_dep = after_deps[name]
+
+            if before_dep[:requirement] != after_dep[:requirement] || before_dep[:type] != after_dep[:type]
+              changes << {
+                manifest_path: manifest_path,
+                ecosystem: after_result[:platform],
+                kind: after_result[:kind],
+                name: name,
+                change_type: "modified",
+                requirement: after_dep[:requirement],
+                previous_requirement: before_dep[:requirement],
+                dependency_type: after_dep[:type]
+              }
+
+              key = [manifest_path, name]
+              new_snapshot[key] = {
+                ecosystem: after_result[:platform],
+                kind: after_result[:kind],
+                requirement: after_dep[:requirement],
+                dependency_type: after_dep[:type]
+              }
+            end
+          end
+        end
+
+        # Process removed manifest files
+        removed_manifests.each do |manifest_path|
+          result = parse_manifest_before_commit(rugged_commit, manifest_path)
+          next unless result
+
+          result[:dependencies].each do |dep|
+            changes << {
+              manifest_path: manifest_path,
+              ecosystem: result[:platform],
+              kind: result[:kind],
+              name: dep[:name],
+              change_type: "removed",
+              requirement: dep[:requirement],
+              dependency_type: dep[:type]
+            }
+
+            key = [manifest_path, dep[:name]]
+            new_snapshot.delete(key)
+          end
+        end
+
+        {
+          changes: changes,
+          snapshot: new_snapshot
+        }
+      end
+
+      # Cache stats for debugging
+      def cache_stats
+        hits = @blob_cache.values.count { |v| v[:hits] > 0 }
+        total = @blob_cache.size
+        { cached_blobs: total, blobs_with_hits: hits }
+      end
+
+      def parse_manifest_at_commit(rugged_commit, manifest_path)
+        blob_oid = repository.blob_oid_at_commit(rugged_commit, manifest_path)
+        return nil unless blob_oid
+
+        parse_manifest_by_oid(blob_oid, manifest_path)
+      end
+
+      def parse_manifest_before_commit(rugged_commit, manifest_path)
+        return nil if rugged_commit.parents.empty?
+
+        blob_oid = repository.blob_oid_at_commit(rugged_commit.parents[0], manifest_path)
+        return nil unless blob_oid
+
+        parse_manifest_by_oid(blob_oid, manifest_path)
+      end
+
+      def parse_manifest_by_oid(blob_oid, manifest_path)
+        cache_key = "#{blob_oid}:#{manifest_path}"
+
+        if @blob_cache.key?(cache_key)
+          @blob_cache[cache_key][:hits] += 1
+          return @blob_cache[cache_key][:result]
+        end
+
+        content = repository.blob_content(blob_oid)
+        return nil unless content
+
+        result = Bibliothecary.analyse_file(manifest_path, content).first
+        @blob_cache[cache_key] = { result: result, hits: 0 }
+        result
+      end
+    end
+  end
+end