git-pkgs 0.4.0 → 0.5.0

data/benchmark_bulk.rb

@@ -1,167 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-require "bundler/setup"
-require "git/pkgs"
-require "benchmark"
-
-repo_path = ARGV[0] || "/Users/andrew/code/octobox"
-sample_size = (ARGV[1] || 500).to_i
-
-# In-memory with WAL mode equivalent (journal_mode=memory for in-memory DB)
-Git::Pkgs::Database.connect_memory
-ActiveRecord::Base.connection.execute("PRAGMA synchronous = OFF")
-ActiveRecord::Base.connection.execute("PRAGMA journal_mode = MEMORY")
-
-repo = Git::Pkgs::Repository.new(repo_path)
-analyzer = Git::Pkgs::Analyzer.new(repo)
-
-walker = repo.walk(repo.default_branch)
-commits = walker.take(sample_size)
-
-puts "Bulk insert benchmark: #{commits.size} commits"
-puts "=" * 60
-
-# Pre-collect all data
-all_commits = []
-all_branch_commits = []
-all_changes = []
-all_snapshots = []
-
-snapshot = {}
-branch = Git::Pkgs::Models::Branch.find_or_create("main")
-position = 0
-manifests_cache = {}
-
-now = Time.now
-
-collect_time = Benchmark.realtime do
-  commits.each do |rugged_commit|
-    next if repo.merge_commit?(rugged_commit)
-    position += 1
-
-    result = analyzer.analyze_commit(rugged_commit, snapshot)
-
-    all_commits << {
-      sha: rugged_commit.oid,
-      message: rugged_commit.message,
-      author_name: rugged_commit.author[:name],
-      author_email: rugged_commit.author[:email],
-      committed_at: rugged_commit.time,
-      has_dependency_changes: result && result[:changes].any?,
-      created_at: now,
-      updated_at: now
-    }
-
-    all_branch_commits << {
-      branch_id: branch.id,
-      commit_position: position,  # placeholder, need to resolve after commit insert
-      commit_sha: rugged_commit.oid
-    }
-
-    next unless result && result[:changes].any?
-
-    result[:changes].each do |change|
-      manifest_key = change[:manifest_path]
-      unless manifests_cache[manifest_key]
-        manifests_cache[manifest_key] = Git::Pkgs::Models::Manifest.find_or_create(
-          path: change[:manifest_path],
-          platform: change[:platform],
-          kind: change[:kind]
-        )
-      end
-
-      all_changes << {
-        commit_sha: rugged_commit.oid,
-        manifest_path: manifest_key,
-        name: change[:name],
-        platform: change[:platform],
-        change_type: change[:change_type],
-        requirement: change[:requirement],
-        previous_requirement: change[:previous_requirement],
-        dependency_type: change[:dependency_type],
-        created_at: now,
-        updated_at: now
-      }
-    end
-
-    snapshot = result[:snapshot]
-
-    snapshot.each do |(manifest_path, name), dep_info|
-      all_snapshots << {
-        commit_sha: rugged_commit.oid,
-        manifest_path: manifest_path,
-        name: name,
-        platform: dep_info[:platform],
-        requirement: dep_info[:requirement],
-        dependency_type: dep_info[:dependency_type],
-        created_at: now,
-        updated_at: now
-      }
-    end
-  end
-end
-
-puts "Collection time: #{collect_time.round(3)}s"
-puts "Data collected:"
-puts "  Commits: #{all_commits.size}"
-puts "  Changes: #{all_changes.size}"
-puts "  Snapshots: #{all_snapshots.size}"
-
-# Bulk insert
-insert_time = Benchmark.realtime do
-  # Insert commits
-  Git::Pkgs::Models::Commit.insert_all(all_commits) if all_commits.any?
-
-  # Build SHA -> ID map
-  commit_ids = Git::Pkgs::Models::Commit.where(sha: all_commits.map { |c| c[:sha] }).pluck(:sha, :id).to_h
-  manifest_ids = Git::Pkgs::Models::Manifest.pluck(:path, :id).to_h
-
-  # Insert branch_commits with resolved IDs
-  branch_commit_records = all_branch_commits.map do |bc|
-    {
-      branch_id: bc[:branch_id],
-      commit_id: commit_ids[bc[:commit_sha]],
-      position: bc[:commit_position]
-    }
-  end
-  Git::Pkgs::Models::BranchCommit.insert_all(branch_commit_records) if branch_commit_records.any?
-
-  # Insert changes with resolved IDs
-  change_records = all_changes.map do |c|
-    {
-      commit_id: commit_ids[c[:commit_sha]],
-      manifest_id: manifest_ids[c[:manifest_path]],
-      name: c[:name],
-      platform: c[:platform],
-      change_type: c[:change_type],
-      requirement: c[:requirement],
-      previous_requirement: c[:previous_requirement],
-      dependency_type: c[:dependency_type],
-      created_at: c[:created_at],
-      updated_at: c[:updated_at]
-    }
-  end
-  Git::Pkgs::Models::DependencyChange.insert_all(change_records) if change_records.any?
-
-  # Insert snapshots with resolved IDs
-  snapshot_records = all_snapshots.map do |s|
-    {
-      commit_id: commit_ids[s[:commit_sha]],
-      manifest_id: manifest_ids[s[:manifest_path]],
-      name: s[:name],
-      platform: s[:platform],
-      requirement: s[:requirement],
-      dependency_type: s[:dependency_type],
-      created_at: s[:created_at],
-      updated_at: s[:updated_at]
-    }
-  end
-  Git::Pkgs::Models::DependencySnapshot.insert_all(snapshot_records) if snapshot_records.any?
-end
-
-puts "Insert time: #{insert_time.round(3)}s"
-
-total = collect_time + insert_time
-puts "\nTotal: #{total.round(3)}s"
-puts "Throughput: #{(all_commits.size / total).round(1)} commits/sec"

data/benchmark_db.rb

@@ -1,138 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-require "bundler/setup"
-require "git/pkgs"
-require "benchmark"
-
-repo_path = ARGV[0] || "/Users/andrew/code/octobox"
-sample_size = (ARGV[1] || 200).to_i
-
-Git::Pkgs::Database.connect_memory
-
-repo = Git::Pkgs::Repository.new(repo_path)
-analyzer = Git::Pkgs::Analyzer.new(repo)
-
-walker = repo.walk(repo.default_branch)
-commits = walker.take(sample_size)
-
-puts "DB operation breakdown: #{commits.size} commits"
-puts "=" * 60
-
-timings = {
-  commit_create: 0.0,
-  branch_commit_create: 0.0,
-  commit_update: 0.0,
-  manifest_find_create: 0.0,
-  change_create: 0.0,
-  snapshot_create: 0.0
-}
-
-counts = {
-  commits: 0,
-  branch_commits: 0,
-  changes: 0,
-  snapshots: 0
-}
-
-snapshot = {}
-branch = Git::Pkgs::Models::Branch.find_or_create("main")
-position = 0
-
-commits.each do |rugged_commit|
-  next if repo.merge_commit?(rugged_commit)
-  position += 1
-
-  result = analyzer.analyze_commit(rugged_commit, snapshot)
-
-  commit = nil
-  timings[:commit_create] += Benchmark.realtime do
-    commit = Git::Pkgs::Models::Commit.find_or_create_from_rugged(rugged_commit)
-  end
-  counts[:commits] += 1
-
-  timings[:branch_commit_create] += Benchmark.realtime do
-    Git::Pkgs::Models::BranchCommit.find_or_create_by(
-      branch: branch,
-      commit: commit,
-      position: position
-    )
-  end
-  counts[:branch_commits] += 1
-
-  next unless result && result[:changes].any?
-
-  timings[:commit_update] += Benchmark.realtime do
-    commit.update(has_dependency_changes: true)
-  end
-
-  result[:changes].each do |change|
-    manifest = nil
-    timings[:manifest_find_create] += Benchmark.realtime do
-      manifest = Git::Pkgs::Models::Manifest.find_or_create(
-        path: change[:manifest_path],
-        platform: change[:platform],
-        kind: change[:kind]
-      )
-    end
-
-    timings[:change_create] += Benchmark.realtime do
-      Git::Pkgs::Models::DependencyChange.create!(
-        commit: commit,
-        manifest: manifest,
-        name: change[:name],
-        platform: change[:platform],
-        change_type: change[:change_type],
-        requirement: change[:requirement],
-        previous_requirement: change[:previous_requirement],
-        dependency_type: change[:dependency_type]
-      )
-    end
-    counts[:changes] += 1
-  end
-
-  snapshot = result[:snapshot]
-
-  snapshot.each do |(manifest_path, name), dep_info|
-    timings[:snapshot_create] += Benchmark.realtime do
-      manifest = Git::Pkgs::Models::Manifest.find_by(path: manifest_path)
-      Git::Pkgs::Models::DependencySnapshot.find_or_create_by(
-        commit: commit,
-        manifest: manifest,
-        name: name
-      ) do |s|
-        s.platform = dep_info[:platform]
-        s.requirement = dep_info[:requirement]
-        s.dependency_type = dep_info[:dependency_type]
-      end
-    end
-    counts[:snapshots] += 1
-  end
-end
-
-total = timings.values.sum
-
-puts "\nDB operation breakdown:"
-puts "-" * 60
-timings.each do |op, time|
-  pct = total > 0 ? (time / total * 100).round(1) : 0
-  puts "  #{op.to_s.ljust(22)} #{time.round(3).to_s.rjust(8)}s  (#{pct}%)"
-end
-puts "-" * 60
-puts "  #{'Total'.ljust(22)} #{total.round(3).to_s.rjust(8)}s"
-
-puts "\nRecord counts:"
-puts "  Commits:        #{counts[:commits]}"
-puts "  BranchCommits:  #{counts[:branch_commits]}"
-puts "  Changes:        #{counts[:changes]}"
-puts "  Snapshots:      #{counts[:snapshots]}"
-
-puts "\nPer-operation averages:"
-puts "  commit_create:        #{(timings[:commit_create] / counts[:commits] * 1000).round(3)}ms"
-puts "  branch_commit_create: #{(timings[:branch_commit_create] / counts[:branch_commits] * 1000).round(3)}ms"
-if counts[:changes] > 0
-  puts "  change_create:        #{(timings[:change_create] / counts[:changes] * 1000).round(3)}ms"
-end
-if counts[:snapshots] > 0
-  puts "  snapshot_create:      #{(timings[:snapshot_create] / counts[:snapshots] * 1000).round(3)}ms"
-end

data/benchmark_detailed.rb

@@ -1,151 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-require "bundler/setup"
-require "git/pkgs"
-require "benchmark"
-
-repo_path = ARGV[0] || "/Users/andrew/code/octobox"
-sample_size = (ARGV[1] || 500).to_i
-
-repo = Git::Pkgs::Repository.new(repo_path)
-analyzer = Git::Pkgs::Analyzer.new(repo)
-
-walker = repo.walk(repo.default_branch)
-commits = walker.take(sample_size)
-
-puts "Benchmarking #{commits.size} commits from #{repo_path}"
-puts "=" * 60
-
-timings = {
-  walk_iteration: 0.0,
-  blob_paths: 0.0,
-  regex_check: 0.0,
-  identify_manifests: 0.0,
-  parse_manifests: 0.0,
-  db_operations: 0.0
-}
-
-counts = {
-  total: 0,
-  merge_commits: 0,
-  regex_passed: 0,
-  identify_passed: 0,
-  has_changes: 0,
-  paths_by_commit: []
-}
-
-platform_times = Hash.new(0.0)
-platform_counts = Hash.new(0)
-
-commits.each do |rugged_commit|
-  counts[:total] += 1
-
-  if repo.merge_commit?(rugged_commit)
-    counts[:merge_commits] += 1
-    next
-  end
-
-  # Phase 1: Extract diff/file paths
-  blob_paths = nil
-  timings[:blob_paths] += Benchmark.realtime do
-    blob_paths = repo.blob_paths(rugged_commit)
-  end
-
-  all_paths = blob_paths.map { |p| p[:path] }
-  counts[:paths_by_commit] << all_paths.size
-
-  # Phase 2: Quick regex check
-  regex_match = nil
-  timings[:regex_check] += Benchmark.realtime do
-    regex_match = analyzer.might_have_manifests?(all_paths)
-  end
-
-  next unless regex_match
-  counts[:regex_passed] += 1
-
-  # Phase 3: Bibliothecary identify_manifests
-  added_paths = blob_paths.select { |p| p[:status] == :added }.map { |p| p[:path] }
-  modified_paths = blob_paths.select { |p| p[:status] == :modified }.map { |p| p[:path] }
-  removed_paths = blob_paths.select { |p| p[:status] == :deleted }.map { |p| p[:path] }
-
-  added_manifests = modified_manifests = removed_manifests = nil
-  timings[:identify_manifests] += Benchmark.realtime do
-    added_manifests = Bibliothecary.identify_manifests(added_paths)
-    modified_manifests = Bibliothecary.identify_manifests(modified_paths)
-    removed_manifests = Bibliothecary.identify_manifests(removed_paths)
-  end
-
-  all_manifests = added_manifests + modified_manifests + removed_manifests
-  next if all_manifests.empty?
-  counts[:identify_passed] += 1
-
-  # Phase 4: Parse manifests (with platform tracking)
-  timings[:parse_manifests] += Benchmark.realtime do
-    all_manifests.each do |manifest_path|
-      start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-
-      blob_oid = repo.blob_oid_at_commit(rugged_commit, manifest_path)
-      if blob_oid
-        content = repo.blob_content(blob_oid)
-        if content
-          result = Bibliothecary.analyse_file(manifest_path, content).first
-          if result
-            platform_counts[result[:platform]] += 1
-            platform_times[result[:platform]] += Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
-          end
-        end
-      end
-    end
-  end
-
-  counts[:has_changes] += 1
-end
-
-total_time = timings.values.sum
-
-puts "\nTiming breakdown:"
-puts "-" * 60
-timings.each do |phase, time|
-  pct = (time / total_time * 100).round(1)
-  puts "  #{phase.to_s.ljust(20)} #{time.round(3).to_s.rjust(8)}s  (#{pct}%)"
-end
-puts "-" * 60
-puts "  #{'Total'.ljust(20)} #{total_time.round(3).to_s.rjust(8)}s"
-
-puts "\nCommit counts:"
-puts "-" * 60
-puts "  Total commits:       #{counts[:total]}"
-puts "  Merge commits:       #{counts[:merge_commits]} (skipped)"
-puts "  Regex passed:        #{counts[:regex_passed]} (#{(counts[:regex_passed].to_f / (counts[:total] - counts[:merge_commits]) * 100).round(1)}%)"
-puts "  Identify passed:     #{counts[:identify_passed]}"
-puts "  Has actual changes:  #{counts[:has_changes]}"
-
-if counts[:paths_by_commit].any?
-  avg_paths = counts[:paths_by_commit].sum.to_f / counts[:paths_by_commit].size
-  max_paths = counts[:paths_by_commit].max
-  puts "\nPaths per commit:"
-  puts "  Average: #{avg_paths.round(1)}"
-  puts "  Max: #{max_paths}"
-end
-
-if platform_times.any?
-  puts "\nTime by platform:"
-  puts "-" * 60
-  platform_times.sort_by { |_, v| -v }.each do |platform, time|
-    count = platform_counts[platform]
-    avg = (time / count * 1000).round(2)
-    puts "  #{platform.ljust(20)} #{time.round(3).to_s.rjust(8)}s  (#{count} files, #{avg}ms avg)"
-  end
-end
-
-puts "\nPer-commit averages:"
-non_merge = counts[:total] - counts[:merge_commits]
-puts "  blob_paths:          #{(timings[:blob_paths] / non_merge * 1000).round(3)}ms"
-puts "  regex_check:         #{(timings[:regex_check] / non_merge * 1000).round(3)}ms"
-if counts[:regex_passed] > 0
-  puts "  identify_manifests:  #{(timings[:identify_manifests] / counts[:regex_passed] * 1000).round(3)}ms (when regex passes)"
-end
-
-commits_per_sec = counts[:total] / total_time
-puts "\nThroughput: #{commits_per_sec.round(1)} commits/sec"

data/benchmark_full.rb

@@ -1,131 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-require "bundler/setup"
-require "git/pkgs"
-require "benchmark"
-
-repo_path = ARGV[0] || "/Users/andrew/code/octobox"
-sample_size = (ARGV[1] || 500).to_i
-
-# Setup in-memory database for fair comparison
-Git::Pkgs::Database.connect_memory
-
-repo = Git::Pkgs::Repository.new(repo_path)
-analyzer = Git::Pkgs::Analyzer.new(repo)
-
-walker = repo.walk(repo.default_branch)
-commits = walker.take(sample_size)
-
-puts "Full pipeline benchmark: #{commits.size} commits"
-puts "=" * 60
-
-timings = {
-  git_diff: 0.0,
-  filtering: 0.0,
-  parsing: 0.0,
-  db_writes: 0.0
-}
-
-snapshot = {}
-branch = Git::Pkgs::Models::Branch.find_or_create("main")
-position = 0
-
-commits.each do |rugged_commit|
-  next if repo.merge_commit?(rugged_commit)
-  position += 1
-
-  # Git diff extraction
-  blob_paths = nil
-  timings[:git_diff] += Benchmark.realtime do
-    blob_paths = repo.blob_paths(rugged_commit)
-  end
-
-  all_paths = blob_paths.map { |p| p[:path] }
-
-  # Filtering (regex + identify_manifests)
-  result = nil
-  timings[:filtering] += Benchmark.realtime do
-    next unless analyzer.might_have_manifests?(all_paths)
-
-    added_paths = blob_paths.select { |p| p[:status] == :added }.map { |p| p[:path] }
-    modified_paths = blob_paths.select { |p| p[:status] == :modified }.map { |p| p[:path] }
-    removed_paths = blob_paths.select { |p| p[:status] == :deleted }.map { |p| p[:path] }
-
-    added_manifests = Bibliothecary.identify_manifests(added_paths)
-    modified_manifests = Bibliothecary.identify_manifests(modified_paths)
-    removed_manifests = Bibliothecary.identify_manifests(removed_paths)
-
-    result = (added_manifests + modified_manifests + removed_manifests).any?
-  end
-
-  # Full analysis with parsing
-  analysis_result = nil
-  if result
-    timings[:parsing] += Benchmark.realtime do
-      analysis_result = analyzer.analyze_commit(rugged_commit, snapshot)
-    end
-  end
-
-  # Database writes
-  timings[:db_writes] += Benchmark.realtime do
-    commit = Git::Pkgs::Models::Commit.find_or_create_from_rugged(rugged_commit)
-    Git::Pkgs::Models::BranchCommit.find_or_create_by(
-      branch: branch,
-      commit: commit,
-      position: position
-    )
-
-    if analysis_result && analysis_result[:changes].any?
-      commit.update(has_dependency_changes: true)
-
-      analysis_result[:changes].each do |change|
-        manifest = Git::Pkgs::Models::Manifest.find_or_create(
-          path: change[:manifest_path],
-          platform: change[:platform],
-          kind: change[:kind]
-        )
-
-        Git::Pkgs::Models::DependencyChange.create!(
-          commit: commit,
-          manifest: manifest,
-          name: change[:name],
-          platform: change[:platform],
-          change_type: change[:change_type],
-          requirement: change[:requirement],
-          previous_requirement: change[:previous_requirement],
-          dependency_type: change[:dependency_type]
-        )
-      end
-
-      snapshot = analysis_result[:snapshot]
-
-      snapshot.each do |(manifest_path, name), dep_info|
-        manifest = Git::Pkgs::Models::Manifest.find_by(path: manifest_path)
-        Git::Pkgs::Models::DependencySnapshot.find_or_create_by(
-          commit: commit,
-          manifest: manifest,
-          name: name
-        ) do |s|
-          s.platform = dep_info[:platform]
-          s.requirement = dep_info[:requirement]
-          s.dependency_type = dep_info[:dependency_type]
-        end
-      end
-    end
-  end
-end
-
-total = timings.values.sum
-
-puts "\nFull pipeline breakdown:"
-puts "-" * 60
-timings.each do |phase, time|
-  pct = total > 0 ? (time / total * 100).round(1) : 0
-  puts "  #{phase.to_s.ljust(15)} #{time.round(3).to_s.rjust(8)}s  (#{pct}%)"
-end
-puts "-" * 60
-puts "  #{'Total'.ljust(15)} #{total.round(3).to_s.rjust(8)}s"
-
-puts "\nThroughput: #{(position / total).round(1)} commits/sec"
-puts "Cache stats: #{analyzer.cache_stats}"