hub_store 0.9.0 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4f3ebc4985063758ffd4fc0fafb5f3be9145c378d442c4682f976b345defec57
-  data.tar.gz: 8aaedc11dc2e93e3f314555af5f6ab0a41e62ce409bfcfa7bef3f0087f7730f6
+  metadata.gz: f201ad7494f23bab6f9eb1938dad969bc434e0d5de26da14cab920414eb9ddae
+  data.tar.gz: 23ed79127208c4a9db714970302bb9c979cdc225fac2482d8a43e7d10d00c687
 SHA512:
-  metadata.gz: 0cbd7373c45054704a77de3e7b2d78f9f576f5462a901bb11649931887957bed64479290d08613821780f10627cf90cad14f24c3436a534cc084b294dffbe6b3
-  data.tar.gz: 6ad266c08cb76a883f8ac02f472387980cf9923d02504b32f9e5615220f0d7b2e949a29cf86abeb63820e419fb258e41c945c3bf9ec0371b922f08898dcdeb3e
+  metadata.gz: e9c54eb576bc9f1739c3be646e200352e3ba77fb73361d4a2f34196c324863fc2f1709212c178273abf2940e3f1956633204441edf85f46610ebe76864a71a24
+  data.tar.gz: 7e06dd0634abbcf883c8f1ee064e7b219ab9d3b3edf562638f73b9de6e2053015fe3b80a4a509616f4a9a9b47d9c434ba7137f9a989b091c840476c9b2a36392

data/.gitignore

@@ -7,7 +7,8 @@
 /spec/reports/
 /tmp/
 .env
-/db
+hub_store_db
 *.csv
 Gemfile.lock
 *.log
+test/fixtures/vcr_cassettes/

data/README.md

@@ -16,12 +16,6 @@ gem install hub_store
 OCTOKIT_ACCESS_TOKEN=<token> hub_store <github_organization/repo_names>
 ```
 
-## Development
-
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
-
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
-
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/hub_store. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.

data/exe/hub_store

@@ -1,6 +1,5 @@
 #!/usr/bin/env ruby
 
 require "hub_store"
-require "hub_store/cli"
 
 HubStore::Cli.run(ARGV)

data/hub_store.gemspec

@@ -31,7 +31,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "activerecord"
   spec.add_dependency "dotenv"
-  spec.add_dependency "hub_link", ">= 0.8"
+  spec.add_dependency "hub_link", ">= 0.12.0"
   spec.add_dependency "sqlite3"
   spec.add_dependency "tty-spinner"
 end

data/lib/hub_store/cli.rb

@@ -1,7 +1,18 @@
+require "hub_store/exporter"
+
+require "hub_store/storage/database"
+require "hub_store/storage/import"
+
 require "hub_store/ui"
 
 module HubStore
   class Cli
+    RESOURCES = {
+      review_requests: Storage::ReviewRequest,
+      reviews: Storage::Review,
+      pull_requests: Storage::PullRequest
+    }.freeze
+
     def self.run(*args)
       new(*args).run
     end
@@ -11,6 +22,7 @@ module HubStore
     end
 
     def run
+      setup_database
       import_data
       export_csv
     end
@@ -19,18 +31,42 @@ module HubStore
 
       attr_reader :argv
 
+      def setup_database
+        ui.start("Preparing database")
+        Storage::Database.new.setup
+        ui.stop("Done.")
+      end
+
       def import_data
         repos.each do |repo|
-          Importer.new(repo: repo).run
+          Storage::Import.new(repo: repo, resources: RESOURCES, start_date: start_date).run do |on|
+            on.init do |query|
+              ui.log "\n-- #{query} --"
+            end
+
+            on.start do |resource|
+              ui.start("Importing #{resource}")
+            end
+
+            on.finish do |count|
+              ui.stop("Total: #{count}")
+            end
+          end
         end
       end
 
       def export_csv
-        [PullRequest, Review, ReviewRequest].each do |resource|
+        RESOURCES.each do |name, resource|
+          ui.start("Exporting #{name}")
           Exporter.new(resource: resource).run
+          ui.stop("Done.")
         end
       end
 
+      def ui
+        @_ui ||= Ui.new
+      end
+
       def repos
         repo_names.split(",")
       end
@@ -39,6 +75,10 @@ module HubStore
         argv[0].presence || stop
       end
 
+      def start_date
+        ENV["START_DATE"]
+      end
+
       def stop
         puts "\nUsage: OCTOKIT_ACCESS_TOKEN=<token> #{$0} <github_org/repo_name>"
         exit

data/lib/hub_store/exporter.rb

@@ -1,24 +1,18 @@
 require "csv"
-require "hub_store/insert"
-require "hub_store/pull_request"
-require "hub_store/review"
 
 module HubStore
   class Exporter
-    def initialize(resource:, ui: Ui.new)
+    def initialize(resource:)
       @resource = resource
-      @ui = ui
     end
 
     def run
-      ui.start("Exporting #{csv_file_name}")
       export_csv
-      ui.stop("Done.")
     end
 
     private
 
-      attr_reader :resource, :ui
+      attr_reader :resource
 
       def export_csv
         CSV.open(csv_file_name, "w", write_headers: true, headers: columns) do |csv|
@@ -28,10 +22,6 @@ module HubStore
         end
       end
 
-    private
-
-      attr_accessor :records, :columns
-
       def csv_file_name
         resource.to_s.demodulize.pluralize.underscore + ".csv"
       end

data/lib/hub_store/storage/database.rb

@@ -0,0 +1,64 @@
+require "active_record"
+
+module HubStore::Storage
+  class Database
+    def initialize(options = { adapter: "sqlite3", database: "hub_store_db" })
+      @options = options
+    end
+
+    def setup
+      connect
+      create_tables
+    end
+
+    private
+
+      attr_reader :options
+
+      def connect
+        ActiveRecord::Base.establish_connection(options)
+      end
+
+      def create_tables
+        ActiveRecord::Schema.define do
+          unless ActiveRecord::Base.connection.table_exists?("pull_requests")
+            create_table "pull_requests", force: :cascade do |t|
+              t.string "submitter"
+              t.string "number"
+              t.string "labels"
+              t.string "repo"
+              t.integer "approval_time"
+              t.integer "time_to_first_review"
+              t.integer "merge_time"
+              t.integer "additions"
+              t.integer "review_count"
+              t.boolean "straight_approval"
+              t.datetime "closed_at"
+              t.datetime "created_at", null: false
+              t.datetime "updated_at", null: false
+            end
+          end
+
+          unless ActiveRecord::Base.connection.table_exists?("reviews")
+            create_table "reviews", force: :cascade do |t|
+              t.bigint "pull_request_id"
+              t.datetime "submitted_at"
+              t.string "reviewer"
+              t.string "state"
+              t.boolean "approval"
+            end
+          end
+
+          unless ActiveRecord::Base.connection.table_exists?("review_requests")
+            create_table "review_requests", force: :cascade, id: false do |t|
+              t.string "digest", limit: 40, primary: true
+              t.bigint "pull_request_id"
+              t.string "requester"
+              t.string "reviewer"
+              t.datetime "created_at", null: false
+            end
+          end
+        end
+      end
+  end
+end

data/lib/hub_store/storage/import.rb

@@ -0,0 +1,26 @@
+require "hub_link/importer"
+require "hub_store/storage/pull_request"
+require "hub_store/storage/review"
+require "hub_store/storage/review_request"
+
+module HubStore::Storage
+  class Import
+    def initialize(repo:, resources:, start_date: nil)
+      @repo = repo
+      @resources = resources
+      @start_date = start_date
+    end
+
+    def run(&block)
+      HubLink::Importer.run(repo: repo, start_date: start_date, resources: resources, &block)
+    end
+
+    private
+
+      attr_reader :repo, :resources
+
+      def start_date
+        @start_date.presence || PullRequest.for(repo).latest_update
+      end
+  end
+end

data/lib/hub_store/{pull_request.rb → storage/pull_request.rb}

@@ -1,12 +1,16 @@
 require "active_record"
 
-module HubStore
+module HubStore::Storage
   class PullRequest < ActiveRecord::Base
     scope :recently_updated_first, -> { order(updated_at: :desc) }
     validates :repo, presence: true
 
     def self.for(repo)
-      where(repo: repo)
+      where(repo: repo.to_s)
+    end
+
+    def self.latest_update
+      recently_updated_first.first&.updated_at
     end
   end
 end

data/lib/hub_store/{review.rb → storage/review.rb}

@@ -1,6 +1,6 @@
 require "active_record"
 
-module HubStore
+module HubStore::Storage
   class Review < ActiveRecord::Base
   end
 end

data/lib/hub_store/{review_request.rb → storage/review_request.rb}

@@ -1,6 +1,7 @@
 require "active_record"
 
-module HubStore
+module HubStore::Storage
   class ReviewRequest < ActiveRecord::Base
+    self.primary_key = :digest
   end
 end

data/lib/hub_store/version.rb

@@ -1,3 +1,3 @@
 module HubStore
-  VERSION = "0.9.0"
+  VERSION = "0.10.0"
 end

data/lib/hub_store.rb

@@ -1,51 +1,6 @@
 require "dotenv/load"
 require "hub_store/version"
-require "hub_store/importer"
-require "hub_store/exporter"
+require "hub_store/cli"
 
 module HubStore
-  class Error < StandardError; end
-	# Set up database
-	ActiveRecord::Base.establish_connection(
-		adapter: "sqlite3",
-		database: "db/hub_store"
-	)
-
-	# Set up database tables and columns
-	ActiveRecord::Schema.define do
-    unless ActiveRecord::Base.connection.table_exists?("pull_requests")
-      create_table "pull_requests", force: :cascade do |t|
-        t.string "submitter"
-        t.string "number"
-        t.string "labels"
-        t.string "repo"
-        t.integer "approval_time"
-        t.integer "time_to_first_review"
-        t.integer "merge_time"
-        t.integer "additions"
-        t.integer "review_count"
-        t.boolean "straight_approval"
-        t.datetime "closed_at"
-        t.datetime "created_at", null: false
-        t.datetime "updated_at", null: false
-      end
-    end
-
-    unless ActiveRecord::Base.connection.table_exists?("reviews")
-      create_table "reviews", force: :cascade do |t|
-        t.bigint "pull_request_id"
-        t.datetime "submitted_at", null: false
-        t.string "reviewer"
-        t.boolean "approval"
-      end
-    end
-
-    unless ActiveRecord::Base.connection.table_exists?("review_requests")
-      create_table "review_requests", force: :cascade do |t|
-        t.string "requester"
-        t.string "reviewer"
-        t.datetime "created_at", null: false
-      end
-    end
-	end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hub_store
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 0.10.0
 platform: ruby
 authors:
 - Jens Balvig
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-03-18 00:00:00.000000000 Z
+date: 2019-05-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -128,14 +128,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: 0.12.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: 0.12.0
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
@@ -187,11 +187,11 @@ files:
 - lib/hub_store.rb
 - lib/hub_store/cli.rb
 - lib/hub_store/exporter.rb
-- lib/hub_store/importer.rb
-- lib/hub_store/insert.rb
-- lib/hub_store/pull_request.rb
-- lib/hub_store/review.rb
-- lib/hub_store/review_request.rb
+- lib/hub_store/storage/database.rb
+- lib/hub_store/storage/import.rb
+- lib/hub_store/storage/pull_request.rb
+- lib/hub_store/storage/review.rb
+- lib/hub_store/storage/review_request.rb
 - lib/hub_store/ui.rb
 - lib/hub_store/version.rb
 homepage: https://balvig.com

data/lib/hub_store/importer.rb

@@ -1,70 +0,0 @@
-require "hub_link"
-require "hub_store/insert"
-require "hub_store/pull_request"
-require "hub_store/review"
-require "hub_store/review_request"
-
-module HubStore
-  class Importer
-    def initialize(repo:, start_date: nil, ui: Ui.new)
-      @repo = repo
-      @start_date = start_date
-      @ui = ui
-    end
-
-    def run
-      stream.in_batches do |batch|
-        ui.log "\n-- #{batch.query} --"
-        import_prs(batch)
-        import_reviews(batch)
-        import_review_requests(batch)
-      end
-    end
-
-    private
-
-      attr_reader :repo, :start_date, :ui
-
-      def start_date
-        @start_date || latest_local_update
-      end
-
-      def latest_local_update
-        PullRequest.for(repo).recently_updated_first.first&.updated_at
-      end
-
-      def import_prs(batch)
-        ui.start("Importing PRs")
-
-        batch.pull_requests.each do |row|
-          Insert.new(row: row, target: PullRequest).run
-        end
-
-        ui.stop("Total: #{PullRequest.count}")
-      end
-
-      def import_reviews(batch)
-        ui.start("Importing reviews")
-
-        batch.reviews.each do |row|
-          Insert.new(row: row, target: Review).run
-        end
-
-        ui.stop("Total: #{Review.count}")
-      end
-
-      def import_review_requests(batch)
-        ui.start("Importing review requests")
-
-        batch.review_requests.each do |row|
-          Insert.new(row: row, target: ReviewRequest).run
-        end
-
-        ui.stop("Total: #{ReviewRequest.count}")
-      end
-
-      def stream
-        @_stream ||= HubLink::Stream.new(repo, start_date: start_date)
-      end
-  end
-end

data/lib/hub_store/insert.rb

@@ -1,32 +0,0 @@
-module HubStore
-  class Insert
-    def initialize(row:, target:)
-      @row = row
-      @target = target
-    end
-
-    def run
-      record.update! importable_attributes
-    end
-
-    private
-
-      def record
-        target.find_or_initialize_by(id: id)
-      end
-
-      def id
-        row[:id]
-      end
-
-      def importable_attributes
-        row.slice(*target_columns)
-      end
-
-      def target_columns
-        target.columns.map(&:name).map(&:to_sym)
-      end
-
-      attr_reader :row, :target
-  end
-end