github-grep 0.0.0 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c7dbab452ca85ef021446d8318c1293623ebbc17e9f2a6a3fd74b32aa2a5a0da
-  data.tar.gz: e5ccd15431cb9f46700aa7f336f68951e71f5b5269d0f97eb13e4a6904632316
+  metadata.gz: bed5f83e7664a3ce11e21935ebc1be098517cd6ab4aa7614b8c7d42309fb0544
+  data.tar.gz: 7b3923830d8148962d77adf72138cd2576d153fcb928a7837f40d3024d29366c
 SHA512:
-  metadata.gz: 994f71f2cd15ca77611c094fb62300fea5446e21939e83e059ff4d15a5cb59bfaec58ecc39c150686dafed72ea4c70225f858c27cad5f7e66e88f233c4b8e3f6
-  data.tar.gz: 39a96320037e21bcfd77b2399733c4a0f15ce40a7a045cd6c2c7153cced54bff5e4d8d52960a9a8077276fc014cd1275e35bea81579cda0433d5cbdc331a62d6
+  metadata.gz: '091a02da44da6fb6b16f01bf82a57a9e690a091da17c702a4c58e24a33f57428dadf0283fa93e92f370f81e857c85e723dacda1b4890543e5fa84fb34aa27cee'
+  data.tar.gz: 0a02ebf19cc013c6ff40f06f70b802875287f81dd32b65c33fe36b48228d96a120a250f9eaa16c1db52c08f516f3d60d793c9d85960975505197e793ec3f87c3

data/README.md

@@ -1,20 +1,18 @@
 makes github search grep and pipeable
 
- - create a [application token](https://github.com/settings/applications) with read access
- - clone repo
- - follow script instructions
+First create a [application token](https://github.com/settings/applications) with read access + enable SSO if available.
 
 ```
-bundle
+gem install github-grep
 
-export GITHUB_TOKEN=xxx
-# or: git config github.token xxx
+export GITHUB_TOKEN=<your token>
+# or: git config github.token <your token>
 
 # search code:
-bin/github-grep 'user:grosser unicorn' | grep 'narrow-it-down' | grep -v 'something good'
+github-grep 'user:grosser unicorn' | grep 'dictionary' | grep -v 'higher'
 
 # search issues and PR comments:
-bin/github-grep 'repo:kubernetes/kubernetes network error' --issues | grep 'narrow-it-down' | grep -v 'something good'
+github-grep 'repo:kubernetes/kubernetes network error' --issues | grep 'narrow-it-down' | grep -v 'something good'
 ```
 
 NOTE: there are random 403 errors on the last page of a search (usually empty anyway), contacted github support about that :/

data/bin/github-grep

@@ -1,11 +1,9 @@
 #!/usr/bin/env ruby
-require 'cgi'
-require 'json'
-require 'shellwords'
-require 'open3'
+$LOAD_PATH << File.expand_path("../../lib", __FILE__)
+require "github_grep"
 
 def usage
-  puts <<-TEXT.gsub(/^    /, "")
+  puts <<~TEXT
     Setup
     -----
     # create a new token at https://github.com/settings/tokens/new with repo access
@@ -18,74 +16,14 @@ def usage
   exit 1
 end
 
-def code_items_to_lines(items)
-  items.flat_map do |item|
-    file = item.fetch('repository').fetch('name') + ":" + item.fetch('path')
-    lines(item).map { |l| "#{file}: #{l}" }
-  end
-end
-
-def issue_items_to_lines(items)
-  items.flat_map do |item|
-    number = item.fetch("number")
-    lines(item).map { |l| "##{number}: #{l}" }
-  end
-end
-
-def lines(item)
-  item.fetch("text_matches").flat_map { |match| match.fetch('fragment').split("\n") }
-end
-
-def search(q, type)
-  per_page = 100
-  page = 1
-
-  loop do
-    response = page(q, type, page, per_page)
-    if page == 1
-      $stderr.puts "Found #{response.fetch("total_count")}"
-    else
-      $stderr.puts "Page #{page}"
-    end
-
-    items = response.fetch('items')
-    yield items
 
-    break if items.size < per_page
-    page += 1
-  end
-end
-
-def page(q, type, page, per_page)
-  github_token = ENV['GITHUB_TOKEN'] || `git config github.token`.strip # TODO: update docs
-  usage if github_token.empty?
-
-  # remove --fail and add -v to see response headers
-  # NOTE: github returns a 403 with a Retry-After: 60 on page 3+ ... talking with support atm but might have to handle it
-  url = "https://api.github.com/search/#{type}?per_page=#{per_page}&page=#{page}&q=#{CGI.escape(q)}"
-  command = ["curl", "-v", "-f", "-H", "Authorization: token #{github_token}", "-H", "Accept: application/vnd.github.v3.text-match+json", url]
-
-  out, err, status = Open3.capture3(*command)
-  if retry_after = err[/Retry-After: (\d+)/, 1] # 403 Abuse rate limit
-    warn "Sleeping #{retry_after} to avoid abuse rate-limit"
-    sleep Integer(retry_after)
-    out, err, status = Open3.capture3(*command)
-  end
-
-  raise "ERROR Request failed\n#{url}\n#{err}\n#{out}" unless status.success?
-
-  JSON.load(out)
-end
+github_token = ENV['GITHUB_TOKEN'] || `git config github.token`.strip # TODO: update docs
+usage if github_token.empty?
 
 type = (ARGV.delete('--issues') ? :issues : :code)
 
 q = ARGV.shift
 usage if ARGV.size != 0
 
-search(q, type) do |items|
-  if type == :issues
-    puts issue_items_to_lines(items)
-  else
-    puts code_items_to_lines(items)
-  end
-end
+grep = GithubGrep.new(github_token)
+grep.render_search(q, type) { |slice| puts slice }

data/lib/github_grep.rb

@@ -0,0 +1,89 @@
+require 'cgi'
+require 'json'
+require 'shellwords'
+require 'open3'
+
+class GithubGrep
+  VERSION = "0.1.0"
+
+  def initialize(token)
+    @token = token
+  end
+
+  def render_search(q, type)
+    search(q, type) do |items|
+      if type == :issues
+        yield issue_items_to_lines(items)
+      else
+        yield code_items_to_lines(items)
+      end
+    end
+  end
+
+  def search(q, type, &block)
+    headers = ["-H", "Accept: application/vnd.github.v3.text-match+json"]
+    url = "https://api.github.com/search/#{type}?q=#{CGI.escape(q)}"
+    all_pages(url, per_page: 100, argv: headers, &block)
+  end
+
+  private
+
+  def code_items_to_lines(items)
+    items.flat_map do |item|
+      file = item.fetch('repository').fetch('name') + ":" + item.fetch('path')
+      lines(item).map { |l| "#{file}: #{l}" }
+    end
+  end
+
+  def issue_items_to_lines(items)
+    items.flat_map do |item|
+      number = item.fetch("number")
+      lines(item).map { |l| "##{number}: #{l}" }
+    end
+  end
+
+  def lines(item)
+    item.fetch("text_matches").flat_map { |match| match.fetch('fragment').split("\n") }
+  end
+
+  def all_pages(url, per_page:, **kwargs)
+    page = 1
+    connector = (url.include?("?") ? "&" : "?")
+    loop do
+      response = request_json("#{url}#{connector}per_page=#{per_page}&page=#{page}", **kwargs)
+      hash = response.is_a?(Hash)
+      if page == 1 && hash && total = response["total_count"]
+        $stderr.puts "Found #{total}"
+      else
+        $stderr.puts "Page #{page}"
+      end
+
+      items = (hash ? response.fetch('items') : response)
+      yield items
+
+      break if items.size < per_page
+      page += 1
+    end
+  end
+
+  def request_json(url, argv: [])
+    # NOTE: github returns a 403 with a Retry-After: 60 on page 3+ ... talking with support atm but might have to handle it
+    command = ["curl", "-sSfv", "-H", "Authorization: token #{@token}", *argv, url]
+
+    out, err, status = Open3.capture3(*command)
+
+    # 403 Abuse rate limit often has no Retry-After
+    retry_after = err[/Retry-After: (\d+)/, 1]
+    abuse_limit = err.include?("returned error: 403")
+    if retry_after || abuse_limit
+      retry_after ||= "20"
+      warn "Sleeping #{retry_after} to avoid abuse rate-limit"
+      sleep Integer(retry_after)
+      out, err, status = Open3.capture3(*command)
+    end
+
+    raise "ERROR Request failed\n#{url}\n#{err}\n#{out}" unless status.success?
+
+    JSON.parse(out)
+  end
+end

metadata

@@ -1,29 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: github-grep
 version: !ruby/object:Gem::Version
-  version: 0.0.0
+  version: 0.1.0
 platform: ruby
 authors:
 - Michael Grosser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-10-13 00:00:00.000000000 Z
+date: 2021-10-18 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: json
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -62,6 +48,7 @@ files:
 - MIT-LICENSE.txt
 - README.md
 - bin/github-grep
+- lib/github_grep.rb
 homepage: https://github.com/grosser/github-grep
 licenses:
 - MIT
@@ -74,7 +61,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.0.0
+      version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="