gist-compile 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f0404315290e1f5a70692d1451eff74a51fc71e9
+  data.tar.gz: b09dd574f835081e4976d7b022e52b79624c3dd2
+SHA512:
+  metadata.gz: b2f3b8f401acb8ff8726ccaddb1e187c9c0fe6a3298536b764290a5eca1ef4ef3180691a88a82a59f9c1a9c6e40c359d14c5284ec361eaf4ccc3c2fceec88fb5
+  data.tar.gz: 07174faeccf7a99f4ec411536e8705de577e143c9a193d05b5289c91d99c34bc1a6b8d30ee1ec4f9022cd746d00933d50e56c42f760c89cfb5386255316f8a38

data/bin/gist-compile

@@ -0,0 +1,179 @@
+#!/usr/bin/env ruby
+# coding: utf-8
+
+require "uri"
+require "json"
+require "thor"
+require "paint"
+require "rubygems"
+require "nokogiri"
+require "rest-client"
+require_relative "../lib/json-serializer.rb"
+
+GIST_BASE_URL             = "https://gist.github.com"
+GIST_USERCONTENT_BASE_URL = "https://gist.githubusercontent.com"
+GIST_XPATH_IDENTIFIER     = "//*[@id=\"js-flash-container\"]/div/div/div/div/div/span/a[3]"
+
+GIST_METADATA_HOOKS = Hash.new
+GIST_METADATA_HOOKS["Title"] = "Title:"
+GIST_METADATA_HOOKS["Authors"] = "Authors:"
+GIST_METADATA_HOOKS["Description"] = "Description:"
+GIST_METADATA_HOOKS["Section"] = "Section:"
+GIST_METADATA_HOOKS["Subsection"] = "Subsection:"
+
+class GistCompile < Thor
+  no_commands do
+    def load_gists(usernames)
+      thread_arr = []
+      metadata = []
+      elements = []
+      gists_data = Hash.new
+      username_urls = []
+
+      usernames.each do |u|
+        username_urls.push(URI::join(GIST_BASE_URL+'/', u.strip()).to_s)
+      end
+
+      username_urls.each do |username_url|
+        page = 1
+        while page < 1000
+          gist_url_current_page = URI::join(username_url+'/', "?page=#{page}").to_s
+          raw_html = Nokogiri::HTML(RestClient.get(gist_url_current_page))
+          new_elements = raw_html.css(GIST_XPATH_IDENTIFIER)
+          if new_elements.length == 0
+            break
+          else
+            elements.concat(new_elements)
+          end
+          page = page + 1
+        end
+      end
+      
+      puts "Processing #{elements.length} gists..."
+      
+      elements.each do |element|
+        thread_arr.push(
+          Thread.new {
+            individual_gist_url = URI::join(GIST_BASE_URL, element['href']+'/').to_s
+            individual_gist_url_raw = URI::join(GIST_USERCONTENT_BASE_URL, element['href']+'/', 'raw').to_s
+            individual_gist_html = Nokogiri::HTML(RestClient.get(individual_gist_url_raw))
+            metadata_for_gist = process_raw_gist(individual_gist_html.xpath("//*/body/p")[0].text)
+            if metadata_for_gist != nil && !metadata_for_gist.empty?
+              $stdout.write("    • #{individual_gist_url}: " + Paint["[", :yellow] + Paint["OK", :green] + Paint["]",:yellow]+"\r\n")
+              $stdout.flush
+              GIST_METADATA_HOOKS.each do |key, value|
+                if !metadata_for_gist.key? key
+                  metadata_for_gist[key] = nil
+                end
+              end
+              metadata_for_gist["URL"] = individual_gist_url
+              metadata_for_gist["URL_RAW"] = individual_gist_url_raw
+              metadata_for_gist["URL_USER"] = /github.com\/[A-Za-z0-9]*/.match(individual_gist_url)
+              metadata.push(metadata_for_gist)
+            else
+              puts "    • #{individual_gist_url}: " + Paint["[", :yellow] + Paint["ERR", :red] + Paint["]",:yellow]
+            end
+          }
+        )
+      end
+      
+      thread_arr.each{ |t| t.join() }
+
+      File.open("./prod/gists.json","w") do |f|
+        pretty_string = JSON.pretty_generate(metadata.sort_by {|item| item["Title"]})
+        f.write(pretty_string)
+        f.close()
+      end
+    end
+    
+    def process_raw_gist (raw_text)
+      metadata = Hash.new
+      
+      comment_char = nil
+      lines = raw_text.lines()
+      last_known_key = nil
+      
+      hooks_remaining = GIST_METADATA_HOOKS.clone
+      
+      for index in 0..lines.length-1
+        stripped_line = lines[index].strip()
+        if stripped_line != nil && !stripped_line.empty?
+          if stripped_line.start_with?("#!")
+            next
+          end
+          
+          if comment_char == nil
+            index = stripped_line.index(GIST_METADATA_HOOKS["Title"])
+            if index == nil
+              return nil
+            else
+              comment_char = stripped_line[0..index-1].strip()
+            end
+          end
+          if stripped_line.start_with?(comment_char)
+            stripped_line = stripped_line.sub(comment_char,'').strip()
+            if stripped_line.empty?
+              last_known_key = nil
+              break
+            end
+            hook_data = search_line_for_hooks(hooks_remaining, last_known_key, stripped_line)
+            if hook_data == nil
+              break
+            else
+              parsed_key = hook_data[0]
+              parsed_value = hook_data[1]
+              
+              last_known_key = parsed_key
+              if metadata.key? parsed_key # is the key already in the hash?
+                metadata[parsed_key] = metadata[parsed_key] + " #{parsed_value}" # append
+              else
+                metadata[parsed_key] = parsed_value
+              end
+            end
+          else
+            last_known_key = nil
+            break
+          end
+        end
+      end
+      
+      return metadata
+    end
+    
+    def search_line_for_hooks(hooks_remaining, last_known_key, line)
+      hooks_remaining.each do |key, value|
+        if line.include? value
+          hooks_remaining.delete(key)
+          return [key, line.sub(value, '')]
+        end
+      end
+      
+      if last_known_key != nil
+        return [last_known_key, line];
+      end
+      
+      return nil
+    end
+  end
+  
+  desc "start", "Run the gist-compile"
+  method_option :username, :aliases => "-u", :desc => "username(s) to run with, seperated by comma: \"claymcleod, othername\""
+  method_option :linkauthor, :aliases => "--link-author", :desc => "When serializing in HTML, the authors page will be linked", :default => false
+  def start
+    usernames = options[:username]
+    linkauthor = options[:linkauthor]
+
+    if usernames == nil
+      puts "You must specify a username! (eg. -u \"claymcleod\")"
+      return
+    end
+    
+    load_gists(usernames.split(","))
+    serializer = JSONSerializer.new(File.open("./prod/gists.json","r").read, linkauthor)
+    serializer.to_md
+    serializer.to_html
+  end
+end
+  
+GistCompile.start(ARGV)
+

data/lib/json-serializer.rb

@@ -0,0 +1,115 @@
+require 'json'
+
+class JSONSerializer
+  attr_accessor :json
+  attr_accessor :linkauthor
+  
+  def initialize(json, linkauthor)
+    @json = json
+    @linkauthor = linkauthor
+  end
+
+  def compile
+    structured_gists = Hash.new{ |h,k| h[k] = Hash.new(&h.default_proc) }
+
+    entries = JSON.parse(json)
+    entries.each do |arr|
+      title = arr["Title"] != nil ? arr["Title"].strip() : nil
+      authors = arr["Authors"] != nil ? arr["Authors"].strip() : nil
+      description = arr["Description"] != nil ? arr["Description"].strip() : nil
+      section = arr["Section"] != nil ? arr["Section"].strip() : nil
+      subsection = arr["Subsection"] != nil ? arr["Subsection"].strip() : nil
+      url = arr["URL"] != nil ? arr["URL"].strip() : nil
+      url_user = arr["URL_USER"] != nil ? arr["URL_USER"].strip() : nil
+
+      if title == nil
+        puts "Error: A title must be included for #{url}"
+        next
+      end
+      
+      if section == nil || subsection == nil
+        puts "Error: #{title} must have a section and subsection to be compiled!"
+        next
+      end
+
+      structured_gists[section][subsection][title] = {"Authors"=>authors, "Description"=>description,
+                                                      "URL"=>url, "URL_USER"=>url_user}
+    end
+    structured_gists = structured_gists.sort.to_h
+    structured_gists.keys.each do |k|
+      structured_gists[k] = structured_gists[k].sort.to_h
+      structured_gists[k].keys.each do |l|
+        structured_gists[k][l] = structured_gists[k][l].sort.to_h
+      end
+    end
+    
+    return structured_gists
+  end
+
+  def to_md
+    structured_gists = compile()
+    File.open("./prod/gists.md","w") do |f|
+      structured_gists.each do |j, v|
+        f.puts("# #{j}\r")
+        structured_gists[j].each do |k, v|
+          f.puts("### #{k}\r")
+          f.puts("\r\n")
+          structured_gists[j][k].each do |t, v|
+            f.puts("##### #{t}\r")
+            structured_gists[j][k][t].each do |l, v|
+              if v != nil
+                f.puts("* #{l}: #{v}\r")
+              end
+            end
+            f.puts("\r\n")
+          end
+        end
+      end
+      f.close()
+    end
+  end
+
+  def to_html
+    structured_gists = compile()
+    File.open("./prod/gists.html","w") do |f|
+      f.puts("<!DOCTYPE HTML>")
+      f.puts("<html>")
+      f.puts("  <head>\n    <meta charset=\"UTF-8\">\n  </head>");
+      f.puts("  <body>")
+      f.puts("    Generated by <a href=\"https://github.com/claymcleod/gist-compile\">gist-compile</a> at "+Time.new.strftime("%Y-%m-%d %H:%M:%S"))
+      structured_gists.each do |j, v|
+        f.puts("    <div class=\"section\">\r")
+        f.puts("    <a id=\"#{j}\"></a>")
+        f.puts("    <h1>#{j}</h1>\r")
+        f.puts("    </div>")
+        structured_gists[j].each do |k, v|
+          f.puts("      <h2>#{k}</h2>\r")
+          f.puts("\r\n")
+          structured_gists[j][k].each do |t, v|
+            url= structured_gists[j][k][t]["URL"]
+            if url == nil
+              f.puts("        <h3>#{t}</h3>\r")
+            else
+              f.puts("        <h3><a href=\'#{url}\'>#{t}</a></h3>")
+            end
+            f.puts("          <ul>");
+            structured_gists[j][k][t].each do |l, v|
+              if l != "URL" && l != "URL_USER" && v != nil
+                if linkauthor && l == "Authors"
+                  f.puts("          <li>#{l}: <a href=\"http://#{structured_gists[j][k][t]["URL_USER"]}\">#{v}</a></li>\r")
+                else
+                  f.puts("          <li>#{l}: #{v}</li>\r")
+                end
+              end
+            end
+            f.puts("\r\n")
+            f.puts("          </ul>\r")
+          end
+        end
+      end
+      f.puts("    </body>");
+      f.puts("</html>");
+      f.close()
+    end
+  end
+end

metadata

@@ -0,0 +1,125 @@
+--- !ruby/object:Gem::Specification
+name: gist-compile
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Clay McLeod
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-02-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.19.1
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.19.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.19.1
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.19.1
+- !ruby/object:Gem::Dependency
+  name: paint
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.0
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.0
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.6.6
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.6.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.6.6
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.6.6
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.7.2
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.7.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.7.2
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.7.2
+description: ''
+email: clay.l.mcleod@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/gist-compile
+- lib/json-serializer.rb
+homepage: http://github.com/claymcleod/gist-compile
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Gist-compile will compile Github gists into an intelligible, book like, form.
+test_files: []