tc-twitterer 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 6547466a301c154b30cb6ddbd3078659001b0713
+  data.tar.gz: ea5b46ce6cfac39b611823a22d7eeabe50a04518
+SHA512:
+  metadata.gz: 308afacfbf5f1a571690d17f762049b1c9cc5dc53cb6e5a78a9a4508d8d0976315d9578092006b1792d5bea29bb43d8c2eb48c5e2ed7b15d7886b69c734efa8f
+  data.tar.gz: 5c38dd3ea1e701b24d3f2435eece396126a471d25476d97b24b1aff9acc1a4ab7893edce6ea24791589e408d4414ba6a44022d806b820571b55ac776d65bca9c

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/Gemfile

@@ -0,0 +1,5 @@
+#FIXME source 'https://rubygems.org'
+source 'http://rubygems.org'
+
+# Specify your gem's dependencies in tc-twitterer.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 tomonocle
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,135 @@
+# twitterer
+Program for tweeting random lines from files hosted on a public GitHub repo.
+
+## Purpose
+I wanted an easy way to drip-feed my [Miscellany](https://github.com/tomonocle/miscellany/) 
+files into the ether, so wrote `twitterer` to do just that.
+
+Essentially you feed it paths to publicly accessible files on GitHub and it'll
+tweet a random line from them (one tweet per file) with a link to the line on
+GitHub. Optionally it can keep track of previous tweets to avoid sounding 
+repetitive.
+
+Note: source files must be in the default branch, but `twitterer` will resolve
+that branch to a commit before tweeting the link.
+
+## Installation
+```
+  $ gem install tc-twitterer
+```
+
+## Configuration
+`twitterer` requires a configuration file to operate. An example is available on 
+[GitHub](https://github.com/tomonocle/tc-twitterer/blob/master/etc/config.toml.example),
+or you can follow the guide below.
+
+### Twitter Consumer API key and secret
+First you need to register a new application with Twitter.
+
+Visit [https://apps.twitter.com/app/new](https://apps.twitter.com/app/new) and follow the steps.
+
+Once you've created the app, hit _Keys and Access Tokens_ to get your... Keys
+and access tokens.
+
+Config entries:
+
+```
+twitter_consumer_key    = "012a3AbcBdefg4ChiDjklEmnF"
+twitter_consumer_secret = "AaBCDEbFGcdHefIgJKhi0jLk1MNOlm2nPQoRS3pTqU4rsVtWuv"
+```
+
+### Twitter User API key and secret
+On the _Keys and Access Tokens_ page for your app, scroll down to _Your Access 
+Token_ and hit _Create my access token_ to create your access token.
+
+Config entries:
+
+```
+twitter_access_token        = "012345678901234567-abAcBdeCDfEghFi0GHIJKjk9l1Lmnop"
+twitter_access_token_secret = "a0AbcB1C2dDEFeGH7If3ghJK4Li15j678kMNlO9PQRmno"
+```
+
+### History file
+`Twitterer` can track its past tweets if you tell it where to store them, which
+prevents it from repeating itself.
+
+Config entry:
+
+```
+history_file = "/path/to/file"
+```
+
+Note: `twitterer` will create the file, but it won't create any directories.
+
+Note 2: history is stored in a CSV in the following format, which lets it 
+double as a log.
+
+```
+timestamp,source,line
+```
+
+### Sources
+Finally you need to specify the locations of the files you want to tweet from.
+
+The following conditions apply:
+
+- Must be hosted on GitHub
+- Must be publicly accessible
+- Must be in the default branch
+
+Paths are specified in the following format: `username/repo/path/to/file.txt`
+
+Config entry:
+
+```
+source = [
+  "tomonocle/tc-twitterer/README.md",
+  "tomonocle/trello-list2card/README.md",
+]
+```
+
+### Final config file
+
+```
+twitter_consumer_key    = "012a3AbcBdefg4ChiDjklEmnF"
+twitter_consumer_secret = "AaBCDEbFGcdHefIgJKhi0jLk1MNOlm2nPQoRS3pTqU4rsVtWuv"
+
+twitter_access_token        = "012345678901234567-abAcBdeCDfEghFi0GHIJKjk9l1Lmnop"
+twitter_access_token_secret = "a0AbcB1C2dDEFeGH7If3ghJK4Li15j678kMNlO9PQRmno"
+
+history_file = "./var/twitterer.history"
+
+source = [
+  "tomonocle/tc-twitterer/README.md",
+  "tomonocle/trello-list2card/README.md",
+]
+```
+
+## Usage
+
+```
+$ twitterer -c config.toml
+```
+
+This will pull down each file in the _source_ list, pick a suitable line then tweet it in the following format:
+
+```
+<first N characters of line> <url>
+```
+
+Where `N` is defined as the maximum tweet length (which will _forever_ be 140 chars) minus the maximum URL length
+from `t.co` (23 as of December 2017). 140-23 = **117**
+
+Log output goes to `STDERR`.
+
+### Logging
+Log level can be adjusted with `-l [debug,info,warn,error,fatal]` (defaults to **warn**).
+
+### Dry-run
+Dry-run (read-only, don't tweet) can be enabled with `-d`. Note: you'll probably want to increase the log level with `-l` for this to be useful.
+
+### Exit codes
+| Code | Meaning |
+|:----:|---------|
+| 0    | Success (wrote output, or nothing to do) |
+| 1    | Failure. Something went wrong. Always accompanied by a FATAL log message. |

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/bin/twitterer

@@ -0,0 +1,34 @@
+#!/usr/bin/env ruby
+require 'optparse'
+require 'bundler/setup'
+require 'tc/twitterer'
+
+# twitterer -c <config> -l <log-level> -d <dry-run>
+
+options = {}
+OptionParser.new do |opts|
+  opts.banner = 'Usage: twitterer -c <config> [options]'
+
+  opts.on( '-c', '--config <config>', 'Path to config file' ) {
+    |c|
+    options[ :config ] = c
+  }
+
+  opts.on( '-l', '--log-level <level>', 'Specify log level (debug,info,warn,error,fatal)' ) {
+    |l|
+    options[ :log_level ] = l
+  }
+
+  opts.on( '-d', '--dry-run', 'Dry run - don\'t change anything' ) {
+    options[ :dry_run ] = true
+  }
+
+  opts.on_tail( '-h', '--help', 'Display help' ) {
+  puts opts
+  exit
+}
+end.parse!
+
+twitterer = TC::Twitterer.new( options[ :config ], options[ :log_level ], options[ :dry_run ] )
+
+twitterer.run

data/etc/config.toml.example

@@ -0,0 +1,30 @@
+# log_level [OPT]
+# The log level to operate at: debug/info/warn/error/fatal
+# log_level = "warn"
+
+# twitter_consumer_key [REQ]
+# Twitter API key for your application (see README.md)
+twitter_consumer_key = ""
+
+# twitter_consumer_secret [REQ]
+# Twitter API secret for your application (see README.md)
+twitter_consumer_secret = ""
+
+# twitter_access_token [REQ]
+# Twitter API key for your user (see README.md)
+twitter_access_token = ""
+
+# twitter_access_token_secret [REQ]
+# Twitter API secret for your user (see README.md)
+twitter_access_token_secret = ""
+
+# history_file [OPT]
+# path to store previous tweets to avoid being repetitive
+# history_file = "./var/twitterer.history"
+
+# source [REQ]
+# array of GitHub-hosted source files to generate tweets from
+# must be on the default branch
+# e.g
+# source = [ "tomonocle/tc-twitterer/README.md", "tomonocle/trello-list2card/README.md" ]
+source = []

data/lib/tc/twitterer/version.rb

@@ -0,0 +1,5 @@
+module TC
+  class Twitterer
+    VERSION = "0.1.0"
+  end
+end

data/lib/tc/twitterer.rb

@@ -0,0 +1,288 @@
+module TC
+  class Twitterer
+    require 'toml'
+    require 'tc/twitterer/version'
+    require 'net/http'
+    require 'json'
+    require 'twitter'
+    require 'redcarpet'
+    require 'redcarpet/render_strip'
+    require 'logger'
+    require 'ostruct'
+    require 'csv'
+
+    MAX_TWEET_LENGTH   = 140
+    MAX_URL_LENGTH     = 23
+    MAX_STRING_LENGTH  = MAX_TWEET_LENGTH - MAX_URL_LENGTH
+    MAX_HISTORY_LENGTH = MAX_STRING_LENGTH
+
+    LOG_LEVEL_MAP = {
+      'debug' => Logger::DEBUG,
+      'info'  => Logger::INFO,
+      'warn'  => Logger::WARN,
+      'error' => Logger::ERROR,
+      'fatal' => Logger::FATAL,
+    }
+
+    DEFAULT_LOG_LEVEL = 'warn'
+
+    def initialize( config_path, log_level, dry_run )
+      @log = Logger.new( STDERR )
+      self.set_log_level( DEFAULT_LOG_LEVEL )
+
+      @log.info 'Starting up'
+
+      begin
+        fail 'config file not specified' unless config_path
+        fail 'config file not found'     unless File.file?( config_path )
+
+        @log.info "Loading config from #{ config_path }"
+        @config = OpenStruct.new( TOML.load_file( config_path ) )
+        @log.info 'Loaded config'
+
+      rescue => e
+        @log.fatal "Failed to load config: #{e.message}"
+        exit 1
+      end
+
+      self.set_log_level( @config[ 'log_level' ] )
+      self.set_log_level( log_level )
+
+      # very basic sanity check of the config
+      [ 'twitter_consumer_key', 'twitter_consumer_secret', 'twitter_access_token', 'twitter_access_token_secret', 'source' ].each do |key|
+        unless @config[ key ]
+          @log.fatal "Required key '#{ key } not present in config"
+          exit 1
+        end
+      end
+
+      begin
+        @twitter = Twitter::REST::Client.new do |config|
+          config.consumer_key        = @config.twitter_consumer_key
+          config.consumer_secret     = @config.twitter_consumer_secret
+          config.access_token        = @config.twitter_access_token
+          config.access_token_secret = @config.twitter_access_token_secret
+        end
+
+        @log.info "Connected to twitter as '#{@twitter.user.screen_name}'"
+
+      rescue => e
+        @log.fatal "Failed to connect to twitter: #{e.message}"
+        exit 1
+      end
+
+      # open and import the history if configured
+      if @config.history_file
+        @log.info( "Loading history from '#{ @config.history_file }'" )
+        @history = {}
+
+        begin
+          unless File.file?( @config.history_file )
+            @log.info 'History not present - creating'
+            File.write( @config.history_file, nil )
+          end
+
+          CSV.foreach( @config.history_file ) do |csv|
+            timestamp, source, line = csv
+
+            @log.debug( "Processing history: #{source}/#{line}/#{timestamp}")
+
+            # apparently ruby doesn't autovivify? Vive la perl!
+            @history[source]    ||= {}
+            @history[source][line] = timestamp
+          end
+
+        rescue => e
+          @log.fatal( "Failed to import history from '#{ @config.history_file }': #{ e }" )
+          exit 1
+        end
+      end
+
+      if dry_run
+        @log.warn 'Dry run mode: ACTIVATED'
+        @dry_run = true
+      end
+    end
+
+    def set_log_level( level )
+      return unless level
+
+      level.downcase!
+
+      unless LOG_LEVEL_MAP[ level ]
+        @log.fatal "Unrecognised log_level '#{ level }'"
+        exit 1
+      end
+
+      @log.level = LOG_LEVEL_MAP[ level ]
+    end
+
+    def resolve_repo( repo )
+      @log.info "Resolving default->hash for '#{repo}'"
+
+      begin
+        response = Net::HTTP.get_response( URI( "https://api.github.com/repos/#{repo}" ) )
+
+        # fail if not 200 OK
+        response.value
+
+        json = JSON.parse( response.body )
+
+        default_branch = json[ 'default_branch' ]
+        @log.info "Found default branch '#{default_branch}'"
+
+        response = Net::HTTP.get_response( URI( "https://api.github.com/repos/#{repo}/git/refs/heads/#{ default_branch }" ) )
+
+        # this will fail unless we get a 200 OK
+        response.value
+
+        json = JSON.parse( response.body )
+
+      rescue => e
+        @log.error "Failed to resolve '#{repo}' default '#{default_branch}': #{e}"
+        raise e
+      end
+
+      hash = json['object']['sha']
+
+      @log.debug "Resolved #{default_branch}->#{hash} for '#{repo}'"
+
+      hash
+    end
+
+    def fetch_file( repo, hash, path )
+      @log.info "Fetching '#{repo}/#{path}' at '#{hash}'"
+
+      begin
+        response = Net::HTTP.get_response( URI( "https://raw.githubusercontent.com/#{repo}/#{hash}/#{path}" ) )
+
+        # this will fail unless we get a 200 OK
+        response.value
+
+      rescue => e
+        @log.error "Failed to fetch '#{repo}/#{path}' at '#{hash}': #{e}"
+        raise e
+      end
+
+      @log.debug "Fetched '#{repo}/#{path}' at '#{hash}'"
+
+      response.body
+    end
+
+    def pick_line( repo, path, contents )
+      n    = 0
+      pick = ''
+      rows = contents.split( "\n" )
+
+      source = "#{repo}/#{path}"
+
+      @log.info "Picking suitable line from '#{source}'"
+
+      for i in 1..rows.count
+        line_number = rand( rows.count )
+        line        = rows[ line_number ]
+
+        # must contain an alpha - don't bother logging as this is specified behavior
+        next unless line =~ /[a-zA-Z]/
+
+        # mustn't've been used before
+        if @config.history_file
+          key = line[ 0 .. MAX_HISTORY_LENGTH ]
+          if @history.key?( source ) and @history[ source ].key?( key )
+            @log.debug "Skipping '#{line}' because we've used it before (#{ @history[ source ][ key ] })"
+            next
+          end
+        end
+
+        # if we're here, we're good to go
+        pick = line
+        n    = line_number + 1
+
+        break
+      end
+
+      if n == 0
+        fail "Failed to pick an entry from '#{source}' - exhausted content?"
+      end
+
+      @log.debug "Picked '#{pick}' [#{n}] from '#{source}'"
+
+      return pick, n
+    end
+
+    def sanitise( line )
+      rc = Redcarpet::Markdown.new( Redcarpet::Render::StripDown )
+
+      # remove any markdown
+      line = rc.render( line ).strip!
+
+      # compress any whitespace
+      line.gsub!( /\s+/, ' ' )
+
+      # truncate to a sane length, add an elipsis if necessary
+      line = ( line.length > MAX_STRING_LENGTH ? "#{ line[ 0 .. MAX_STRING_LENGTH ] }..." : line )
+
+      # just in case we truncated after a space
+      line.gsub!( /\s\.\.\./, '...' )
+
+      line
+    end
+
+    def tweet( repo, hash, path, line, line_number )
+      link = "https://github.com/#{repo}/blame/#{hash}/#{path}#L#{line_number}"
+
+      tweet = sprintf '%s %s', sanitise( line ), link
+
+      @log.warn sprintf "%sTweeting '%s' [%d] from %s/%s", ( @dry_run == true ? '[DRYRUN] ' : '' ), tweet, tweet.length, repo, path
+      @twitter.update( tweet ) unless @dry_run
+    end
+
+    def update_history( source, line )
+      # limit the length to avoid bloat in the history file
+      line = line[ 0 .. MAX_HISTORY_LENGTH ]
+
+      @log.info sprintf "%sAdding '%s' to history for '%s'", ( @dry_run == true ? '[DRYRUN] ' : '' ), line, source
+      return if @dry_run
+
+      CSV.open( @config.history_file, 'a' ) do |csv|
+        csv << [ Time.now.strftime( '%FT%T%z' ), source, line ]
+      end
+    end
+
+    def run
+      @config.source.each do |source|
+        @log.info "Processing '#{source}'"
+
+        begin
+          if m = source.match( %r{(.*?/.*?)/(.*)} )
+            repo, path = m.captures
+          else
+            fail "Couldn't extract repo and path from '#{source}' - skipping"
+            next
+          end
+
+          # convert default->hash
+          hash = resolve_repo( repo )
+
+          # fetch file
+          file_body = fetch_file( repo, hash, path )
+
+          # extract suitable line
+          line, line_number = pick_line( repo, path, file_body )
+
+          # generate the tweet and send it
+          tweet( repo, hash, path, line, line_number )
+
+          # store in history
+          if @config.history_file
+            update_history( source, line )
+          end
+
+          # all done!
+        rescue => e
+          @log.error "Failed '#{source}': #{e}"
+        end
+      end
+    end
+  end
+end

data/tc-twitterer.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'tc/twitterer/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'tc-twitterer'
+  spec.version       = TC::Twitterer::VERSION
+  spec.authors       = ['tomonocle']
+  spec.email         = ['github@woot.co.uk']
+
+  spec.summary       = 'Program for tweeting random lines from files hosted on a public GitHub repo'
+  spec.description   = 'This program pulls a specified file from a GitHub repo, then tweets a random line with a link to the line on GitHub'
+  spec.homepage      = 'https://github.com/tomonocle/tc-twitterer'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'twitter', '~> 6.2'
+  spec.add_dependency 'toml', '~> 0.2'
+  spec.add_dependency 'redcarpet', '~> 3.4'
+
+  spec.add_development_dependency 'bundler', '~> 1.11'
+  spec.add_development_dependency 'rake', '~> 10.0'
+end

metadata

@@ -0,0 +1,125 @@
+--- !ruby/object:Gem::Specification
+name: tc-twitterer
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- tomonocle
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-12-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: twitter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.2'
+- !ruby/object:Gem::Dependency
+  name: toml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: redcarpet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: This program pulls a specified file from a GitHub repo, then tweets a
+  random line with a link to the line on GitHub
+email:
+- github@woot.co.uk
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/twitterer
+- etc/config.toml.example
+- lib/tc/twitterer.rb
+- lib/tc/twitterer/version.rb
+- tc-twitterer.gemspec
+homepage: https://github.com/tomonocle/tc-twitterer
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Program for tweeting random lines from files hosted on a public GitHub repo
+test_files: []