nntp_scrape 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in nntp_scrape.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Scott Fleckenstein
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,56 @@
+# NntpScrape
+
+Various usenet related utilities
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'nntp_scrape'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install nntp_scrape
+
+## Usage
+
+First, write a config file at ~/.nntp_scrape
+
+```yaml
+host: unlimited.newshosting.com
+port: 563 #default 
+ssl: true #default
+user: foobar
+pass: bazbak
+
+```
+
+Then you can:
+
+```bash
+#/usr/bin/env bash
+# watches the provided group, outputting a line of json everytime a new article is posted
+nntp_scrape headers alt.binaries.teevee
+
+# starts a repl to interact with the usenet server directly
+nntp_scrape repl
+>> run Capabilities
+=> #<NntpScrape::Commands::Capabilities:0x007ffc56914e20
+ @caps=
+  ["VERSION 1",
+   "MODE-READER",
+   "READER",
+   ...
+```
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/nntp_scrape

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+# 
+require 'nntp_scrape'
+
+NntpScrape::Cli.start(ARGV)

data/lib/nntp_scrape.rb

@@ -0,0 +1,24 @@
+require "nntp_scrape/version"
+require 'active_support/core_ext/string'
+
+module NntpScrape
+  autoload :Cli,      "nntp_scrape/cli"
+  autoload :Config,   "nntp_scrape/config"
+  autoload :NNTP,     "nntp_scrape/nntp"
+  autoload :Repl,     "nntp_scrape/repl"
+  
+  module Commands
+    autoload :Base,         "nntp_scrape/commands/base"
+    autoload :AuthInfo,     "nntp_scrape/commands/auth_info"
+    autoload :Group,        "nntp_scrape/commands/group"
+    autoload :Next,         "nntp_scrape/commands/next"
+    autoload :Head,         "nntp_scrape/commands/head"
+    autoload :Xhdr,         "nntp_scrape/commands/xhdr"
+    autoload :Xover,        "nntp_scrape/commands/xover"
+    autoload :Capabilities, "nntp_scrape/commands/capabilities"
+    
+  end
+  
+ 
+  
+end

data/lib/nntp_scrape/cli.rb

@@ -0,0 +1,47 @@
+require 'thor'
+require 'active_support'
+
+module NntpScrape
+  
+  class Cli < Thor
+    class_option :config, 
+      :desc => "Configuration file location",
+      :type => :string, 
+      :default => ENV["HOME"] + "/.nntp_scrape"
+    
+    desc "headers GROUP", "watches GROUP, outputting lines of json as new headers are found"
+    def headers(group)
+      setup
+      
+      @client.watch_new(group) do |result|
+        p result
+      end
+
+    end  
+    
+    desc "repl", "Starts an interactive repl"
+    def repl
+      setup
+      r = Repl.new(@client)
+      r.start
+    end
+    
+    
+    private
+    def setup
+      config, status = *Config.load(options[:config])
+      
+      unless status == true
+        puts status
+        exit 1
+      end
+
+      @client = NNTP.new(config.host, config.port, config.ssl, config.user, config.pass)
+      
+      unless @client.logged_in?
+        puts "login failed"
+        exit 1
+      end
+    end
+  end
+end

data/lib/nntp_scrape/commands/auth_info.rb

@@ -0,0 +1,16 @@
+module NntpScrape
+  module Commands
+    class AuthInfo < Base
+      
+      def initialize(type, value)
+        @type = type
+        @value = value
+      end
+      
+      def execute(client)
+        run_short client, "AUTHINFO", @type, @value
+      end
+      
+    end
+  end
+end

data/lib/nntp_scrape/commands/base.rb

@@ -0,0 +1,46 @@
+module NntpScrape
+  module Commands
+    class Base
+      attr_reader :response, :status_line, :lines
+      attr_accessor :timeout
+      
+      def initialize
+        @timeout = 5
+      end
+      
+      def self.supported?(client)
+        true
+      end
+      
+      def ran?
+        status_line.present?
+      end
+    
+      def success?
+        return false unless ran?
+        status_code[0] == "1" || status_code[0] == "2"
+      end
+      
+      def continue?
+        success? || status_code[0] == "3"
+      end
+    
+      def status_code
+        @status_code ||= status_line.split.first
+      end
+    
+      def execute(client)
+        raise "implement in subclass"
+      end
+    
+      def run_short(client, cmd, *params)
+        @status_line = client.run_short(cmd, *params).strip
+        
+      end
+    
+      def run_long(client, cmd, *params)
+        @status_line, @lines = *client.run_long(cmd, *params)
+      end
+    end
+  end
+end

data/lib/nntp_scrape/commands/capabilities.rb

@@ -0,0 +1,18 @@
+module NntpScrape
+  module Commands
+    class Capabilities < Base
+      attr_reader :caps
+      
+      def initialize
+        @caps = []
+      end
+      
+      def execute(client)
+        run_long client, "CAPABILITIES" 
+        return unless success?
+        
+        @caps = lines
+      end
+    end
+  end
+end

data/lib/nntp_scrape/commands/group.rb

@@ -0,0 +1,27 @@
+module NntpScrape
+  module Commands
+    class Group < Base
+      attr_reader :article_count
+      attr_reader :low_id
+      attr_reader :high_id 
+      
+      PARSER = /^(\d+)\s+(\d+)\s+(\d+)\s+(\d+)/     
+      
+      def initialize(group)
+        @group = group
+      end
+      
+      def execute(client)
+        run_short client, "GROUP", @group
+        
+        if match = PARSER.match(status_line)
+          @article_count = match[2]
+          @low_id = match[3]
+          @high_id = match[4]
+        end
+        
+      end
+      
+    end
+  end
+end

data/lib/nntp_scrape/commands/head.rb

@@ -0,0 +1,28 @@
+module NntpScrape
+  module Commands
+    class Head < Base
+      attr_reader :message_id
+      attr_reader :data
+      
+      def initialize(number=nil)
+        @number = number
+      end
+      
+      def execute(client)
+        if @number.present?
+          run_long client, "HEAD", @number
+        else
+          run_long client, "HEAD"          
+        end
+        
+        
+        @message_id = status_line.split.last
+        @data = {}
+        lines.each do |line|
+          key, value = *line.split(": ", 2)
+          @data[key] = value
+        end        
+      end
+    end
+  end
+end

data/lib/nntp_scrape/commands/next.rb

@@ -0,0 +1,9 @@
+module NntpScrape
+  module Commands
+    class Next < Base
+      def execute(client)
+        run_short client, "NEXT"
+      end
+    end
+  end
+end

data/lib/nntp_scrape/commands/xhdr.rb

@@ -0,0 +1,26 @@
+module NntpScrape
+  module Commands
+    class Xhdr < Base
+      attr_reader :results
+      
+      def self.supported?(client)
+        client.caps.include? "XHDR"
+      end
+      
+      def initialize(field, range)
+        @field = field
+        @range = range
+      end
+      
+      def execute(client)
+        if @range.is_a? Range
+          run_long client, "XHDR", @field, "#{@range.begin}-#{@range.end}"
+        else
+          run_long client, "XHDR", @field, @range         
+        end
+        
+        @results = @lines.map{|l| l.split(" ")}
+      end
+    end
+  end
+end

data/lib/nntp_scrape/commands/xover.rb

@@ -0,0 +1,25 @@
+module NntpScrape
+  module Commands
+    class Xover < Base
+      attr_reader :results
+      
+      def self.supported?(client)
+        client.caps.include? "XOVER"
+      end
+      
+      def initialize(range)
+        @range = range
+      end
+      
+      def execute(client)
+        if @range.is_a? Range
+          run_long client, "XOVER", "#{@range.begin}-#{@range.end}"
+        else
+          run_long client, "XOVER", @range         
+        end
+        
+        @results = @lines.map{|l| l.split("\t")}
+      end
+    end
+  end
+end

data/lib/nntp_scrape/config.rb

@@ -0,0 +1,24 @@
+require 'yaml'
+require 'hashie'
+
+module NntpScrape
+  class Config < Hashie::Trash
+    property :host, :required => true
+    property :ssl,  :default => true
+    property :port, :default => 563
+    property :user, :required => true
+    property :pass, :required => true
+        
+    def self.load(config_path)
+      return [nil, :path_doesnt_exist] unless File.exist?(config_path)
+  
+      yaml =  YAML.load_file(config_path)
+      [new(yaml), true]
+    rescue Psych::SyntaxError
+      [nil, :invalid_yaml]
+    rescue ArgumentError => e
+      [nil, e]
+    end
+    
+  end
+end

data/lib/nntp_scrape/nntp.rb

@@ -0,0 +1,163 @@
+require 'socket'
+require 'openssl'
+require 'timeout'
+
+module NntpScrape
+  class NNTP
+    attr_reader :socket
+    attr_reader :caps
+    
+    def initialize(host, port, ssl, user, pass)
+      @host = host
+      @port = port
+      @ssl  = ssl
+      @user = user
+      @pass = pass
+      @caps = []
+      open
+    end
+    
+    def logged_in?
+      @logged_in
+    end
+    
+    def debug?
+      @debug
+    end
+    
+    def debug=(val)
+      @debug = val
+    end
+    
+    def open
+      @socket = make_socket
+      login
+    end
+    
+    def watch(group, start_id=nil)
+      watch = Commands::Group.new(group)
+      run watch
+      return false unless watch.success?
+      
+      puts watch.high_id
+      head = Commands::Head.new(start_id || watch.high_id)
+      run head
+      return false unless head.success?
+            
+      loop do
+        commands = [Commands::Next.new, Commands::Head.new]
+        
+        unless run *commands
+          puts "pausing..."
+          sleep 1
+          next
+        end
+        puts commands[1].message_id
+      end
+    end
+    
+    def watch_new(group, start_id=nil)
+      if start_id.nil?
+        watch = Commands::Group.new(group)
+        run watch
+        return false unless watch.success?
+        start_id = watch.high_id
+      end
+      
+      loop do
+        watch = Commands::Group.new(group)
+        run watch
+        next unless watch.success?
+        
+        end_id = watch.high_id
+        
+        if start_id == end_id
+          sleep 1
+          next
+        end
+        
+        xhdr = Commands::Xhdr.new "Message-ID", start_id...end_id
+        run xhdr
+        next unless xhdr.success?
+        
+        xhdr.results.each{|v| yield v}
+        start_id = end_id
+      end
+      
+    end
+    
+    ##
+    # Executes an array of commands until one fails
+    # 
+    # @return [Boolean] true if all commands succeeded
+    def run(*commands)
+      commands.each do |cmd|
+        Timeout::timeout(cmd.timeout) do
+          cmd.execute(self)
+        end
+        return false unless cmd.continue?
+      end
+      true
+    rescue Errno::EPIPE, Errno::ECONNRESET, OpenSSL::SSL::SSLError, Timeout::Error
+      open
+      retry
+    end
+    
+    
+    def run_short(cmd, *params)
+      command = "#{cmd} #{params.join " "}"
+        
+      puts command if debug?
+        
+      socket.print "#{command}\r\n"
+      status_line = socket.gets
+        
+      puts status_line if debug?
+        
+      status_line
+    end
+    
+    def run_long(cmd, *params)
+      status_line = run_short(cmd, *params)      
+      lines = []
+        
+      return [status_line, lines] unless ["2", "1"].include? status_line[0]  #i.e. success
+
+      loop do
+        next_line = socket.gets
+        break if next_line.strip == "."
+        lines << next_line.strip
+      end  
+      [status_line, lines]      
+    end
+    
+    private
+    def make_socket
+      socket = TCPSocket.open @host, @port
+      
+      if @ssl
+        cert_store = OpenSSL::X509::Store.new
+        cert_store.set_default_paths
+        ssl_context = OpenSSL::SSL::SSLContext.new
+        ssl_context.cert_store = cert_store
+        socket = OpenSSL::SSL::SSLSocket.new socket, ssl_context
+        socket.connect
+      end
+      
+      socket.gets
+      socket
+    end
+    
+    def login
+      login_commands = [
+        Commands::AuthInfo.new("USER", @user),
+        Commands::AuthInfo.new("PASS", @pass),
+        Commands::Capabilities.new,
+      ]
+      @logged_in = run *login_commands
+      
+      @caps = login_commands.last.caps
+      
+    end
+  end
+end

data/lib/nntp_scrape/nntp_command.rb

@@ -0,0 +1,40 @@
+module NntpScrape
+  class NNTPCommand
+    attr_reader :response, :status_line
+    
+    def success?
+      status_code[0] == "2"
+    end
+    
+    def status_code
+      @status_code ||= status_line.split.first
+    end
+    
+    def execute(client)
+      raise "implement in subclass"
+    end
+  end
+  
+  class ShortCommand < NNTPCommand
+    def execute(client)
+      socket = client.socket
+      socket.print "#{cmd} #{params.join " "}\r\n"
+      @status_line = socket.gets
+    end
+  end
+  
+  class LongCommand < ShortCommand
+    attr_reader :lines
+    
+    def execute(client)
+      super client
+            
+      @lines = []
+      loop do
+        next_line = socket.gets
+        break if next_line.strip == "."
+        @lines << next_line
+      end
+    end
+  end
+end

data/lib/nntp_scrape/repl.rb

@@ -0,0 +1,41 @@
+require 'pry'
+
+module NntpScrape
+  class Repl 
+    include Commands
+    
+    attr_reader :client
+    
+    def initialize(client)
+      @client = client
+    end
+    
+    def start
+      my_prompt = [ 
+        proc { |obj, *| ">> " },
+        proc { |obj, *| "*> "} 
+      ]
+      binding.pry :quiet => true, :cli => true, :prompt => my_prompt
+    end
+    
+    def run(klass, *params)
+      cmd = klass.new(*params)
+      @client.run(cmd)
+      cmd
+    end
+    
+    def short(cmd, *params)
+      @client.run_short(cmd, *params)
+    rescue Errno::EPIPE, Errno::ECONNRESET, OpenSSL::SSL::SSLError
+      @client.open
+      retry
+    end 
+       
+    def long(cmd, *params)
+      @client.run_long(cmd, *params)
+    rescue Errno::EPIPE, Errno::ECONNRESET, OpenSSL::SSL::SSLError
+      @client.open
+      retry
+    end
+  end
+end

data/lib/nntp_scrape/version.rb

@@ -0,0 +1,3 @@
+module NntpScrape
+  VERSION = "0.0.1"
+end

data/nntp_scrape.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'nntp_scrape/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "nntp_scrape"
+  gem.version       = NntpScrape::VERSION
+  gem.authors       = ["Scott Fleckenstein"]
+  gem.email         = ["nullstyle@gmail.com"]
+  gem.description   = %q{A series of command line tools to interact with usenet}
+  gem.summary       = %q{A series of command line tools to interact with usenet}
+  gem.homepage      = ""
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+  
+  gem.add_dependency "thor",          ">= 0.16.0"
+  gem.add_dependency "activesupport", ">= 3.2.9"
+  gem.add_dependency "hashie",        ">= 1.2.0"
+  gem.add_dependency "pry",           ">= 0.9.10"
+end

metadata

@@ -0,0 +1,133 @@
+--- !ruby/object:Gem::Specification
+name: nntp_scrape
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Scott Fleckenstein
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-12-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.16.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.16.0
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 3.2.9
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 3.2.9
+- !ruby/object:Gem::Dependency
+  name: hashie
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.10
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.10
+description: A series of command line tools to interact with usenet
+email:
+- nullstyle@gmail.com
+executables:
+- nntp_scrape
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/nntp_scrape
+- lib/nntp_scrape.rb
+- lib/nntp_scrape/cli.rb
+- lib/nntp_scrape/commands/auth_info.rb
+- lib/nntp_scrape/commands/base.rb
+- lib/nntp_scrape/commands/capabilities.rb
+- lib/nntp_scrape/commands/group.rb
+- lib/nntp_scrape/commands/head.rb
+- lib/nntp_scrape/commands/next.rb
+- lib/nntp_scrape/commands/xhdr.rb
+- lib/nntp_scrape/commands/xover.rb
+- lib/nntp_scrape/config.rb
+- lib/nntp_scrape/nntp.rb
+- lib/nntp_scrape/nntp_command.rb
+- lib/nntp_scrape/repl.rb
+- lib/nntp_scrape/version.rb
+- nntp_scrape.gemspec
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: A series of command line tools to interact with usenet
+test_files: []
+has_rdoc: