pluto 1.0.1 → 1.1.0

data/lib/pluto/fetcher.rb

@@ -1,292 +0,0 @@
-module Pluto
-
-
-class Fetcher
-
-  include LogUtils::Logging
-
-  include Models   # for easy convenience access for Activity etc.
-
-  def initialize
-    @worker  = ::Fetcher::Worker.new
-  end
-
-  def debug=(value)  @debug = value;   end
-  def debug?()       @debug || false;  end
-
-
-  def fetch_feed( url )
-    response = @worker.get( url )
-
-    ## if debug?
-      puts "http status #{response.code} #{response.message}"
-
-      puts "http header - server: #{response.header['server']} - #{response.header['server'].class.name}"
-      puts "http header - etag: #{response.header['etag']} - #{response.header['etag'].class.name}"
-      puts "http header - last-modified: #{response.header['last-modified']} - #{response.header['last-modified'].class.name}"
-    ## end
-
-    xml = response.body
-
-    ###
-    # NB: Net::HTTP will NOT set encoding UTF-8 etc.
-    # will mostly be ASCII
-    # - try to change encoding to UTF-8 ourselves
-    logger.debug "xml.encoding.name (before): #{xml.encoding.name}"
-
-    #####
-    # NB: ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
-
-    ## NB:
-    # for now "hardcoded" to utf8 - what else can we do?
-    # - note: force_encoding will NOT change the chars only change the assumed encoding w/o translation
-    xml = xml.force_encoding( Encoding::UTF_8 )
-    logger.debug "xml.encoding.name (after): #{xml.encoding.name}"      
-
-    xml
-  end
-
-
-  def feed_by_rec( feed_rec )
-    # simple feed fetcher; use for debugging (only/mostly)
-    #  -- will NOT change db records in any way
-
-    feed_url = feed_rec.feed_url
-    feed_key = feed_rec.key
-
-    feed_xml = fetch_feed( feed_url )
-
-    logger.debug "feed_xml:"
-    logger.debug feed_xml[ 0..500 ] # get first 500 chars
-
-    #  if opts.verbose?  # also write a copy to disk
-    if debug?
-      logger.debug "saving feed to >./#{feed_key}.xml<..."
-      File.open( "./#{feed_key}.xml", 'w' ) do |f|
-        f.write( feed_xml )
-      end
-    end
-      
-    puts "Before parsing feed >#{feed_key}<..."
-
-    ## fix/todo: check for feed.nil?   -> error parsing!!!
-    #    or throw exception
-    feed = FeedUtils::Parser.parse( feed_xml )
-  end
-
-
-  def feed_by_rec_if_modified( feed_rec )   # try smart http update; will update db records
-    feed_url = feed_rec.feed_url
-    feed_key = feed_rec.key
-
-    ### todo/fix: normalize/unifiy feed_url
-    ##  - same in fetcher - use shared utitlity method or similar
-
-    @worker.use_cache = true
-    @worker.cache[ feed_url ] = {
-      'etag'          => feed_rec.http_etag,
-      'last-modified' => feed_rec.http_last_modified
-    }
-
-    begin
-      response = @worker.get( feed_url )
-    rescue SocketError => e
-      ## catch socket error for unknown domain names (e.g. pragdave.blogs.pragprog.com)
-      ###  will result in SocketError -- getaddrinfo: Name or service not known
-      puts "*** error: fetching feed '#{feed_key}' - #{e.to_s}"
-      Activity.create!( text: "*** error: fetching feed '#{feed_key}' - #{e.to_s}" )
-
-      ### todo/fix: update feed rec in db
-      @worker.use_cache = false   # fix/todo: restore old use_cache setting instead of false
-      return nil
-    end
-
-    @worker.use_cache = false   # fix/todo: restore old use_cache setting instead of false
-
-    if response.code == '304'  # not modified (conditional GET - e.g. using etag/last-modified)
-      puts "OK - fetching feed '#{feed_key}' - HTTP status #{response.code} #{response.message}"
-      puts "no change; request returns not modified (304); skipping parsing feed"
-      return nil   # no updates available; nothing to do
-    end
-
-    feed_fetched = Time.now
-
-    if response.code != '200'   # note Net::HTTP response.code is a string in ruby
-
-      puts "*** error: fetching feed '#{feed_key}' - HTTP status #{response.code} #{response.message}"
-      
-      feed_attribs = {
-        http_code:          response.code.to_i,
-        http_server:        response.header[ 'server' ],
-        http_etag:          nil,
-        http_last_modified: nil,
-        body:               nil,
-        md5:                nil,
-        fetched:            feed_fetched 
-      }
-      feed_rec.update_attributes!( feed_attribs )
-      
-      ## add log error activity -- in future add to error log - better - why? why not?
-      Activity.create!( text: "*** error: fetching feed '#{feed_key}' - HTTP status #{response.code} #{response.message}" )
-
-      return nil  #  sorry; no feed for parsing available
-    end
-
-    puts "OK - fetching feed '#{feed_key}' - HTTP status #{response.code} #{response.message}"
-
-    feed_xml = response.body
-    ###
-    # NB: Net::HTTP will NOT set encoding UTF-8 etc.
-    # will mostly be ASCII
-    # - try to change encoding to UTF-8 ourselves
-    logger.debug "feed_xml.encoding.name (before): #{feed_xml.encoding.name}"
-
-    #####
-    # NB: ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
-
-    ## NB:
-    # for now "hardcoded" to utf8 - what else can we do?
-    # - note: force_encoding will NOT change the chars only change the assumed encoding w/o translation
-    feed_xml = feed_xml.force_encoding( Encoding::UTF_8 )
-    logger.debug "feed_xml.encoding.name (after): #{feed_xml.encoding.name}"
-
-    ## check for md5 hash for response.body
-
-    last_feed_md5 = feed_rec.md5
-    feed_md5 = Digest::MD5.hexdigest( feed_xml )
-    
-    if last_feed_md5 && last_feed_md5 == feed_md5
-      # not all servers handle conditional gets, so while not much can be
-      # done about the bandwidth, but if the response body is identical
-      # the downstream processing (parsing, caching, ...) can be avoided.
-      #  - thanks to planet mars -fido.rb for the idea, cheers.
-      
-      puts "no change; md5 digests match; skipping parsing feed"
-      return nil   # no updates available; nothing to do
-    end
-
-    feed_attribs = {
-      http_code:          response.code.to_i,
-      http_server:        response.header[ 'server' ],
-      http_etag:          response.header[ 'etag' ],
-      http_last_modified: response.header[ 'last-modified' ], ## note: last_modified header gets stored as plain text (not datetime)
-      body:               feed_xml,
-      md5:                feed_md5,
-      fetched:            feed_fetched
-    }
-
-    ## if debug?
-      puts "http header - server: #{response.header['server']} - #{response.header['server'].class.name}"
-      puts "http header - etag: #{response.header['etag']} - #{response.header['etag'].class.name}"
-      puts "http header - last-modified: #{response.header['last-modified']} - #{response.header['last-modified'].class.name}"
-    ## end
-
-    feed_rec.update_attributes!( feed_attribs )
-
-    logger.debug "feed_xml:"
-    logger.debug feed_xml[ 0..300 ] # get first 300 chars
-      
-    puts "Before parsing feed >#{feed_key}<..."
-
-    ### move to feedutils
-    ### logger.debug "using stdlib RSS::VERSION #{RSS::VERSION}"
-
-    ## fix/todo: check for feed.nil?   -> error parsing!!!
-    #    or throw exception
-    feed = FeedUtils::Parser.parse( feed_xml )
-  end
-
-
-  def site_by_rec_if_modified( site_rec )   # try smart http update; will update db records
-    site_url = site_rec.url
-    site_key = site_rec.key
-
-    ### todo/fix: normalize/unifiy feed_url
-    ##  - same in fetcher - use shared utitlity method or similar
-
-    @worker.use_cache = true
-    @worker.cache[ site_url ] = {
-      'etag'          => site_rec.http_etag,
-      'last-modified' => site_rec.http_last_modified
-    }
-
-    response = @worker.get( site_url )
-    @worker.use_cache = false   # fix/todo: restore old use_cache setting instead of false
-
-    if response.code == '304'  # not modified (conditional GET - e.g. using etag/last-modified)
-      puts "OK - fetching site '#{site_key}' - HTTP status #{response.code} #{response.message}"
-      puts "no change; request returns not modified (304); skipping parsing site config"
-      return nil   # no updates available; nothing to do
-    end
-
-    site_fetched = Time.now
-
-    if response.code != '200'   # note Net::HTTP response.code is a string in ruby
-
-      puts "*** error: fetching site '#{site_key}' - HTTP status #{response.code} #{response.message}"
-
-      site_attribs = {
-        http_code:          response.code.to_i,
-        http_server:        response.header[ 'server' ],
-        http_etag:          nil,
-        http_last_modified: nil,
-        body:               nil,
-        md5:                nil,
-        fetched:            feed_fetched 
-      }
-      site_rec.update_attributes!( site_attribs )
-      
-      ## add log error activity -- in future add to error log - better - why? why not?
-      Activity.create!( text: "*** error: fetching site '#{site_key}' - HTTP status #{response.code} #{response.message}" )
-
-      return nil  #  sorry; no feed for parsing available
-    end
-
-    puts "OK - fetching site '#{site_key}' - HTTP status #{response.code} #{response.message}"
-
-    site_text = response.body
-    
-    ###
-    # NB: Net::HTTP will NOT set encoding UTF-8 etc.
-    # will mostly be ASCII
-    # - try to change encoding to UTF-8 ourselves
-    logger.debug "site_text.encoding.name (before): #{site_text.encoding.name}"
-
-    #####
-    # NB: ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
-
-    ## NB:
-    # for now "hardcoded" to utf8 - what else can we do?
-    # - note: force_encoding will NOT change the chars only change the assumed encoding w/o translation
-    site_text = site_text.force_encoding( Encoding::UTF_8 )
-    logger.debug "site_text.encoding.name (after): #{site_text.encoding.name}"
-
-    site_attribs = {
-      http_code:          response.code.to_i,
-      http_server:        response.header[ 'server' ],
-      http_etag:          response.header[ 'etag' ],
-      http_last_modified: response.header[ 'last-modified' ], ## note: last_modified header gets stored as plain text (not datetime)
-      fetched:            site_fetched
-    }
-
-    ## if debug?
-      puts "http header - server: #{response.header['server']} - #{response.header['server'].class.name}"
-      puts "http header - etag: #{response.header['etag']} - #{response.header['etag'].class.name}"
-      puts "http header - last-modified: #{response.header['last-modified']} - #{response.header['last-modified'].class.name}"
-    ## end
-
-    site_rec.update_attributes!( site_attribs )
-
-    ## logger.debug "site_text:"
-    ## logger.debug site_text[ 0..300 ] # get first 300 chars
-
-
-    puts "Before parsing site config >#{site_key}<..."
-    
-    # assume ini format for now
-    site_config = INI.load( site_text )
-  end
-
-end # class Fetcher
-
-end # module Pluto

data/lib/pluto/formatter.rb

@@ -1,81 +0,0 @@
-module Pluto
-
-class Formatter
-
-  include LogUtils::Logging
-
-  include Models
-  include ManifestHelper
-  
-  include TextUtils::DateHelper  # e.g. lets us use time_ago_in_words
-  include TextUtils::HypertextHelper # e.g. lets us use link_to, strip_tags, sanitize, textify, etc.
-
-  def initialize( opts, config )
-    @opts    = opts
-    @config  = config
-  end
-
-  attr_reader :opts, :config, :site
-
-
-  def run( arg )
-    ### remove - always use make( site_key )
-    ## fix: change arg to planet_key or just key or similar
-    #  todo: rename run to some less generic  - merge/build/etc. ??
-    
-    site_key = arg
-    site_key = site_key.downcase.gsub('.ini','').gsub('.yml','')  # remove .ini|.yml extension if present
-
-    manifest_name = opts.manifest
-    output_path   = opts.output_path
-
-    make_for(site_key, manifest_name, output_path )
-  end
-
-
-  def make_for( site_key, manifest_name, output_path )
-
-    ## fix: remove reference to opts
-    ##  - e.g. now still used for auto-installer
-
-    manifest_name = manifest_name.downcase.gsub('.txt', '' )  # remove .txt if present
-
-    logger.debug "manifest=#{manifest_name}"
-
-    # check for matching manifests
-    manifests = installed_template_manifests.select { |m| m[0] == manifest_name+'.txt' }
-        
-    if manifests.empty?
-      
-      ### try - autodownload
-      puts "*** template pack '#{manifest_name}' not found; trying auto-install..."
-      
-      Installer.new( opts ).install( manifest_name )
-
-      ### try again
-      
-      # check for matching manifests
-      manifests = installed_template_manifests.select { |m| m[0] == manifest_name+'.txt' }
-      
-      if manifests.empty?
-         puts "*** error: unknown template pack '#{manifest_name}'; use pluto ls to list installed template packs"
-         exit 2
-      end
-    end
-
-    manifestsrc = manifests[0][1]
-    pakpath     = output_path
-
-    @site = Site.find_by_key( site_key )
-    if @site.nil?
-      puts "*** warn: no site with key '#{site_key}' found; using untitled site record"
-      @site = Site.new
-      @site.title = 'Planet Untitled'
-    end
-
-    Pakman::Templater.new.merge_pak( manifestsrc, pakpath, binding, site_key )
-  end
-
-end # class Formatter
-
-end # module Pluto

data/lib/pluto/installer.rb

@@ -1,58 +0,0 @@
-module Pluto
-
-class Installer
-
-### fix: remove opts, use config (wrapped!!)
-  
-  include LogUtils::Logging
-  
-  def initialize( opts )
-    @opts    = opts
-  end
-
-  attr_reader :opts
-
-
-  def install( shortcut_or_source )
-
-    logger.debug "fetch >#{shortcut_or_source}<"
-    
-    ## check for builtin shortcut (assume no / or \) 
-    if shortcut_or_source.index( '/' ).nil? && shortcut_or_source.index( '\\' ).nil?
-      shortcut = shortcut_or_source
-      sources = opts.map_fetch_shortcut( shortcut )
-
-      if sources.empty?
-        puts "** Error: No mapping found for shortcut '#{shortcut}'."
-        return
-      end
-      puts "  Mapping fetch shortcut '#{shortcut}' to: #{sources.join(',')}"
-    else
-      sources = [shortcut_or_source]  # pass arg through unmapped
-    end
-
-    sources.each do |source|
-      install_template( source )
-    end
-
-  end # method run
-
-
-  def install_template( src )
-    # src = 'http://github.com/geraldb/slideshow/raw/d98e5b02b87ee66485431b1bee8fb6378297bfe4/code/templates/fullerscreen.txt'
-    # src = 'http://github.com/geraldb/sandbox/raw/13d4fec0908fbfcc456b74dfe2f88621614b5244/s5blank/s5blank.txt'
-    uri = URI.parse( src )
-    logger.debug "scheme: #{uri.scheme}, host: #{uri.host}, port: #{uri.port}, path: #{uri.path}"
-
-    pakname = File.basename( uri.path ).downcase.gsub('.txt','')
-    pakpath = File.expand_path( "#{opts.config_path}/#{pakname}" )
-    
-    logger.debug "packname >#{pakname}<"
-    logger.debug "pakpath >#{pakpath}<"
-  
-    Pakman::Fetcher.new.fetch_pak( src, pakpath )
-  end
-
-end # class Installer
-
-end # module Pluto

data/lib/pluto/lister.rb

@@ -1,42 +0,0 @@
-module Pluto
-
-class Lister
-
-  include LogUtils::Logging
-
-  include ManifestHelper
-
-  def initialize( opts )
-    @opts    = opts
-  end
-
-  attr_reader :opts
-
-  def list
-    home = Env.home
-    ## replace home w/ ~ (to make out more readable (shorter))
-    ## e.g. use gsub( home, '~' )
-
-    puts ''
-    puts 'Installed template packs in search path'
-    
-    installed_template_manifest_patterns.each_with_index do |pattern,i|
-      puts "    [#{i+1}] #{pattern.gsub(home,'~')}"
-    end
-    puts '  include:'
-    
-    manifests = installed_template_manifests
-    if manifests.empty?
-      puts "    -- none --"
-    else
-      manifests.each do |manifest|
-        pakname      = manifest[0].gsub('.txt','')
-        manifestpath = manifest[1].gsub(home,'~')
-        puts "%16s (%s)" % [pakname,manifestpath]
-      end
-    end
-  end # method list
-
-end # class Lister
-
-end # module Pluto