gsa-feeds 0.0.1

data/History.txt

@@ -0,0 +1,4 @@
+== 0.0.1 2007-11-16
+
+* Initial release
+  * worksforme

data/License.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2007 Jesse Newland
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Manifest.txt

@@ -0,0 +1,22 @@
+History.txt
+License.txt
+Manifest.txt
+README.txt
+Rakefile
+config/hoe.rb
+config/requirements.rb
+lib/gsa_feeds.rb
+lib/multipart.rb
+lib/gsa_feeds/version.rb
+lib/gsa_feeds/base.rb
+lib/gsa_feeds/record.rb
+log/debug.log
+script/destroy
+script/generate
+script/txt2html
+setup.rb
+tasks/deployment.rake
+tasks/environment.rake
+tasks/website.rake
+test/test_gsa_feeds.rb
+test/test_helper.rb

data/README.txt

@@ -0,0 +1,4 @@
+GSA Feeds
+=========
+
+A Ruby wrapper for the Google Search Appliance Feeds Protocol. This protocol exists to push content and metadata to the search appliance for processing, indexing, and serving as search results. Feeds are available in GSA software versions 4.2 and later.

data/Rakefile

@@ -0,0 +1,4 @@
+require 'config/requirements'
+require 'config/hoe' # setup Hoe + all gem configuration
+
+Dir['tasks/**/*.rake'].each { |rake| load rake }

data/config/hoe.rb

@@ -0,0 +1,71 @@
+require 'gsa_feeds/version'
+
+AUTHOR = 'Jesse Newlnad'  # can also be an array of Authors
+EMAIL = "jnewland@gmail.com"
+DESCRIPTION = "Wrap the Google Search Appliance Feeds API"
+GEM_NAME = 'gsa-feeds' # what ppl will type to install your gem
+RUBYFORGE_PROJECT = 'gsa-feeds' # The unix name for your project
+HOMEPATH = "http://#{RUBYFORGE_PROJECT}.rubyforge.org"
+DOWNLOAD_PATH = "http://rubyforge.org/projects/#{RUBYFORGE_PROJECT}"
+
+@config_file = "~/.rubyforge/user-config.yml"
+@config = nil
+RUBYFORGE_USERNAME = "jnewland"
+def rubyforge_username
+  unless @config
+    begin
+      @config = YAML.load(File.read(File.expand_path(@config_file)))
+    rescue
+      puts <<-EOS
+ERROR: No rubyforge config file found: #{@config_file}
+Run 'rubyforge setup' to prepare your env for access to Rubyforge
+ - See http://newgem.rubyforge.org/rubyforge.html for more details
+      EOS
+      exit
+    end
+  end
+  RUBYFORGE_USERNAME.replace @config["username"]
+end
+
+
+REV = nil 
+# UNCOMMENT IF REQUIRED: 
+# REV = `svn info`.each {|line| if line =~ /^Revision:/ then k,v = line.split(': '); break v.chomp; else next; end} rescue nil
+VERS = GsaFeeds::VERSION::STRING + (REV ? ".#{REV}" : "")
+RDOC_OPTS = ['--quiet', '--title', 'gsa_feeds documentation',
+    "--opname", "index.html",
+    "--line-numbers", 
+    "--main", "README",
+    "--inline-source"]
+
+class Hoe
+  def extra_deps 
+    @extra_deps.reject! { |x| Array(x).first == 'hoe' } 
+    @extra_deps
+  end 
+end
+
+# Generate all the Rake tasks
+# Run 'rake -T' to see list of generated tasks (from gem root directory)
+hoe = Hoe.new(GEM_NAME, VERS) do |p|
+  p.author = AUTHOR 
+  p.description = DESCRIPTION
+  p.email = EMAIL
+  p.summary = DESCRIPTION
+  p.url = HOMEPATH
+  p.rubyforge_name = RUBYFORGE_PROJECT if RUBYFORGE_PROJECT
+  p.test_globs = ["test/**/test_*.rb"]
+  p.clean_globs |= ['**/.*.sw?', '*.gem', '.config', '**/.DS_Store']  #An array of file patterns to delete on clean.
+  
+  # == Optional
+  p.changes = p.paragraphs_of("History.txt", 0..1).join("\n\n")
+  p.extra_deps = [['builder', '>= 2.1.1']]     # An array of rubygem dependencies [name, version], e.g. [ ['active_support', '>= 1.3.1'] ]
+  
+  #p.spec_extras = {}    # A hash of extra values to set in the gemspec.
+  
+end
+
+CHANGES = hoe.paragraphs_of('History.txt', 0..1).join("\\n\\n")
+PATH    = (RUBYFORGE_PROJECT == GEM_NAME) ? RUBYFORGE_PROJECT : "#{RUBYFORGE_PROJECT}/#{GEM_NAME}"
+hoe.remote_rdoc_dir = File.join(PATH.gsub(/^#{RUBYFORGE_PROJECT}\/?/,''), 'rdoc')
+hoe.rsync_args = '-av --delete --ignore-errors'

data/config/requirements.rb

@@ -0,0 +1,17 @@
+require 'fileutils'
+include FileUtils
+
+require 'rubygems'
+%w[rake hoe newgem rubigen].each do |req_gem|
+  begin
+    require req_gem
+  rescue LoadError
+    puts "This Rakefile requires the '#{req_gem}' RubyGem."
+    puts "Installation: gem install #{req_gem} -y"
+    exit
+  end
+end
+
+$:.unshift(File.join(File.dirname(__FILE__), %w[.. lib]))
+
+require 'gsa_feeds'

data/lib/gsa_feeds.rb

@@ -0,0 +1,8 @@
+$:.unshift File.dirname(__FILE__)
+require 'multipart'
+require 'builder'
+module GsaFeeds
+  TIMEOUT = 60
+end
+require 'gsa_feeds/base'
+require 'gsa_feeds/record'

data/lib/gsa_feeds/base.rb

@@ -0,0 +1,95 @@
+class GsaFeeds::Base
+  def initialize(hostname, datasource = 'web', feedtype = 'incremental', timeout = false)
+    @hostname = hostname
+    @timeout = timeout || GsaFeeds::TIMEOUT
+    @records = []
+    @datasource = datasource
+    @feedtype = feedtype
+  end
+  attr_accessor :records, :datasource, :feedtype, :timeout, :hostname
+  alias_method :to_a, :records
+  
+  def clear!
+    @records = []
+  end
+
+  def add_record(record)
+    @records << record
+  end
+
+  def build_record(url, options = {}, metadata = nil, content = nil)
+    @records << GsaFeeds::Record.new(url, options, metadata, content)
+  end
+
+  def commit!
+    push_records(@records)
+    clear!
+    true
+  end
+  
+  def to_xml
+    build_xml(@records)
+  end
+  
+private
+  
+  # def post_form(query, headers, url = "http://#{@hostname}:19900/xmlfeed")
+  #   url = URI.parse(url)
+  #   Net::HTTP.start(url.host, url.port) do |con|
+  #     con.read_timeout = @timeout
+  #     return con.post(url.path, query, headers)
+  #   end
+  # end 
+  
+  def push_records(records)
+    params = Hash.new
+
+    params["data"] = build_xml(records)
+    params["datasource"] = @datasource
+    params["feedtype"] = @feedtype
+
+    url = URI.parse("http://#{@hostname}:19900/xmlfeed")
+    res = Net::HTTP.multi_post_form(url, params)
+
+    # res holds the response to the POST
+    case res
+    when Net::HTTPSuccess
+     return true
+    else
+      raise "Unknown error #{res}: #{res.inspect}\n#{res.body}"
+    end
+  end
+  
+  def build_xml(records)
+    string = ''
+    builder = Builder::XmlMarkup.new(:target=>string, :indent=>2)
+    builder.instruct!
+    builder.declare! :DOCTYPE, :gsafeed, :PUBLIC, "-//Google//DTD GSA Feeds//EN", ""
+    builder.gsafeed do |feed|
+      feed.header do |header|
+        header.datasource @datasource
+        header.feedtype @feedtype
+      end
+      feed.group do |r|
+        records.each do |record_data|
+          if (!record_data.content.nil? || !record_data.metadata.nil?)
+            r.record(record_data.to_h) do |record_details|
+              unless record_data.metadata.nil?
+                record_details.metadata do |meta_tags|
+                  record_data.metadata.each do |meta_info|
+                    meta_tags.meta(meta_info)
+                  end
+                end
+              end
+              record_details.content record_data.content unless record_data.content.nil?
+            end
+          else
+            r.record(record_data.to_h)
+          end
+        end
+      end
+    end
+    string
+  end
+  
+end

data/lib/gsa_feeds/record.rb

@@ -0,0 +1,18 @@
+class GsaFeeds::Record
+  def initialize(url, options = {}, metadata = nil, content = nil)
+    options = {:mimetype => 'text/html'}.merge(options)
+    @hash = options.merge({:url => url})
+    @metadata = metadata
+    @content = content
+  end
+  attr_accessor :hash, :metadata, :content
+  
+  #automap the metadata hash into an array name / content pairs
+  def metadata
+    @metadata.map { |key, value| {:name => key, :content => value } }
+  end
+  
+  def to_h
+    @hash
+  end
+end

data/lib/gsa_feeds/version.rb

@@ -0,0 +1,9 @@
+module GsaFeeds #:nodoc:
+  module VERSION #:nodoc:
+    MAJOR = 0
+    MINOR = 0
+    TINY  = 1
+
+    STRING = [MAJOR, MINOR, TINY].join('.')
+  end
+end

data/lib/multipart.rb

@@ -0,0 +1,47 @@
+#kastner is effing awesome. that is all.
+class Net::HTTP
+  def self.multi_post_form(url, params)
+    req = Post.new(url.path)
+    req.multipart_params = params
+    req.basic_auth url.user, url.password if url.user
+    new(url.host, url.port).start {|http|
+      http.request(req)
+    }
+  end
+end
+  
+module Net
+  class HTTP
+    class Post
+      def multipart_params=(param_hash={})
+        boundary_token = [Array.new(8) {rand(256)}].join
+        self.content_type = "multipart/form-data; boundary=#{boundary_token}"
+        boundary_marker = "--#{boundary_token}\r\n"
+        
+        self.body = param_hash.map { |param_name, param_value|
+          boundary_marker + text_to_multipart(param_name, param_hash.delete(param_name).to_s) unless param_value.respond_to?(:read)
+        }.join('') 
+        
+        self.body += param_hash.map { |param_name, param_value|
+          boundary_marker + file_to_multipart(param_name, param_value)
+        }.join('') 
+        
+        self.body += "--#{boundary_token}--\r\n"
+      end
+
+    protected
+  
+      def file_to_multipart(key,file)
+        filename = File.basename(file.path || file.base_uri.to_s)
+        mime_type = "application/octet-stream"
+        part = %Q|Content-Disposition: form-data; name=\"#{key}\"; filename="#{filename}"\r\n|
+        part += "Content-Transfer-Encoding: binary\r\n"
+        part += "Content-Type: #{mime_type}\r\n\r\n#{file.read}\r\n"
+      end
+
+      def text_to_multipart(key,value)
+        "Content-Disposition: form-data; name=\"#{key}\"\r\n\r\n#{value}\r\n"
+      end
+    end
+  end
+end

data/script/destroy

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.join(File.dirname(__FILE__), '..')
+
+begin
+  require 'rubigen'
+rescue LoadError
+  require 'rubygems'
+  require 'rubigen'
+end
+require 'rubigen/scripts/destroy'
+
+ARGV.shift if ['--help', '-h'].include?(ARGV[0])
+RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit]
+RubiGen::Scripts::Destroy.new.run(ARGV)

data/script/generate

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.join(File.dirname(__FILE__), '..')
+
+begin
+  require 'rubigen'
+rescue LoadError
+  require 'rubygems'
+  require 'rubigen'
+end
+require 'rubigen/scripts/generate'
+
+ARGV.shift if ['--help', '-h'].include?(ARGV[0])
+RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit]
+RubiGen::Scripts::Generate.new.run(ARGV)

data/script/txt2html

@@ -0,0 +1,74 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+begin
+  require 'newgem'
+rescue LoadError
+  puts "\n\nGenerating the website requires the newgem RubyGem"
+  puts "Install: gem install newgem\n\n"
+  exit(1)
+end
+require 'redcloth'
+require 'syntax/convertors/html'
+require 'erb'
+require File.dirname(__FILE__) + '/../lib/gsa_feeds/version.rb'
+
+version  = GsaFeeds::VERSION::STRING
+download = 'http://rubyforge.org/projects/gsa_feeds'
+
+class Fixnum
+  def ordinal
+    # teens
+    return 'th' if (10..19).include?(self % 100)
+    # others
+    case self % 10
+    when 1: return 'st'
+    when 2: return 'nd'
+    when 3: return 'rd'
+    else    return 'th'
+    end
+  end
+end
+
+class Time
+  def pretty
+    return "#{mday}#{mday.ordinal} #{strftime('%B')} #{year}"
+  end
+end
+
+def convert_syntax(syntax, source)
+  return Syntax::Convertors::HTML.for_syntax(syntax).convert(source).gsub(%r!^<pre>|</pre>$!,'')
+end
+
+if ARGV.length >= 1
+  src, template = ARGV
+  template ||= File.join(File.dirname(__FILE__), '/../website/template.rhtml')
+  
+else
+  puts("Usage: #{File.split($0).last} source.txt [template.rhtml] > output.html")
+  exit!
+end
+
+template = ERB.new(File.open(template).read)
+
+title = nil
+body = nil
+File.open(src) do |fsrc|
+  title_text = fsrc.readline
+  body_text = fsrc.read
+  syntax_items = []
+  body_text.gsub!(%r!<(pre|code)[^>]*?syntax=['"]([^'"]+)[^>]*>(.*?)</\1>!m){
+    ident = syntax_items.length
+    element, syntax, source = $1, $2, $3
+    syntax_items << "<#{element} class='syntax'>#{convert_syntax(syntax, source)}</#{element}>"
+    "syntax-temp-#{ident}"
+  }
+  title = RedCloth.new(title_text).to_html.gsub(%r!<.*?>!,'').strip
+  body = RedCloth.new(body_text).to_html
+  body.gsub!(%r!(?:<pre><code>)?syntax-temp-(\d+)(?:</code></pre>)?!){ syntax_items[$1.to_i] }
+end
+stat = File.stat(src)
+created = stat.ctime
+modified = stat.mtime
+
+$stdout << template.result(binding)