knjrbfw 0.0.21 → 0.0.22

data/VERSION

@@ -1 +1 @@
-0.0.21
+0.0.22

data/knjrbfw.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{knjrbfw}
-  s.version = "0.0.21"
+  s.version = "0.0.22"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Kasper Johansen"]
-  s.date = %q{2012-03-04}
+  s.date = %q{2012-03-16}
   s.description = %q{Including stuff for HTTP, SSH and much more.}
   s.email = %q{k@spernj.org}
   s.extra_rdoc_files = [

data/lib/knj/arrayext.rb

@@ -79,10 +79,24 @@ module Knj::ArrayExt
       end
     end
     
-    adds.each do |key, value|
-      hash_given[key] = value
+    hash_given.merge!(adds)
+    
+    return hash_given
+  end
+  
+  #Converts all keys in the given hash to strings.
+  def self.hash_str(hash_given)
+    adds = {}
+    
+    hash_given.each do |key, val|
+      if !key.is_a?(String)
+        adds[key.to_s] = val
+        hash_given.delete(key)
+      end
     end
     
+    hash_given.merge!(adds)
+    
     return hash_given
   end
   
@@ -201,7 +215,7 @@ module Knj::ArrayExt
     Knj::Php.foreach(hash) do |key, val|
       if val.is_a?(String)
         begin
-          hash[key] = Knj::Php.utf8_encode(encoding)
+          hash[key] = Knj::Php.utf8_encode(val)
         rescue Encoding::UndefinedConversionError => e
           if args["ignore_encoding_errors"]
             next

data/lib/knj/cmd_parser.rb

@@ -4,15 +4,37 @@ class Knj::Cmd_parser
     
     str.lines.each do |line|
       next if line.match(/^total ([\d\.,]+)(M|k|G|)$/)
-      match = line.match(/^(.)(.)(.)(.)(.)(.)(.)(.)(.)(.)\s+(\d+)\s+(.+)\s+([^\W].+?)\s+([\d\.,]+)(M|k|G|K|)\s+((\d+)-(\d+)-(\d+)|(([A-z]{3})\s+(\d+)))\s+(\d+):(\d+)\s+(.+)$/)
+      match = line.match(/^(.)(.)(.)(.)(.)(.)(.)(.)(.)(.)\s+(\d+)\s+(.+)\s+([^\W].+?)\s+([\d\.,]+)(M|k|G|K|)\s+((\d+)-(\d+)-(\d+)|(([A-z]{3})\s+(\d+)))\s+((\d+):(\d+)|(\d{4}))\s+(.+)$/)
       raise "Could not match: '#{line}'." if !match
       
+      year = nil
+      
+      if match[17].to_i > 0
+        year = match[17].to_i
+      elsif match[26].to_i > 0
+        year = match[26].to_i
+      end
+      
+      hour = match[24].to_i
+      min = match[25].to_i
+      
       if match[17] and match[18] and match[19]
-        time = Time.local(match[17].to_i, match[18].to_i, match[19].to_i, match[23].to_i, match[24].to_i)
+        month = match[18].to_i
+        date = match[19].to_i
       elsif match[20] and match[21] and match[22]
-        time = Time.local(Time.now.year, match[21], match[22].to_i, match[23].to_i, match[24].to_i)
+        month = Knj::Datet.month_str_to_no(match[21])
+        date = match[22].to_i
+      end
+      
+      if !year
+        if month > Time.now.month
+          year = Time.now.year - 1
+        else
+          year = Time.now.year
+        end
       end
       
+      time = Time.local(year, month, date, hour, min)
       bytes = match[14].gsub(",", ".").to_f
       
       size_match = match[15]
@@ -50,7 +72,7 @@ class Knj::Cmd_parser
         :grp => match[13],
         :size => bytes.to_i,
         :time => time,
-        :file => match[25]
+        :file => match[27]
       }
     end
     

data/lib/knj/csv.rb

@@ -1,5 +1,8 @@
 class Knj::Csv
   def self.arr_to_csv(arr, del, encl)
+    raise "No delimiter given." if !del
+    raise "No enclosure given." if !encl
+    
     str = ""
     first = true
     arr.each do |val|

data/lib/knj/datarow.rb

@@ -11,6 +11,11 @@ class Knj::Datarow
     return @depending_data
   end
   
+  def self.autodelete_data
+    @autodelete_data = [] if !@autodelete_data
+    return @autodelete_data
+  end
+  
   #This helps various parts of the framework determine if this is a datarow class without requiring it.
   def is_knj?
     return true
@@ -33,7 +38,14 @@ class Knj::Datarow
         methodname = val[:method]
         
         if val[:depends]
-          depending_data << {
+          self.depending_data << {
+            :colname => colname,
+            :classname => classname
+          }
+        end
+        
+        if val[:autodelete]
+          self.autodelete_data << {
             :colname => colname,
             :classname => classname
           }
@@ -106,7 +118,7 @@ class Knj::Datarow
         
         if val[:required]
           colname = "#{classname.to_s.downcase}_id".to_sym if !colname
-          required_data << {
+          self.required_data << {
             :col => colname,
             :class => classname
           }
@@ -378,6 +390,7 @@ class Knj::Datarow
       sql = "SELECT #{table_str}.*"
     end
     
+    qargs = nil
     ret = self.list_helper(d)
     
     sql << " FROM #{table_str}"
@@ -389,6 +402,8 @@ class Knj::Datarow
       case key
         when "return_sql"
           #ignore
+        when :cloned_ubuf
+          qargs = {:cloned_ubuf => true}
         else
           raise "Invalid key: '#{key}' for '#{self.name}'. Valid keys are: '#{@columns_sqlhelper_args[:cols].keys.sort}'."
       end
@@ -418,7 +433,7 @@ class Knj::Datarow
     
     if select_col_as_array
       ids = [] if !block
-      d.db.q(sql) do |data|
+      d.db.q(sql, qargs) do |data|
         if block
           block.call(data[:id])
         else
@@ -437,7 +452,7 @@ class Knj::Datarow
       return 0
     end
     
-    return d.ob.list_bysql(self.classname, sql, &block)
+    return d.ob.list_bysql(self.classname, sql, qargs, &block)
   end
   
   def self.list_helper(d)

data/lib/knj/datet.rb

@@ -129,7 +129,7 @@ class Knj::Datet
   #Is a year a leap year in the Gregorian calendar? Copied from Date-class.
   def self.gregorian_leap?(y)
     if Date.respond_to?("gregorian_leap?")
-      return Date.gregorian_leap?
+      return Date.gregorian_leap?(y)
     elsif y % 4 == 0 && y % 100 != 0
       return true
     elsif y % 400 == 0

data/lib/knj/gettext_threadded.rb

@@ -30,11 +30,11 @@ class Knj::Gettext_threadded
                 
                 cont = nil
                 File.open(pofn, {:encoding => @args[:encoding]}) do |fp|
-                  cont = fp.read
+                  cont = fp.read.encode("utf-8")
                 end
                 
                 cont.scan(/msgid\s+\"(.+)\"\nmsgstr\s+\"(.+)\"\n\n/) do |match|
-                  @langs[file][match[0]] = match[1]
+                  @langs[file][match[0]] = match[1].to_s.encode("utf-8")
                 end
               end
             end

data/lib/knj/google_sitemap.rb

@@ -2,58 +2,97 @@ class Knj::Google_sitemap
   attr_reader :doc
   
   def initialize(args = {})
+    raise "No block given." if !block_given?
+    
     @args = args
     
     #used for Time.iso8601.
     require "time"
     
-    @doc = REXML::Document.new
-    @doc << REXML::XMLDecl.new("1.0", "UTF-8")
+    #REXML is known to leak memory - use subprocess.
+    @subproc = Knj::Process_meta.new("id" => "google_sitemap", "debug_err" => true)
     
-    @urlset = @doc.add_element("urlset")
-    @urlset.add_attributes("xmlns" => "http://www.sitemaps.org/schemas/sitemap/0.9")
+    begin
+      @subproc.static("Object", "require", "rexml/rexml")
+      @subproc.static("Object", "require", "rexml/document")
+      
+      @doc = @subproc.new("REXML::Document")
+      
+      xmldecl = @subproc.new("REXML::XMLDecl", "1.0", "UTF-8")
+      @doc._pm_send_noret("<<", xmldecl)
+      
+      urlset = @subproc.proxy_from_call(@doc, "add_element", "urlset")
+      urlset._pm_send_noret("add_attributes", {"xmlns" => "http://www.sitemaps.org/schemas/sitemap/0.9"})
+      
+      @root = @subproc.proxy_from_call(@doc, "root")
+      yield(self)
+    ensure
+      @doc = nil
+      @root = nil
+      @subproc.destroy
+      @subproc = nil
+    end
   end
   
   def add_url(url_value, lastmod_value, cf_value = nil, priority_value = nil)
-    el = REXML::Element.new("url")
-    
-    loc = el.add_element("loc")
-    loc.text = url_value
-    
     if !lastmod_value or lastmod_value.to_i == 0
       raise sprintf("Invalid date: %1$s, url: %2$s", lastmod_value.to_s, url_value)
     end
     
-    lm = el.add_element("lastmod")
+    el = @subproc.new("REXML::Element", "url")
+    
+    loc = @subproc.proxy_from_call(el, "add_element", "loc")
+    loc._pm_send_noret("text=", url_value)
+    
+    lm = @subproc.proxy_from_call(el, "add_element", "lastmod")
     if @args.key?(:date_min) and @args[:date_min] > lastmod_value
       lastmod_value = @args[:date_min]
     end
     
-    lm.text = lastmod_value.iso8601
+    lm._pm_send_noret("text=", lastmod_value.iso8601)
     
     if cf_value
-      cf = el.add_element("changefreq")
-      cf.text = cf_value
+      cf = @subproc.proxy_from_call(el, "add_element", "changefreq")
+      cf._pm_send_noret("text=", cf_value)
     end
     
     if priority_value
-      priority = el.add_element("priority")
-      priority.text = priority_value
+      priority = @subproc.proxy_from_call("el", "add_element", "priority")
+      priority._pm_send_noret("text=", priority_value)
     end
     
-    @doc.root << el
+    @root._pm_send_noret("<<", el)
   end
   
+  #This will return a non-human-readable XML-string.
   def to_xml
     return @doc.to_s
   end
   
+  #This will return a non-human-readable XML-string.
   def to_s
     return @doc.to_s
   end
   
+  #This will print the result.
   def write
-    writer = REXML::Formatters::Pretty.new(5)
-    writer.write(@doc, $stdout)
+    #Require and spawn StringIO in the subprocess.
+    @subproc.static("Object", "require", "stringio")
+    string_io = @subproc.spawn_object("StringIO")
+    
+    #We want a human-readable print.
+    writer = @subproc.spawn_object("REXML::Formatters::Pretty", 5)
+    writer._pm_send_noret("write", @doc, string_io)
+    
+    #Prepare printing by rewinding StringIO to read from beginning.
+    string_io._pm_send_noret("rewind")
+    
+    #Buffer results from subprocess in order to speed up printing.
+    string_io._process_meta_block_buffer_use = true
+    
+    #Print out the result in bits to avoid raping the memory (subprocess is already raped - no question there...).
+    string_io._pm_send_noret("each", 4096) do |str|
+      print str
+    end
   end
 end

data/lib/knj/http2.rb

@@ -33,6 +33,14 @@ class Knj::Http2
     
     raise "No host was given." if !@args[:host]
     self.reconnect
+    
+    if block_given?
+      begin
+        yield(self)
+      ensure
+        self.destroy
+      end
+    end
   end
   
   def socket_working?
@@ -49,6 +57,25 @@ class Knj::Http2
     return true
   end
   
+  def destroy
+    @args = nil
+    @cookies = nil
+    @debug = nil
+    @mutex = nil
+    @uagent = nil
+    @keepalive_timeout = nil
+    @request_last = nil
+    
+    @sock.close if @sock and !@sock.closed?
+    @sock = nil
+    
+    @sock_plain.close if @sock_plain and !@sock_plain.closed?
+    @sock_plain = nil
+    
+    @sock_ssl.close if @sock_ssl and !@sock_ssl.closed?
+    @sock_ssl = nil
+  end
+  
   #Reconnects to the host.
   def reconnect
     print "Http2: Reconnect.\n" if @debug
@@ -106,19 +133,25 @@ class Knj::Http2
   end
   
   def get(addr, args = {})
-    @mutex.synchronize do
-      header_str = "GET /#{addr} HTTP/1.1#{@nl}"
-      header_str << self.header_str(self.default_headers(args), args)
-      header_str << "#{@nl}"
-      
-      print "Http2: Writing headers.\n" if @debug
-      self.write(header_str)
-      
-      print "Http2: Reading response.\n" if @debug
-      resp = self.read_response(args)
+    begin
+      @mutex.synchronize do
+        header_str = "GET /#{addr} HTTP/1.1#{@nl}"
+        header_str << self.header_str(self.default_headers(args), args)
+        header_str << "#{@nl}"
+        
+        print "Http2: Writing headers.\n" if @debug
+        self.write(header_str)
+        
+        print "Http2: Reading response.\n" if @debug
+        resp = self.read_response(args)
+        
+        print "Http2: Done with get request.\n" if @debug
+        return resp
+      end
+    rescue Knj::Errors::Retry => e
+      print "Redirecting to: #{e.message}\n"
       
-      print "Http2: Done with get request.\n" if @debug
-      return resp
+      return self.get(e.message, args)
     end
   end
   
@@ -159,59 +192,67 @@ class Knj::Http2
   end
   
   def post(addr, pdata = {}, args = {})
-    @mutex.synchronize do
-      praw = ""
-      pdata.each do |key, val|
-        praw << "&" if praw != ""
-        praw << "#{Knj::Web.urlenc(key)}=#{Knj::Web.urlenc(val)}"
+    begin
+      @mutex.synchronize do
+        praw = ""
+        pdata.each do |key, val|
+          praw << "&" if praw != ""
+          praw << "#{Knj::Web.urlenc(key)}=#{Knj::Web.urlenc(val)}"
+        end
+        
+        header_str = "POST /#{addr} HTTP/1.1#{@nl}"
+        header_str << self.header_str(self.default_headers(args).merge("Content-Length" => praw.length), args)
+        header_str << "#{@nl}"
+        header_str << praw
+        
+        self.write(header_str)
+        return self.read_response(args)
       end
-      
-      header_str = "POST /#{addr} HTTP/1.1#{@nl}"
-      header_str << self.header_str(self.default_headers(args).merge("Content-Length" => praw.length), args)
-      header_str << "#{@nl}"
-      header_str << praw
-      
-      self.write(header_str)
-      return self.read_response(args)
+    rescue Knj::Errors::Retry => e
+      return self.get(e.message, args)
     end
   end
   
   def post_multipart(addr, pdata, args = {})
-    @mutex.synchronize do
-      boundary = Digest::MD5.hexdigest(Time.now.to_f.to_s)
-      
-      praw = ""
-      pdata.each do |key, val|
-        praw << "--#{boundary}#{@nl}"
+    begin
+      @mutex.synchronize do
+        boundary = Digest::MD5.hexdigest(Time.now.to_f.to_s)
         
-        if val.class.name == "Tempfile" and val.respond_to?("original_filename")
-          praw << "Content-Disposition: form-data; name=\"#{key}\"; filename=\"#{val.original_filename}\";#{@nl}"
-        else
-          praw << "Content-Disposition: form-data; name=\"#{key}\";#{@nl}"
+        praw = ""
+        pdata.each do |key, val|
+          praw << "--#{boundary}#{@nl}"
+          
+          if val.class.name == "Tempfile" and val.respond_to?("original_filename")
+            praw << "Content-Disposition: form-data; name=\"#{key}\"; filename=\"#{val.original_filename}\";#{@nl}"
+          else
+            praw << "Content-Disposition: form-data; name=\"#{key}\";#{@nl}"
+          end
+          
+          praw << "Content-Type: text/plain#{@nl}"
+          praw << "Content-Length: #{val.length}#{@nl}"
+          praw << @nl
+          
+          if val.is_a?(StringIO)
+            praw << val.read
+          else
+            praw << val.to_s
+          end
+          
+          praw << @nl
         end
         
-        praw << "Content-Type: text/plain#{@nl}"
-        praw << "Content-Length: #{val.length}#{@nl}"
-        praw << @nl
+        header_str = "POST /#{addr} HTTP/1.1#{@nl}"
+        header_str << "Content-Type: multipart/form-data; boundary=#{boundary}#{@nl}"
+        header_str << self.header_str(self.default_headers(args).merge("Content-Length" => praw.length), args)
+        header_str << "#{@nl}"
+        header_str << praw
+        header_str << "--#{boundary}--"
         
-        if val.is_a?(StringIO)
-          praw << val.read
-        else
-          praw << val.to_s
-        end
-        
-        praw << @nl
+        self.write(header_str)
+        return self.read_response(args)
       end
-      
-      header_str = "POST /#{addr} HTTP/1.1#{@nl}"
-      header_str << "Content-Type: multipart/form-data; boundary=#{boundary}#{@nl}"
-      header_str << self.header_str(self.default_headers(args).merge("Content-Length" => praw.length), args)
-      header_str << "#{@nl}"
-      header_str << praw
-      header_str << "--#{boundary}--"
-      
-      self.write(header_str)
-      return self.read_response(args)
+    rescue Knj::Errors::Retry => e
+      return self.get(e.message, args)
     end
   end
   
@@ -313,13 +354,15 @@ class Knj::Http2
       args[:port] = uri.port if uri.port
       
       if !args[:host] or args[:host] == @args[:host]
-        return self.get(resp.header("location"))
+        raise Knj::Errors::Retry, resp.header("location")
       else
         http = Knj::Http2.new(args)
         return http.get(uri.path)
       end
     elsif resp.args[:code].to_s == "500"
       raise "500 - Internal server error."
+    elsif resp.args[:code].to_s == "403"
+      raise Knj::Errors::NoAccess
     else
       return resp
     end
@@ -428,6 +471,7 @@ class Knj::Http2::Response
     return @args[:headers][key].first.to_s
   end
   
+  #Returns true if a header of the given string exists.
   def header?(key)
     return true if @args[:headers].key?(key) and @args[:headers][key].first.to_s.length > 0
     return false