noms-command 0.5.0 → 2.1.1

data/lib/noms/command/base.rb

@@ -12,8 +12,18 @@ end
 
 class NOMS::Command::Base
 
+    attr_accessor :logger
+
+    def logger
+        @log
+    end
+
+    def logger=(new_logger)
+        @log = new_logger
+    end
+
     def default_logger
-        log = Logger.new $stdin
+        log = Logger.new $stderr
         log.level = Logger::WARN
         log.level = Logger::DEBUG if ENV['NOMS_DEBUG']
         log

data/lib/noms/command/formatter.rb

@@ -14,10 +14,12 @@ class NOMS::Command
 
 end
 
-class NOMS::Command::Formatter
+class NOMS::Command::Formatter < NOMS::Command::Base
 
     def initialize(data=nil, opt={})
+        @log = opt[:logger] || default_logger
         @data = data
+        @log.debug("Created formatter for: #{@data.inspect}");
         @format_raw_object = opt[:format_raw_object] || lambda { |o| o.to_yaml }
     end
 
@@ -60,7 +62,6 @@ class NOMS::Command::Formatter
     end
 
     def render_object_list(objlist)
-        objlist['$labels'] ||= true
         objlist['$format'] ||= 'lines'
         raise NOMS::Command::Error.new("objectlist ('lines' format) must contain '$columns' list") unless
             objlist['$columns'] and objlist['$columns'].respond_to? :map
@@ -160,7 +161,7 @@ class NOMS::Command::Formatter
 
     def render_object_record(object)
         labels = object.has_key?('$labels') ? object['$labels'] : true
-        fields = (object['$fields'] || object['$data'].keys).sort
+        fields = (object['$fields'].nil? or object['$fields'].empty?) ? object['$data'].keys.sort : object['$fields']
         data = object['$data']
         fields.map do |field|
             (labels ? (field + ': ') : '' ) + _string(data[field])
@@ -168,7 +169,7 @@ class NOMS::Command::Formatter
     end
 
     def filter_object(object)
-        if object['$fields']
+        if (object['$fields'] and ! object['$fields'].empty?)
             Hash[object['$fields'].map { |f| [f, object['$data'][f]] }]
         else
             object['$data']

data/lib/noms/command/home.rb

@@ -0,0 +1,21 @@
+#!ruby
+
+require 'noms/command/version'
+
+class NOMS
+
+end
+
+class NOMS::Command
+
+    @@home = (ENV.has_key?('NOMS_HOME') and ! ENV['NOMS_HOME'].empty?) ? ENV['NOMS_HOME'] : File.join(ENV['HOME'], '.noms')
+
+    def self.home=(value)
+        @@home = value
+    end
+
+    def self.home
+        @@home
+    end
+
+end

data/lib/noms/command/useragent.rb

@@ -7,8 +7,10 @@ require 'httpclient'
 require 'uri'
 require 'highline/import'
 
-require 'noms/command/auth'
-require 'noms/command/base'
+require 'noms/command'
+require 'noms/command/useragent/cache'
+require 'noms/command/useragent/requester'
+require 'noms/command/useragent/response'
 
 class NOMS
 
@@ -20,24 +22,34 @@ end
 
 class NOMS::Command::UserAgent < NOMS::Command::Base
 
+    attr_accessor :cache, :cacher, :auth
+
     def initialize(origin, attrs={})
-        @origin = origin
-        @client = HTTPClient.new :agent_name => "noms/#{NOMS::Command::VERSION}"
+        @origin = origin.respond_to?(:scheme) ? origin : URI.parse(origin)
+        # httpclient
         # TODO Replace with TOFU implementation
-        @client.ssl_config.verify_mode = OpenSSL::SSL::VERIFY_NONE
-        @redirect_checks = [ ]
         @log = attrs[:logger] || default_logger
+        cookies = attrs.has_key?(:cookies) ? attrs[:cookies] : true
+
+        @client = NOMS::Command::UserAgent::Requester.new :logger => @log, :cookies => cookies
+
+        @redirect_checks = [ ]
+        @plaintext_identity = attrs[:plaintext_identity] || nil
+
+        @cache = attrs.has_key?(:cache) ? attrs[:cache] : true
+        @max_age = attrs[:max_age] || 3600
 
-        @log.debug "(UserAgent) specified identities = #{attrs[:specified_identities]}"
-        @auth = NOMS::Command::Auth.new(:logger => @log,
-                                        :specified_identities => (attrs[:specified_identities] || []))
-        # TODO: Set cookie jar to something origin-specific
-        # TODO: Set user-agent to something nomsy
-        # caching
-        @client.redirect_uri_callback = lambda do |uri, res|
-            raise NOMS::Command::Error.new "Bad redirect URL #{url}" unless check_redirect(uri)
-            @client.default_redirect_uri_callback(uri, res)
+        if @cache
+            @cacher = NOMS::Command::UserAgent::Cache.new
         end
+
+        @auth = attrs[:auth] || NOMS::Command::Auth.new(:logger => @log,
+                                                        :specified_identities =>
+                                                        (attrs[:specified_identities] || []))
+    end
+
+    def clear_cache!
+        @cacher.clear! unless @cacher.nil?
     end
 
     def auth
@@ -45,17 +57,14 @@ class NOMS::Command::UserAgent < NOMS::Command::Base
     end
 
     def check_redirect(url)
-        @log.debug "Running #{@redirect_checks.size} redirect checks on #{url}" unless @redirect_checks.empty?
         @redirect_checks.all? { |check| check.call(url) }
     end
 
     def origin=(new_origin)
-        @log.debug "Setting my origin to #{new_origin}"
         @origin = new_origin
     end
 
     def absolute_url(url)
-        @log.debug "Calculating absolute url of #{url} in context of #{@origin}"
         begin
             url = URI.parse url unless url.respond_to? :scheme
             url = URI.join(@origin, url) unless url.absolute?
@@ -65,43 +74,114 @@ class NOMS::Command::UserAgent < NOMS::Command::Base
         end
     end
 
-    def request(method, url, data=nil, headers={}, tries=10, identity=nil)
+    # Calculate a key for caching based on the method and URL
+    def request_key(method, url, opt={})
+        OpenSSL::Digest::SHA1.new([method, url.to_s].join(' ')).hexdigest
+    end
+
+    def request(method, url, data=nil, headers={}, tries=10, identity=nil, cached=nil)
         req_url = absolute_url(url)
         @log.debug "#{method} #{req_url}" + (headers.empty? ? '' : headers.inspect)
-        response = @client.request(method.to_s.upcase, req_url, '', data, headers)
-        @log.debug "-> #{response.status} #{response.reason} (#{response.content.size} bytes of #{response.contenttype})"
-        @log.debug JSON.pretty_generate(response.headers)
+
+        # TODO: check Vary
+        if @cache and cached.nil? and method.to_s.upcase == 'GET'
+            key = request_key('GET', req_url)
+            cached_response = NOMS::Command::UserAgent::Response.from_cache(@cacher.get(key), :logger => @log)
+            if cached_response and cached_response.is_a? NOMS::Command::UserAgent::Response
+                cached_response.logger = @log
+
+                if cached_response.age < @max_age
+                    if (cached_response.auth_hash.nil? or (identity and identity.auth_verify? cached_response.auth_hash))
+                        if cached_response.current?
+                            return [cached_response, req_url]
+                        else
+                            # Maybe we can revalidate it
+                            if cached_response.etag
+                                headers = { 'If-None-Match' => cached_response.etag }.merge headers
+                                return self.request(method, url, data, headers, tries, identity, cached_response)
+                            elsif cached_response.last_modified
+                                headers = { 'If-Modified-Since' => cached_response.last_modified.httpdate }.merge headers
+                                return self.request(method, url, data, headers, tries, identity, cached_response)
+                            end
+                        end
+                    end
+                end
+            end
+        end
+
+        begin
+            response = @client.request :method => method,
+                                       :url => req_url,
+                                       :body => data,
+                                       :headers => headers
+        rescue StandardError => e
+            @log.debug e.backtrace.join("\n")
+            raise NOMS::Command::Error.new "Couldn't retrieve #{req_url} (#{e.class}): #{e.message}"
+        end
+        @log.debug "-> #{response.statusText} (#{response.body.size} bytes of #{response.content_type})"
+        @log.debug { JSON.pretty_generate(response.header) }
+
         case response.status
         when 401
-            @log.debug "   handling unauthorized"
             if identity
-                @log.debug "   we have an identity #{identity} but are trying again"
+                # The identity we got was no good, try again
+                identity.clear
                 if tries > 0
-                    @log.debug "loading authentication identity for #{url}"
-                    identity = @auth.load(url, response)
+                    identity = @auth.load(req_url, response)
+                    # httpclient
                     @client.set_auth(identity['domain'], identity['username'], identity['password'])
-                    response, req_url = self.request(method, url, data, headers, tries - 1, identity)
+                    return self.request(method, url, data, headers, tries - 1, identity)
                 end
             else
-                identity = @auth.load(url, response)
-                @client.set_auth(identity['domain'], identity['username'], identity['password'])
-                response, req_url = self.request(method, url, data, headers, 2, identity)
+                identity = @auth.load(req_url, response)
+                # httpclient
+                if identity
+                    @client.set_auth(identity['domain'], identity['username'], identity['password'])
+                    return self.request(method, url, data, headers, 2, identity)
+                end
+            end
+            identity = nil
+        when 304
+            # The cached response has been revalidated
+            if cached
+                # TODO: Update Date: and Expires:/Cache-Control: headers
+                # in cached response and re-cache, while maintaining
+                # original_date
+                key = request_key(method, req_url)
+                @cacher.freshen key
+                response, req_url = [cached, req_url]
+            else
+                raise NOMS::Command::Error.new "Server returned 304 Not Modified for #{new_url}, " +
+                    "but we were not revalidating a cached copy"
             end
         when 302, 301
-            new_url = response.header['location'].first
+            new_url = response.header('Location')
             if check_redirect new_url
-                @log.debug "redirect to #{new_url}"
                 raise NOMS::Command::Error.new "Can't follow redirect to #{new_url}: too many redirects" if tries <= 0
-                response, req_url = self.request(method, new_url, data, headers, tries - 1)
+                return self.request(method, new_url, data, headers, tries - 1, identity)
             end
         end
 
-        if identity and response.ok?
-            @log.debug "Login succeeded, saving #{identity}"
-            identity.save
+        if identity and response.success?
+            @log.debug "Login succeeded, saving #{identity['username']} @ #{identity}"
+            if @plaintext_identity
+                identity.save :file => @plaintext_identity, :encrypt => false
+            else
+                identity.save :encrypt => true
+            end
+        end
+
+        if @cache and method.to_s.upcase == 'GET' and response.cacheable?
+            cache_object = response.cacheable_copy
+            cache_object.cached!
+            if identity
+                cache_object.auth_hash = identity.verification_hash
+            end
+            key = request_key(method, req_url)
+            @cacher.set(key, cache_object.to_cache)
         end
 
-        @log.debug "<- #{response.status} #{response.reason} <- #{req_url}"
+        @log.debug "<- #{response.statusText} <- #{req_url}"
         [response, req_url]
     end
 
@@ -116,18 +196,15 @@ class NOMS::Command::UserAgent < NOMS::Command::Base
     end
 
     def add_redirect_check(&block)
-        @log.debug "Adding #{block} to redirect checks"
         @redirect_checks << block
     end
 
     def clear_redirect_checks
-        @log.debug "Clearing redirect checks"
         @redirect_checks = [ ]
     end
 
     def pop_redirect_check
         unless @redirect_checks.empty?
-            @log.debug "Popping redirect check: #{@redirect_checks[-1]}"
             @redirect_checks.pop
         end
     end

data/lib/noms/command/useragent/cache.rb

@@ -0,0 +1,124 @@
+#!ruby
+
+require 'noms/command/version'
+require 'noms/command/home'
+
+require 'pstore'
+require 'fileutils'
+
+require 'noms/command'
+
+class NOMS
+
+end
+
+class NOMS::Command
+
+end
+
+class NOMS::Command::UserAgent < NOMS::Command::Base
+
+end
+
+class NOMS::Command::UserAgent::Cache
+
+    @@format_version = '0'
+    @@max_cache_size = 10 * 1024 * 1024
+    @@trim_cache_size = 8 * 1024 * 1024
+    @@location = File.join(NOMS::Command.home, 'cache', @@format_version)
+
+    def self.clear!
+        FileUtils.rm_r @@location if File.directory? @@location
+        ensure_dir @@location
+        meta = PStore.new(File.join(@@location, 'metadata.db'))
+        meta.transaction do
+            meta[:cache_size] = 0
+            meta[:file_count] = 0
+        end
+    end
+
+    def ensure_dir(dir)
+        FileUtils.mkdir_p dir unless File.directory? dir
+    end
+
+    def initialize
+        ensure_dir @@location
+        @meta = PStore.new File.join(@@location, 'metadata.db')
+        @meta.transaction do
+            @meta[:cache_size] ||= 0
+            @meta[:file_count] ||= 0
+
+            if @meta[:cache_size] > @@max_cache_size
+                trim!
+            end
+        end
+    end
+
+    def clear!
+        FileUtils.rm_r @@location if File.directory? @@location
+        ensure_dir @@location
+        @meta.transaction do
+            @meta[:cache_size] = 0
+            @meta[:file_count] = 0
+        end
+    end
+
+    def cache_dir(key=nil)
+        if key
+            File.join(@@location, 'data', key[0 .. 1])
+        else
+            File.join(@@location, 'data')
+        end
+    end
+
+    def set(key, item_data)
+        size = 0
+        ensure_dir cache_dir(key)
+        File.open(File.join(cache_dir(key), key), 'w') do |fh|
+            fh.write item_data
+            size = fh.stat.size
+        end
+        @meta.transaction do
+            @meta[:cache_size] += size
+            @meta[:file_count] += 1
+        end
+    end
+
+    def freshen(key)
+        cache_file = File.join(cache_dir(key), key)
+        File.utime(Time.now, Time.now, cache_file)
+    end
+
+    def get(key)
+        cache_file = File.join(cache_dir(key), key)
+        obj = nil
+        if File.exist? cache_file
+            obj = File.open(cache_file, 'r') do |fh|
+                fh.read
+                # Marshal.load(fh)
+            end
+        end
+        obj
+    end
+
+    def trim!
+        # Trim cache, all within PStore transaction
+        cache_size = 0
+        files = Dir["#{@@location}/*/*"].map do |file|
+            stat = File.stat(file)
+            size = stat.size
+            mtime = stat.mtime
+            total += size
+            [file, size, mtime]
+        end.sort { |a, b| a[2] <=> b[2] }
+        file_count = files.length
+        while cache_size > @@trim_cache_size
+            file, size, = files.shift
+            File.unlink(file)
+            cache_size -= size
+        end
+        @meta[:cache_size] = cache_size
+        @meta[:file_count] = file_count
+    end
+
+end

data/lib/noms/command/useragent/requester.rb

@@ -0,0 +1,48 @@
+#!ruby
+
+require 'noms/command/version'
+
+require 'noms/command/base'
+
+class NOMS
+
+end
+
+class NOMS::Command
+
+end
+
+class NOMS::Command::UserAgent < NOMS::Command::Base
+
+end
+
+class NOMS::Command::UserAgent::Requester < NOMS::Command::Base
+
+    @@requester_class = 'httpclient'
+
+    def self.new(opts={})
+        case @@requester_class
+        when 'httpclient'
+            require 'noms/command/useragent/requester/httpclient'
+            NOMS::Command::UserAgent::Requester::HTTPClient.new(opts)
+        when 'typhoeus'
+            require 'noms/command/useragent/requester/typhoeus'
+            NOMS::Command::UserAgent::Requester::Typhoeus.new(opts)
+        else
+            raise NOMS::Command::Error.new "Internal error - no requester class #{@@requester_class}"
+        end
+    end
+
+    def initialize(opt={})
+
+    end
+
+    def request(req_attr={})
+
+    end
+
+    def set_auth(domain, user, password)
+
+    end
+
+end