alf 0.9.3 → 0.10.0

data/lib/alf/environment/base.rb

@@ -0,0 +1,37 @@
+module Alf
+  class Environment
+    module Base
+
+      #
+      # Returns a dataset whose name is provided.
+      #
+      # This method resolves named datasets to tuple enumerables. When the 
+      # dataset exists, this method must return an Iterator, typically a 
+      # Reader instance. Otherwise, it must throw a NoSuchDatasetError.
+      #
+      # @param [Symbol] name the name of a dataset
+      # @return [Iterator] an iterator, typically a Reader instance
+      # @raise [NoSuchDatasetError] when the dataset does not exists
+      #
+      def dataset(name)
+      end
+      undef :dataset
+      
+      #
+      # Branches this environment and puts some additional explicit 
+      # definitions.
+      #
+      # This method is provided for (with ...) expressions and should not
+      # be overriden by subclasses.
+      #
+      # @param [Hash] a set of (name, Iterator) pairs.
+      # @return [Environment] an environment instance with new definitions set
+      #
+      def branch(defs)
+        Explicit.new(defs, self)
+      end
+
+    end # module Base
+    include(Base)
+  end # class Environment
+end # module Alf

data/lib/alf/environment/class_methods.rb

@@ -0,0 +1,93 @@
+module Alf
+  class Environment
+    module ClassMethods
+
+      #
+      # Returns registered environments
+      #
+      def environments
+        @environments ||= []
+      end
+      
+      #
+      # Register an environment class under a specific name. 
+      #
+      # Registered class must implement a recognizes? method that takes an array
+      # of arguments; it must returns true if an environment instance can be built
+      # using those arguments, false otherwise. Please be very specific in the 
+      # implementation for returning true. See also autodetect and recognizes?
+      #
+      # @param [Symbol] name name of the environment kind
+      # @param [Class] clazz class that implemented the environment
+      #
+      def register(name, clazz)
+        environments << [name, clazz]
+        (class << self; self; end).
+          send(:define_method, name) do |*args|
+            clazz.new(*args)
+          end
+      end
+      
+      #
+      # Auto-detect the environment to use for specific arguments.
+      #
+      # This method returns an instance of the first registered Environment class 
+      # that returns true to an invocation of recognizes?(args). It raises an 
+      # ArgumentError if no such class can be found.    
+      #
+      # @return [Environment] an environment instance
+      # @raise [ArgumentError] when no registered class recognizes the arguments
+      #
+      def autodetect(*args)
+        if (args.size == 1) && args.first.is_a?(Environment)
+          return args.first
+        else
+          environments.each do |name,clazz|
+            return clazz.new(*args) if clazz.recognizes?(args)
+          end
+        end
+        raise ArgumentError, "Unable to auto-detect Environment with #{args.inspect}"
+      end
+      
+      #
+      # (see Environment.autodetect)
+      #
+      def coerce(*args)
+        autodetect(*args)
+      end
+      
+      #
+      # Returns true _args_ can be used for building an environment instance,
+      # false otherwise.
+      #
+      # When returning true, an immediate invocation of new(*args) should 
+      # succeed. While runtime exception are admitted (no such database, for 
+      # example), argument errors should not occur (missing argument, wrong 
+      # typing, etc.).
+      #
+      # Please be specific in the implementation of this extension point, as 
+      # registered environments for a chain and each of them should have a 
+      # chance of being selected.
+      #
+      def recognizes?(args)
+        false
+      end
+      
+      #
+      # Returns the default environment
+      #
+      def default
+        examples
+      end
+      
+      #
+      # Returns the examples environment
+      #
+      def examples
+        folder File.expand_path('../../../../examples/operators', __FILE__)
+      end
+    
+    end # module ClassMethods
+    extend(ClassMethods)
+  end # class Environment
+end # module Alf

data/lib/alf/environment/explicit.rb

@@ -0,0 +1,38 @@
+module Alf
+  class Environment
+    #
+    # Specialization of Environment that works with explicitely defined 
+    # datasources and allow branching and unbranching.
+    #
+    class Explicit < Environment
+      
+      #
+      # Creates a new environment instance with initial definitions
+      # and optional child environment.
+      #
+      def initialize(defs = {}, child = nil)
+        @defs = defs
+        @child = child
+      end
+      
+      # 
+      # Unbranches this environment and returns its child
+      #
+      def unbranch
+        @child
+      end
+      
+      # (see Environment#dataset)
+      def dataset(name)
+        if @defs.has_key?(name)
+          @defs[name]
+        elsif @child
+          @child.dataset(name)
+        else
+          raise "No such dataset #{name}"
+        end 
+      end
+      
+    end # class Explicit
+  end # class Environment
+end # module Alf

data/lib/alf/environment/folder.rb

@@ -0,0 +1,62 @@
+module Alf
+  class Environment
+    #
+    # Specialization of Environment to work on files of a given folder.
+    #
+    # This kind of environment resolves datasets by simply looking at 
+    # recognized files in a specific folder. "Recognized" files are simply
+    # those for which a Reader subclass has been previously registered.
+    # This environment then serves reader instances.
+    #
+    class Folder < Environment
+      
+      # 
+      # (see Environment.recognizes?)
+      #
+      # Returns true if args contains onely a String which is an existing
+      # folder.
+      #
+      def self.recognizes?(args)
+        (args.size == 1) && 
+        args.first.is_a?(String) && 
+        File.directory?(args.first.to_s)
+      end
+      
+      #
+      # Creates an environment instance, wired to the specified folder.
+      #
+      # @param [String] folder path to the folder to use as dataset source
+      #
+      def initialize(folder)
+        @folder = folder
+      end
+      
+      # (see Environment#dataset)
+      def dataset(name)
+        if file = find_file(name)
+          Reader.reader(file, self)
+        else
+          raise "No such dataset #{name} (#{@folder})"
+        end
+      end
+      
+      protected
+      
+      def find_file(name)
+        # TODO: refactor this, because it allows getting out of the folder
+        if File.exists?(name.to_s)
+          name.to_s
+        elsif File.exists?(explicit = File.join(@folder, name.to_s)) &&
+              File.file?(explicit)
+          explicit
+        else
+          Dir[File.join(@folder, "#{name}.*")].find do |f|
+            File.file?(f)
+          end
+        end
+      end
+      
+      Environment.register(:folder, self)
+    end # class Folder
+  end # class Environment
+end # module Alf

data/lib/alf/extra/csv.rb

@@ -0,0 +1,104 @@
+module Alf
+  module CSV
+    
+    #
+    # Provides common tooling to CSV renderer and readers
+    #
+    module Commons
+
+      DEFAULT_OPTIONS = {
+        :headers => true
+      }
+      
+      private
+      
+      #
+      # Returns CSV in ruby 1.9 and FasterCSV for ruby < 1.9.
+      # This method handles require as well.
+      #
+      def get_csv_class
+        if RUBY_VERSION >= "1.9"
+          require 'csv'
+          ::CSV
+        else
+          ::Alf::Tools::friendly_require('fastercsv')
+          ::FasterCSV
+        end
+      end
+      
+      #
+      # Returns a CSV instance bound to a given io and options
+      #
+      def get_csv(io)
+        get_csv_class.new(io, options)
+      end
+      
+    end # module Commons
+    
+    #
+    # Implements Alf::Renderer contract for outputting CSV files. 
+    #
+    class Renderer < Alf::Renderer
+      include CSV::Commons
+      
+      protected
+      
+      # (see Renderer#render)
+      def render(input, output)
+        csv = get_csv(output)
+        header = nil
+        input.each do |tuple|
+          unless header 
+            header = extract_header(tuple)
+            csv << header.collect{|k| k.to_s}
+          end
+          csv << extract_row(tuple, header)
+        end
+        output
+      end
+      
+      private
+      
+      def extract_header(tuple)
+        tuple.keys
+      end
+      
+      def extract_row(tuple, header)
+        header.collect{|k| tuple[k]}
+      end
+  
+      ::Alf::Renderer.register(:csv, "as a csv file", self)
+    end # class Renderer
+    
+    #
+    # Implements Alf::Reader contract for reading CSV files.
+    #
+    class Reader < Alf::Reader
+      include CSV::Commons
+      
+      def each
+        with_input_io do |io|
+          block = Proc.new{|row|
+            next if row.header_row?
+            yield(symbolize_keys(row.to_hash))
+          }
+          case io
+          when StringIO
+            get_csv_class.parse(io.string, options, &block) 
+          else
+            get_csv(io).each(&block)
+          end
+        end
+      end
+      
+      private
+      
+      def symbolize_keys(h)
+        Hash[h.collect{|k,v| [k.to_sym,v] }]
+      end
+      
+      ::Alf::Reader.register(:csv, [".csv"], self)
+    end # class Reader
+    
+  end # module CSV
+end # module Alf

data/lib/alf/extra/logs.rb

@@ -0,0 +1,100 @@
+module Alf
+  module Logs
+        
+    #
+    # Implements Alf::Reader contract for reading log files.
+    #
+    class Reader < Alf::Reader
+      
+      DEFAULT_OPTIONS = {
+        :file_format => nil,
+        :line_def    => :access
+      }
+      
+      attr_reader :options
+      
+      def initialize(*args)
+        Alf::Tools::friendly_require('request_log_analyzer')
+        super(*args)
+        @options[:file_format] = coerce_file_format(@options[:file_format])
+      end
+      
+      def each
+        parser = infer_parser(input_path)
+        with_input_io do |io|
+          parser.parse_stream(io) do |req|
+            yield request_to_tuple(req)
+          end
+        end
+      end
+      
+      private
+      
+      def coerce_file_format(file_format)
+        case file_format
+        when NilClass
+          nil
+        when RequestLogAnalyzer::FileFormat
+          file_format
+        when Symbol
+          RequestLogAnalyzer::FileFormat.load(file_format)
+        when Array
+          RequestLogAnalyzer::FileFormat.load(*file_format)
+        else 
+          raise ArgumentError, "Invalid file format: #{file_format}"
+        end
+      end
+      
+      def infer_parser(path)
+        file_format = @options[:file_format] || begin
+          unless path
+            raise NotImplementedError, "Logs::Reader does not work on IO for now"
+          end
+          RequestLogAnalyzer::FileFormat.autodetect(path)
+        end
+        RequestLogAnalyzer::Source::LogParser.new(file_format)
+      end
+      
+      def request_to_tuple(req)
+        req.attributes
+      end
+      
+      LOG_TYPES = {
+        # RequestLogAnalyzer::Request::Converters
+        :string          => String,
+        :float           => Float,
+        :decimal         => Float,
+        :int             => Integer,
+        :integer         => Integer,
+        :sym             => Symbol,
+        :symbol          => Symbol,
+        :timestamp       => Integer,
+        :traffic         => Integer,
+        :duration        => Float,
+        :epoch           => Integer,
+        # AmazonS3
+        :nillable_string => String,
+        :referer         => String,
+        :user_agent      => String,
+        # Apache
+        :path            => String,
+        # MySQL
+        :sql             => String
+      }
+      
+      def infer_heading(format, line_def = :access)
+        h = Hash[format.line_definitions[line_def].captures.collect{|capt|
+          [ capt[:name], to_type(capt[:type]) ]
+        }]
+        Alf::Heading.new(h)
+      end
+      
+      def to_type(log_type)
+        LOG_TYPES[log_type] || String
+      end
+      
+      ::Alf::Reader.register(:logs, [".log"], self)
+    end # class Reader
+    
+  end # module Logs
+end # module Alf

data/lib/alf/extra/sequel.rb

@@ -0,0 +1,77 @@
+module Alf
+  module Sequel
+    
+    #
+    # Specialization of Alf::Environment to distribute Sequel datasets
+    #
+    class Environment < ::Alf::Environment
+      
+      #
+      # (see Alf::Environment.recognizes?)
+      #
+      # Returns true if args contains one String that can be interpreted as
+      # a valid database uri.
+      #
+      def self.recognizes?(args)
+        require 'uri'
+        return false unless (args.size == 1) && args.first.is_a?(String)
+        uri = URI::parse(args.first)
+        if uri.scheme || looks_a_sqlite_file?(args.first)
+          true
+        else
+          false
+        end
+      rescue ::URI::Error
+        false
+      end
+      
+      def self.looks_a_sqlite_file?(f)
+        (File.file?(f) && File.extname(f).==(".db"))
+      end
+      
+      # Creates an Environment instance
+      def initialize(uri, options = {})
+        @uri = self.class.looks_a_sqlite_file?(uri) ? "sqlite://#{uri}" : uri
+        @options = options
+      end
+      
+      #
+      # (see Alf::Environment#dataset)
+      # 
+      def dataset(name)
+        Iterator.new(connect[name])
+      end
+      
+      private 
+      
+      # Creates a database connection
+      def connect
+        Alf::Tools::friendly_require('sequel')
+        @db ||= ::Sequel.connect(@uri, @options)
+      end
+      
+      ::Alf::Environment.register(:sequel, self)
+    end # class Environment
+    
+    # Specialization of Alg::Iterator to work on a Sequel dataset
+    class Iterator
+      include ::Alf::Iterator
+      
+      def initialize(dataset)
+        @dataset = dataset
+      end
+      
+      # (see Alf::Iterator#each)
+      def each
+        @dataset.each(&Proc.new)
+      end
+      
+      # (see Alf::Iterator#pipe)
+      def pipe(input, env = nil)
+        self
+      end
+      
+    end # class Iterator
+    
+  end # module Sequel
+end # module Alf

data/lib/alf/extra.rb

@@ -0,0 +1,5 @@
+require 'alf/extra/yaml'
+require 'alf/extra/csv'
+require 'alf/extra/logs'
+require 'alf/extra/sequel'
+

data/lib/alf/iterator/base.rb

@@ -0,0 +1,38 @@
+module Alf
+  module Iterator
+    module Base
+
+      #
+      # Wire the iterator input and an optional execution environment.
+      #
+      # Iterators (typically Reader and Operator instances) work from input data 
+      # that come from files, or other operators, and so on. This method wires 
+      # this input data to the iterator. Wiring is required before any attempt
+      # to call each, unless autowiring occurs at construction. The exact kind of
+      # input object is left at discretion of Iterator implementations.
+      #
+      # @param [Object] input the iterator input, at discretion of the Iterator
+      #        implementation.
+      # @param [Environment] environment an optional environment for resolving
+      #        named datasets if needed.
+      # @return [Object] self
+      #
+      def pipe(input, environment = nil)
+        self
+      end
+      undef :pipe
+      
+      #
+      # Converts this iterator to an in-memory Relation.
+      #
+      # @return [Relation] a relation instance, as the set of tuples
+      #         that would be yield by this iterator.
+      #
+      def to_rel
+        Relation::coerce(self)
+      end
+
+    end # module Base
+    include(Base)
+  end # module Iterator
+end # module Alf

data/lib/alf/iterator/class_methods.rb

@@ -0,0 +1,22 @@
+module Alf
+  module Iterator
+    module ClassMethods
+
+      # 
+      # Coerces something to an iterator
+      #
+      def coerce(arg, environment = nil)
+        case arg
+        when Iterator, Array
+          arg
+        when String, Symbol
+          Proxy.new(environment, arg.to_sym)
+        else
+          Reader.coerce(arg, environment)
+        end
+      end
+
+    end # module ClassMethods
+    extend(ClassMethods)
+  end # module Iterator
+end # module Alf

data/lib/alf/iterator/proxy.rb

@@ -0,0 +1,33 @@
+module Alf
+  module Iterator
+    class Proxy
+      include Iterator
+
+      # @return [Symbol] name of the dataset to request to environment
+      attr_reader :dataset
+
+      #
+      # Creates a proxy instance.
+      #
+      # @param [Environment] env the environment serving iterator instances
+      # @param [Symbol] dataset named dataset to rely on
+      #
+      def initialize(env, dataset)
+        @environment, @dataset = env, dataset
+      end
+
+      # (see Iterator#pipe)
+      def pipe(input, environment = nil)
+        @environment ||= environment 
+        @dataset ||= input
+        self
+      end
+
+      # (see Iterator#each)
+      def each(&block)
+        @environment.dataset(@dataset).each(&block)
+      end
+
+    end # class Proxy
+  end # module Iterator
+end # module Alf