bespoke 0.2.0 → 0.2.1

data/bespoke.gemspec

@@ -19,11 +19,12 @@ Gem::Specification.new do |gem|
 
   gem.add_development_dependency "bundler", ">= 1.0.0"
   gem.add_development_dependency "rspec", "~> 2.6"
-  gem.add_development_dependency "debugger"
+  gem.add_development_dependency 'guard'
+  gem.add_development_dependency 'guard-rspec'
 
   gem.add_dependency 'rake'
-  gem.add_dependency 'docile'
   gem.add_dependency 'mustache'
   gem.add_dependency 'sqlite3'
   gem.add_dependency 'sequel'
+  gem.add_dependency 'activesupport'
 end

data/lib/bespoke.rb

@@ -1,6 +1,20 @@
 require "bespoke/version"
 
+module Bespoke
+  class Error < StandardError
+    unless instance_methods.include? :cause
+      attr_reader :cause
+
+      def initialize(msg, cause = $!)
+        super(msg)
+        @cause = cause
+      end
+    end
+  end
+end
+
 require "bespoke/template"
 require "bespoke/xsltproc"
 require "bespoke/projection"
 require "bespoke/join"
+require 'bespoke/export'

data/lib/bespoke/dsl.rb

@@ -1,6 +1,5 @@
 require "bespoke"
-require "docile"
 
 def projection(name, xpath_root=nil, &block)
-  Docile.dsl_eval(Bespoke::Projection.new(name, xpath_root), &block)
-end
+  Bespoke::Projection.new(name, xpath_root, &block)
+end

data/lib/bespoke/export.rb

@@ -0,0 +1,7 @@
+module Bespoke
+  class Export
+
+  end
+end
+
+require_relative 'export/filter'

data/lib/bespoke/export/filter.rb

@@ -0,0 +1,154 @@
+require_relative 'filter/select_template'
+
+module Bespoke
+  class Export
+    class Filter
+      attr_reader :templates, :data_set
+
+      # Constructs a new filter
+      #
+      # @param dataset [Sequel::Dataset] a Sequel::Dataset, this is genrally
+      #   obtained using #[] on an instance of a Sequel::Database but, can also
+      #   be the output of another filter's `#apply` method.
+      #
+      # @param templates [Hash{Symbol => String, Array, Hash{String, Symbol => String, Symbol}}]
+      #   A hash of hashes specifying columns/aliases for  select clause output,
+      #   joins to perform (only performs left joins right now, this needs some
+      #   work) and finally a where clause to filter the results. The selects,
+      #   joins and where keys MUST be symbols, the nested hashes (when used)
+      #   MAY have either String or Symbol keys.
+      #
+      # :selects key ==
+      #    The selects key is most useful as a hash, the keys will get used as aliases
+      #    for the specified template strings or symbol column defniitions.
+      #
+      #    The following examples cover most use cases:
+      #    ```
+      #    # basic column specifications
+      #    { first_name: :students__first_name  } # => `students`.`first_name` AS 'first_name'
+      #    { 'first_name' => '<students.first_name>'  } # => `students`.`first_name` AS 'first_name'
+      #    { 'first_name' => :first_name  } # => `first_name` AS 'first_name'
+      #
+      #    # generate arbitray output strings:
+      #    { 'user_id' => 'student_<students.id>'  } # => ('student_' || `students`.`first_name`) AS 'first_name'
+      #
+      #    # use periods in table/column names by quoting
+      #    { 'first_name' => '<students."name.first_name">'  } # => `students`.`name.first_name` AS 'first_name'
+      #
+      #    # use backticks as quotes
+      #    { 'first_name' => '<students.`name.first_name`>'  } # => `students`.`name.first_name` AS 'first_name'
+      #
+      #    # coalesce values using a pipe `|` and supply a default value using single quotes
+      #    { 'course_name' =>  "<staff.admin_username | staff.teacher_username | 'default_value'>" }
+      #    # => COALESCE(`staff`.`admin_username`, `staff`.`teacher_username`, 'default_value') AS 'course_name'
+      #
+      #    # use SQL functions (N.B. this currently does no manipulation on arguments so they remain strings)
+      #    { 'course_length' =>  "<length('foobar')>" } # => LENGTH('foobar') AS 'course_length'
+      #
+      #    # delimited arrays are specially handled using concat_ws (which rejects NULLS before concatenating the results):
+      #    { 'course_name' => '<courses.name>-<terms.abbreviation>-<staff."name.last_name">'  }
+      #    # => concat_ws(`courses`.`name`, `terms`.`abbreviation`, `staff`.`name.last_name`, '-') AS 'course_name'
+      #
+      #    # in delimited arrays the delimiter can be any arbitrary sequence:
+      #    { 'course_name' => '<courses.name>=+=<terms.abbreviation>=+=<staff."name.last_name">'  }
+      #    # => concat_ws(`courses`.`name`, `terms`.`abbreviation`, `staff`.`name.last_name`, '=+=') AS 'course_name'
+      #    ```
+      #
+      # :joins key ==
+      #   The joins key is somewhat less flexible in what it will accept and
+      #   generate but it still meets current needs.
+      #
+      #   ```
+      #   # in hash specifications symbols and strings are interchangeable
+      #   # assuming we're selecting from a table named `foos`
+      #   { bars: [:id, :foo_id] } # => LEFT JOIN `bars` ON `foos`.`id` = `bars`.`foo_id`
+      #   [:bars, :quxes] # => LEFT JOIN `bars` LEFT JOIN `quxes`
+      #   ```
+      #
+      # :where key ==
+      #   This is even more limited than the joins key and I don't anticipate
+      #   it being extended because for these transforms there isn't much
+      #   filtering to be done.
+      #   This key will only accept a string that gets inserted directly into
+      #   the SQL, after extensive testing this doesn't appear to present any
+      #   SQLi potential since only the first statement gets executed and no
+      #   additional information can be pulled into the query at this point
+      #   due to theis being the last segment we support. A user can still,
+      #   however, cause the query to be invalid or reduce the amount of data
+      #   returned.
+
+      def initialize(data_set, templates)
+        @data_set = data_set
+        @set_name = data_set.opts[:from].first
+        @templates = templates
+      end
+
+      # Applys the selects and joins to the passed dataset, returns a
+      # new Dataset which can be interacted with using Enumerable methods
+      # as well as any other methods on Sequel::Dataset
+      def apply
+        ast = apply_selects(data_set, templates[:selects], templates[:functions])
+        ast = apply_joins(ast, templates[:joins])
+        apply_where(ast, templates[:where])
+      end
+
+      # #sql gives access to the SQL generated by applying the filters to the
+      # dataset, this is mostly for diagnositcs and testing since other code
+      # can interact with the dataset returned from #apply using Enumerable
+      # methods
+      def sql
+        apply.sql
+      end
+
+      private
+
+      # TODO: figure out how to allow specification of what type of join we're doing
+      #       as well as being able to join tables to ones we've already joined, maybe
+      #       we can do something like:
+      #       {table_three: ['<table_two.id>', '<table_three.two_id>', 'inner']}
+      #       with the last part being optional with the default being left
+      def apply_joins(ast, joins)
+        return ast unless joins
+        case joins
+        when String
+          ast.join(joins)
+        when Array
+          joins.inject(ast) {|ast, join|
+            ast.left_join(join)
+          }
+        when Hash
+          joins.inject(ast) {|ast, (joined_table, columns)|
+            ast.left_join(joined_table, Sequel.qualify(@set_name, columns[0]) => Sequel.qualify(joined_table, columns[1]))
+          }
+        else
+          raise "Unsupported type (#{ joins.class }) in joins key for filter template"
+        end
+      end
+
+      def apply_selects(ast, selects, functions)
+        return ast unless selects
+        case selects
+        when String
+          ast.select(selects)
+        when Hash
+          compiled_selects = selects.map do |(select_alias, template_string)|
+            SelectTemplate.compile(select_alias, template_string, functions || {})
+          end
+          ast.select(*compiled_selects)
+        else
+          raise "Unsupported type (#{ selects.class }) in selects key for filter template"
+        end
+      end
+
+      def apply_where(ast, where)
+        return ast unless where
+        case where
+        when String
+          ast.where(where)
+        else
+          raise "Unsupported type (#{ where.class }) in where key for filter template"
+        end
+      end
+    end
+  end
+end

data/lib/bespoke/export/filter/function_call.rb

@@ -0,0 +1,61 @@
+require 'active_support/core_ext/hash'
+
+module Bespoke
+  class Export
+    class Filter
+      class FunctionCall
+        EXTRACTION_REGEXP = /\A(?<function>[^\(]+)\((?<args>[^\)]*)\)\z/
+
+        DEFAULT_CUSTOM_FUNCTIONS = {
+          'random_str' => ->(args) { "LOWER(HEX(RANDOMBLOB(#{ args[0].to_i })))" },
+        }.with_indifferent_access
+
+        attr_reader :custom_functions
+
+        def self.match?(string)
+          !!EXTRACTION_REGEXP.match(string.strip)
+        end
+
+        def self.extract_from(string, custom_functions={})
+          new(string, custom_functions).extract
+        end
+
+        def initialize(string, custom_functions={})
+          @string = string.strip
+          @custom_functions = DEFAULT_CUSTOM_FUNCTIONS.merge(custom_functions)
+        end
+
+        def extract
+          if custom_function?
+            Sequel.lit(compile_custom_function)
+          else
+            Sequel.function(function.upcase, *arguments)
+          end
+        end
+
+        def function
+          call_parts[:function]
+        end
+
+        # TODO: Make this type cast integers, booleans and possibly identifiers
+        def arguments
+          @arguments ||= call_parts[:args].split(',').map(&:strip)
+        end
+
+        private
+
+        def compile_custom_function
+          @custom_functions[function.downcase].call(arguments)
+        end
+
+        def custom_function?
+          @custom_functions.keys.include? function.downcase
+        end
+
+        def call_parts
+          @function_parts ||= EXTRACTION_REGEXP.match(@string)
+        end
+      end
+    end
+  end
+end

data/lib/bespoke/export/filter/identifier.rb

@@ -0,0 +1,66 @@
+require 'strscan'
+
+module Bespoke
+  class Export
+    class Filter
+      class Identifier
+        def self.extract_from(value)
+          new(value).sequel_identifier
+        end
+
+        def initialize(unparsed_value)
+          @unparsed_value = unparsed_value.dup.strip.freeze
+        end
+
+        def sequel_identifier
+          table.nil? ? Sequel.identifier(column) : Sequel.qualify(table, column)
+        end
+
+        def column
+          parts[:column]
+        end
+
+        def table
+          parts[:table]
+        end
+
+        QUOTED_PART_EXTRACTION_REGEXP = /(?<quote>["`])(?<value>\S+?)\k<quote>/
+        def parts
+          return @parts if @parts
+          if quoted_identifier?
+            parts_array = extract_parts_from_quoted_identifier
+          else
+            parts_array = @unparsed_value.split('.')
+          end
+          @parts = {
+            table: (parts_array.length == 1 ? nil : parts_array.first),
+            column: parts_array.last,
+          }
+        end
+
+        private
+
+        def quoted_identifier?
+          !!QUOTED_PART_EXTRACTION_REGEXP.match(@unparsed_value)
+        end
+
+        def extract_parts_from_quoted_identifier
+          scanner = StringScanner.new(@unparsed_value)
+          template = @unparsed_value.dup
+          identifiers = []
+
+          while scanner.scan_until(QUOTED_PART_EXTRACTION_REGEXP)
+            quoted_part = scanner[0]
+            bare_identifier = scanner[2]
+            identifiers << bare_identifier
+            template.sub!(quoted_part, identifiers.index(bare_identifier).to_s)
+          end
+
+          parts_array = template.split('.').map{|part|
+            /\A\d+\z/.match(part) ? identifiers[part.to_i] : part
+          }
+        end
+      end
+    end
+  end
+end

data/lib/bespoke/export/filter/select_template.rb

@@ -0,0 +1,109 @@
+%w{function_call identifier string_literal}.each do |file|
+  require_relative file
+end
+
+module Bespoke
+  class Export
+    class Filter
+      class SelectTemplate
+        attr_accessor :template_string, :select_alias
+
+        TEMPLATE_CHECK_REGEXP = /<[^>]+>/
+
+        def self.compile(select_alias, template_string, custom_functions={})
+          new(select_alias, template_string, custom_functions).compile
+        end
+
+        def initialize(select_alias, template_string, custom_functions={})
+          @select_alias = select_alias
+          @template_string = template_string
+          @custom_functions = custom_functions
+        end
+
+        def compile
+          return Sequel.as(template_string, select_alias) if simple_string_template?
+          return column_definitions.first.as(select_alias) if single_column_template?
+          return Sequel.function(:concat_ws, *column_definitions, array_separator).as(select_alias) if joined_array_template?
+
+          concatenated_literals_and_columns
+        end
+
+        private
+
+        def array_separator
+          unique_string_literal_parts.first
+        end
+
+        def column_definitions
+          @column_definitions ||= extract_columns_from_template
+        end
+
+        def complex_template?
+          !!TEMPLATE_CHECK_REGEXP.match(template_string)
+        end
+
+        def concatenated_literals_and_columns
+          joined_parts = split_template_parts.zip(column_definitions).flatten.reject{|part| part == '' || part.nil?}
+
+          first_part = (joined_parts.first.is_a?(String) ? Sequel.expr(joined_parts.shift) : joined_parts.shift)
+
+          joined_parts.inject(first_part) {|ast, part|
+            ast + part
+          }.as(select_alias)
+        end
+
+        def extract_columns_from_template
+          template_string.scan(TEMPLATE_CHECK_REGEXP).map {|template_part|
+            unbracketed_template = template_part[1..-2].strip
+            identifiers = extract_identifiers(unbracketed_template)
+            (identifiers.length == 1 ? identifiers[0] : Sequel.function(:COALESCE, *identifiers))
+          }
+        end
+
+        def extract_identifiers(template_part)
+          template_part.split(/\s*\|\s*/).map {|string|
+            if StringLiteral.match?(string)
+              StringLiteral.extract_from(string)
+            elsif FunctionCall.match?(string)
+              FunctionCall.extract_from(string, @custom_functions)
+            else
+              Identifier.extract_from(string)
+            end
+          }
+        end
+
+        def joined_array_template?
+          single_unique_string_literal_part? && multiple_column_definitions?
+        end
+
+        def multiple_column_definitions?
+          column_definitions.size > 1
+        end
+
+        def simple_string_template?
+          !complex_template?
+        end
+
+        def single_column_template?
+          column_definitions.size == 1 && template_string_literals.empty?
+        end
+
+        def single_unique_string_literal_part?
+          unique_string_literal_parts.size == 1
+        end
+
+        def split_template_parts
+          @split_template_parts ||= template_string.split(TEMPLATE_CHECK_REGEXP)
+        end
+
+        def template_string_literals
+          @template_string_literals ||= split_template_parts.reject{|part| part == ""}
+        end
+
+        def unique_string_literal_parts
+          @unique_string_literal_parts ||= template_string_literals.uniq
+        end
+      end
+    end
+  end
+end