masamune 0.11.9 → 0.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e71d5026f78a0babee52533490fe3d79da306451
-  data.tar.gz: 4e78ab8efb0c9697984707dd4f242b061a50b858
+  metadata.gz: a12e8b8ce6cb92c5d551e9024053d50ff59f3cd6
+  data.tar.gz: 72bdc252c7b9b6f9e787b211eae46bc3edb69358
 SHA512:
-  metadata.gz: 16462d0f049ee2caac1f540dbe2266b268b58f601762edb46d69d7b3db676ff9af173d212bf20c3526f4c40307bed4b9d02ccce4c01adde2c5891e73c421c581
-  data.tar.gz: 1f7fc256a7561808e36390f0bad66c6add14f89b71d36d3ea91e8650ed5f5c719b7cfdc851991e4f4c6676c7206b6e513a842a83bf2b797085978cc00d2f799b
+  metadata.gz: 96be3850b76ec158f8a13be46ec2a5108c9ec01f99987f0a9414602b4072b768e072b72906a28ac9b8aa101940ece9e0f1255c2bcf25bc6cdf2726d9c7783c36
+  data.tar.gz: 31863312d8fdbbad89e1e0a57fcbbe51b754feca838891d1f02cefe09f547da431f6089e36e6dafc56aa20cabea8a4d0cec432b0c0b07e11c496714668388942

data/lib/masamune/actions/transform.rb

@@ -49,21 +49,10 @@ module Masamune::Actions
     FILE_MODE = 0777 - File.umask
 
     def load_dimension(source_files, source, target)
-      output = Tempfile.new('masamune')
-      FileUtils.chmod(FILE_MODE, output.path)
-
-      if source.respond_to?(:map) and map = source.map(to: target)
-        result = map.apply(source_files, output)
-      else
-        output = source_files
-        result = source
+      optional_apply_map(source_files, source, target) do |intermediate_files, intermediate|
+        transform = Wrapper.load_dimension(intermediate_files, intermediate, target)
+        postgres file: transform.to_file, debug: (source.debug || target.debug || intermediate.debug)
       end
-
-      transform = Wrapper.load_dimension(output, result, target)
-      logger.debug(File.read(output)) if (source.debug || map.debug)
-      postgres file: transform.to_file, debug: (source.debug || target.debug || map.debug)
-    ensure
-      output.unlink
     end
 
     def consolidate_dimension(target)
@@ -77,13 +66,39 @@ module Masamune::Actions
     end
 
     def load_fact(source_files, source, target, date)
-      transform = Wrapper.load_fact(source_files, source, target, date)
-      postgres file: transform.to_file, debug: (source.debug || target.debug)
+      optional_apply_map(source_files, source, target) do |intermediate_files, intermediate|
+        transform = Wrapper.load_fact(intermediate_files, intermediate, target, date)
+        postgres file: transform.to_file, debug: (source.debug || target.debug || intermediate.debug)
+      end
     end
 
     def rollup_fact(source, target, date)
       transform = Wrapper.rollup_fact(source, target, date)
       postgres file: transform.to_file, debug: (source.debug || target.debug)
     end
+
+    private
+
+    def optional_apply_map(source_files, source, target, &block)
+      if source.respond_to?(:map) and map = source.map(to: target)
+        apply_map(map, source_files, source, target, &block)
+      else
+        yield source_files, source
+      end
+    end
+
+    def apply_map(map, source_files, source, target)
+      Tempfile.open('masamune') do |output|
+        begin
+          FileUtils.chmod(FILE_MODE, output.path)
+          result = map.apply(source_files, output)
+          result.debug = map.debug
+          logger.debug(File.read(output)) if (source.debug || result.debug)
+          yield output, result
+        ensure
+          output.unlink
+        end
+      end
+    end
   end
 end

data/lib/masamune/schema.rb

@@ -30,7 +30,6 @@ module Masamune
     require 'masamune/schema/fact'
     require 'masamune/schema/column'
     require 'masamune/schema/row'
-    require 'masamune/schema/event'
     require 'masamune/schema/map'
   end
 end

data/lib/masamune/schema/catalog.rb

@@ -173,7 +173,7 @@ module Masamune::Schema
     end
 
     def file(id, options = {})
-      format_options = options.extract!(:format, :headers)
+      format_options = options.extract!(:format, :headers, :json_encoding)
       @context.push(options)
       yield if block_given?
       store = HasFormat.new(@context, format_options)
@@ -182,14 +182,6 @@ module Masamune::Schema
       @context.pop
     end
 
-    def event(id, options = {})
-      @context.push(options)
-      yield if block_given?
-      @context.events[id] = HasMap.new Masamune::Schema::Event.new(@context.options.merge(id: id))
-    ensure
-      @context.pop
-    end
-
     def attribute(id, options = {})
       @context.options[:attributes] << Masamune::Schema::Event::Attribute.new(options.merge(id: id))
     end
@@ -200,7 +192,7 @@ module Masamune::Schema
       raise ArgumentError, "invalid map, from: is missing" unless from && from.try(:id)
       raise ArgumentError, "invalid map from: '#{from.id}', to: is missing" unless to
       @context.push(options)
-      @context.options[:function] = block.to_proc
+      @context.options[:function] = block.to_proc if block
       from.maps[to] ||= Masamune::Schema::Map.new(@context.options.merge(source: from, target: to))
     ensure
       @context.pop

data/lib/masamune/schema/column.rb

@@ -133,7 +133,7 @@ module Masamune::Schema
     def compact_name
       if reference
         # XXX once columns only reference columns, this can be cleaned up
-        if @id == reference.surrogate_key.reference_name(reference.label)
+        if reference.surrogate_key && @id == reference.surrogate_key.reference_name(reference.label)
           "#{reference.id}.#{reference.surrogate_key.id}".to_sym
         else
           "#{reference.id}.#{@id}".to_sym
@@ -184,12 +184,12 @@ module Masamune::Schema
       array_value? ? "#{elem}[]" : elem
     end
 
-    def hql_type
+    def hql_type(for_surrogate_key = false)
       elem =
       case type
       when :integer
-        'INT'
-      when :string
+        for_surrogate_key ? 'STRING' : 'INT'
+      when :string, :enum, :key_value, :timestamp
         'STRING'
       else
         sql_type
@@ -264,7 +264,9 @@ module Masamune::Schema
         when Date, DateTime
           value.to_time
         when String
-          if value =~ /\A\d+\z/
+          if value.blank?
+            nil
+          elsif value =~ /\A\d+\z/
             Time.at(value.to_i)
           else
             Time.parse(value)
@@ -281,7 +283,7 @@ module Masamune::Schema
         when Hash
           value
         when String
-          ruby_key_value(YAML.load(value))
+          YAML.load(value)
         when nil
           {}
         end
@@ -290,7 +292,7 @@ module Masamune::Schema
         when Hash
           value
         when String
-          ruby_key_value(JSON.load(value))
+          JSON.load(value)
         when nil
           {}
         end
@@ -366,6 +368,10 @@ module Masamune::Schema
       [name, sql_type(surrogate_key), *sql_constraints, reference_constraint, sql_default].compact.join(' ')
     end
 
+    def as_hql
+      [name, hql_type(surrogate_key)].compact.join(' ')
+    end
+
     def as_hash
       {id: id}.tap do |hash|
         DEFAULT_ATTRIBUTES.keys.each do |attr|
@@ -462,11 +468,9 @@ module Masamune::Schema
     end
 
     def required_value?
-      if reference
-        !(reference.null || reference.default)
-      else
-        surrogate_key || natural_key || !(null || default)
-      end
+      return false if reference && (reference.null || !reference.default.nil?)
+      return false if null || !default.nil?
+      true
     end
 
     private
@@ -495,24 +499,6 @@ module Masamune::Schema
       end
     end
 
-    def ruby_key_value(hash)
-      case sub_type
-      when :boolean
-        Hash[hash.map { |key, value| ruby_boolean_key_value(key, value) }.compact]
-      else
-        hash
-      end
-    end
-
-    def ruby_boolean_key_value(key, value)
-      case value
-      when true, '1', 1
-        [key, true]
-      when false, '0', 0
-        [key, false]
-      end
-    end
-
     def csv_array(value)
       case value
       when Array

data/lib/masamune/schema/dimension.rb

@@ -77,15 +77,8 @@ module Masamune::Schema
         next if column.surrogate_key
         next if reserved_column_ids.include?(column.id)
 
-        if column.type == :key_value
-          column_now, column_was = column.dup, column.dup
-          column_now.id, column_was.id = "#{column.id}_now", "#{column.id}_was"
-          column_now.strict, column_was.strict = false, false
-          [column_now, column_was]
-        else
-          column.dup.tap do |column_copy|
-            column_copy.strict = false unless column.surrogate_key || column.natural_key || (column.reference && column.reference.surrogate_key.auto)
-          end
+        column.dup.tap do |column_copy|
+          column_copy.strict = false unless column.surrogate_key || column.natural_key || (column.reference && column.reference.surrogate_key.auto)
         end
       end.flatten
     end

data/lib/masamune/schema/fact.rb

@@ -82,10 +82,6 @@ module Masamune::Schema
       @partition_tables[partition_range] ||= self.class.new(id: @id, store: store, columns: partition_table_columns, parent: self, range: partition_range, grain: grain, inherit: true)
     end
 
-    def partitions
-      columns.select { |_, column| column.partition }
-    end
-
     def measures
       columns.select { |_, column| column.measure }
     end

data/lib/masamune/schema/map.rb

@@ -239,7 +239,7 @@ module Masamune::Schema
     def default_row(columns)
       {}.tap do |row|
         columns.each do |_, column|
-          row[column.name] = column.default_ruby_value
+          row[column.compact_name] = column.default_ruby_value
         end
       end
     end

data/lib/masamune/schema/row.rb

@@ -89,7 +89,7 @@ module Masamune::Schema
     end
 
     def headers
-      values.keys
+      @columns.map { |_, column| column.name }
     end
 
     def serialize
@@ -121,8 +121,8 @@ module Masamune::Schema
       values.each do |key, value|
         next unless key
         if column = parent.dereference_column_name(key)
-          @columns[column.name] = column
-          result[column.name] = column.ruby_value(value)
+          @columns[column.compact_name] = column
+          result[column.compact_name] = column.ruby_value(value)
         elsif strict
           raise ArgumentError, "#{@values} contains undefined columns #{key}"
         end

data/lib/masamune/schema/store.rb

@@ -26,7 +26,7 @@ module Masamune::Schema
   class Store
     include Masamune::HasEnvironment
 
-    SUPPORTED_ATTRIBUTES = %(table dimension fact file event)
+    SUPPORTED_ATTRIBUTES = %(table dimension fact file)
 
     DEFAULT_ATTRIBUTES =
     {
@@ -45,7 +45,6 @@ module Masamune::Schema
     attr_accessor :dimensions
     attr_accessor :facts
     attr_accessor :files
-    attr_accessor :events
     attr_accessor :references
 
     class << self
@@ -67,7 +66,6 @@ module Masamune::Schema
       @dimensions = {}.with_indifferent_access
       @facts      = {}.with_indifferent_access
       @files      = {}.with_indifferent_access
-      @events     = {}.with_indifferent_access
       @references = {}.with_indifferent_access
       @extra      = []
     end

data/lib/masamune/schema/table.rb

@@ -39,7 +39,8 @@ module Masamune::Schema
       columns:         {},
       rows:            [],
       inherit:         false,
-      debug:           false
+      debug:           false,
+      properties:      {}
     }
 
     DEFAULT_ATTRIBUTES.keys.each do |attr|
@@ -111,7 +112,7 @@ module Masamune::Schema
     end
 
     def defined_columns
-      columns.values
+      columns.values.reject { |column| column.partition }
     end
     method_with_last_element :defined_columns
 
@@ -154,6 +155,10 @@ module Masamune::Schema
       columns.values.select { | column| column.reference && column.reference.foreign_key }
     end
 
+    def partitions
+      columns.select { |_, column| column.partition }
+    end
+
     def insert_rows
       rows.select { |row| row.insert_values.any? }
     end
@@ -174,6 +179,27 @@ module Masamune::Schema
       columns.reject { |_, column| reserved_column_ids.include?(column.id) }
     end
 
+    def denormalized_columns
+      columns.map do |_, column|
+        next if column.surrogate_key || column.ignore
+        if column.reference
+          column.reference.natural_keys.any? ? column.reference.natural_keys : column.reference.denormalized_columns
+        else
+          column
+        end
+      end.flatten.compact
+    end
+
+    def denormalized_column_names
+      denormalized_columns.map do |column|
+        if column.parent == self
+          column.name.to_s
+        else
+          [column.parent.id, column.name].join('.')
+        end
+      end
+    end
+
     def stage_table(options = {})
       selected = options[:columns] if options[:columns]
       selected ||= options[:target].columns.values.map(&:compact_name) if options[:target]

data/lib/masamune/transform.rb

@@ -25,7 +25,6 @@ module Masamune
     require 'masamune/transform/operator'
 
     require 'masamune/transform/define_table'
-    require 'masamune/transform/define_event_view'
     require 'masamune/transform/define_schema'
     require 'masamune/transform/denormalize_table'
 

data/lib/masamune/transform/define_schema.rb

@@ -21,12 +21,10 @@
 #  THE SOFTWARE.
 
 require 'masamune/transform/define_table'
-require 'masamune/transform/define_event_view'
 
 module Masamune::Transform
   module DefineSchema
     include DefineTable
-    include DefineEventView
 
     extend ActiveSupport::Concern
 
@@ -44,10 +42,6 @@ module Masamune::Transform
         operators << define_table(fact)
       end
 
-      context.events.each do |_, event|
-        operators << define_event_view(event)
-      end
-
       operators += context.extra(:post)
 
       Operator.new __method__, *operators, source: context

data/lib/masamune/transform/define_table.hql.erb

@@ -22,13 +22,14 @@
 
 CREATE TABLE IF NOT EXISTS <%= target.name %>
 (
-  <%- target.reference_columns.each do |column| -%>
-  <%= column.name %> <%= column.hql_type %>,
+  <%- target.defined_columns.each do |column, last| -%>
+  <%= column.as_hql %><%= ',' unless last %>
   <%- end -%>
-  <%- target.measures.each do |_, measure| -%>
-  <%= measure.name %> <%= measure.hql_type %>,
-  <%- end -%>
-  <%= target.time_key.name %> <%= target.time_key.hql_type %>
 )
+<%- if target.partition_by -%>
 PARTITIONED BY (<%= target.partition_by %>)
+<%- end -%>
+<%- if target.properties[:format] == :tsv -%>
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
+<%- end -%>
 TBLPROPERTIES ('serialization.null.format' = '');

data/lib/masamune/transform/define_table.rb

@@ -33,6 +33,7 @@ module Masamune::Transform
 
     class Hive < SimpleDelegator
       def partition_by
+        return unless partitions.any?
         partitions.map { |_, column| "#{column.name} #{column.hql_type}" }.join(', ')
       end
     end

data/lib/masamune/transform/{define_event_view.hql.erb → denormalize_table.hql.erb}

@@ -20,32 +20,14 @@
 --  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 --  THE SOFTWARE.
 
-DROP VIEW IF EXISTS <%= target.view_name %>;
-CREATE VIEW IF NOT EXISTS <%= target.view_name %> (
-  uuid,
-  type,
-  <%- target.view_columns.each do |value| -%>
-  <%= value %>,
+SELECT
+  <%- target.select_columns(columns).each do |column, last| -%>
+  <%= column %><%= ',' unless last %>
   <%- end -%>
-  delta,
-  created_at,
-  y, m, d ,h
-) PARTITIONED ON (y, m, d, h) AS
-SELECT DISTINCT
-  uuid,
-  type,
-  <%- target.view_values.each do |value| -%>
-  <%= value %>,
-  <%- end -%>
-  IF(type = '<%= target.update_type %>', 1, 0) AS delta,
-  ctime_iso8601 AS created_at,
-  y, m, d ,h
 FROM
-  events
-LATERAL VIEW
-  json_tuple(events.json, <%= target.view_columns.map { |value| "'#{value}'" }.join(', ') %>) event_data AS <%= target.view_columns.join(', ') %>
-WHERE
-  type = '<%= target.create_type %>' OR
-  type = '<%= target.update_type %>' OR
-  type = '<%= target.delete_type %>'
+  <%= target.name %>
+ORDER BY
+  <%- target.order_by_columns(order_by).each do |column, last| -%>
+  <%= column %><%= ',' unless last %>
+  <%- end -%>
 ;