masamune 0.11.9 → 0.12.0

data/lib/masamune/transform/denormalize_table.psql.erb

@@ -33,7 +33,7 @@ ON
   <%= condition %>
 <%- end -%>
 ORDER BY
-  <%- target.order_by_columns(columns).each do |column, last| -%>
+  <%- target.order_by_columns(order_by).each do |column, last| -%>
   <%= column %><%= ',' unless last %>
   <%- end -%>
 ;

data/lib/masamune/transform/denormalize_table.rb

@@ -24,20 +24,29 @@ module Masamune::Transform
   module DenormalizeTable
     extend ActiveSupport::Concern
 
-    def denormalize_table(target, columns = [])
-      Operator.new(__method__, target: target, columns: columns, presenters: { postgres: Postgres })
+    def denormalize_table(target, options = {})
+      options.symbolize_keys!
+      columns = options[:include] || []
+      columns += options[:columns] || target.denormalized_column_names
+      columns -= options[:except] || []
+      order_by = options[:order] || columns
+      Operator.new(__method__, target: target, columns: columns, order_by: order_by, presenters: { postgres: Common, hive: Common })
     end
 
     private
 
-    class Postgres < SimpleDelegator
+    class Common < SimpleDelegator
       include Masamune::LastElement
 
       def select_columns(column_names)
         column_names.map do |column_name|
           next unless column = dereference_column_name(column_name)
           if column.reference
-            "#{column.foreign_key_name} AS #{column.name}"
+            if column.reference.implicit || column.reference.degenerate
+              "#{column.name} AS #{column.name}"
+            else
+              "#{column.foreign_key_name} AS #{column.name}"
+            end
           else
             column.qualified_name
           end

data/lib/masamune/transform/snapshot_dimension.rb

@@ -56,7 +56,7 @@ module Masamune::Transform
           if column.natural_key
             "#{column.name} AS #{column.name}"
           elsif column.type == :key_value
-            "hstore_merge(#{column.name}_now) OVER #{window} - hstore_merge(#{column.name}_was) OVER #{window} AS #{column.name}"
+            "hstore_merge(#{column.name}) OVER #{window} AS #{column.name}"
           else
             "coalesce_merge(#{column.name}) OVER #{window} AS #{column.name}"
           end

data/lib/masamune/transform/stage_fact.rb

@@ -77,7 +77,7 @@ module Masamune::Transform
 
             if column.reference && !column.reference.default.nil?
               coalesce_values << column.reference.default(column.adjacent) if column.adjacent.natural_key
-            elsif !column.adjacent.default.nil?
+            elsif column.adjacent && !column.adjacent.default.nil?
               coalesce_values << column.adjacent.sql_value(column.adjacent.default)
             end
 

data/lib/masamune/version.rb

@@ -21,5 +21,5 @@
 #  THE SOFTWARE.
 
 module Masamune
-  VERSION = '0.11.9'
+  VERSION = '0.12.0'
 end

data/spec/masamune/actions/transform_spec.rb

@@ -46,16 +46,6 @@ describe Masamune::Actions::Transform do
         column 'updated_at', type: :timestamp
       end
 
-      map from: postgres.user_file, to: postgres.user_dimension do |row|
-        {
-          user_id: row[:id],
-          tenant_id: row[:tenant_id],
-          source_kind: 'users',
-          start_at: row[:updated_at],
-          delta: 1
-        }
-      end
-
       fact 'visits', partition: 'y%Ym%m', grain: %w(hourly daily monthly) do
         references :date
         references :user
@@ -89,13 +79,37 @@ describe Masamune::Actions::Transform do
   end
 
   describe '.load_dimension' do
-    before do
-      mock_command(/\Apsql/, mock_success)
+    subject { instance.load_dimension(source_file, postgres.user_file, postgres.user_dimension) }
+
+    context 'without :map' do
+      before do
+        expect_any_instance_of(Masamune::Schema::Map).to_not receive(:apply)
+        mock_command(/\Apsql/, mock_success)
+      end
+
+      it { is_expected.to be_success }
     end
 
-    subject { instance.load_dimension(source_file, postgres.user_file, postgres.user_dimension) }
+    context 'with :map' do
+      before do
+        catalog.schema :postgres do
+          map from: postgres.user_file, to: postgres.user_dimension do |row|
+            {
+              user_id: row[:id],
+              tenant_id: row[:tenant_id],
+              source_kind: 'users',
+              start_at: row[:updated_at],
+              delta: 1
+            }
+          end
+        end
+
+        expect_any_instance_of(Masamune::Schema::Map).to receive(:apply).and_call_original
+        mock_command(/\Apsql/, mock_success)
+      end
 
-    it { is_expected.to be_success }
+      it { is_expected.to be_success }
+    end
   end
 
   describe '.relabel_dimension' do
@@ -121,13 +135,31 @@ describe Masamune::Actions::Transform do
   describe '.load_fact' do
     let(:date) { DateTime.civil(2014, 8) }
 
-    before do
-      mock_command(/\Apsql/, mock_success)
+    context 'without :map' do
+      before do
+        expect_any_instance_of(Masamune::Schema::Map).to_not receive(:apply)
+        mock_command(/\Apsql/, mock_success)
+      end
+
+      subject { instance.load_fact(source_file, postgres.visits_hourly_file, postgres.visits_hourly_fact, date) }
+
+      it { is_expected.to be_success }
     end
 
-    subject { instance.load_fact(source_file, postgres.visits_hourly_file, postgres.visits_hourly_fact, date) }
+    context 'with :map' do
+      before do
+        catalog.schema :postgres do
+          map from: postgres.visits_hourly_file, to: postgres.visits_hourly_fact, distinct: true
+        end
 
-    it { is_expected.to be_success }
+        expect_any_instance_of(Masamune::Schema::Map).to receive(:apply).and_call_original
+        mock_command(/\Apsql/, mock_success)
+      end
+
+      subject { instance.load_fact(source_file, postgres.visits_hourly_file, postgres.visits_hourly_fact, date) }
+
+      it { is_expected.to be_success }
+    end
   end
 
   describe '.rollup_fact' do

data/spec/masamune/schema/catalog_spec.rb

@@ -332,30 +332,6 @@ describe Masamune::Schema::Catalog do
       it { expect(visits_monthly.measures).to include :count }
     end
 
-    context 'when schema contains events' do
-      before do
-        instance.schema :hive do
-          event 'event_one' do
-            attribute 'attribute_one'
-            attribute 'attribute_two'
-          end
-
-          event 'event_two' do
-            attribute 'attribute_three'
-            attribute 'attribute_four'
-          end
-        end
-      end
-
-      let(:event_one) { hive.event_one_event }
-      let(:event_two) { hive.event_two_event }
-
-      it { expect(event_one.attributes).to include :attribute_one }
-      it { expect(event_one.attributes).to include :attribute_two }
-      it { expect(event_two.attributes).to include :attribute_three }
-      it { expect(event_two.attributes).to include :attribute_four }
-    end
-
     context 'when schema contains file' do
       before do
         instance.schema :postgres do
@@ -456,35 +432,6 @@ describe Masamune::Schema::Catalog do
       end
     end
 
-    context 'when schema contains map from: event' do
-      before do
-        instance.schema :postgres do
-          dimension 'user', type: :mini do
-            column 'user_id', type: :integer, natural_key: true
-            column 'name', type: :string
-          end
-
-          event 'users' do
-            attribute 'id', type: :integer, immutable: true
-            attribute 'name', type: :string
-          end
-
-          map from: postgres.users_event, to: postgres.user_dimension do |row|
-            {
-              'user_id' => row[:id],
-              'name'    => row[:name_now]
-            }
-          end
-        end
-      end
-
-      subject(:map) { postgres.users_event.map(to: postgres.user_dimension) }
-
-      it 'constructs map' do
-        expect(map.function).to_not be_nil
-      end
-    end
-
     context 'when schema contains map missing the from: field' do
       subject(:schema) do
         instance.schema :postgres do

data/spec/masamune/schema/column_spec.rb

@@ -433,31 +433,6 @@ describe Masamune::Schema::Column do
         it { is_expected.to eq({"k" => "v"}) }
       end
     end
-
-    context 'with type :yaml and sub_type :boolean' do
-      let(:column) { described_class.new(id: 'yaml', type: :yaml, sub_type: :boolean) }
-      let(:value) do
-        {
-          'true'         => true,
-          'one'          => '1',
-          'zero'         => '0',
-          'false'        => false,
-          'string'       => 'string',
-          'one_integer'  => 1,
-          'zero_integer' => 0
-        }.to_yaml
-      end
-
-      it 'should cast yaml to ruby' do
-        expect(result['true']).to eq(true)
-        expect(result['false']).to eq(false)
-        expect(result['one']).to eq(true)
-        expect(result['zero']).to eq(false)
-        expect(result['one_integer']).to eq(true)
-        expect(result['zero_integer']).to eq(false)
-        expect(result.key?('string')).to eq(false)
-      end
-    end
   end
 
   describe '#default_ruby_value' do
@@ -665,8 +640,13 @@ describe Masamune::Schema::Column do
       it { is_expected.to eq(false) }
     end
 
-    context 'when surrogate_key' do
-      let(:column) { described_class.new id: 'name', type: :string, surrogate_key: true }
+    context 'when column has default of false' do
+      let(:column) { described_class.new id: 'flag', type: :boolean, default: false }
+      it { is_expected.to eq(false) }
+    end
+
+    context 'when column has reference' do
+      let(:column) { described_class.new id: 'name', type: :string }
       it { is_expected.to eq(true) }
 
       context 'when reference allow null' do
@@ -682,22 +662,10 @@ describe Masamune::Schema::Column do
         end
         it { is_expected.to eq(false) }
       end
-    end
-
-    context 'when natural_key' do
-      let(:column) { described_class.new id: 'name', type: :string, natural_key: true }
-      it { is_expected.to eq(true) }
 
-      context 'when reference allow null' do
+      context 'when reference has default of false' do
         before do
-          allow(column).to receive(:reference).and_return(double(null: true, default: nil))
-        end
-        it { is_expected.to eq(false) }
-      end
-
-      context 'when reference has default' do
-        before do
-          allow(column).to receive(:reference).and_return(double(null: false, default: 'Unknown'))
+          allow(column).to receive(:reference).and_return(double(null: false, default: false))
         end
         it { is_expected.to eq(false) }
       end

data/spec/masamune/schema/fact_spec.rb

@@ -47,7 +47,9 @@ describe Masamune::Schema::Fact do
         Masamune::Schema::TableReference.new(user_dimension)
       ],
       columns: [
-        Masamune::Schema::Column.new(id: 'total', type: :integer)
+        Masamune::Schema::Column.new(id: 'total', type: :integer),
+        Masamune::Schema::Column.new(id: 'y', type: :integer, partition: true),
+        Masamune::Schema::Column.new(id: 'm', type: :integer, partition: true)
       ]
   end
 

data/spec/masamune/schema/map_spec.rb

@@ -52,32 +52,6 @@ describe Masamune::Schema::Map do
         column 'deleted_at', type: :timestamp, null: true
       end
     end
-
-    catalog.schema :hive do
-      event 'user' do
-        attribute 'id', type: :integer, immutable: true
-        attribute 'tenant_id', type: :integer, immutable: true
-        attribute 'admin', type: :boolean
-        attribute 'preferences', type: :json
-      end
-
-      dimension 'tenant', type: :two, implicit: true do
-        column 'tenant_id'
-      end
-
-      fact 'user' do
-        references :tenant
-        measure 'delta'
-      end
-
-      file 'user' do
-        column 'id', type: :integer
-        column 'tenant_id', type: :integer
-        column 'admin', type: :boolean
-        column 'preferences', type: :json
-        column 'deleted_at', type: :timestamp, null: true
-      end
-    end
   end
 
   context 'without source' do
@@ -136,22 +110,26 @@ describe Masamune::Schema::Map do
     end
 
     context 'with undefined function' do
-      let(:source) { catalog.hive.user_event }
-      let(:target) { catalog.hive.user_fact }
-      let(:source_data) { '' }
-      let(:target_data) { '' }
-
       before do
-        catalog.schema :hive do
-          map from: hive.user_event, to: hive.user_fact do |row|
+        catalog.schema :files do
+          file 'input'
+          file 'output'
+
+          map from: files.input , to: files.output do |row|
+            # Empty
           end
         end
       end
 
+      let(:source) { catalog.files.input }
+      let(:target) { catalog.files.output }
+      let(:source_data) { '' }
+      let(:target_data) { '' }
+
       it { expect { subject }.to raise_error ArgumentError, /function for map between .* does not return output for default input/ }
     end
 
-    context 'from csv file to dimension' do
+    context 'from csv file to postgres dimension' do
       before do
         catalog.schema :files do
           map from: postgres.user_file, to: postgres.user_dimension, distinct: true do |row|
@@ -161,7 +139,7 @@ describe Masamune::Schema::Map do
               'user_account_state.name'    => row[:deleted_at] ? 'deleted' :  'active',
               'hr_user_account_state.name' => row[:deleted_at] ? 'deleted' :  'active',
               'admin'                      => row[:admin],
-              'preferences_now'            => row[:preferences],
+              'preferences'                => row[:preferences],
               'source'                     => 'users_file',
               'cluster_id'                 => 100
             }
@@ -193,7 +171,7 @@ describe Masamune::Schema::Map do
 
       let(:target_data) do
         <<-EOS.strip_heredoc
-          tenant_id,user_id,user_account_state_type_name,hr_user_account_state_type_name,admin,preferences_now,source,cluster_id
+          tenant_id,user_id,user_account_state_type_name,hr_user_account_state_type_name,admin,preferences,source,cluster_id
           30,1,active,active,FALSE,{},users_file,100
           40,2,deleted,deleted,TRUE,"{""enabled"":true}",users_file,100
         EOS
@@ -210,19 +188,26 @@ describe Masamune::Schema::Map do
       it_behaves_like 'apply input/output'
     end
 
-    context 'from event to postgres dimension with quoted json' do
+    context 'from tsv file to postgres dimension' do
       before do
         catalog.schema :files do
-          map from: hive.user_event, to: postgres.user_dimension do |row|
+          file 'input', format: :tsv, headers: false do
+            column 'id', type: :integer
+            column 'tenant_id', type: :integer
+            column 'admin', type: :boolean
+            column 'preferences', type: :json
+            column 'deleted_at', type: :timestamp, null: true
+          end
+
+          map from: files.input, to: postgres.user_dimension do |row|
             raise if row[:tenant_id] == 42
             {
               'tenant_id'               => row[:tenant_id],
               'user_id'                 => row[:id],
-              'user_account_state.name' => row[:type] =~ /delete/ ? 'deleted' : 'active',
-              'admin'                   => row[:type] =~ /delete/ ? row[:admin_was] : row[:admin_now],
-              'preferences_now'         => row[:preferences_now],
-              'preferences_was'         => row[:preferences_was],
-              'source'                  => 'user_event',
+              'user_account_state.name' => row[:deleted_at] ? 'deleted' : 'active',
+              'admin'                   => row[:admin],
+              'preferences'             => row[:preferences],
+              'source'                  => 'user_file',
               'cluster_id'              => 100
             }
           end
@@ -230,7 +215,7 @@ describe Masamune::Schema::Map do
       end
 
       let(:source) do
-        catalog.hive.user_event
+        catalog.files.input
       end
 
       let(:target) do
@@ -242,72 +227,113 @@ describe Masamune::Schema::Map do
         expect(environment.logger).to receive(:warn).with(/failed to parse '{.*}' for #{source.name}/).ordered
       end
 
-      let(:source_data) do
-        <<-EOS.strip_heredoc
-          X	user_create	1	30	0	\\N	\\N	\\N
-          # NOTE intentional duplicate record
-          X	user_create	1	30	0	\\N	\\N	\\N
-          A	user_create	1	42	0	\\N	\\N	\\N
-          Y	user_delete	2	40	0	1	"{""enabled"":true}"	\\N
-          # NOTE record is intentionally invalid
-          Z	user_create	3	50	0	1	INVALID_JSON	\\N
-        EOS
-      end
-
       let(:target_data) do
         <<-EOS.strip_heredoc
-          tenant_id,user_id,user_account_state_type_name,admin,preferences_now,preferences_was,source,cluster_id
-          30,1,active,FALSE,{},{},user_event,100
-          30,1,active,FALSE,{},{},user_event,100
-          40,2,deleted,TRUE,"{""enabled"":true}",{},user_event,100
+          tenant_id,user_id,user_account_state_type_name,admin,preferences,source,cluster_id
+          30,1,active,FALSE,{},user_file,100
+          30,1,active,FALSE,{},user_file,100
+          40,2,deleted,TRUE,"{""enabled"":true}",user_file,100
         EOS
       end
 
-      it 'should match target data' do
-        is_expected.to eq(target_data)
+      context 'with quoted json' do
+        let(:source_data) do
+          <<-EOS.strip_heredoc
+            1	30	0
+            # NOTE intentional duplicate record
+            1	30	0
+            1	42	0
+            2	40	1	"{""enabled"":true}"	2015-07-19 00:00:00
+            # NOTE record is intentionally invalid
+            3	50	0	INVALID_JSON
+          EOS
+        end
+
+        it 'should match target data' do
+          is_expected.to eq(target_data)
+        end
+
+        it_behaves_like 'apply input/output'
       end
 
-      it_behaves_like 'apply input/output'
+      context 'with raw json' do
+        let(:source_data) do
+          <<-EOS.strip_heredoc
+            1	30	0
+            # NOTE intentional duplicate record
+            1	30	0
+            1	42	0
+            2	40	1	{"enabled":true}	2015-07-19 00:00:00
+            # NOTE record is intentionally invalid
+            3	50	0	INVALID_JSON
+          EOS
+        end
+
+        it 'should match target data' do
+          is_expected.to eq(target_data)
+        end
+
+        it_behaves_like 'apply input/output'
+      end
     end
 
-    context 'from event to tsv file' do
+    context 'from tsv file to csv file' do
       before do
         catalog.schema :files do
-          map from: hive.user_event, to: hive.user_file do |row|
+          file 'input', format: :tsv, headers: false do
+            column 'id', type: :integer
+            column 'tenant_id', type: :integer
+            column 'admin', type: :boolean
+            column 'preferences', type: :json
+            column 'deleted_at', type: :timestamp, null: true
+          end
+
+          file 'output', format: :csv, headers: true do
+            column 'id', type: :integer
+            column 'tenant_id', type: :integer
+            column 'admin', type: :boolean
+            column 'preferences', type: :yaml
+            column 'deleted_at', type: :timestamp, null: true
+          end
+
+          map from: files.input, to: files.output do |row|
             {
               'id'          => row[:id],
               'tenant_id'   => row[:tenant_id],
-              'deleted_at'  => row[:type] =~ /delete/ ? row[:created_at] : nil,
-              'admin'       => row[:admin_now],
-              'preferences' => row[:preferences_now]
+              'deleted_at'  => row[:deleted_at],
+              'admin'       => row[:admin],
+              'preferences' => row[:preferences]
             }
           end
         end
       end
 
       let(:source) do
-        catalog.hive.user_event
+        catalog.files.input
       end
 
       let(:target) do
-        catalog.hive.user_file
+        catalog.files.output
       end
 
       let(:source_data) do
         <<-EOS.strip_heredoc
-          X	user_create	1	30	0	\\N	\\N	\\N	0	\\N
-          Y	user_delete	2	40	0	1	"{""enabled"":true}"	\\N	0	2014-02-26T18:15:51Z
+          1	30	0
+          2	40	0	"{""enabled"":true}"	2014-02-26T18:15:51.000Z
         EOS
       end
 
       let(:target_data) do
         <<-EOS.strip_heredoc
-          1	30			{}
-          2	40	2014-02-26T18:15:51.000Z		"{""enabled"":true}"
+          id,tenant_id,deleted_at,admin,preferences
+          1,30,,FALSE,"--- {}
+          "
+          2,40,2014-02-26T18:15:51.000Z,FALSE,"---
+          enabled: true
+          "
         EOS
       end
 
-
       it 'should match target data' do
         is_expected.to eq(target_data)
       end
@@ -315,37 +341,46 @@ describe Masamune::Schema::Map do
       it_behaves_like 'apply input/output'
     end
 
-    context 'from event to csv file' do
+    context 'from csv file to tsv file' do
       before do
         catalog.schema :files do
-          map from: hive.user_event, to: postgres.user_file do |row|
+          file 'input', format: :csv, headers: true, json_encoding: :quoted do
+            column 'id', type: :integer
+            column 'tenant_id', type: :integer
+            column 'admin', type: :boolean
+            column 'preferences', type: :yaml
+            column 'deleted_at', type: :timestamp, null: true
+          end
+
+          file 'output', format: :tsv, headers: false do
+            column 'id', type: :integer
+            column 'tenant_id', type: :integer
+            column 'admin', type: :boolean
+            column 'preferences', type: :json
+            column 'deleted_at', type: :timestamp, null: true
+          end
+
+          map from: files.input, to: files.output do |row|
             {
               'id'          => row[:id],
               'tenant_id'   => row[:tenant_id],
-              'deleted_at'  => row[:type] =~ /delete/ ? row[:created_at] : nil,
-              'admin'       => row[:admin_now],
-              'preferences' => row[:preferences_now]
+              'deleted_at'  => row[:deleted_at],
+              'admin'       => row[:admin],
+              'preferences' => row[:preferences]
             }
           end
         end
       end
 
       let(:source) do
-        catalog.hive.user_event
+        catalog.files.input
       end
 
       let(:target) do
-        catalog.postgres.user_file
+        catalog.files.output
       end
 
       let(:source_data) do
-        <<-EOS.strip_heredoc
-          X	user_create	1	30	0	\\N	\\N	\\N	0	\\N
-          Y	user_delete	2	40	0	1	"{""enabled"":true}"	\\N	0	2014-02-26T18:15:51Z
-        EOS
-      end
-
-      let(:target_data) do
         <<-EOS.strip_heredoc
           id,tenant_id,deleted_at,admin,preferences
           1,30,,FALSE,"--- {}
@@ -356,6 +391,13 @@ describe Masamune::Schema::Map do
         EOS
       end
 
+      let(:target_data) do
+        <<-EOS.strip_heredoc
+          1	30		FALSE	{}
+          2	40	2014-02-26T18:15:51.000Z	FALSE	"{""enabled"":true}"
+        EOS
+      end
+
       it 'should match target data' do
         is_expected.to eq(target_data)
       end
@@ -363,56 +405,44 @@ describe Masamune::Schema::Map do
       it_behaves_like 'apply input/output'
     end
 
-    context 'from event to fact' do
+    context 'with multiple outputs' do
       before do
         catalog.schema :files do
-          map from: hive.user_event, to: hive.user_fact do |row|
-            if row[:type] =~ /update/
-              [
-                {
-                  'tenant.tenant_id' => row[:tenant_id],
-                  'delta'            => 0,
-                  'time_key'         => row[:created_at]
-                },
-                {
-                  'tenant.tenant_id' => row[:tenant_id],
-                  'delta'            => 0,
-                  'time_key'         => row[:created_at]
-                }
-              ]
-            else
-              {
-                'tenant.tenant_id' => row[:tenant_id],
-                'delta'            => row[:type] =~ /create/ ? 1 :  -1,
-                'time_key'         => row[:created_at]
-              }
-            end
+          file 'input' do
+            column 'id', type: :integer
+          end
+
+          file 'output' do
+            column 'id', type: :integer
+          end
+
+          map from: files.input, to: files.output do |row|
+            [row, row]
           end
         end
       end
 
       let(:source) do
-        catalog.hive.user_event
+        catalog.files.input
       end
 
       let(:target) do
-        catalog.hive.user_fact
+        catalog.files.output
       end
 
       let(:source_data) do
         <<-EOS.strip_heredoc
-          X	user_create	3	10	0	1	"{""enabled"":true}"	\\N	\\N	2015-01-01T00:10:00Z
-          Y	user_update	3	10	0	1	"{""enabled"":true}"	\\N	\\N	2015-01-01T00:20:00Z
-          Z	user_delete	3	10	0	1	"{""enabled"":true}"	\\N	\\N	2015-01-01T00:30:00Z
+          1
+          2
         EOS
       end
 
       let(:target_data) do
         <<-EOS.strip_heredoc
-          10	1	1420071000
-          10	0	1420071600
-          10	0	1420071600
-          10	-1	1420072200
+          1
+          1
+          2
+          2
         EOS
       end
 
@@ -423,75 +453,40 @@ describe Masamune::Schema::Map do
       it_behaves_like 'apply input/output'
     end
 
-    context 'from event with array attribute to fact' do
+    context 'without block' do
       before do
-        catalog.clear!
-        catalog.schema :hive do
-          event 'user' do
-            attribute 'id', type: :integer, immutable: true
-            attribute 'group_id', type: :integer, array: true
-          end
-
-          dimension 'group', type: :two, implicit: true do
-            column 'group_id'
+        catalog.schema :files do
+          file 'input' do
+            column 'id', type: :integer
           end
 
-          fact 'user' do
-            references :group
-            column 'junk'
-            measure 'total'
+          file 'output' do
+            column 'id', type: :integer
           end
 
-          map from: hive.user_event, to: hive.user_fact, columns: %w(group.group_id total time_key) do |row|
-            result = []
-            (row[:group_id_now] - row[:group_id_was]).each do |group_id|
-              result <<
-                {
-                  'group.group_id' => group_id,
-                  'total'          => 1,
-                  'time_key'       => row[:created_at]
-                }
-            end
-            (row[:group_id_was] - row[:group_id_now]).each do |group_id|
-              result <<
-                {
-                  'group.group_id' => group_id,
-                  'total'          => -1,
-                  'time_key'       => row[:created_at]
-                }
-            end
-            result
-          end
+          map from: files.input, to: files.output
         end
       end
 
       let(:source) do
-        catalog.hive.user_event
+        catalog.files.input
       end
 
       let(:target) do
-        catalog.hive.user_fact
+        catalog.files.output
       end
 
       let(:source_data) do
         <<-EOS.strip_heredoc
-          # new lines and comments should be skipped
-
-          X	user_create	3	[1,2]	[]	0	2015-01-01T00:10:00Z
-          Y	user_update	3	[1,2,3]	[1,2]	1	2015-01-01T00:20:00Z
-          Y	user_update	3	[1,2]	[1,2,3]	1	2015-01-01T00:30:00Z
-          Z	user_delete	3	[]	[1,2]	0	2015-01-01T00:40:00Z
+          1
+          2
         EOS
       end
 
       let(:target_data) do
         <<-EOS.strip_heredoc
-          1	1	1420071000
-          2	1	1420071000
-          3	1	1420071600
-          3	-1	1420072200
-          1	-1	1420072800
-          2	-1	1420072800
+          1
+          2
         EOS
       end
 
@@ -502,44 +497,47 @@ describe Masamune::Schema::Map do
       it_behaves_like 'apply input/output'
     end
 
-    context 'from event to postgres dimension with raw json' do
+    context 'from file to table' do
       before do
-        catalog.schema :files do
-          map from: hive.user_event, to: postgres.user_dimension do |row|
-            {
-              'tenant_id'               => row[:tenant_id],
-              'user_id'                 => row[:id],
-              'user_account_state.name' => row[:type] =~ /delete/ ? 'deleted' : 'active',
-              'admin'                   => row[:type] =~ /delete/ ? row[:admin_was] : row[:admin_now],
-              'preferences_now'         => row[:preferences_now],
-              'preferences_was'         => row[:preferences_was],
-              'source'                  => 'user_event',
-              'cluster_id'              => 100
-            }
+        catalog.schema :postgres do
+          table 'parent' do
+            column 'id', type: :integer
           end
+
+          file 'input', format: :csv, headers: false do
+            column 'parent.id', type: :integer
+            column 'id', type: :integer
+          end
+
+          table 'output' do
+            references :parent
+            column 'id', type: :integer
+          end
+
+          map from: postgres.input_file, to: postgres.output_table
         end
       end
 
       let(:source) do
-        catalog.hive.user_event
+        catalog.postgres.input_file
       end
 
       let(:target) do
-        catalog.postgres.user_dimension
+        catalog.postgres.output_table
       end
 
       let(:source_data) do
         <<-EOS.strip_heredoc
-          X	user_create	1	30	0	\\N	\\N	\\N
-          Y	user_delete	2	40	0	1	{"enabled":true}	\\N
+          10,1
+          10,2
         EOS
       end
 
       let(:target_data) do
         <<-EOS.strip_heredoc
-          tenant_id,user_id,user_account_state_type_name,admin,preferences_now,preferences_was,source,cluster_id
-          30,1,active,FALSE,{},{},user_event,100
-          40,2,deleted,TRUE,"{""enabled"":true}",{},user_event,100
+          parent_table_id,id
+          10,1
+          10,2
         EOS
       end
 
@@ -566,7 +564,7 @@ describe Masamune::Schema::Map do
       it { is_expected.to eq(%Q{"{}","{}"}) }
     end
 
-    context 'with raw quoted json' do
+    context 'with quoted empty json' do
       before do
         io.write '"{}","{}"'
         io.rewind