activefacts-compositions 1.9.17 → 1.9.18

data/lib/activefacts/compositions/staging.rb

@@ -17,44 +17,80 @@ module ActiveFacts
       def self.options
         datavault_options.
         merge({
+          cdc: [%w{record satellite all}, "Add computed hash fields for change detection"],
+          persistent: ['Boolean', "Allow multiple batches to be loaded into the same tables"],
           stgname: ['String', "Suffix or pattern for naming staging tables. Include a + to insert the name. Default 'STG'"],
-          fk: ['Boolean', "Retain foreign keys in the output (by default they are deleted)"]
         }).
-        merge(Relational.options).
-        reject{|k,v| [:surrogates].include?(k) }
+        merge(Relational.options)
       end
 
       def initialize constellation, name, options = {}
         # Extract recognised options:
-        datavault_initialize options
+        @option_cdc = options.delete('cdc')
+
+        @option_persistent = options.delete('persistent')
+        options = {'surrogates'=>'Record GUID'}.    # You must have surrogates, but can call them what you wish
+          merge(options).merge({'fk'=>'natural', 'audit'=>'batch'}) if @option_persistent          # Must be batch auditing
+
         @option_stg_name = options.delete('stgname') || 'STG'
         @option_stg_name.sub!(/^/,'+ ') unless @option_stg_name =~ /\+/
 
-        @option_keep_fks = options.delete('fk') || false
-        @option_keep_fks = ['', true, 'true', 'yes'].include?(@option_keep_fks)
+        datavault_initialize options
 
+        @option_fk = :natural      # Default value
         super constellation, name, options, 'Staging'
       end
 
-      def generate
-        create_loadbatch if @option_loadbatch
+      def complete_foreign_keys
         super
+
+        augment_keys if @option_persistent
       end
 
-      def inject_value_fields
+      def needs_surrogate(composite)
+        @option_surrogates && composite.mapping.object_type != @loadbatch_entity_type
+      end
+
+      def generate
+        create_loadbatch if @option_audit == 'batch'
         super
-        inject_loadbatch_relationships if @option_loadbatch
       end
 
-      def inject_all_datetime_recordsource
+      # Find the leaf absorption of the LoadBatchID in composite
+      def load_batch_field composite
+        load_batch_role =
+          composite.mapping.object_type.all_role.detect do |role|
+            c = role.counterpart and c.object_type == @loadbatch_entity_type
+          end
+        trace :index, "Found LoadBatch role in #{load_batch_role.fact_type.default_reading}" if load_batch_role
+        # There can only be one absorption of LoadBatch, because we added it,
+        # but if you have separate subtypes, we need to select the one for the right composite:
+        absorptions = load_batch_role.
+          counterpart.
+          all_absorption_as_child_role.
+          select{|a| a.root == composite}
+        # There should now always be exactly one.
+        raise "Missing or ambiguous FK to LoadBatch from #{composite.inspect}" if absorptions.size != 1
+        absorptions[0].all_leaf[0]
+        # This is the absorption of LoadBatchID
+      end
+
+      def inject_all_audit_fields
+        inject_loadbatch_relationships if @option_audit == 'batch'
+      end
+
+      def apply_all_audit_transformations
         composites = @composition.all_composite.to_a
         return if composites.empty?
 
         trace :staging, "Injecting load datetime and record source" do
           @composition.all_composite.each do |composite|
-            next if composite.mapping.object_type.name == @option_loadbatch
-            inject_datetime_recordsource composite.mapping
-            composite.mapping.re_rank
+            is_loadbatch_composite = composite.mapping.object_type == @loadbatch_entity_type
+            composite.mapping.injection_annotation = 'loadbatch' if is_loadbatch_composite
+            if @option_audit == 'record' || is_loadbatch_composite
+              inject_audit_fields composite
+              composite.mapping.re_rank
+            end
           end
         end
       end
@@ -63,25 +99,51 @@ module ActiveFacts
         # Rename composites with STG prefix
         apply_composite_name_pattern
 
-        inject_all_datetime_recordsource
-      end
+        apply_all_audit_transformations
 
-      def complete_foreign_keys
-        if @option_keep_fks
-          super
-        else
-          retract_foreign_keys
-        end
+        super
       end
 
-      def retract_foreign_keys
-        trace :relational_paths, "Retracting foreign keys" do
+      def augment_keys
+        trace :index, "Augmenting keys" do
           @composition.all_composite.each do |composite|
+            next if composite.mapping.object_type == @loadbatch_entity_type
+            target_batch_id = load_batch_field(composite)
             composite.all_access_path.each do |path|
-              next if MM::Index === path
-              trace :relational_paths, "Retracting #{path.inspect}" do
-                path.retract
+              # Ignore foreign keys and non-unique indices:
+              next unless MM::Index === path and path.is_unique
+
+              # Don't meddle with the surrogate
+              next if path.all_index_field.size == 1 && MM::SurrogateKey === path.all_index_field.single.component
+
+              trace :index, "Add LoadBatchID to #{path.inspect}" do
+                @constellation.IndexField(access_path: path, ordinal: path.all_index_field.size, component: target_batch_id)
+              end
+
+              # Note that the RecordGUID will have become the primary key.
+              # Foreign keys would be enforced onto the natural key, but we
+              # want the natural key to be clustering, so we switch them around
+              if composite.natural_index == path
+                pk = composite.primary_index
+                composite.primary_index = composite.natural_index
+                composite.natural_index = pk
+
+                # Fix the foreign keys that use this changed natural key:
+                trace :index, "Appending LoadBatch to foreign keys to #{composite.mapping.name}" do
+                  composite.
+                  all_foreign_key_as_target_composite.
+                  each do |fk|
+                    trace :index, "Appending LoadBatch to #{fk.inspect}" do
+                      source_batch_id = load_batch_field(fk.source_composite)
+                      trace :index, "ForeignKeyField is #{source_batch_id.root.mapping.name}.#{source_batch_id.inspect}"
+                      trace :index, "IndexField is #{target_batch_id.root.mapping.name}.#{target_batch_id.inspect}"
+                      @constellation.ForeignKeyField(foreign_key: fk, ordinal: fk.all_foreign_key_field.size, component: source_batch_id)
+                      @constellation.IndexField(access_path: fk, ordinal: fk.all_index_field.size, component: target_batch_id)
+                    end
+                  end
+                end
               end
+
             end
           end
         end

data/lib/activefacts/compositions/traits/datavault.rb

@@ -4,66 +4,133 @@ module ActiveFacts
       module DataVault
         def datavault_options
           {
-            datestamp: ['String', "Data type name to use for data vault date stamps (default: DateTime)"],
-            recordsource: ['String', "Data type name to use for data vault record source (default: String)"],
-            loadbatch: ['String', "Create a load batch table using this name, default LoadBatch"],
+            # Structural options:
+            audit: [%w{record batch}, "Add date/source auditing fields to each record (hub/link/staging) or via a LoadBatch table"],
+            # Variation options:
+            loadbatch: ['String', "Change the name of the load batch table from LoadBatch"],
+            datestamp: ['String', "Data type name to use for audit date stamps (default: DateTime)"],
+            source: ['String', "Data type name to use for audit source (default: String)"],
           }
         end
 
         def datavault_initialize options
-          @option_datestamp = options.delete('datestamp')
-          @option_datestamp = 'DateTime' if [true, '', 'true', 'yes', nil].include?(@option_datestamp)
-          @option_recordsource = options.delete('recordsource')
-          @option_recordsource = 'String' if [true, '', 'true', 'yes', nil].include?(@option_recordsource)
-          @option_loadbatch = options.delete('loadbatch')
-          @option_loadbatch = 'LoadBatch' if [true, 'true', 'yes'].include?(@option_loadbatch)
-          @option_loadbatch = nil if [false, 'false', ''].include?(@option_loadbatch)
+          @option_audit = options.delete('audit')
+
+          case @option_loadbatch = options.delete('loadbatch')
+          when true, 'true', 'yes'
+            @option_loadbatch = 'LoadBatch'
+          when false, 'false'
+            @option_loadbatch = nil
+          end
+          @option_loadbatch ||= 'LoadBatch' if @option_audit == 'batch'
+
+          case @option_datestamp = options.delete('datestamp')
+          when true, '', 'true', 'yes', nil
+            @option_datestamp = 'DateTime'
+          when false, 'false'
+            @option_datestamp = nil
+          end
+
+          case @option_source = options.delete('source')
+          when true, '', 'true', 'yes', nil
+            @option_source = 'String'
+          when false, 'false'
+            @option_source = nil
+          end
+
+        end
+
+        def compile text
+          @vocabulary = @constellation.Vocabulary.values[0]
+          @compiler ||= ActiveFacts::CQL::Compiler.new(@vocabulary, constellation: @constellation)
+          @compiler.compile("schema #{@vocabulary.name};\n"+text)
         end
 
         def create_loadbatch
-          vocabulary = @constellation.Vocabulary.values[0]
+          return unless @option_audit == 'batch' && @option_loadbatch
 
           schema_text = %Q{
-            schema #{vocabulary.name};
-
             each #{@option_loadbatch} ID is written as an Auto Counter auto-assigned at commit;
             each #{@option_loadbatch} is independent identified by its ID;
-            each #{@option_datestamp} is written as a #{@option_datestamp};
-            #{@option_loadbatch} began at one start-#{@option_datestamp};
           }
-          @compiler = ActiveFacts::CQL::Compiler.new(@vocabulary, constellation: @constellation)
-          @compiler.compile(schema_text)
-          @loadbatch_entity_type = @constellation.EntityType[[vocabulary.identifying_role_values, @option_loadbatch]]
+          compile(schema_text)
+          @loadbatch_entity_type = @constellation.EntityType[[@vocabulary.identifying_role_values, @option_loadbatch]]
+        end
+
+        # This method only works after the LoadBatch composite has been asserted, of course
+        def loadbatch_composite
+          @loadbatch_composite ||= @composition.
+            all_composite.detect{|c| c.mapping.object_type == @loadbatch_entity_type}
         end
 
         def inject_loadbatch_relationships
-          return unless @option_loadbatch
-          @composition.all_composite.each do |composite|
-            next if composite.mapping.object_type.name == @option_loadbatch
-            @compiler.compile("#{composite.mapping.name} was loaded in one #{@option_loadbatch};")
+          return unless @option_audit == 'batch'
+          trace :batch, "Injecting LoadBatch relationships" do
+            @composition.all_composite.each do |composite|
+              inject_loadbatch_relationship composite
+            end
           end
-          @loadbatch_entity_type.all_role.each do |role|
-            populate_reference role
-            populate_reference role.counterpart
+        end
+
+        def inject_loadbatch_relationship composite
+          return if composite == loadbatch_composite
+          trace :batch, "Injecting LoadBatch relationship into #{composite.mapping.name}" do
+            compile("#{composite.mapping.name} was loaded in one #{@option_loadbatch};")
+
+            role = composite.mapping.object_type.all_role.detect{|r| c = r.counterpart and c.object_type == @loadbatch_entity_type}
+            version_field = populate_reference(role)
+            version_field.injection_annotation = 'loadbatch'
+            composite.mapping.re_rank
+            loadbatch_composite.mapping.re_rank
+            version_field
           end
         end
 
-        def inject_datetime_recordsource mapping
-          # Add a load DateTime value
-          date_field = @constellation.ValidFrom(:new,
-            parent: mapping,
-            name: "LoadTime",
-            object_type: datestamp_type
-          )
-
-          # Add a load DateTime value
-          recsrc_field = @constellation.ValueField(:new,
-            parent: mapping,
-            name: "RecordSource",
-            object_type: recordsource_type
-          )
-          mapping.re_rank
-          date_field
+        def inject_audit_fields composite, copy_from = nil
+          version_field = begin
+            if @option_audit == 'batch' && composite != @loadbatch_composite
+              if copy_from
+                # Our hub or link has an FK to LoadBatch already. Fork a copy to use on the satellite
+                trace :batch, "Copying LoadBatchID from #{copy_from.mapping.name} into #{composite.mapping.name}" do
+                  parent_version_field = @version_fields[copy_from]
+                  load_batch = parent_version_field.fork_to_new_parent composite.mapping
+                  leaves = parent_version_field.all_leaf
+                  raise "WARNING: unexpected audit structure" unless leaves.size == 1
+                  version_field = leaves[0].fork_to_new_parent load_batch
+                  load_batch.injection_annotation =
+                  version_field.injection_annotation =
+                    parent_version_field.injection_annotation
+                  version_field
+                end
+              else
+                inject_loadbatch_relationship composite
+              end
+            else
+              if datestamp_type
+                # Add a load DateTime value
+                version_field = @constellation.ValidFrom(:new,
+                  parent: composite.mapping,
+                  name: "LoadTime",
+                  object_type: datestamp_type,
+                  injection_annotation: "datavault"
+                )
+              end
+
+              if recordsource_type
+                # Add a load DateTime value
+                recsrc_field = @constellation.Mapping(:new,
+                  parent: composite.mapping,
+                  name: "RecordSource",
+                  object_type: recordsource_type,
+                  injection_annotation: "datavault"
+                )
+              end
+              version_field
+            end
+
+          end
+          composite.mapping.re_rank
+          (@version_fields ||= {})[composite] = version_field
         end
 
         def datestamp_type_name
@@ -72,10 +139,10 @@ module ActiveFacts
 
         def datestamp_type
           @datestamp_type ||= begin
-            vocabulary = @composition.all_composite.to_a[0].mapping.object_type.vocabulary
-            @constellation.ObjectType[[[vocabulary.name], datestamp_type_name]] or
+            @vocabulary ||= @composition.all_composite.to_a[0].mapping.object_type.vocabulary
+            @constellation.ObjectType[[[@vocabulary.name], datestamp_type_name]] or
               @constellation.ValueType(
-                vocabulary: vocabulary,
+                vocabulary: @vocabulary,
                 name: datestamp_type_name,
                 concept: [:new, :implication_rule => "datestamp injection"]
               )
@@ -83,15 +150,15 @@ module ActiveFacts
         end
 
         def recordsource_type_name
-          @option_recordsource
+          @option_source
         end
 
         def recordsource_type
           @recordsource_type ||= begin
-            vocabulary = @composition.all_composite.to_a[0].mapping.object_type.vocabulary
-            @constellation.ObjectType[[[vocabulary.name], recordsource_type_name]] or
+            @vocabulary ||= @composition.all_composite.to_a[0].mapping.object_type.vocabulary
+            @constellation.ObjectType[[[@vocabulary.name], recordsource_type_name]] or
               @constellation.ValueType(
-                vocabulary: vocabulary,
+                vocabulary: @vocabulary,
                 name: recordsource_type_name,
                 concept: [:new]
               )
@@ -107,7 +174,7 @@ module ActiveFacts
 
         def apply_composite_name_pattern
           @composites.each do |key, composite|
-            next if composite.mapping.object_type.name == @option_loadbatch
+            next if composite.mapping.name == @option_loadbatch
             composite.mapping.name = apply_name_pattern(@option_stg_name, composite.mapping.name)
           end
         end

data/lib/activefacts/compositions/traits/rails.rb

@@ -145,11 +145,11 @@ module ActiveFacts
           # respective Composites. The name of a foreign key takes this into account.
 
           def from_association_name
-            absorption.column_name.snakecase
+            mapping.column_name.snakecase
           end
 
           def to_association
-            if absorption && absorption.child_role.is_unique
+            if mapping && mapping.child_role.is_unique
               [ "has_one", source_composite.rails.singular_name]
             else
               [ "has_many", source_composite.rails.plural_name]

data/lib/activefacts/compositions/version.rb

@@ -1,5 +1,5 @@
 module ActiveFacts
   module Compositions
-    VERSION = "1.9.17"
+    VERSION = "1.9.18"
   end
 end

data/lib/activefacts/generator/doc/cwm.rb

@@ -142,7 +142,7 @@ module ActiveFacts
             nsdef(index)
             index.all_index_field.map{|idf| idf.component.index_xmiid = index.xmiid}
           end
-          table.all_foreign_key_as_source_composite.sort_by{|fk| [fk.source_composite.mapping.name, fk.absorption.inspect] }.map do |fk|
+          table.all_foreign_key_as_source_composite.sort_by{|fk| [fk.source_composite.mapping.name, fk.mapping.inspect] }.map do |fk|
             nsdef(fk)
           end
         end
@@ -226,7 +226,7 @@ module ActiveFacts
                 generate_index(depth+2, table.xmiid, index, name, table.all_foreign_key_as_target_composite)
               end
             ) * "" +
-            (table.all_foreign_key_as_source_composite.sort_by{|fk| [fk.source_composite.mapping.name, fk.absorption.inspect] }.map do |fk|
+            (table.all_foreign_key_as_source_composite.sort_by{|fk| [fk.source_composite.mapping.name, fk.mapping.inspect] }.map do |fk|
                 generate_foreign_key(depth+2, table.xmiid, fk)
               end
             ) * "" +
@@ -386,7 +386,7 @@ module ActiveFacts
 
         # Return CWM type, typenum for the passed base type
         def normalise_type_cwm(type_name, length)
-          type = MM::DataType.normalise(type_name)
+          type = MM::DataType.intrinsic_type(type_name)
 
           case type
           when MM::DataType::TYPE_Boolean;  ['boolean', 16]
@@ -408,8 +408,8 @@ module ActiveFacts
             ['real', 7, data_type_context.default_length(type, type_name)]
           when MM::DataType::TYPE_Decimal;  ['decimal', 3]
           when MM::DataType::TYPE_Money;    ['decimal', 3]
-          when MM::DataType::TYPE_Char;     ['char', 12, length || data_type_context.char_default_length]
-          when MM::DataType::TYPE_String;   ['varchar', 12, length || data_type_context.varchar_default_length]
+          when MM::DataType::TYPE_Char;     ['char', 12, length]
+          when MM::DataType::TYPE_String;   ['varchar', 12, length]
           when MM::DataType::TYPE_Text;     ['text', 2005, length || 'MAX']
           when MM::DataType::TYPE_Date;     ['date', 91]
           when MM::DataType::TYPE_Time;     ['time', 92]
@@ -490,14 +490,10 @@ module ActiveFacts
           end
 
           def surrogate_type
-            type_name, = choose_integer_type(0, 2**(default_surrogate_length-1)-1)
+            type_name, = choose_integer_range(0, 2**(default_surrogate_length-1)-1)
             type_name
           end
 
-          def valid_from_type
-            'TIMESTAMP'
-          end
-
           def date_time_type
             'TIMESTAMP'
           end
@@ -512,14 +508,6 @@ module ActiveFacts
             'VARCHAR'
           end
 
-          def char_default_length
-            nil
-          end
-
-          def varchar_default_length
-            nil
-          end
-
           def default_surrogate_length
             64
           end

data/lib/activefacts/generator/doc/ldm.rb

@@ -517,7 +517,7 @@ module ActiveFacts
             }#{
               value_constraint ? check_clause(column_name, value_constraint) : ''
             }"
-          when MM::Injection
+          when MM::ValidFrom
             component.object_type.name
           else
             raise "Can't make a column from #{component}"