activefacts-compositions 1.9.8 → 1.9.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c3f30aa198a18c360d4fee63de36a3764df7d799
-  data.tar.gz: 28db19dcd307a9ab3ab1e7a50fada7023651d6e6
+  metadata.gz: 21764b88dbad036917cb41736a88158dfe3a0557
+  data.tar.gz: c6e8dac27a2114b466b03e6707489280af4103ca
 SHA512:
-  metadata.gz: 0b179e371a8f8b47a308d5790c6a21e4dec2c490eb7bd76da9bab58a0c8ae201fbf50c27a6a6b882e3395a9118b75ab16568353e076a17be1fd5197c4b78ae2f
-  data.tar.gz: cb28e3eb273607b31fb7e55fa2e826a5cc965764345a4b80c8096531d33be0dc97e05b7103424b5afd9bee1bc68d7fb9d51341a87967909eba52eaddd1964ed7
+  metadata.gz: 6dc9d157c1ff522b0b529d3693b35149d1d4a917648d46a909ff32a3003164fbb86decc0db1823072b51fdb513d820857d835a3946418ddb6f4ed64678677757
+  data.tar.gz: 52e636f04f8acd46418c0ab3def5bfacc99a0802714a2f07dc1cc4baf730d926ecd34a5eaa7d4cb33a42f936081290a6dbf514d7cc2822f56523bbbd2368029c

data/activefacts-compositions.gemspec

@@ -26,7 +26,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "activefacts", "~> 1", ">= 1.8"
 
   spec.add_runtime_dependency("activefacts-api", "~> 1", ">= 1.9.11")
-  spec.add_runtime_dependency("activefacts-metamodel", "~> 1", ">= 1.9.10")
+  spec.add_runtime_dependency("activefacts-metamodel", "~> 1", ">= 1.9.12")
   spec.add_runtime_dependency "tracing", "~> 2", ">= 2.0.6"
 
   spec.add_runtime_dependency "activefacts-cql", "~> 1", ">= 1.8"

data/bin/schema_compositor

@@ -23,22 +23,26 @@ class SchemaCompositor
     @options = {}
     while argv[0] =~ /^-/
       option, value = argv.shift.split(/=/, 2)
-      @options[option.sub(/^-*/,'')] =
-	(value =~ /,/ ? value.split(',') : Array(value)).
-	inject({}){|h,s| k, v = s.split(/=/, 2); h[k] = v || true; h }
+      csv = (value =~ /,/ ? value.split(',') : Array(value))
+      modes = csv.inject({}){|h,s| k, v = s.split(/=/, 2); h[k] = v || true; h }
+      @options[option.sub(/^-*/,'')] = modes
     end
   end
 
   # Load and enumerate all available modules in this path
   def enumerate_available path
-    Loadable.new(path).
-    enumerate.
-    select do |filename|
-      begin
-	require(pathname = path+"/"+filename)
-      rescue LoadError => e
-      rescue Exception => e
-	$stderr.puts "Can't load #{pathname}: #{e.class}: #{e.message} #{e.backtrace[0]}"
+    trace :loading, "Enumerating under #{path.inspect}" do
+      Loadable.new(path).
+      enumerate.
+      select do |filename|
+	begin
+	  require(pathname = path+"/"+filename)
+	  trace :loading, "Loaded #{pathname}"
+	rescue LoadError => e
+	  trace :loading, "Can't load #{pathname}: #{e.class}: #{e.message} #{e.backtrace[0]}"
+	rescue Exception => e
+	  $stderr.puts "Can't load #{pathname}: #{e.class}: #{e.message} #{e.backtrace[0]}"
+	end
       end
     end
   end
@@ -47,21 +51,21 @@ class SchemaCompositor
     # Arrange the requested compositors and generators:
     @compositors = []
     @generators = []
-    @options.clone.each do |option, mode|
+    @options.clone.each do |option, modes|
       if action = ActiveFacts::Compositions.compositors[option]
-	options.delete(option)
-	check_options(action, mode)
-	@compositors << [action, mode]
+        options.delete(option)
+        check_options(action, modes)
+        @compositors << [action, modes]
       elsif action = ActiveFacts::Generators.generators[option]
-	options.delete(option)
-	check_options(action, mode)
-	@generators << [action, mode]
+        options.delete(option)
+        check_options(action, modes)
+        @generators << [action, modes]
       else
-	$stderr.puts "Action --#{option} is not recognised"
-	exit 1
+        $stderr.puts "Action --#{option} is not recognised"
+        exit 1
       end
-      if mode && mode['help']
-	puts "Help for #{option} is not yet available"
+      if modes['help']
+        puts "Help for #{option} is not yet available"
       end
     end
   end
@@ -82,32 +86,33 @@ class SchemaCompositor
 
       # Read the input file:
       vocabulary =
-	if input_klass
-	  begin
-	    input_klass.readfile(filename, *input_options)
-	  rescue => e
-	    $stderr.puts "#{e.message}"
-	    if trace :exception
-	      $stderr.puts "\t#{e.backtrace*"\n\t"}"
-	    else
-	      $stderr.puts "\t#{e.backtrace[0]}"
-	    end
-	    exit 1
-	  end
-	end
+        if input_klass
+          begin
+            input_klass.readfile(filename, *input_options)
+          rescue => e
+            $stderr.puts "#{e.message}"
+            if trace :exception
+              $stderr.puts "\t#{e.backtrace*"\n\t"}"
+            else
+              $stderr.puts "\t#{e.backtrace[0]}"
+            end
+            exit 1
+          end
+        end
       exit 0 unless vocabulary
       vocabulary.finalise unless vocabulary == true
 
-      # Run each compositor
-      @compositors.each do |compositor_klass, mode|
-	compositor = compositor_klass.new(vocabulary.constellation, basename, mode||{})
-	compositor.generate
+      # Run one or more compositors
+      compositions = @compositors.map do |compositor_klass, modes|
+        compositor = compositor_klass.new(vocabulary.constellation, basename, modes)
+        compositor.generate
+        compositor.composition
+      end
 
-	# Run each generator
-	@generators.each do |generator, mode|
-	  output = generator.new(compositor.composition, mode||{}).generate
-	  puts output if output
-	end
+      # Run each generator
+      @generators.each do |generator_klass, modes|
+	output = generator_klass.new(compositions, modes).generate
+        puts output if output
       end
     end
   end
@@ -124,33 +129,33 @@ class SchemaCompositor
     else
       stream.puts "Options for --#{name} (say e.g. --#{action_name action}=option1=value,option2)"
       options.keys.sort.each do |key|
-	type, description = *options[key]
-	tag =
-	  key.to_s +
-	  case type
-	  when NilClass,'Boolean', TrueClass
-	    ''
-	  when Numeric
-	    ' num'
-	  when Pathname
-	    ' file'
-	  else
-	    ' str'
-	  end
-
-	stream.puts "\t#{tag}#{' '*(24-tag.size)}#{description}"
+        type, description = *options[key]
+        tag =
+          key.to_s +
+          case type
+          when NilClass,'Boolean', TrueClass
+            ''
+          when Numeric
+            ' num'
+          when Pathname
+            ' file'
+          else
+            ' str'
+          end
+
+        stream.puts "\t#{tag}#{' '*(24-tag.size)}#{description}"
       end
     end
   end
 
   # Ensure that the options provided are supported by the action
-  def check_options action, mode
-    if mode['help']
+  def check_options action, modes
+    if modes['help']
       display_options(action)
       exit
     end
     options = action.options
-    unsupported = mode.keys.select{|k| !options.has_key?(k.to_sym)}
+    unsupported = modes.keys.select{|k| !options.has_key?(k.to_sym)}
     return if unsupported.empty?
     $stderr.puts "Action --#{action_name action} does not support #{unsupported.size >1 ? 'these options' : 'this option'}: #{unsupported*', '}"
     display_options(action, $stderr)

data/lib/activefacts/compositions/datavault.rb

@@ -27,14 +27,14 @@ module ActiveFacts
         # Extract recognised options:
         @option_reference = options.delete('reference')
         @option_datestamp = options.delete('datestamp')
-        @option_id = ' ' + (options.delete('id') || 'VID')
+        @option_id = ' ' + (options.delete('id') || 'HID')
         @option_hub_name = options.delete('hubname') || 'HUB'
         @option_hub_name.sub!(/^/,'+ ') unless @option_hub_name =~ /\+/
         @option_link_name = options.delete('linkname') || 'LINK'
         @option_link_name.sub!(/^/,'+ ') unless @option_link_name =~ /\+/
         @option_sat_name = options.delete('satname') || 'SAT'
         @option_sat_name.sub!(/^/,'+ ') unless @option_sat_name =~ /\+/
-        @option_ref_name = options.delete('refname') || 'SAT'
+        @option_ref_name = options.delete('refname') || 'REF'
         @option_ref_name.sub!(/^/,'+ ') unless @option_ref_name =~ /\+/
 
         super constellation, name, options
@@ -161,6 +161,8 @@ module ActiveFacts
 
         rename_parents
 
+        inject_all_datetime_recordsource
+
         unless @option_reference
           if trace :reference_retraction
             # Add a logger so we can trace the resultant retractions:
@@ -301,17 +303,12 @@ module ActiveFacts
 
           # Add a Surrogate foreign Key to the parent composite
           fk_target = composite.primary_index.all_index_field.single
-          fk_field = fork_component_to_new_parent(satellite.mapping, fk_target.component)
+          fk_field = fk_target.component.fork_to_new_parent satellite.mapping
 
-          # Add a load DateTime value
-          date_field = @constellation.ValidFrom(
-            :new,
-            parent: satellite.mapping,
-            name: "Load"+datestamp_type_name,
-            object_type: datestamp_type
-          )
+          # Add a load DateTime value and record source
+          date_field = inject_datetime_recordsource(satellite.mapping)
 
-          # Add a natural key:
+          # Add a primary and natural key:
           natural_index =
             @constellation.Index(:new, composite: satellite, is_unique: true,
               presence_constraint: nil, composite_as_natural_index: satellite, composite_as_primary_index: satellite)
@@ -338,24 +335,6 @@ module ActiveFacts
         end
       end
 
-      def datestamp_type_name
-        @datestamp_type_name ||= begin
-          [true, '', 'true', 'yes', nil].include?(t = @option_datestamp) ? 'DateTime' : t
-        end
-      end
-
-      def datestamp_type
-        @datestamp_type ||= begin
-          vocabulary = @composition.all_composite.to_a[0].mapping.object_type.vocabulary
-          @constellation.ObjectType[[[vocabulary.name], datestamp_type_name]] or
-            @constellation.ValueType(
-              vocabulary: vocabulary,
-              name: datestamp_type_name,
-              concept: [:new, :implication_rule => "datestamp injection"]
-            )
-        end
-      end
-
       # Decide what to call a new satellite that will adopt this component
       def name_satellite component
         satellite_name =
@@ -457,12 +436,12 @@ module ActiveFacts
           # We have no fact type for this absorption; it should be the LinkFactType of the notional objectification
           # This affects the absorption path comment on the related SQL coliumn, for example.
           # REVISIT: Add the LinkFactType for the notional objectification, and use that.
-          fk1_component = fork_component_to_new_parent(mapping, fk1_target.component)
+          fk1_component = fk1_target.component.fork_to_new_parent mapping
 
           fk2_target = link_to.primary_index.all_index_field.single
           if make_copy
-            # See the above comment for fk1_component; it aplies here also
-            fk2_component = fork_component_to_new_parent(mapping, fk2_target.component)
+            # See the above comment for fk1_component; it applies here also
+            fk2_component = fk2_target.component.fork_to_new_parent mapping
           else
             # We're using the leaf component of the absorption we moved across
           end
@@ -507,13 +486,9 @@ module ActiveFacts
           debugger if issues > 0
 =end
 
-          # Add a load DateTime value
-          date_field = @constellation.ValidFrom(:new,
-            parent: mapping,
-            name: "FirstLoad"+datestamp_type_name,
-            object_type: datestamp_type
-          )
-          mapping.re_rank
+
+          # # Add a load DateTime value
+          # date_field = add_datetime_recordsource(mapping)
 
           #link.mapping.re_rank
 
@@ -538,6 +513,15 @@ module ActiveFacts
         end
       end
 
+      def inject_all_datetime_recordsource
+        @link_composites.each do |composite|
+          inject_datetime_recordsource(composite.mapping)
+        end
+        @hub_composites.each do |composite|
+          inject_datetime_recordsource(composite.mapping)
+        end
+      end
+
     end
 
     publish_compositor(DataVault)

data/lib/activefacts/compositions/relational.rb

@@ -46,6 +46,9 @@ module ActiveFacts
           # Inject surrogate keys if the options ask for that
           inject_surrogates if @option_surrogates
 
+          # Inject load date time and record source if building staging
+          inject_all_datetime_recordsource
+
           # Remove the un-used absorption paths
           delete_reverse_absorptions
 
@@ -139,6 +142,7 @@ module ActiveFacts
             end
 
             # Rule 3: If there's more than one absorption path and any functional dependencies that can't absorb us, it's a table
+#            trace :relational_optimiser, "From-references for #{object_type.name}(#{pi_roles.map(&:object_type).map(&:name)*', '}) are #{references_from.map(&:inspect)*', '}"
             non_identifying_refs_from =
               candidate.references_from.reject do |member|
                 case member
@@ -392,6 +396,67 @@ module ActiveFacts
         return true
       end
 
+      # This function is over-written in the Staging subclass
+      def inject_all_datetime_recordsource
+      end
+
+      #
+      # Datetime and recordsource functions defined here because they are used in both Staging and Datavault subclasses
+      #
+      def inject_datetime_recordsource mapping
+        # Add a load DateTime value
+        date_field = @constellation.ValidFrom(:new,
+          parent: mapping,
+          name: "Load"+datestamp_type_name,
+          object_type: datestamp_type
+        )
+        # Add a load DateTime value
+        recsrc_field = @constellation.ValueField(:new,
+          parent: mapping,
+          name: "RecordSource",
+          object_type: recordsource_type
+        )
+        mapping.re_rank
+        date_field
+      end
+
+      def datestamp_type_name
+        @datestamp_type_name ||= begin
+          [true, '', 'true', 'yes', nil].include?(t = @option_datestamp) ? 'DateTime' : t
+        end
+      end
+
+      def datestamp_type
+        @datestamp_type ||= begin
+          vocabulary = @composition.all_composite.to_a[0].mapping.object_type.vocabulary
+          @constellation.ObjectType[[[vocabulary.name], datestamp_type_name]] or
+            @constellation.ValueType(
+              vocabulary: vocabulary,
+              name: datestamp_type_name,
+              concept: [:new, :implication_rule => "datestamp injection"]
+            )
+        end
+      end
+
+      def recordsource_type_name
+        @recordsource_type_name ||= begin
+          [true, '', 'true', 'yes', nil].include?(t = @option_recordsource) ? 'RecordSource' : t
+        end
+      end
+
+      def recordsource_type
+        @recordsource_type ||= begin
+          vocabulary = @composition.all_composite.to_a[0].mapping.object_type.vocabulary
+          @constellation.ObjectType[[[vocabulary.name], recordsource_type_name]] or
+            @constellation.ValueType(
+              vocabulary: vocabulary,
+              name: recordsource_type_name,
+              concept: [:new]
+            )
+        end
+      end
+
+
       def clean_unused_mappings
         @candidates.keys.to_a.each do |object_type|
           candidate = @candidates[object_type]
@@ -482,8 +547,21 @@ module ActiveFacts
         accumulator
       end
 
+      # Overwritten by Staging and Datavault subclasses
       def devolve_all
-        # Data Vaults have satellites, not normal relational schemas.
+      end
+
+      #
+      # Rename parents functions defined because they are used in both Staging and Datavault subclasses
+      #
+      def apply_name pattern, name
+        pattern.sub(/\+/, name)
+      end
+
+      def rename_parents
+        @composites.each do |key, composite|
+          composite.mapping.name = apply_name(@option_stg_name, composite.mapping.name)
+        end
       end
 
       # This member is an Absorption. Process it recursively, absorbing all its members or just a key
@@ -545,7 +623,7 @@ module ActiveFacts
           if rank == MM::Component::RANK_SURROGATE && prefer_natural
             next
           end
-          member = fork_component_to_new_parent mapping, member
+          member = member.fork_to_new_parent mapping
           augment_paths paths, member
           if rank == MM::Component::RANK_SURROGATE && !prefer_natural
             break   # Will always be first (higher rank), and usurps others
@@ -592,7 +670,7 @@ module ActiveFacts
         ordered.each do |member|
           trace :relational_columns, proc {"#{top_level ? 'Existing' : 'Absorbing'} #{member.inspect}"} do
             unless top_level    # Top-level members are already instantiated
-              member = fork_component_to_new_parent(mapping, member)
+              member = member.fork_to_new_parent mapping
             end
             rel = paths.merge(relevant_paths(newpaths, member))
             augment_paths rel, member
@@ -758,17 +836,6 @@ module ActiveFacts
         end
       end
 
-      def fork_component_to_new_parent parent, component
-        case component
-        # A place to put more special cases.
-        when MM::ValueField
-          # When we fork from a ValueField, we want to use the name of the ValueType, not the ValueField name
-          @constellation.fork component, guid: :new, parent: parent, name: component.object_type.name
-        else
-          @constellation.fork component, guid: :new, parent: parent
-        end
-      end
-
       # Make a new Absorption in the reverse direction from the one given
       def mirror absorption, parent
         @constellation.fork(