activefacts-compositions 1.9.16 → 1.9.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1f365db9787609b86e8184f61d9fc4c2d0b6e76e
-  data.tar.gz: 66f3cd28bbd6eeadb7a9ee48fb1afd2774697193
+  metadata.gz: 629d91efc99ba1f54872c6c668a1ee0f86bf7d04
+  data.tar.gz: b2e8c929798553e24d7bf1e8fe0e81718c0b811b
 SHA512:
-  metadata.gz: 1b8237ff0549bd7d36c8c8bcae1ad5272a3d13357c61ae4b7bcd4f9c1cd6c88bc46525844128d472231034c4df2c999230c7e964eee40a730515549ab94d2fcc
-  data.tar.gz: 8057016adea612072f44aa65a84496d6e80660394ae9199157868475dca6e9a39ed96679c0b71cc62408f3a964bca2ebe9fbc1511d4fbb1b8fda9aeeeed81143
+  metadata.gz: a5ebe59af26c890d1f5771400d3dced9c123c96b252b10ee0805c04612fed3f7eac0ef4018ecfbdfe09882b43605fa6109307b831c2fce15541c0a5ff43311be
+  data.tar.gz: bd9f2f20c79718881cf6a8f7c42b21cb37f6b0d5346302f77d9453341f6aff7768acc9f0e495c7b4e1850879935bddecf72d14e5fb98ce0740b62fa23e7973d1

data/lib/activefacts/compositions/staging.rb

@@ -18,6 +18,7 @@ module ActiveFacts
         datavault_options.
         merge({
           stgname: ['String', "Suffix or pattern for naming staging tables. Include a + to insert the name. Default 'STG'"],
+          fk: ['Boolean', "Retain foreign keys in the output (by default they are deleted)"]
         }).
         merge(Relational.options).
         reject{|k,v| [:surrogates].include?(k) }
@@ -29,6 +30,9 @@ module ActiveFacts
         @option_stg_name = options.delete('stgname') || 'STG'
         @option_stg_name.sub!(/^/,'+ ') unless @option_stg_name =~ /\+/
 
+        @option_keep_fks = options.delete('fk') || false
+        @option_keep_fks = ['', true, 'true', 'yes'].include?(@option_keep_fks)
+
         super constellation, name, options, 'Staging'
       end
 
@@ -39,7 +43,7 @@ module ActiveFacts
 
       def inject_value_fields
         super
-        inject_loadbatch_relationships
+        inject_loadbatch_relationships if @option_loadbatch
       end
 
       def inject_all_datetime_recordsource
@@ -61,6 +65,28 @@ module ActiveFacts
 
         inject_all_datetime_recordsource
       end
+
+      def complete_foreign_keys
+        if @option_keep_fks
+          super
+        else
+          retract_foreign_keys
+        end
+      end
+
+      def retract_foreign_keys
+        trace :relational_paths, "Retracting foreign keys" do
+          @composition.all_composite.each do |composite|
+            composite.all_access_path.each do |path|
+              next if MM::Index === path
+              trace :relational_paths, "Retracting #{path.inspect}" do
+                path.retract
+              end
+            end
+          end
+        end
+      end
+
     end
 
     publish_compositor(Staging)

data/lib/activefacts/compositions/staging/persistent.rb

@@ -0,0 +1,107 @@
+#
+# ActiveFacts Compositions, Staging Compositor.
+#
+#       Computes a Persistent Staging schema for Data Vault.
+#
+# Copyright (c) 2017 Clifford Heath. Read the LICENSE file.
+#
+# This style of staging area contains a table for every source table,
+# with injected foreign keys to a LoadBatch table, as for transient
+# staging.
+#
+# Where it differs is that all unique constraints are disabled, and
+# replaced by a primary key that is the source schema's PK appended
+# with the LoadBatchID. Each record also has a ValidUntil timestamp,
+# which is NULL for the most recent record. This allows a load batch
+# to load a new version of any record, and the key value will be
+# adjacent to the previous versions of that record.
+#
+# After a batch is complete, any updated record keys will address
+# more than one record with a NULL ValidUntil timestamp, and this
+# allows delta detection. As a delta is processed, the older record
+# can be marked as valid only until the current LoadBatch time, or
+# it can be deleted. If retained, a purge process can remove records
+# that have exceeded their useful lifetime.
+#
+# Note that this approach doesn't directly detect deleted records.
+# If the source system uses soft deletion, this generator can be
+# configured with the name and value of the deleted flag field(s).
+# This approach also works to manage CDC systems, which provide a
+# record-deleted flag.
+#
+
+require "activefacts/compositions/staging"
+
+module ActiveFacts
+  module Compositions
+    class Staging
+      class Persistent < Staging
+      public
+        def self.options
+          super.
+          merge({
+          }).
+          merge(Relational.options).
+          reject{|k,v| [:surrogates].include?(k) }
+        end
+
+        def initialize constellation, name, options = {}
+          # Extract recognised options:
+          super(constellation, name, {"loadbatch"=>true}.merge(options))
+
+          raise "--staging/persistent requires the loadbatch option (you can't disable it)" unless @option_loadbatch
+        end
+
+        def complete_foreign_keys
+          super
+
+          remake_primary_keys
+        end
+
+        def remake_primary_keys
+          trace :relational_paths, "Remaking primary keys" do
+            @composition.all_composite.each do |composite|
+              next if composite.mapping.object_type == @loadbatch_entity_type
+              composite.all_access_path.each do |path|
+                # Ignore foreign keys:
+                next unless MM::Index === path
+
+                # Don't meddle with the LoadBatch table:
+                next if composite.mapping.object_type == @loadbatch_entity_type
+                if composite.natural_index == path
+                  # Add LoadBatchID to the natural index:
+                  trace :relational_paths, "Appending LoadBatch to primary key #{path.inspect}" do
+                    load_batch_role =
+                      composite.mapping.object_type.all_role.detect do |role|
+                        c = role.counterpart and c.object_type == @loadbatch_entity_type
+                      end
+                    trace :relational_paths, "Found LoadBatch role in #{load_batch_role.fact_type.default_reading}" if load_batch_role
+                    # There can only be one absorption of LoadBatch, because we added it,
+                    # but if you have separate subtypes, we need to select the one for the right composite:
+                    absorptions = load_batch_role.
+                      counterpart.
+                      all_absorption_as_child_role.
+                      select{|a| a.root == composite}
+                    # There should now always be exactly one.
+                    raise "Missing or ambiguous FK to LoadBatch from #{composite.inspect}" if absorptions.size != 1
+                    absorption = absorptions[0]
+                    absorption.all_leaf.each do |leaf|
+                      @constellation.IndexField(access_path: path, ordinal: path.all_index_field.size, component: leaf)
+                    end
+                  end
+                elsif path.is_unique
+                  # Retract other unique keys:
+                  trace :relational_paths, "Retracting unique secondary index #{path.inspect}" do
+                    # REVISIT: Or just make it non-unique?
+                    path.retract
+                  end
+                end
+              end
+            end
+          end
+        end
+      end # Persistent
+    end   # Staging
+    publish_compositor(Staging::Persistent)
+  end
+end

data/lib/activefacts/compositions/traits/datavault.rb

@@ -16,7 +16,8 @@ module ActiveFacts
           @option_recordsource = options.delete('recordsource')
           @option_recordsource = 'String' if [true, '', 'true', 'yes', nil].include?(@option_recordsource)
           @option_loadbatch = options.delete('loadbatch')
-          @option_loadbatch = 'LoadBatch' if [true, '', 'true', 'yes'].include?(@option_loadbatch)
+          @option_loadbatch = 'LoadBatch' if [true, 'true', 'yes'].include?(@option_loadbatch)
+          @option_loadbatch = nil if [false, 'false', ''].include?(@option_loadbatch)
         end
 
         def create_loadbatch
@@ -36,6 +37,7 @@ module ActiveFacts
         end
 
         def inject_loadbatch_relationships
+          return unless @option_loadbatch
           @composition.all_composite.each do |composite|
             next if composite.mapping.object_type.name == @option_loadbatch
             @compiler.compile("#{composite.mapping.name} was loaded in one #{@option_loadbatch};")

data/lib/activefacts/compositions/version.rb

@@ -1,5 +1,5 @@
 module ActiveFacts
   module Compositions
-    VERSION = "1.9.16"
+    VERSION = "1.9.17"
   end
 end

data/lib/activefacts/generator/sql.rb

@@ -60,6 +60,12 @@ module ActiveFacts
         when "raw"
           @restrict = "rdv"
         end
+
+        # Do not (yet) expose the closed-world vs open world problem.
+        # Closed World vs Open World uniqueness is a semantic issue,
+        # and so is OW, CW or CW with negation for unary fact types.
+        # We need an overall strategy for handling it.
+        @closed_world_indices = false   # Allow for SQL Server's non-standard NULL indexing
       end
 
       def generate
@@ -184,13 +190,16 @@ module ActiveFacts
           end
         contains_nullable_columns = nullable_columns.size > 0
 
+        # The index can only be emitted as PRIMARY if it has no nullable columns:
         primary = index.composite_as_primary_index && !contains_nullable_columns
+
         column_names =
           index.all_index_field.map do |ixf|
             column_name(ixf.component)
           end
 
-        if contains_nullable_columns
+        if contains_nullable_columns and @closed_world_indices
+          # Implement open-world uniqueness using a filtered index:
           table_name = safe_table_name(index.composite)
           delayed_indices <<
             'CREATE UNIQUE'+index_kind(index)+' INDEX '+

data/lib/activefacts/generator/sql/server.rb

@@ -23,6 +23,11 @@ module ActiveFacts
           })
         end
 
+        def initialize composition, options = {}
+          super
+          @closed_world_indices = true
+        end
+
         def table_name_max
           128
         end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: activefacts-compositions
 version: !ruby/object:Gem::Version
-  version: 1.9.16
+  version: 1.9.17
 platform: ruby
 authors:
 - Clifford Heath
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-11-10 00:00:00.000000000 Z
+date: 2017-11-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -209,6 +209,7 @@ files:
 - lib/activefacts/compositions/names.rb
 - lib/activefacts/compositions/relational.rb
 - lib/activefacts/compositions/staging.rb
+- lib/activefacts/compositions/staging/persistent.rb
 - lib/activefacts/compositions/traits/datavault.rb
 - lib/activefacts/compositions/traits/rails.rb
 - lib/activefacts/compositions/version.rb