activefacts-compositions 1.9.17 → 1.9.18

data/lib/activefacts/generator/etl/unidex.rb

@@ -0,0 +1,341 @@
+#
+#       ActiveFacts Unified Index Schema Generator
+#
+# Copyright (c) 2009-2016 Clifford Heath. Read the LICENSE file.
+#
+require 'digest/sha1'
+require 'activefacts/metamodel'
+require 'activefacts/metamodel/datatypes'
+require 'activefacts/compositions'
+require 'activefacts/compositions/names'
+require 'activefacts/generator'
+require 'activefacts/generator/traits/sql'
+
+module ActiveFacts
+  module Generators
+    module ETL
+      class Unidex
+
+        MM = ActiveFacts::Metamodel unless const_defined?(:MM)
+        def self.options
+          # REVISIT: There's no way to support SQL dialect options here
+          sql_trait = ActiveFacts::Generators::Traits::SQL
+          Class.new.extend(sql_trait).  # Anonymous class to enable access to traits module instance methods
+          options.
+          merge(
+            {
+              dialect: [String, "SQL Dialect to use"],
+              value_width: [Integer, "Number of characters to index from long values"],
+              phonetic_confidence: [Integer, "Percentage confidence for a phonetic match"],
+            }
+          )
+        end
+
+        def initialize composition, options = {}
+          @composition = composition
+          @options = options
+
+          @trait = ActiveFacts::Generators::Traits::SQL
+          if @dialect = options.delete("dialect")
+            require 'activefacts/generator/traits/sql/'+@dialect
+            trait_name = ActiveFacts::Generators::Traits::SQL.constants.detect{|c| c.to_s =~ %r{#{@dialect}}i}
+            @trait = @trait.const_get(trait_name)
+          end
+          self.class.include @trait
+          self.class.extend @trait
+          extend @trait
+
+          process_options options
+        end
+
+        def process_options options
+          @value_width = (options.delete('value_width') || 32).to_i
+          @phonetic_confidence = (options.delete('phonetic_confidence') || 70).to_i
+
+          super
+        end
+
+        def generate
+          @all_table_unions = []
+          header +
+          @composition.
+            all_composite.
+            sort_by{|c| c.mapping.name}.
+            map{|c| generate_composite c}.
+            concat([all_union(@all_table_unions)]).
+            compact*"\n" +
+          trailer
+        end
+
+        def all_union unions
+          return '' if unions.empty?
+          create_or_replace("#{schema_name}_unidex", 'VIEW') + " AS\n" +
+          unions.compact.map{|s| "SELECT * FROM "+s } *
+          "\nUNION ALL " +
+          ";\n"
+        end
+
+        def header
+          schema_prefix
+        end
+
+        def generate_composite composite
+          return nil if composite.mapping.injection_annotation
+          return nil if composite.mapping.object_type.is_static
+
+          trace :unidex, "Generating view for #{table_name(composite)}" do
+            union =
+            composite.mapping.all_member.to_a.sort_by{|m| m.name}.flat_map do |member|
+              next nil if member.injection_annotation
+              rank_key = member.rank_key
+
+              case key_type = rank_key[0]
+              when MM::Component::RANK_SURROGATE,     # A surrogate key; these do not get indexed
+                  MM::Component::RANK_DISCRIMINATOR,  # Replacement for exclusive indicators, often subtypes
+                  MM::Component::RANK_MULTIPLE        # Nested absorption
+                trace :unidex, "Ignoring #{MM::DataTypes::TypeNames[key_type]} #{column_name member}"
+                next nil
+              when MM::Component::RANK_INJECTION      # ValueField (index), ValidFrom (don't) or an Absorption with an injection annotation
+                if MM::ValueField === member
+                  generate_value leaf
+                else
+                  nil
+                end
+              when MM::Component::RANK_INDICATOR
+                generate_indicator member
+              else
+                raise "Unexpected non-Absorption" unless MM::Absorption === member
+                if member.foreign_key
+                  # Index this record by the natural key of the FK target record, if possible
+                  generate_joined_value member
+                # elsif member.full_absorption  # REVISIT: Anything special to do here?
+                else
+                  (member.all_member.size > 0 ? member.all_leaf : [member]).flat_map do |leaf|
+                    generate_value leaf
+                  end
+                end
+              end
+            end.compact * "\nUNION ALL"
+
+            if union.size > 0
+              union_name = "#{table_name(composite)}_unidex"
+              @all_table_unions << union_name
+
+              "/*\n"+
+              " * View to extract unified index values for #{table_name(composite)}\n"+
+              " */\n"+
+              create_or_replace("#{union_name}", 'VIEW') + " AS" +
+              union +
+              ";\n"
+            else
+              ''
+            end
+          end
+        end
+
+        def trailer
+          ''
+        end
+
+        def generate_indicator leaf
+          nil # REVISIT: Do we need anything here?
+          # select leaf.root, safe_column_name(leaf), 1, column_name(leaf), 1
+        end
+
+        # This foreign key connects two composites (tables)
+        def generate_joined_value member
+          foreign_key = member.foreign_key
+          # REVISIT: Is this restriction even necessary?
+          return nil unless foreign_key.composite.mapping.object_type.is_static
+
+          # Index the source table by the natural key of the target, if we can find one
+          indices = foreign_key.composite.all_index
+          return null if indices.empty?
+
+          search_index_by = {}
+          searchable_indices =
+            indices.select do |ix|
+              next false if !ix.is_unique
+              non_fk_components = ix.all_index_field.map(&:component) - foreign_key.all_index_field.map(&:component)
+              next unless non_fk_components.size == 1
+              component = non_fk_components[0]
+              next unless MM::Absorption === component
+              value_type = component.object_type
+              search_methods = value_type.applicable_parameter_restrictions('Search')
+              search_methods.reject!{|vtpr| m = vtpr.value_range.minimum_bound and m.value == 'none'}
+              search_methods.map!{|sm| sm.value_range.minimum_bound.value.effective_value}
+              if search_methods.empty?
+                false
+              else
+                search_index_by[ix] = search_methods
+              end
+            end
+          return nil if search_index_by.empty?
+
+          search_index_by.flat_map do |search_index, search_methods|
+            trace :unidex, "Search #{table_name foreign_key.source_composite} via #{table_name search_index.composite}.#{column_name search_index.all_index_field.to_a[0].component} using #{search_methods.map(&:inspect)*', '}"
+
+            fk_pairs =
+                  foreign_key.all_foreign_key_field.to_a.
+              zip foreign_key.all_index_field.to_a
+            leaf = search_index.all_index_field.to_a[0].component         # Returning this natural index value
+            source_table = table_name(foreign_key.composite)
+            source_field = safe_column_name(member)
+            type_name, options = leaf.data_type(data_type_context)        # Which has this type_name
+            intrinsic_type = MM::DataType.intrinsic_type(type_name)       # Which corresponds to this intrinsic type
+
+            col_expr = Expression.new(
+              %Q{
+                (SELECT  #{safe_column_name(leaf)}
+                 FROM    #{source_table} AS f
+                 WHERE   #{
+                  fk_pairs.map do |fkf, ixf|
+                    "#{table_name foreign_key.source_composite}.#{safe_column_name(fkf.component)} = f.#{safe_column_name(ixf.component)}"
+                  end*' AND '
+                 })}.
+              gsub(/\s+/,' '),
+              intrinsic_type,
+              foreign_key.all_foreign_key_field.to_a.all?{|fkf| fkf.component.path_mandatory}
+            )
+            search_expr foreign_key.source_composite, intrinsic_type, col_expr, search_methods, source_field
+          end
+        end
+
+        def generate_value leaf
+          return nil unless leaf.is_a?(MM::Absorption)
+
+          value_type = leaf.object_type
+          type_name, options = leaf.data_type(data_type_context)
+          length = options[:length]
+          value_constraint = options[:value_constraint]
+
+          # Look for instructions on how to index this leaf for search:
+          search_methods = value_type.applicable_parameter_restrictions('Search')
+          search_methods.reject!{|vtpr| m = vtpr.value_range.minimum_bound and m.value == 'none'}
+          return nil if search_methods.empty?
+          search_methods.map!{|sm| sm.value_range.minimum_bound.value.effective_value}
+
+          # Convert from the model's data type to a metamodel type, if possible
+          intrinsic_type = MM::DataType.intrinsic_type(type_name)
+          data_type_name = intrinsic_type ? MM::DataType::TypeNames[intrinsic_type] : type_name
+          trace :unidex, "Search #{table_name leaf.root}.#{column_name(leaf)} as #{data_type_name} using #{search_methods.map(&:inspect)*', '}"
+
+          col_expr = Expression.new(safe_column_name(leaf), intrinsic_type, leaf.is_mandatory)
+          source_field = safe_column_name(leaf)
+
+          search_expr leaf.root, intrinsic_type, col_expr, search_methods, source_field
+        end
+
+        def search_expr composite, intrinsic_type, col_expr, search_methods, source_field
+          case intrinsic_type
+          when MM::DataType::TYPE_Char,
+               MM::DataType::TYPE_String,
+               MM::DataType::TYPE_Text
+            # Produce a truncated value with the requested search
+            search_methods.flat_map do |sm|
+              case sm
+              when 'none'         # Do not index this value
+                nil
+              when 'simple'       # Disregard white-space only
+                select(composite, truncate(col_expr, @value_width), 'simple', source_field, 1.0)
+
+              when 'alpha'        # Strip white space and punctuation, just use alphabetic characters
+                select(composite, truncate(as_alpha(col_expr), @value_width), 'alpha', source_field, 0.9)
+
+              when 'words'        # Break the text into words and match each word like alpha
+                nil # REVISIT: Implement this type
+
+              # when 'phrases'    # Words, but where adjacent sequences of words matter
+              when 'typo'         # Use trigram similarity to detect typographic errors
+                # REVISIT: Implement this type properly
+                select(composite, trigram(as_alpha(col_expr)), 'typo', source_field, 0.9)
+
+              when 'phonetic'     # Use phonetic matching as well as trigrams
+                phonetics(col_expr).map do |p|
+                  select(composite, p, 'phonetic', source_field, @phonetic_confidence/100.0)
+                end
+
+              when 'names'        # Break the text into words and match each word like phonetic
+                nil # REVISIT: Implement this type
+
+              when 'text'         # Index a large text field using significant words and phrases
+                nil # REVISIT: Implement this type
+              else
+                raise "Unknown search method #{sm}"
+              end
+            end
+
+          when MM::DataType::TYPE_Boolean
+            nil # REVISIT: Implement this type
+
+          when MM::DataType::TYPE_Integer,
+               MM::DataType::TYPE_Real,
+               MM::DataType::TYPE_Decimal,
+               MM::DataType::TYPE_Money
+            # Produce a right-justified value
+            select(composite, lexical_decimal(col_expr, @value_width, value_type.scale), 'simple', source_field, 1)
+
+          when MM::DataType::TYPE_Date
+            # Produce an ISO representation that sorts lexically (YYYY-MM-DD)
+            # REVISIT: Support search methods here
+            select(composite, lexical_date(col_expr), 'simple', source_field, 1)
+
+          when MM::DataType::TYPE_DateTime,
+               MM::DataType::TYPE_Timestamp
+            # Produce an ISO representation that sorts lexically (YYYY-MM-DD HH:mm:ss)
+            # REVISIT: Support search methods here
+            select(composite, lexical_datetime(col_expr), 'simple', source_field, 1)
+
+          when MM::DataType::TYPE_Time
+            # Produce an ISO representation that sorts lexically (YYYY-MM-DD HH:mm:ss)
+            select(composite, lexical_time(col_expr), 'simple', source_field, 1)
+
+          when MM::DataType::TYPE_Binary
+            nil   # No indexing applied
+          when nil   # Data Type is unknown
+          else
+          end
+        end
+
+        def stylise_column_name name
+          name.words.send(@column_case)*@column_joiner
+        end
+
+        def select composite, expression, processing, source_field, confidence = 1, where = []
+          # These fields are in order of index precedence, to co-locate
+          # comparable values regardless of source record type or column
+          where << 'Value IS NOT NULL' if expression.to_s =~ /\bNULL\b/
+          processing_name = stylise_column_name("Processing")
+          value_name = stylise_column_name("Value")
+          load_batch_id_name = stylise_column_name("LoadBatchID")
+          record_guid_name = stylise_column_name("RecordGUID")
+          confidence_name = stylise_column_name("Confidence")
+          source_table_name = stylise_column_name("SourceTable")
+          source_field_name = stylise_column_name("SourceField")
+          expression_text = expression.to_s
+          expression_text = "ARRAY[#{expression_text}]" unless expression.is_array
+          select = %Q{
+            SELECT  '#{processing}' AS #{processing_name},
+                    #{expression_text} AS #{value_name},
+                    #{load_batch_id_name},
+                    #{"%.2f" % confidence} AS #{confidence_name},
+                    #{record_guid_name},
+                    '#{safe_table_name(composite)}' AS #{source_table_name},
+                    '#{source_field}' AS #{source_field_name}
+            FROM    #{safe_table_name(composite)}
+            WHERE   COALESCE(#{expression},'') <> ''}.
+            unindent
+
+          if where.empty?
+            select
+          else
+            "\nSELECT * FROM (#{select}\n) AS s WHERE #{where*' AND '}"
+          end
+
+        end
+
+      end
+    end
+    publish_generator ETL::Unidex
+  end
+end

data/lib/activefacts/generator/oo.rb

@@ -25,7 +25,8 @@ module ActiveFacts
       end
 
       def generate
-        @composites_emitted = {}
+        @composites_started = {}
+        @composites_finished = {}
 
         retract_intrinsic_types
 
@@ -58,7 +59,7 @@ module ActiveFacts
         return true if o.name == "_ImplicitBooleanValueType"
         return false if o.supertype
         # A value type with no supertype must be emitted if it is the child in any absorption:
-        return !composite.mapping.all_member.detect{|m| m.forward_absorption}
+        return !composite.mapping.all_member.detect{|m| m.forward_mapping}
       end
 
       def retract_intrinsic_types
@@ -92,8 +93,25 @@ module ActiveFacts
         "REVISIT: override class_finale\n"
       end
 
-      def generate_class composite, predefine_role_players = true
-        return nil if @composites_emitted[composite]
+      def exclude_as_counterpart member
+        # If emitting a role also creates its counterpart, we can exclude the counterparts.
+        # This depends on the subtype, so centralise the knowledge here where it can be overridden.
+        # Non-Mappings always get emitted:
+        return false unless MM::Mapping === member
+
+        # TypeInheritance always gets skipped:
+        return true if MM::Absorption === member && member.child_role.fact_type.is_a?(MM::TypeInheritance)
+
+        # Don't skip the mandatory counterpart of a one-to-one:
+        return false if member.is_one_to_one and member.is_mandatory
+
+        # Otherwise skip all reverse mappings (these have a forward mapping)
+        return member.forward_mapping
+      end
+
+      def generate_class composite, precursor_to = [], predefine_role_players = true
+        return nil if @composites_started[composite]
+        @composites_started[composite] = true
 
         mapping = composite.mapping
         object_type = mapping.object_type
@@ -104,9 +122,7 @@ module ActiveFacts
         supertype_composites =
           object_type.all_supertype.map{|s| composite_for(s) }.compact
         forward_declarations +=
-          supertype_composites.map{|c| generate_class(c, false)}.compact
-
-        @composites_emitted[composite] = true
+          supertype_composites.map{|c| generate_class(c, precursor_to+[composite], false)}.compact
 
         # Select the members that will be declared as O-O roles:
         mapping.re_rank
@@ -114,23 +130,22 @@ module ActiveFacts
           all_member.
           sort_by{|m| m.ordinal}.
           reject do |m|
-            m.is_a?(MM::Absorption) and
-              m.forward_absorption || m.child_role.fact_type.is_a?(MM::TypeInheritance)
+            exclude_as_counterpart(m)
           end
 
         if predefine_role_players
-          # The idea was good, but we need to avoid triggering a forward reference problem.
-          # We only do it when we're not dumping a supertype dependency.
-          #
-          # For those roles that derive from Mappings, produce class definitions to avoid forward references:
+          # To reduce forward references, emit any classes that play a role
+          # we're about to emit, unless one is a subtype of any class that
+          # is currently being processed.
           forward_composites =
             members.
               select{ |m| m.is_a?(MM::Mapping) }.
               map{ |m| composite_for m.object_type }.
+              reject{|c| precursor_to.detect{|p| c.mapping.object_type.supertypes_transitive.include?(p.mapping.object_type) }}.
               compact.
               sort_by{|c| c.mapping.name}
           forward_declarations +=
-            forward_composites.map{|c| generate_class(c)}.compact
+            forward_composites.map{|c| generate_class(c, precursor_to+[composite])}.compact
         end
 
         forward_declarations = forward_declarations.map{|f| "#{f}\n"}*''
@@ -166,6 +181,8 @@ module ActiveFacts
             value_type_declaration object_type
           end
 
+        @composites_finished[composite] = true
+
         forward_declarations +
         class_prelude(object_type, primary_supertype) +
           type_declaration +

data/lib/activefacts/generator/rails/models.rb

@@ -172,9 +172,9 @@ module ActiveFacts
             end
 
             [
-            fk.absorption ? "    \# #{fk.absorption.comment}" : nil,
+            fk.mapping ? "    \# #{fk.mapping.comment}" : nil,
             "    belongs_to :#{association_name}#{class_name}#{foreign_key}",
-            fk.absorption ? '' : nil,
+            fk.mapping ? '' : nil,
             ]
           end.compact
         end
@@ -193,13 +193,13 @@ module ActiveFacts
 
             [
               # REVISIT: We want the reverse-order comment here really
-              fk.absorption ? "    \# #{fk.absorption.comment}" : nil,
+              fk.mapping ? "    \# #{fk.mapping.comment}" : nil,
               %Q{    #{association_type} :#{association_name}} +
               %Q{, :class_name => '#{fk.source_composite.rails.class_name}'} +
               %Q{, :foreign_key => :#{fk.all_foreign_key_field.single.component.column_name.snakecase}} +
               %Q{, :dependent => :destroy}
             ] +
-            # If fk.absorption.source_composite is a join table, we can emit a has_many :through for each other key
+            # If fk.mapping.source_composite is a join table, we can emit a has_many :through for each other key
             # REVISIT: We could alternately do this for all belongs_to's in the source composite
             if fk.source_composite.primary_index.all_index_field.size > 1
               fk.source_composite.primary_index.all_index_field.map(&:component).flat_map do |ic|
@@ -212,7 +212,7 @@ module ActiveFacts
             else
               []
             end +
-            [fk.absorption ? '' : nil]
+            [fk.mapping ? '' : nil]
           end
         end
 
@@ -221,6 +221,7 @@ module ActiveFacts
           ccs =
             composite.mapping.all_leaf.flat_map do |component|
               next unless component.path_mandatory && !component.is_a?(Metamodel::Indicator)
+              next if composite.primary_index != composite.natural_index && composite.primary_index.all_index_field.detect{|ixf| ixf.component == component}
               next if component.is_a?(Metamodel::Mapping) && component.object_type.is_a?(Metamodel::ValueType) && component.is_auto_assigned
               [ "    validates :#{component.column_name.snakecase}, :presence => true" ]
             end.compact

data/lib/activefacts/generator/rails/schema.rb

@@ -18,7 +18,7 @@ module ActiveFacts
         HEADER = "# Auto-generated from CQL, edits will be lost"
         def self.options
           ({
-            exclude_fks:      ['Boolean', "Don't generate foreign key definitions"],
+            fks:              ['Boolean', "Generate foreign key definitions"],
             include_comments: ['Boolean', "Generate a comment for each column showing the absorption path"],
             closed_world:     ['Boolean', "Set this if your DBMS only allows one null in a unique index (MS SQL)"],
           })
@@ -27,7 +27,7 @@ module ActiveFacts
         def initialize composition, options = {}
           @composition = composition
           @options = options
-          @option_exclude_fks = options.delete("exclude_fks")
+          @option_exclude_fks = [false, 'f', 'n', 'no'].include?(options.delete("fks"))
           @option_include_comments = options.delete("include_comments")
           @option_closed_world = options.delete("closed_world")
         end
@@ -147,7 +147,7 @@ module ActiveFacts
                       ixf.ordinal == 0                              # It's first in this index
                     end
                   already_indexed = from_index_needed.any?{|ixf| @indexes_generated[ixf.access_path]}
-                  is_one_to_one = fk.absorption && fk.absorption.child_role.is_unique
+                  is_one_to_one = fk.mapping && fk.mapping.child_role.is_unique
 
                   index_name = ACTR::name_trunc("index_#{ar_table_name}_on_#{from_column_names[0]}")
                   [
@@ -244,14 +244,10 @@ module ActiveFacts
           end
 
           def surrogate_type
-            type_name, = choose_integer_type(0, 2**(default_surrogate_length-1)-1)
+            type_name, = choose_integer_range(0, 2**(default_surrogate_length-1)-1)
             type_name
           end
 
-          def valid_from_type
-            date_time_type
-          end
-
           def date_time_type
             'datetime'
           end
@@ -271,7 +267,7 @@ module ActiveFacts
 
         # Return SQL type and (modified?) length for the passed base type
         def normalise_type type_name
-          type = MM::DataType.normalise(type_name)
+          type = MM::DataType.intrinsic_type(type_name)
 
           [
             type,

data/lib/activefacts/generator/ruby.rb

@@ -69,7 +69,7 @@ module ActiveFacts
         if component.is_a?(MM::Absorption) and
             counterpart = component.object_type and
             counterpart_composite = composite_for(counterpart)
-          counterpart_class_emitted = @composites_emitted[counterpart_composite]
+          counterpart_class_emitted = @composites_finished[counterpart_composite]
 
           counterpart_class_name = ruby_class_name counterpart_composite
           counterpart_default_role = ruby_role_name counterpart_composite.mapping
@@ -79,7 +79,7 @@ module ActiveFacts
 
           # Does the reverse role need explicit specification?
           implied_reverse_role_name = ruby_role_name(component.root.mapping)
-          actual_reverse_role_name = ruby_role_name component.reverse_absorption
+          actual_reverse_role_name = ruby_role_name(component.reverse_mapping || component.forward_mapping)
 
           if implied_reverse_role_name != actual_reverse_role_name
             counterpart_spec = ", counterpart: :#{actual_reverse_role_name}"