activefacts-compositions 1.9.18 → 1.9.19

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9264168bcd4294c87277a11fde73ba06c0e490e6
-  data.tar.gz: 40f2da7b5f7a72066b13ed7bc2de790e3717af82
+  metadata.gz: 2dd043b17bcbdb5bf4f525fe24022ad95a32946e
+  data.tar.gz: 95fb5b1b17e005cc0bd90f6f9c527e6455557a87
 SHA512:
-  metadata.gz: ecb7fb4562dbe1e8bef8924abc26a19c2aa4816eca5d09fd609b34a060acd11bef1d72642fe32ef631a5af2c1bd82bc968fbe8c48d386c0f409d41aba624c5bc
-  data.tar.gz: 46e7e8d45dadc6154867d2ce6ccd605bbcd897ab86f063481b8680b0fec4284ccd529edbd38fa86cabadeaad1584e82090fd89fc3f3a430da6dce47d61950106
+  metadata.gz: 528cbe042208d8dd290ef5cd5e11ae390261f25c04c4e5558ef111f7fc18aab8a82ea1421ee2147c71d3659c077c93af632ba7372fc337b2797f2a6c69846b30
+  data.tar.gz: fdf3d9610a0751f91d8b2b8fb022918aa81c31b99a73a359100622cef6f87d04338428253172921bde29e48906b25622915ea89d20fa416e0d156e8fd1695fff

data/activefacts-compositions.gemspec

@@ -20,7 +20,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_development_dependency "bundler", ">= 1.11"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake", "> 10"
   spec.add_development_dependency "rspec", "~> 3.3"
 
   spec.add_runtime_dependency "activesupport", ">= 4.2.7"

data/lib/activefacts/compositions/datavault.rb

@@ -630,13 +630,8 @@ module ActiveFacts
             if absorption.foreign_key
               trace :datavault, "Setting new source composite for #{absorption.foreign_key.inspect}"
               absorption.foreign_key.source_composite = link
-              if absorption.foreign_key.all_foreign_key_field.single
-                fk2_component = absorption.foreign_key.all_foreign_key_field.single.component
-              else
-                p absorption.foreign_key
-                debugger
-                absorption.foreign_key.retract
-              end
+              debugger unless absorption.foreign_key.all_foreign_key_field.single
+              fk2_component = absorption.foreign_key.all_foreign_key_field.single.component
             end
           end
 

data/lib/activefacts/compositions/staging.rb

@@ -51,6 +51,16 @@ module ActiveFacts
         @option_surrogates && composite.mapping.object_type != @loadbatch_entity_type
       end
 
+      def inject_surrogates
+        assign_groups
+        super
+      end
+
+      def assign_groups
+        @composites.values.each{|composite| composite.composite_group = 'base' }
+        loadbatch_composite.composite_group = 'batch' if @option_audit == 'batch'
+      end
+
       def generate
         create_loadbatch if @option_audit == 'batch'
         super

data/lib/activefacts/compositions/version.rb

@@ -1,5 +1,5 @@
 module ActiveFacts
   module Compositions
-    VERSION = "1.9.18"
+    VERSION = "1.9.19"
   end
 end

data/lib/activefacts/generator/doc/glossary.rb

@@ -8,7 +8,6 @@
 require 'activefacts/metamodel'
 require 'activefacts/compositions'
 require 'activefacts/generator'
-require 'byebug'
 
 module ActiveFacts
   module Generators #:nodoc:

data/lib/activefacts/generator/etl/unidex.rb

@@ -50,7 +50,7 @@ module ActiveFacts
 
         def process_options options
           @value_width = (options.delete('value_width') || 32).to_i
-          @phonetic_confidence = (options.delete('phonetic_confidence') || 70).to_i
+          @phonetic_confidence = (options.delete('phonetic_confidence') || 40).to_i
 
           super
         end
@@ -236,32 +236,55 @@ module ActiveFacts
               case sm
               when 'none'         # Do not index this value
                 nil
+
               when 'simple'       # Disregard white-space only
                 select(composite, truncate(col_expr, @value_width), 'simple', source_field, 1.0)
 
-              when 'alpha'        # Strip white space and punctuation, just use alphabetic characters
-                select(composite, truncate(as_alpha(col_expr), @value_width), 'alpha', source_field, 0.9)
-
-              when 'words'        # Break the text into words and match each word like alpha
-                nil # REVISIT: Implement this type
-
-              # when 'phrases'    # Words, but where adjacent sequences of words matter
-              when 'typo'         # Use trigram similarity to detect typographic errors
-                # REVISIT: Implement this type properly
-                select(composite, trigram(as_alpha(col_expr)), 'typo', source_field, 0.9)
+              when 'alpha',        # Strip white space and punctuation, just use alphabetic characters
+                   'typo'          # Use trigram similarity to detect typographic errors, over the same values
+                truncated = truncate(as_alpha(col_expr), @value_width)
+                select(
+                  composite, truncated, sm, source_field,
+                  "CASE WHEN #{truncated} = #{col_expr} THEN 1.0 ELSE 0.95 END"	# Maybe exact match.
+                )
 
               when 'phonetic'     # Use phonetic matching as well as trigrams
-                phonetics(col_expr).map do |p|
-                  select(composite, p, 'phonetic', source_field, @phonetic_confidence/100.0)
-                end
+                search_expr(composite, intrinsic_type, col_expr, ['typo'], source_field) <<
+                select(composite, phonetics(col_expr), 'phonetic', source_field, @phonetic_confidence/100.0, true)
+
+              when 'words'        # Break the text into words and match each word like alpha
+                truncated = truncate(unnest(as_words(col_expr)), @value_width)
+                select(composite, truncated, sm, source_field, 0.90, true)
 
               when 'names'        # Break the text into words and match each word like phonetic
-                nil # REVISIT: Implement this type
+                truncated = truncate(unnest(as_words(col_expr, "''-")), @value_width)   # N.B. ' is doubled for SQL
+                search_expr(composite, intrinsic_type, col_expr, ['words'], source_field) <<
+                phonetics(truncated).map do |phonetic|
+                  select(composite, phonetic, 'names', source_field, @phonetic_confidence/100.0, true)
+                end
 
               when 'text'         # Index a large text field using significant words and phrases
                 nil # REVISIT: Implement this type
+
+              when 'number'       # Cast to number and back to text to canonicalise the value;
+                # If the number doesn't match this regexp, we don't index it.
+                # This doesn't handle all valid Postgres numeric literals (e.g. 2.3e-4)
+                select(composite, col_expr, 'number', source_field, number_or_null(col_expr))
+
+              when 'phone'        # Phone numbers; split, strip each to digits, take the last 8 of each
+                select(composite, phone_numbers(col_expr), 'phone', source_field, 1)
+
+              when 'email'        # Use a regexp to find email addresses in this field
+                select(composite, truncate(email_addresses(col_expr), @value_width), 'email', source_field, 1)
+
+              when 'date'         # Convert string to standard date format if it looks like a date, NULL otherwise
+                select(
+                  composite, col_expr, 'date', source_field, 1,
+                  %Q{CASE WHEN #{col_expr} ~ '^ *[0-9]+[.]?[0-9]*|[.][0-9]+) *$' THEN (#{col_expr}::numeric):text ELSE NULL END}
+                )
+
               else
-                raise "Unknown search method #{sm}"
+                $stderrs.puts "Unknown search method #{sm}"
               end
             end
 
@@ -273,6 +296,7 @@ module ActiveFacts
                MM::DataType::TYPE_Decimal,
                MM::DataType::TYPE_Money
             # Produce a right-justified value
+            # REVISIT: This is a dumb thing to do.
             select(composite, lexical_decimal(col_expr, @value_width, value_type.scale), 'simple', source_field, 1)
 
           when MM::DataType::TYPE_Date
@@ -301,7 +325,7 @@ module ActiveFacts
           name.words.send(@column_case)*@column_joiner
         end
 
-        def select composite, expression, processing, source_field, confidence = 1, where = []
+        def select composite, expression, processing, source_field, confidence = 1, distinct = false, where = []
           # These fields are in order of index precedence, to co-locate
           # comparable values regardless of source record type or column
           where << 'Value IS NOT NULL' if expression.to_s =~ /\bNULL\b/
@@ -313,17 +337,16 @@ module ActiveFacts
           source_table_name = stylise_column_name("SourceTable")
           source_field_name = stylise_column_name("SourceField")
           expression_text = expression.to_s
-          expression_text = "ARRAY[#{expression_text}]" unless expression.is_array
           select = %Q{
-            SELECT  '#{processing}' AS #{processing_name},
+            SELECT#{distinct ? ' DISTINCT' : ''}
+                    '#{processing}' AS #{processing_name},
                     #{expression_text} AS #{value_name},
                     #{load_batch_id_name},
-                    #{"%.2f" % confidence} AS #{confidence_name},
+                    #{confidence} AS #{confidence_name},
                     #{record_guid_name},
                     '#{safe_table_name(composite)}' AS #{source_table_name},
                     '#{source_field}' AS #{source_field_name}
-            FROM    #{safe_table_name(composite)}
-            WHERE   COALESCE(#{expression},'') <> ''}.
+            FROM    #{safe_table_name(composite)}}.
             unindent
 
           if where.empty?

data/lib/activefacts/generator/rails/models.rb

@@ -16,6 +16,7 @@ module ActiveFacts
         HEADER = "# Auto-generated from CQL, edits will be lost"
         def self.options
           ({
+            keep:       ['Boolean', "Keep stale model files"],
             output:     [String,    "Overwrite model files into this output directory"],
             concern:    [String,    "Namespace for the concerns"],
             validation: ['Boolean', "Disable generation of validations"],
@@ -25,6 +26,7 @@ module ActiveFacts
         def initialize composition, options = {}
           @composition = composition
           @options = options
+          @option_keep = options.delete("keep")
           @option_output = options.delete("output")
           @option_concern = options.delete("concern")
           @option_validations = options.include?('validations') ? options.delete("validations") : true
@@ -35,7 +37,7 @@ module ActiveFacts
         end
 
         def generate
-          list_extant_files if @option_output
+          list_extant_files if @option_output && !@option_keep
 
           @ok = true
           models =
@@ -46,7 +48,7 @@ module ActiveFacts
             compact*"\n"
 
           warn "\# #{@composition.name} generated with errors" unless @ok
-          delete_old_generated_files if @option_output
+          delete_old_generated_files if @option_output && !@option_keep
 
           models
         end
@@ -158,6 +160,7 @@ module ActiveFacts
           composite.all_foreign_key_as_source_composite.
           sort_by{ |fk| fk.all_foreign_key_field.map(&:component).flat_map(&:path).map(&:rank_key) }.
           flat_map do |fk|
+            next nil if fk.all_foreign_key_field.size > 1
             association_name = fk.rails.from_association_name
 
             if association_name != fk.composite.rails.singular_name
@@ -171,9 +174,14 @@ module ActiveFacts
               foreign_key = ''
             end
 
+            single_fk_field = fk.all_foreign_key_field.single.component
+            if !single_fk_field.path_mandatory
+              optional = ", :optional => true"
+            end
+
             [
             fk.mapping ? "    \# #{fk.mapping.comment}" : nil,
-            "    belongs_to :#{association_name}#{class_name}#{foreign_key}",
+            "    belongs_to :#{association_name}#{class_name}#{foreign_key}#{optional}",
             fk.mapping ? '' : nil,
             ]
           end.compact
@@ -184,9 +192,10 @@ module ActiveFacts
           composite.all_foreign_key_as_target_composite.
           sort_by{ |fk| fk.all_foreign_key_field.map(&:component).flat_map(&:path).map(&:rank_key) }.
           flat_map do |fk|
+            next nil if fk.all_foreign_key_field.size > 1
 
             if fk.all_foreign_key_field.size > 1
-              raise "Can't emit Rails associations for multi-part foreign key with #{fk.references.inspect}. Did you mean to use --surrogate?"
+              raise "Can't emit Rails associations for multi-part foreign key with #{fk.all_foreign_key_field.inspect}. Did you mean to use --surrogate?"
             end
 
             association_type, association_name = *fk.rails.to_association
@@ -205,15 +214,22 @@ module ActiveFacts
               fk.source_composite.primary_index.all_index_field.map(&:component).flat_map do |ic|
                 next nil if ic.is_a?(MM::Indicator)      # or use rails.plural_name(ic.references[0].to_names) ?
                 onward_fks = ic.all_foreign_key_field.map(&:foreign_key)
-                next nil if onward_fks.size == 0 or onward_fks.detect{|fk| fk.composite == composite} # Skip the back-reference
-                # REVISIT: This far association name needs to be augmented for its role name
-                "    has_many :#{onward_fks[0].composite.rails.plural_name}, :through => :#{association_name}"
+                next nil if onward_fks.size == 0 or onward_fks.detect{|ofk| ofk.composite == composite} # Skip the back-reference
+                # This far association name needs to be augmented for its role name
+                # so the reverse associations still work for customised association names
+                source =
+                  if composite.rails.singular_name != fk.rails.from_association_name
+                    ", :source => :#{fk.rails.from_association_name}"
+                  else
+                    ''
+                  end
+                "    has_many :#{onward_fks[0].composite.rails.plural_name}, :through => :#{association_name}#{source}"
               end.compact
             else
               []
             end +
             [fk.mapping ? '' : nil]
-          end
+          end.compact
         end
 
         def column_constraints composite
@@ -223,7 +239,9 @@ module ActiveFacts
               next unless component.path_mandatory && !component.is_a?(Metamodel::Indicator)
               next if composite.primary_index != composite.natural_index && composite.primary_index.all_index_field.detect{|ixf| ixf.component == component}
               next if component.is_a?(Metamodel::Mapping) && component.object_type.is_a?(Metamodel::ValueType) && component.is_auto_assigned
-              [ "    validates :#{component.column_name.snakecase}, :presence => true" ]
+              if component.all_foreign_key_field.size == 0
+                [ "    validates :#{component.column_name.snakecase}, :presence => true" ]
+              end
             end.compact
           ccs.unshift("") unless ccs.empty?
           ccs

data/lib/activefacts/generator/traits/expr.rb

@@ -19,6 +19,7 @@ module ActiveFacts
       attr_reader :type_num       # ActiveFacts::Metamodel::DataType number
       attr_reader :value          # String representation of the expression
       attr_reader :is_mandatory   # false if nullable
+                  # This doesn't handle Postgres expressions, which can include a sub-table (e.g. via unnest)
       attr_reader :is_array       # the expression returns an array of the specified type
 
       # Construct an expression that addresses a field from a Metamodel::Component

data/lib/activefacts/generator/traits/sql/postgres.rb

@@ -146,22 +146,24 @@ module ActiveFacts
               "'|'::text || " +
               expressions.map(&:to_s) * " || '|'::text || " +
               " || '|'::text",
-              MM::DataType::TYPE_String,
-              true
+              MM::DataType::TYPE_String
             )
           end
 
           # Return an expression that yields a hash of the given expression
           def hash expr, algo = 'sha1'
-            Expression.new("digest(#{expr}, '#{algo}')", MM::DataType::TYPE_Binary, expr.is_mandatory)
+            Expression.new("digest(#{expr}, '#{algo}')", MM::DataType::TYPE_Binary, expr.is_mandatory, expr.is_array)
           end
 
           def truncate expr, length
-            Expression.new("substring(#{expr} for #{length})", MM::DataType::TYPE_String, expr.is_mandatory)
+            Expression.new("left(#{expr}, #{length})", MM::DataType::TYPE_String, expr.is_mandatory, expr.is_array)
           end
 
           def trigram expr
-            Expression.new("show_trgm(#{expr})", MM::DataType::TYPE_String, expr.is_mandatory, true)
+            # This is not a useful way to handle trigrams. Instead, create a trigram index
+            # over an ordinary text index value, and use a similarity search over that.
+            # Expression.new("show_trgm(#{expr})", MM::DataType::TYPE_String, expr.is_mandatory, true)
+            expr
           end
 
           # Produce a lexically-sortable decimal representation of the given numeric expression, to the overall specified length and scale
@@ -174,6 +176,39 @@ module ActiveFacts
             )
           end
 
+          def number_or_null expr
+            Expression.new(
+              %Q{CASE WHEN #{expr} ~ '^ *[-+]?([0-9]+[.]?[0-9]*|[.][0-9]+) *$' THEN #{expr}::numeric ELSE NULL END},
+              MM::DataType::TYPE_Real,
+              false
+            )
+          end
+
+          def split_on_separators expr, seps = ',\\\\|'
+            Expression.new(
+              %Q{regexp_split_to_table(#{expr}, E'#{seps}')},
+              MM::DataType::TYPE_String, true, true
+            )
+          end
+
+          # Extract separated numbers, remove non-digits, take the last 8 (removing area codes etc)
+          def phone_numbers expr
+            Expression.new(
+              %Q{right(#{split_on_separators(%Q{regexp_replace(#{expr}, '[^0-9]+', '', 'g')})}, 8)},
+              MM::DataType::TYPE_String,
+              true
+            )
+          end
+
+          # Extract separated numbers, remove non-digits, take the last 8 (removing area codes etc)
+          def email_addresses expr
+            Expression.new(
+              %Q{unnest(regexp_matches(#{expr}, E'[-_.[:alnum:]]+@[-_.[:alnum:]]+'))},
+              MM::DataType::TYPE_String,
+              true
+            )
+          end
+
           def lexical_date expr
             Expression.new("to_char(#{expr}, 'YYYY-MM-DD')", MM::DataType::TYPE_String, expr.is_mandatory)
           end
@@ -190,13 +225,38 @@ module ActiveFacts
             Expression.new("btrim(lower(regexp_replace(#{expr}, '[^[:alnum:]]+', ' ', 'g')))", MM::DataType::TYPE_String, expr.is_mandatory)
           end
 
+          def as_words expr, extra_word_chars = ''
+            Expression.new(
+              "regexp_split_to_array(lower(#{expr}), E'[^[:alnum:]#{extra_word_chars}]+')",
+              MM::DataType::TYPE_String, expr.is_mandatory, true
+            )
+          end
+
+          def unnest expr
+            Expression.new("unnest(#{expr})", MM::DataType::TYPE_String, expr.is_mandatory, true)
+          end
+
           def phonetics expr
-            dmetaphone = "dmetaphone(#{expr})"
-            dmetaphone_alt = "dmetaphone_alt(#{expr})"
-            [
-              Expression.new(dmetaphone, MM::DataType::TYPE_String, expr.is_mandatory),
-              Expression.new("CASE WHEN #{dmetaphone} <> #{dmetaphone_alt} THEN #{dmetaphone_alt} ELSE NULL END", MM::DataType::TYPE_String, expr.is_mandatory)
-            ]
+            if expr.is_array
+              [
+                Expression.new(
+                  %Q{dmetaphone(#{expr})},
+                  MM::DataType::TYPE_String,
+                  expr.is_mandatory
+                ),
+                Expression.new(
+                  %Q{dmetaphone_alt(#{expr})},
+                  MM::DataType::TYPE_String,
+                  expr.is_mandatory
+                )
+              ]
+            else
+              Expression.new(
+                %Q{unnest(ARRAY[dmetaphone(#{expr}), dmetaphone_alt(#{expr})])},
+                MM::DataType::TYPE_String,
+                expr.is_mandatory
+              )
+            end
           end
 
           # Reserved words cannot be used anywhere without quoting.

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: activefacts-compositions
 version: !ruby/object:Gem::Version
-  version: 1.9.18
+  version: 1.9.19
 platform: ruby
 authors:
 - Clifford Heath
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-01-15 00:00:00.000000000 Z
+date: 2018-02-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -28,16 +28,16 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '10'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '10'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -257,7 +257,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: Create and represent composite schemas, schema transforms and data transforms