brick 1.0.246 → 1.0.248

data/lib/generators/brick/migrations_builder.rb

@@ -16,6 +16,17 @@ module Brick
                   'double precision' => 'float',
                   'smallint' => 'integer', # %%% Need to put in "limit: 2"
                   'ARRAY' => 'string', # Note that we'll also add ", array: true"
+                  # MySQL data types
+                  'tinytext' => 'text',   # 255 bytes
+                  # 'text' => 'text',     # 64kb
+                  'mediumtext' => 'text', # 16mb
+                  'longtext' => 'text',   # 4gb
+                  'tinyblob' => 'binary',
+                  'blob' => 'binary',
+                  'mediumblob' => 'binary',
+                  'longblob' => 'binary',
+                  'enum' => 'string',
+                  'set' => 'string', # Note that we'll also add ", array: true"
                   # Oracle data types
                   'VARCHAR2' => 'string',
                   'CHAR' => 'string',
@@ -119,7 +130,8 @@ module Brick
         stuck = {}
         indexes = {} # Track index names to make sure things are unique
         built_schemas = {} # Track all built schemas so we can place an appropriate drop_schema command only in the first
-                          # migration in which that schema is referenced, thereby allowing rollbacks to function properly.
+                           # migration in which that schema is referenced, thereby allowing rollbacks to function properly.
+        is_airtable = nil
         versions_to_create = [] # Resulting versions to be used when updating the schema_migrations table
         # Start by making migrations for fringe tables (those with no foreign keys).
         # Continue layer by layer, creating migrations for tables that reference ones already done, until
@@ -148,8 +160,9 @@ module Brick
             mig = gen_migration_columns(relations, tbl, (tbl_parts = tbl.split('.')), (add_fks = []), built_schemas, mig_path, current_mig_time,
                                         key_type, is_4x_rails, ar_version, do_fks_last, versions_to_create)
             after_fks.concat(add_fks) if do_fks_last
-            current_mig_time[0] += 1.minute
-            versions_to_create << migration_file_write(mig_path, "create_#{::Brick._brick_index(tbl, nil, separator, relations[tbl])}", current_mig_time, ar_version, mig)
+            increment_time(mig_path, current_mig_time)
+            is_airtable ||= (relation = relations[tbl])&.fetch(:airtable_table, nil)
+            versions_to_create << migration_file_write(mig_path, "create_#{::Brick._brick_index(tbl, nil, separator, relation, true)}", current_mig_time, ar_version, mig)
           end
           done.concat(fringe)
           chosen -= done
@@ -161,15 +174,15 @@ module Brick
             mig = gen_migration_columns(relations, tbl, (tbl_parts = tbl.split('.')), (add_fks = []), built_schemas, mig_path, current_mig_time,
                                         key_type, is_4x_rails, ar_version, do_fks_last, versions_to_create)
             after_fks.concat(add_fks)
-            current_mig_time[0] += 1.minute
+            increment_time(mig_path, current_mig_time)
             versions_to_create << migration_file_write(mig_path, "create_#{
-              ::Brick._brick_index(tbl, :migration, separator, relations[tbl])
+              ::Brick._brick_index(tbl, :migration, separator, relations[tbl], true)
             }", current_mig_time, ar_version, mig)
           end
           done.concat(chosen)
           chosen.clear
 
-          case do_fks_last
+          case after_fks.present? && do_fks_last
           when 'Separate' # Add a final migration to create all the foreign keys
             mig = +"  def change\n"
             after_fks.each do |add_fk|
@@ -187,7 +200,7 @@ module Brick
             # disable the final FK migration by using a ".rbx" file extension.
             fks_extension = if after_fks.length > 500
               minutes = (after_fks.length + 1000) / 1500
-              mig << "    if ActiveRecord::Base.connection.adapter_name == 'PostgreSQL'\n"
+              mig << "    if ['PostgreSQL', 'PostGIS'].include?(ActiveRecord::Base.connection.adapter_name)\n"
               mig << "      puts 'NOTE:  It could take around #{minutes} #{'minute'.pluralize(minutes)} on a FAST machine for Postgres to do all the final processing for these foreign keys.  Please be patient!'\n"
 
               mig << "      # Vacuum takes only about ten seconds when all the tables are empty,
@@ -201,9 +214,10 @@ module Brick
               'rb'
             end
             mig << +"  end\n"
-            current_mig_time[0] += 1.minute
-            versions_to_create << migration_file_write(mig_path, "create_brick_fks.#{fks_extension}", current_mig_time, ar_version, mig)
-            puts "Have written out a final migration called 'create_brick_fks.#{fks_extension}' which creates #{after_fks.length} foreign keys."
+            increment_time(mig_path, current_mig_time)
+            base_name = "_#{relations[:base_name].tr(' ', '').underscore}" if relations.key?(:base_name)
+            versions_to_create << migration_file_write(mig_path, "create_brick_fks#{base_name}.#{fks_extension}", current_mig_time, ar_version, mig)
+            puts "Have written out a final migration called 'create_brick_fks#{base_name}.#{fks_extension}' which creates #{after_fks.length} foreign keys."
             if fks_extension == 'rbx'
               puts "  This file extension (.rbx) will cause it not to run yet when you do a 'rails db:migrate'.
   The idea here is to do all data loading first, and then rename that migration file back
@@ -247,7 +261,7 @@ module Brick
             ".  Here's the top 5 blockers" if stuck_sorted.length > 5
           }:"
           pp stuck_sorted[0..4]
-        elsif do_schema_migrations # Successful, and now we can update the schema_migrations table accordingly
+        elsif do_schema_migrations && !is_airtable # Successful, and now we can update the schema_migrations table accordingly
           smtn = ::Brick._schema_migrations_table_name
           if ActiveRecord::Migration.table_exists?(smtn)
             # Remove to_delete - to_create
@@ -298,7 +312,7 @@ module Brick
                  end
         unless schema.blank? || built_schemas.key?(schema)
           mig = +"  def change\n    create_schema(:#{schema}) unless schema_exists?(:#{schema})\n  end\n"
-          current_mig_time[0] += 1.minute
+          increment_time(mig_path, current_mig_time)
           versions_to_create << migration_file_write(mig_path, "create_db_schema_#{schema.underscore}", current_mig_time, ar_version, mig)
           built_schemas[schema] = nil
         end
@@ -352,12 +366,13 @@ module Brick
         mig = +"  def change\n    return unless reverting? || !table_exists?(#{tbl_code})\n\n"
         mig << "    create_table #{tbl_code}#{id_option} do |t|\n"
         possible_ts = [] # Track possible generic timestamps
-        relation[:cols].each do |col, col_type|
+        (relation.key?(:col_order) && relation[:col_order].present? && relation[:col_order].map { |col_name| [col_name, relation[:cols][col_name]] } ||
+         relation[:cols]).each do |col, col_type|
           sql_type = SQL_TYPES[col_type.first] || SQL_TYPES[col_type[0..1]] ||
                      SQL_TYPES.find { |r| r.first.is_a?(Regexp) && col_type.first =~ r.first }&.last ||
                      col_type.first
           suffix = col_type[3] || pkey_cols&.include?(col) ? +', null: false' : +''
-          suffix << ', array: true' if (col_type.first == 'ARRAY')
+          suffix << ', array: true' if (['ARRAY', 'set'].include?(col_type.first))
           if !is_4x_rails && klass && (comment = klass.columns_hash.fetch(col, nil)&.comment)&.present?
             suffix << ", comment: #{comment.inspect}"
           end
@@ -424,7 +439,7 @@ module Brick
         mig << "    end\n"
         if pk_is_also_fk
           mig << "    reversible do |dir|\n"
-          mig << "      dir.up { execute('ALTER TABLE #{tbl} ADD PRIMARY KEY (#{pk_is_also_fk})') }\n"
+          mig << "      dir.up { execute('ALTER TABLE #{tbl} ADD PRIMARY KEY (#{pkey_cols.map { |pk| "\"#{pk}\"" }.join(', ')})') }\n"
           mig << "    end\n"
         end
         add_fks.each do |add_fk|
@@ -449,7 +464,16 @@ module Brick
         "      t.#{type.start_with?('numeric') ? 'decimal' : type} :#{name}#{suffix}\n"
       end
 
+      def increment_time(mig_path, current_mig_time)
+        loop do
+          current_mig_time[0] += 1.minute
+          break if Dir.glob("#{mig_path}/#{current_mig_time.first.strftime('%Y%m%d%H%M00')}_*.*").empty?
+
+        end
+      end
+
       def migration_file_write(mig_path, name, current_mig_time, ar_version, mig)
+        name.gsub!(/[^a-zA-Z0-9_\.\p{L}\p{N}]/, 'x') # Strip out any weird unicode characters
         File.open("#{mig_path}/#{version = current_mig_time.first.strftime('%Y%m%d%H%M00')}_#{name}#{'.rb' unless name.index('.')}", "w") do |f|
           f.write "class #{name.split('.').first.camelize} < ActiveRecord::Migration#{ar_version}\n"
           f.write mig

data/lib/generators/brick/seeds_builder.rb

@@ -13,9 +13,10 @@ module Brick
       end
 
       def generate_seeds(relations = nil)
+        action = 'Overwrite'
         if File.exist?((seed_file_path = "#{::Rails.root}/db/seed") + 's.rb')
-          puts "WARNING: seeds file #{seed_file_path}s.rb appears to already be present.\nOverwrite?"
-          return unless gets_list(list: ['No', 'Yes']) == 'Yes'
+          puts "WARNING: seeds file #{seed_file_path}s.rb appears to already be present.\nAppend or Overwrite?"
+          return if (action = gets_list(list: ['Cancel', 'Append', 'Overwrite'])) == 'Cancel'
 
           puts "\n"
         end
@@ -24,7 +25,7 @@ module Brick
           is_airtable = true # So far the only thing that feeds us relations is Airtable
           require 'generators/brick/airtable_api_caller'
           # include ::Brick::MigrationsBuilder
-          chosen = relations.map { |k, v| SeedModel.new(k, nil, false, v[:airtable_table]) }
+          chosen = relations.each_with_object([]) { |v, s| s << SeedModel.new(v.first, nil, false, v.last[:airtable_table]) unless v.first.is_a?(Symbol) }
         else
           ::Brick.mode = :on
           ActiveRecord::Base.establish_connection
@@ -57,11 +58,14 @@ module Brick
           end
         end
         batch_num = 1
-        seeds_file = File.open("#{seed_file_path}s.rb", 'w')
+        seeds_file = File.open("#{seed_file_path}s.rb", action == 'Overwrite' ? 'w' : 'a+')
+        seeds_file.write("\n") if action == 'Append'
         seeds_len = lines_len = 0
         seeds_file.write('# Seeds file for ')
         seed_rows = nil
-        if (arbc = ActiveRecord::Base.connection).respond_to?(:current_database) # SQLite3 can't do this!
+        if is_airtable
+          seeds_file.write("Airtable Base \"#{relations[:base_name]}\":\n")
+        elsif (arbc = ActiveRecord::Base.connection).respond_to?(:current_database) # SQLite3 can't do this!
           seeds_file.write("#{arbc.current_database}:\n")
         elsif (filename = arbc.instance_variable_get(:@connection_parameters)&.fetch(:database, nil))
           seeds_file.write("#{filename}:\n")
@@ -146,19 +150,38 @@ module Brick
 
             has_rows = false
             is_empty = true
-            klass_name = is_airtable ? ::Brick::AirtableApiCaller.sane_table_name(relation[:airtable_table]&.name)&.singularize&.camelize : klass.name
-            # Pull the records
-            collection = if is_airtable
-              if (airtable_table = relation[:airtable_table])
-                ::Brick::AirtableApiCaller.https_get("https://api.airtable.com/v0/#{airtable_table.base_id}/#{airtable_table.id}").fetch('records', nil)
-              end
-            else
-              # Must convert to a symbol when doing ORDER BY due to this ActiveRecord nuance:
-              # https://github.com/rails/rails/issues/2601
-              klass.order(*pkey_cols.map(&:to_sym)) unless relation[:pkey].empty?
-            end
+            collection = nil
+            klass_name = if is_airtable
+                           airtable_table = relation[:airtable_table]
+                           ::Brick.config.table_name_prefixes&.fetch(tbl, nil) ||
+                             tbl.singularize.camelize
+                         else
+                           # Must convert to a symbol when doing ORDER BY due to this ActiveRecord nuance:
+                           # https://github.com/rails/rails/issues/2601
+                           collection = klass.order(*pkey_cols.map(&:to_sym)).enum_for unless relation[:pkey].empty?
+                           klass.name
+                         end
             seed_rows = +''
-            collection&.each do |obj|
+            airtable_idx = offset = nil
+            while collection || airtable_table do
+              obj = if is_airtable
+                if collection.nil? || (airtable_idx += 1) >= collection.length
+                  break if collection && !offset
+
+                  offset_param = "?offset=#{offset}" if offset
+                  result = ::Brick::AirtableApiCaller.https_get("https://api.airtable.com/v0/#{airtable_table.base_id}/#{airtable_table.id}#{offset_param}")
+                  offset = result.fetch('offset', nil)
+                  collection = result.fetch('records', nil)
+                  airtable_idx = 0
+                end
+                collection[airtable_idx]
+              else
+                begin
+                  collection.next
+                rescue StopIteration
+                  break
+                end
+              end
               if is_airtable
                 fields = obj['fields'].each_with_object({}) do |field, s|
                   if relation[:cols].keys.include?(col_name = ::Brick::AirtableApiCaller.sane_name(field.first))
@@ -178,7 +201,7 @@ module Brick
                     end
                   end
                 end
-                objects = relation[:airtable_table].objects
+                objects = airtable_table.objects
                 obj = objects[airtable_id = obj['id']] = AirtableObject.new(seed_model, obj['fields'], obj['createdTime'])
               end
               unless has_rows
@@ -195,16 +218,21 @@ module Brick
               data = []
               updates = []
               relation[:cols].each do |col, _col_type|
-                # Skip primary key columns, unless they are part of a foreign key.
+                # Skip primary key columns, unless they are part of a foreign key or a composite key.
                 # (But always add all columns if it's Airtable!)
                 next if !(fk = fkeys.find { |assoc| col == assoc[:fk] }) &&
                         !is_airtable &&
-                        pkey_cols.include?(col) && orig_pk_val.is_a?(Integer)
+                        pkey_cols.first == col && orig_pk_val.is_a?(Integer)
 
                 # Used to be:  obj.send(col)
                 # (and with that it was possible to raise ActiveRecord::Encryption::Errors::Configuration...)
                 #  %%% should test further and see if that is possible with this code!)
-                if (val = obj.attributes_before_type_cast[col])
+                val = if is_airtable && col == 'airtable_id'
+                        pk_val
+                      else
+                        obj.attributes_before_type_cast[col]
+                      end
+                if val
                   if (val.is_a?(Time) || val.is_a?(Date))
                     val = val.to_s
                   elsif val.is_a?(Numeric)
@@ -283,28 +311,42 @@ end\n"
               end
               seed_rows = +''
             end
-            unless has_rows || klass_name.nil?
+            unless has_rows || (is_airtable && !airtable_table)
               seeds_file.write("  # (Skipping #{klass_name} as it has no rows)\n")
             end
           end
           done.concat(fringe)
           chosen -= done
         end
-        airtable_assocs_done = {}
-        airtable_assoc_recids.each do |table_name, assoc_pairs| # N:M links
-          # Make note of any other tables which have exactly duplicated data
-          dupes = airtable_assocs_done.select do |dupe_table, dupe_pairs|
-            dupe_table != table_name && assoc_pairs.length > 0 && assoc_pairs.length == dupe_pairs.length &&
-            dupe_pairs.all? { |pair_name, _v3| assoc_pairs.keys.include?(pair_name) }
+
+        if airtable_assoc_recids.present?
+          seed_rows << "\n# Seeding #{airtable_assoc_recids.count} sets of Airtable associations:\n"
+          airtable_assocs_done = {}
+          dupe_list = []
+          airtable_assoc_recids.each do |table_name, assoc_pairs| # N:M links
+            # Make note of any other tables which have exactly duplicated data
+            dupes = airtable_assocs_done.select do |dupe_table, dupe_pairs|
+              dupe_table != table_name && assoc_pairs.length > 0 && assoc_pairs.length == dupe_pairs.length &&
+              dupe_pairs.all? { |pair_name, _v3| assoc_pairs.keys.include?(pair_name) }
+            end
+            seed_rows << "  if ActiveRecord::Migration.table_exists?('#{table_name}')\n"
+            unless dupes.empty?
+              seed_rows << "    # Duplicate data found in: #{dupes.keys.join(', ')}\n"
+              dupes.keys.each { |k| dupe_list << "#{table_name} | #{k}" }
+            end
+            seed_rows << "    puts 'Airtable associations for #{table_name.camelize.singularize}'\n"
+            assoc_pairs.each do |_k2, pair_values|
+              seed_rows << "#{table_name.singularize.camelize}.create(#{pair_values})\n"
+            end
+            seed_rows << "  end\n"
+            airtable_assocs_done[table_name] = assoc_pairs
           end
-          seed_rows << "  if ActiveRecord::Migration.table_exists?('#{table_name}')\n"
-          seed_rows << "    # Duplicate data found in: #{dupes.keys.join(', ')}\n" unless dupes.empty?
-          seed_rows << "    puts 'Seeding Airtable associations for #{table_name.camelize.singularize}'\n"
-          assoc_pairs.each do |_k2, pair_values|
-            seed_rows << "#{table_name.singularize.camelize}.create(#{pair_values})\n"
+
+          if dupe_list.present?
+            puts "\nAssociative tables with duplicate data:"
+            dupe_list.each { |entry| puts "  #{entry}" }
+            puts '(You can safely remove one of the two tables in each of the pairs listed above.)'
           end
-          seed_rows << "  end\n"
-          airtable_assocs_done[table_name] = assoc_pairs
         end
         seeds_file.write(seed_rows) unless seed_rows.blank?
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: brick
 version: !ruby/object:Gem::Version
-  version: 1.0.246
+  version: 1.0.248
 platform: ruby
 authors:
 - Lorin Thwaits