arql 0.2.10 → 0.3.0

data/lib/arql/connection.rb

@@ -4,6 +4,9 @@ module Arql
       def open(options)
         ActiveRecord::Base.establish_connection(options)
         $C = ActiveRecord::Base.connection
+        $C.define_singleton_method(:dump) do |filename, no_create_db=false|
+          Arql::Mysqldump.new.dump_database(filename, no_create_db)
+        end
       end
     end
   end

data/lib/arql/definition.rb

@@ -1,3 +1,4 @@
+require 'arql/concerns'
 module Arql
   module Extension
     extend ActiveSupport::Concern
@@ -34,6 +35,10 @@ module Arql
       [self].write_excel(filename, *fields, **options)
     end
 
+    def dump(filename, batch_size=500)
+      [self].dump(filename, batch_size)
+    end
+
     included do
     end
 
@@ -65,6 +70,10 @@ module Arql
       def to_create_sql
         ActiveRecord::Base.connection.exec_query("show create table #{table_name}").rows.last.last
       end
+
+      def dump(filename, no_create_table=false)
+        Arql::Mysqldump.new.dump_table(filename, table_name, no_create_table)
+      end
     end
   end
 
@@ -145,6 +154,7 @@ module Arql
       @@models = []
       ActiveRecord::Base.connection.tap do |conn|
         Object.const_set('ArqlModel', Class.new(ActiveRecord::Base) do
+                           include ::Arql::Concerns::TableDataDefinition
                            self.abstract_class = true
 
                            define_singleton_method(:indexes) do
@@ -158,230 +168,6 @@ module Arql
                                }
                              end.t
                            end
-                           # Add a new +type+ column named +column_name+ to +table_name+.
-                           #
-                           # The +type+ parameter is normally one of the migrations native types,
-                           # which is one of the following:
-                           # <tt>:primary_key</tt>, <tt>:string</tt>, <tt>:text</tt>,
-                           # <tt>:integer</tt>, <tt>:bigint</tt>, <tt>:float</tt>, <tt>:decimal</tt>, <tt>:numeric</tt>,
-                           # <tt>:datetime</tt>, <tt>:time</tt>, <tt>:date</tt>,
-                           # <tt>:binary</tt>, <tt>:boolean</tt>.
-                           #
-                           # You may use a type not in this list as long as it is supported by your
-                           # database (for example, "polygon" in MySQL), but this will not be database
-                           # agnostic and should usually be avoided.
-                           #
-                           # Available options are (none of these exists by default):
-                           # * <tt>:limit</tt> -
-                           #   Requests a maximum column length. This is the number of characters for a <tt>:string</tt> column
-                           #   and number of bytes for <tt>:text</tt>, <tt>:binary</tt>, and <tt>:integer</tt> columns.
-                           #   This option is ignored by some backends.
-                           # * <tt>:default</tt> -
-                           #   The column's default value. Use +nil+ for +NULL+.
-                           # * <tt>:null</tt> -
-                           #   Allows or disallows +NULL+ values in the column.
-                           # * <tt>:precision</tt> -
-                           #   Specifies the precision for the <tt>:decimal</tt>, <tt>:numeric</tt>,
-                           #   <tt>:datetime</tt>, and <tt>:time</tt> columns.
-                           # * <tt>:scale</tt> -
-                           #   Specifies the scale for the <tt>:decimal</tt> and <tt>:numeric</tt> columns.
-                           # * <tt>:collation</tt> -
-                           #   Specifies the collation for a <tt>:string</tt> or <tt>:text</tt> column. If not specified, the
-                           #   column will have the same collation as the table.
-                           # * <tt>:comment</tt> -
-                           #   Specifies the comment for the column. This option is ignored by some backends.
-                           #
-                           # Note: The precision is the total number of significant digits,
-                           # and the scale is the number of digits that can be stored following
-                           # the decimal point. For example, the number 123.45 has a precision of 5
-                           # and a scale of 2. A decimal with a precision of 5 and a scale of 2 can
-                           # range from -999.99 to 999.99.
-                           #
-                           # Please be aware of different RDBMS implementations behavior with
-                           # <tt>:decimal</tt> columns:
-                           # * The SQL standard says the default scale should be 0, <tt>:scale</tt> <=
-                           #   <tt>:precision</tt>, and makes no comments about the requirements of
-                           #   <tt>:precision</tt>.
-                           # * MySQL: <tt>:precision</tt> [1..63], <tt>:scale</tt> [0..30].
-                           #   Default is (10,0).
-                           # * PostgreSQL: <tt>:precision</tt> [1..infinity],
-                           #   <tt>:scale</tt> [0..infinity]. No default.
-                           # * SQLite3: No restrictions on <tt>:precision</tt> and <tt>:scale</tt>,
-                           #   but the maximum supported <tt>:precision</tt> is 16. No default.
-                           # * Oracle: <tt>:precision</tt> [1..38], <tt>:scale</tt> [-84..127].
-                           #   Default is (38,0).
-                           # * DB2: <tt>:precision</tt> [1..63], <tt>:scale</tt> [0..62].
-                           #   Default unknown.
-                           # * SqlServer: <tt>:precision</tt> [1..38], <tt>:scale</tt> [0..38].
-                           #   Default (38,0).
-                           #
-                           # == Examples
-                           #
-                           #  User.add_column(:picture, :binary, limit: 2.megabytes)
-                           #  # ALTER TABLE "users" ADD "picture" blob(2097152)
-                           #
-                           #  Article.add_column(:status, :string, limit: 20, default: 'draft', null: false)
-                           #  # ALTER TABLE "articles" ADD "status" varchar(20) DEFAULT 'draft' NOT NULL
-                           #
-                           #  Answer.add_column(:bill_gates_money, :decimal, precision: 15, scale: 2)
-                           #  # ALTER TABLE "answers" ADD "bill_gates_money" decimal(15,2)
-                           #
-                           #  Measurement.add_column(:sensor_reading, :decimal, precision: 30, scale: 20)
-                           #  # ALTER TABLE "measurements" ADD "sensor_reading" decimal(30,20)
-                           #
-                           #  # While :scale defaults to zero on most databases, it
-                           #  # probably wouldn't hurt to include it.
-                           #  Measurement.add_column(:huge_integer, :decimal, precision: 30)
-                           #  # ALTER TABLE "measurements" ADD "huge_integer" decimal(30)
-                           #
-                           #  # Defines a column that stores an array of a type.
-                           #  User.add_column(:skills, :text, array: true)
-                           #  # ALTER TABLE "users" ADD "skills" text[]
-                           #
-                           #  # Defines a column with a database-specific type.
-                           #  Shape.add_column(:triangle, 'polygon')
-                           #  # ALTER TABLE "shapes" ADD "triangle" polygon
-                           define_singleton_method(:add_column) do |column_name, type, **options|
-                             conn.add_column(table_name, column_name, type, **options)
-                           end
-
-                           # Changes the column's definition according to the new options.
-                           # See TableDefinition#column for details of the options you can use.
-                           #
-                           #   Supplier.change_column(:name, :string, limit: 80)
-                           #   Post.change_column(:description, :text)
-                           #
-                           define_singleton_method(:change_column) do |column_name, type, options = {}|
-                             conn.change_column(table_name, column_name, type, options)
-                           end
-
-                           # Removes the column from the table definition.
-                           #
-                           #   Supplier.remove_column(:qualification)
-                           #
-                           # The +type+ and +options+ parameters will be ignored if present. It can be helpful
-                           # to provide these in a migration's +change+ method so it can be reverted.
-                           # In that case, +type+ and +options+ will be used by #add_column.
-                           # Indexes on the column are automatically removed.
-                           define_singleton_method(:remove_column) do |column_name, type = nil, **options|
-                             conn.remove_column(table_name, column_name, type, **options)
-                           end
-
-                           # Adds a new index to the table. +column_name+ can be a single Symbol, or
-                           # an Array of Symbols.
-                           #
-                           # The index will be named after the table and the column name(s), unless
-                           # you pass <tt>:name</tt> as an option.
-                           #
-                           # ====== Creating a simple index
-                           #
-                           #   Supplier.add_index(:name)
-                           #
-                           # generates:
-                           #
-                           #   CREATE INDEX suppliers_name_index ON suppliers(name)
-                           #
-                           # ====== Creating a unique index
-                           #
-                           #   Account.add_index([:branch_id, :party_id], unique: true)
-                           #
-                           # generates:
-                           #
-                           #   CREATE UNIQUE INDEX accounts_branch_id_party_id_index ON accounts(branch_id, party_id)
-                           #
-                           # ====== Creating a named index
-                           #
-                           #   Account.add_index([:branch_id, :party_id], unique: true, name: 'by_branch_party')
-                           #
-                           # generates:
-                           #
-                           #  CREATE UNIQUE INDEX by_branch_party ON accounts(branch_id, party_id)
-                           #
-                           # ====== Creating an index with specific key length
-                           #
-                           #   Account.add_index(:name, name: 'by_name', length: 10)
-                           #
-                           # generates:
-                           #
-                           #   CREATE INDEX by_name ON accounts(name(10))
-                           #
-                           # ====== Creating an index with specific key lengths for multiple keys
-                           #
-                           #   Account.add_index([:name, :surname], name: 'by_name_surname', length: {name: 10, surname: 15})
-                           #
-                           # generates:
-                           #
-                           #   CREATE INDEX by_name_surname ON accounts(name(10), surname(15))
-                           #
-                           # Note: SQLite doesn't support index length.
-                           #
-                           # ====== Creating an index with a sort order (desc or asc, asc is the default)
-                           #
-                           #   Account.add_index([:branch_id, :party_id, :surname], order: {branch_id: :desc, party_id: :asc})
-                           #
-                           # generates:
-                           #
-                           #   CREATE INDEX by_branch_desc_party ON accounts(branch_id DESC, party_id ASC, surname)
-                           #
-                           # Note: MySQL only supports index order from 8.0.1 onwards (earlier versions accepted the syntax but ignored it).
-                           #
-                           # ====== Creating a partial index
-                           #
-                           #   Account.add_index([:branch_id, :party_id], unique: true, where: "active")
-                           #
-                           # generates:
-                           #
-                           #   CREATE UNIQUE INDEX index_accounts_on_branch_id_and_party_id ON accounts(branch_id, party_id) WHERE active
-                           #
-                           # Note: Partial indexes are only supported for PostgreSQL and SQLite 3.8.0+.
-                           #
-                           # ====== Creating an index with a specific method
-                           #
-                           #   Developer.add_index(:name, using: 'btree')
-                           #
-                           # generates:
-                           #
-                           #   CREATE INDEX index_developers_on_name ON developers USING btree (name) -- PostgreSQL
-                           #   CREATE INDEX index_developers_on_name USING btree ON developers (name) -- MySQL
-                           #
-                           # Note: only supported by PostgreSQL and MySQL
-                           #
-                           # ====== Creating an index with a specific operator class
-                           #
-                           #   Developer.add_index(:name, using: 'gist', opclass: :gist_trgm_ops)
-                           #   # CREATE INDEX developers_on_name ON developers USING gist (name gist_trgm_ops) -- PostgreSQL
-                           #
-                           #   Developer.add_index([:name, :city], using: 'gist', opclass: { city: :gist_trgm_ops })
-                           #   # CREATE INDEX developers_on_name_and_city ON developers USING gist (name, city gist_trgm_ops) -- PostgreSQL
-                           #
-                           #   Developer.add_index([:name, :city], using: 'gist', opclass: :gist_trgm_ops)
-                           #   # CREATE INDEX developers_on_name_and_city ON developers USING gist (name gist_trgm_ops, city gist_trgm_ops) -- PostgreSQL
-                           #
-                           # Note: only supported by PostgreSQL
-                           #
-                           # ====== Creating an index with a specific type
-                           #
-                           #   Developer.add_index(:name, type: :fulltext)
-                           #
-                           # generates:
-                           #
-                           #   CREATE FULLTEXT INDEX index_developers_on_name ON developers (name) -- MySQL
-                           #
-                           # Note: only supported by MySQL.
-                           #
-                           # ====== Creating an index with a specific algorithm
-                           #
-                           #  Developer.add_index(:name, algorithm: :concurrently)
-                           #  # CREATE INDEX CONCURRENTLY developers_on_name on developers (name)
-                           #
-                           # Note: only supported by PostgreSQL.
-                           #
-                           # Concurrently adding an index is not supported in a transaction.
-                           #
-                           # For more information see the {"Transactional Migrations" section}[rdoc-ref:Migration].
-                           define_singleton_method(:add_index) do |column_name, options = {}|
-                             conn.add_index(table_name, column_name, options)
-                           end
                          end)
         conn.tables.each do |table_name|
           table_comment = conn.table_comment(table_name)
@@ -459,6 +245,9 @@ module Arql
         records.write_excel(filename, *fields, **options)
       end
 
+      def dump(filename, batch_size=500)
+        records.dump(filename, batch_size)
+      end
     end
   end
 end

data/lib/arql/ext/array.rb

@@ -83,11 +83,17 @@ class Array
         end
         csv << fields
       end
+      if size > 0 && first.is_a?(Hash)
+        if fields.empty?
+          fields = first.keys
+        end
+        csv << fields
+      end
       each do |row|
         if row.is_a?(Array)
           csv << row.map(&:to_s)
         else
-          csv << row.slice(fields).values.map(&:to_s)
+          csv << row.slice(*fields).values.map(&:to_s)
         end
       end
     end
@@ -105,14 +111,29 @@ class Array
           end
           sheet.add_row(fields, types: [:string] * fields.size)
         end
+        if size > 0 && first.is_a?(Hash)
+          if fields.empty?
+            fields = first.keys
+          end
+          sheet.add_row(fields, types: [:string] * fields.size)
+        end
         each do |row|
           if row.is_a?(Array)
             sheet.add_row(row.map(&:to_s), types: [:string] * row.size)
           else
-            sheet.add_row(row.slice(fields).values.map(&:to_s), types: [:string] * fields.size)
+            sheet.add_row(row.slice(*fields).values.map(&:to_s), types: [:string] * fields.size)
           end
         end
       end
     end
   end
+
+  def dump(filename, batch_size=500)
+    File.open(File.expand_path(filename), 'w') do |file|
+      group_by(&:class).each do |(klass, records)|
+        file.puts(klass.to_upsert_sql(records, batch_size))
+      end
+    end
+    {size: size, file: File.expand_path(filename)}
+  end
 end

data/lib/arql/ext/hash.rb

@@ -3,19 +3,35 @@ class Hash
     generate_excel(filename) do |workbook|
       each do |sheet_name, sheet_data|
         workbook.add_worksheet(name: sheet_name) do |sheet|
-          if sheet_data.is_a?(Hash) && sheet_data[:fields].present?
-              fields = sheet_data[:fields].map(&:to_s)
-            else
-              fields = sheet_data[:data].first.attributes.keys
-            end
+          if sheet_data.is_a?(Hash)
+            fields = sheet_data[:fields].map(&:to_s)
             sheet.add_row(fields, types: [:string] * fields.size)
-            sheet_data = sheet_data[:data]
+            sheet_data[:data].each do |row|
+              sheet.add_row(row.slice(*fields).values.map(&:to_s), types: [:string] * fields.size)
+            end
           end
-          sheet_data.each do |row|
-            if row.is_a?(Array)
-              sheet.add_row(row.map(&:to_s), types: [:string] * row.size)
-            else
-              sheet.add_row(row.slice(fields).values.map(&:to_s), types: [:string] * fields.size)
+
+          if sheet_data.is_a?(Array)
+            if sheet_data.size > 0 && sheet_data.first.is_a?(ActiveModel::Base)
+              fields = sheet_data.first.attributes.keys
+              sheet.add_row(fields, types: [:string] * fields.size)
+              sheet_data.each do |row|
+                sheet.add_row(row.slice(*fields).values.map(&:to_s), types: [:string] * fields.size)
+              end
+            end
+
+            if sheet_data.size > 0 && sheet_data.first.is_a?(Hash)
+              fields = sheet_data.first.keys
+              sheet.add_row(fields, types: [:string] * fields.size)
+              sheet_data.each do |row|
+                sheet.add_row(row.slice(*fields).values.map(&:to_s), types: [:string] * fields.size)
+              end
+            end
+
+            if sheet_data.size > 0 && sheet_data.first.is_a?(Array)
+              sheet_data.each do |row|
+                sheet.add_row(row.map(&:to_s), types: [:string] * fields.size)
+              end
             end
           end
         end

data/lib/arql/ext/kernel.rb

@@ -1,6 +1,9 @@
+require 'arql/concerns'
 module Kernel
   CSV_BOM = "\xef\xbb\xbf"
 
+  include ::Arql::Concerns::GlobalDataDefinition
+
   def sql(sql)
     ActiveRecord::Base.connection.exec_query(sql)
   end
@@ -25,7 +28,7 @@ module Kernel
           Terminal::Table.new { |t|
           t.headings = ['PK', 'Name', 'SQL Type', 'Limit', 'Precision', 'Scale', 'Default', 'Nullable', 'Comment']
           t.rows = table[:columns].map { |column|
-            pk = if [::ActiveRecord::Base.connection.primary_key(table_name)].flatten.include?(column_name)
+            pk = if [::ActiveRecord::Base.connection.primary_key(table_name)].flatten.include?(column)
                    'Y'
                  else
                    ''
@@ -44,7 +47,7 @@ module Kernel
           Terminal::Table.new { |t|
           t.headings = ['PK', 'Name', 'SQL Type', 'Limit', 'Precision', 'Scale', 'Default', 'Nullable', 'Comment']
           t.rows = table[:columns].map { |column|
-            pk = if [::ActiveRecord::Base.connection.primary_key(table_name)].flatten.include?(column_name)
+            pk = if [::ActiveRecord::Base.connection.primary_key(table_name)].flatten.include?(column)
                    'Y'
                  else
                    ''
@@ -92,7 +95,7 @@ module Kernel
   def generate_excel(filename)
     Axlsx::Package.new do |package|
       yield(package.workbook)
-      package.serialize(filename)
+      package.serialize(File.expand_path(filename))
     end
   end
 
@@ -106,159 +109,4 @@ module Kernel
     end
   end
 
-  # Example:
-  #
-  #   create_table :post, id: false, primary_key: :id do |t|
-  #     t.column :id, :bigint, precison: 19, comment: 'ID'
-  #     t.column :name, :string, comment: '名称'
-  #     t.column :gmt_created, :datetime, comment: '创建时间'
-  #     t.column :gmt_modified, :datetime, comment: '最后修改时间'
-  #   end
-  #
-  # Creates a new table with the name +table_name+. +table_name+ may either
-  # be a String or a Symbol.
-  #
-  # There are two ways to work with #create_table. You can use the block
-  # form or the regular form, like this:
-  #
-  # === Block form
-  #
-  #   # create_table() passes a TableDefinition object to the block.
-  #   # This form will not only create the table, but also columns for the
-  #   # table.
-  #
-  #   create_table(:suppliers) do |t|
-  #     t.column :name, :string, limit: 60
-  #     # Other fields here
-  #   end
-  #
-  # === Block form, with shorthand
-  #
-  #   # You can also use the column types as method calls, rather than calling the column method.
-  #   create_table(:suppliers) do |t|
-  #     t.string :name, limit: 60
-  #     # Other fields here
-  #   end
-  #
-  # === Regular form
-  #
-  #   # Creates a table called 'suppliers' with no columns.
-  #   create_table(:suppliers)
-  #   # Add a column to 'suppliers'.
-  #   add_column(:suppliers, :name, :string, {limit: 60})
-  #
-  # The +options+ hash can include the following keys:
-  # [<tt>:id</tt>]
-  #   Whether to automatically add a primary key column. Defaults to true.
-  #   Join tables for {ActiveRecord::Base.has_and_belongs_to_many}[rdoc-ref:Associations::ClassMethods#has_and_belongs_to_many] should set it to false.
-  #
-  #   A Symbol can be used to specify the type of the generated primary key column.
-  # [<tt>:primary_key</tt>]
-  #   The name of the primary key, if one is to be added automatically.
-  #   Defaults to +id+. If <tt>:id</tt> is false, then this option is ignored.
-  #
-  #   If an array is passed, a composite primary key will be created.
-  #
-  #   Note that Active Record models will automatically detect their
-  #   primary key. This can be avoided by using
-  #   {self.primary_key=}[rdoc-ref:AttributeMethods::PrimaryKey::ClassMethods#primary_key=] on the model
-  #   to define the key explicitly.
-  #
-  # [<tt>:options</tt>]
-  #   Any extra options you want appended to the table definition.
-  # [<tt>:temporary</tt>]
-  #   Make a temporary table.
-  # [<tt>:force</tt>]
-  #   Set to true to drop the table before creating it.
-  #   Set to +:cascade+ to drop dependent objects as well.
-  #   Defaults to false.
-  # [<tt>:if_not_exists</tt>]
-  #   Set to true to avoid raising an error when the table already exists.
-  #   Defaults to false.
-  # [<tt>:as</tt>]
-  #   SQL to use to generate the table. When this option is used, the block is
-  #   ignored, as are the <tt>:id</tt> and <tt>:primary_key</tt> options.
-  #
-  # ====== Add a backend specific option to the generated SQL (MySQL)
-  #
-  #   create_table(:suppliers, options: 'ENGINE=InnoDB DEFAULT CHARSET=utf8mb4')
-  #
-  # generates:
-  #
-  #   CREATE TABLE suppliers (
-  #     id bigint auto_increment PRIMARY KEY
-  #   ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
-  #
-  # ====== Rename the primary key column
-  #
-  #   create_table(:objects, primary_key: 'guid') do |t|
-  #     t.column :name, :string, limit: 80
-  #   end
-  #
-  # generates:
-  #
-  #   CREATE TABLE objects (
-  #     guid bigint auto_increment PRIMARY KEY,
-  #     name varchar(80)
-  #   )
-  #
-  # ====== Change the primary key column type
-  #
-  #   create_table(:tags, id: :string) do |t|
-  #     t.column :label, :string
-  #   end
-  #
-  # generates:
-  #
-  #   CREATE TABLE tags (
-  #     id varchar PRIMARY KEY,
-  #     label varchar
-  #   )
-  #
-  # ====== Create a composite primary key
-  #
-  #   create_table(:orders, primary_key: [:product_id, :client_id]) do |t|
-  #     t.belongs_to :product
-  #     t.belongs_to :client
-  #   end
-  #
-  # generates:
-  #
-  #   CREATE TABLE order (
-  #       product_id bigint NOT NULL,
-  #       client_id bigint NOT NULL
-  #   );
-  #
-  #   ALTER TABLE ONLY "orders"
-  #     ADD CONSTRAINT orders_pkey PRIMARY KEY (product_id, client_id);
-  #
-  # ====== Do not add a primary key column
-  #
-  #   create_table(:categories_suppliers, id: false) do |t|
-  #     t.column :category_id, :bigint
-  #     t.column :supplier_id, :bigint
-  #   end
-  #
-  # generates:
-  #
-  #   CREATE TABLE categories_suppliers (
-  #     category_id bigint,
-  #     supplier_id bigint
-  #   )
-  #
-  # ====== Create a temporary table based on a query
-  #
-  #   create_table(:long_query, temporary: true,
-  #     as: "SELECT * FROM orders INNER JOIN line_items ON order_id=orders.id")
-  #
-  # generates:
-  #
-  #   CREATE TEMPORARY TABLE long_query AS
-  #     SELECT * FROM orders INNER JOIN line_items ON order_id=orders.id
-  #
-  # See also TableDefinition#column for details on how to create columns.
-
-  def create_table(table_name, **options, &blk)
-    ActiveRecord::Base.connection.create_table(table_name, **options, &blk)
-  end
 end