flydata 0.6.14 → 0.7.0

data/lib/flydata/source_postgresql/query_based_sync/resource_requester.rb

@@ -0,0 +1,86 @@
+require 'flydata/query_based_sync/resource_requester.rb'
+require 'flydata/source_postgresql/query_based_sync/response.rb'
+require 'flydata/source_postgresql/query_based_sync/diff_query_generator'
+require 'flydata-core/postgresql/pg_client'
+
+module Flydata
+module SourcePostgresql
+module QueryBasedSync
+
+class ResourceRequester < Flydata::QueryBasedSync::ResourceRequester
+  RESPONSE_CLASS = Flydata::SourcePostgresql::QueryBasedSync::Response
+
+  # Override
+  def create_resource_client(context)
+    FlydataCore::Postgresql::PGClient.new(context.dbconf)
+  end
+
+  # Override
+  def fetch_responses_once(table_name, latest_src_pos)
+    table_sym = table_name.to_sym
+    t_meta = context.table_meta[table_sym]
+
+    table_src_pos = context.table_src_pos_files[table_sym].pos
+    master_src_pos = context.cur_src_pos_file.pos
+
+    # Set from_sid, to_sid and pk_values
+
+    to_sid = latest_src_pos.snapshot
+    pk_values = nil
+
+    # - table.binlog.pos does not exist
+    if table_src_pos.nil?
+      from_sid = master_src_pos.snapshot
+    # - pk_values exists -> continue from the last transaction
+    elsif table_src_pos.pk_values
+      from_sid = table_src_pos.snapshot
+      to_sid = table_src_pos.to_snapshot
+      pk_values = table_src_pos.pk_values
+    else
+      from_sid = table_src_pos.snapshot
+    end
+
+    if from_sid == to_sid
+      log_debug("Skip fetching records (No changes)")
+      return
+    end
+
+    log_debug("Start query", from_sid: from_sid, to_sid: to_sid, pk_values: pk_values)
+
+    # Create and execute query
+    query = generate_query(t_meta, from_sid, to_sid, pk_values)
+
+    result = @resource_client.query(query)
+
+    # Create response object based on result
+    responses = build_responses(table_name, result, from_sid: from_sid, to_sid: to_sid, pk_values: pk_values)
+
+    Array(responses)
+  end
+
+  private
+
+  def generate_query(t_meta, from_sid, to_sid, pk_values)
+    last_pks = if pk_values
+                 pk_values.collect{|h| h.values.first}
+               else
+                 nil
+               end
+    DiffQueryGenerator.new(t_meta[:table_name], t_meta[:table_schema],
+      to_sid: to_sid,
+      from_sid: from_sid,
+      columns: t_meta[:columns],
+      pk_columns: t_meta[:primary_keys],
+      last_pks: last_pks,
+      limit: t_meta[:max_num_rows_per_query],
+    ).build_query
+  end
+
+  def build_responses(table_name, raw_result, query_cond = {})
+    RESPONSE_CLASS.create_responses(context, table_name, raw_result, query_cond)
+  end
+end
+
+end
+end
+end

data/lib/flydata/source_postgresql/query_based_sync/response.rb

@@ -0,0 +1,12 @@
+require 'flydata/query_based_sync/response'
+
+module Flydata
+module SourcePostgresql
+module QueryBasedSync
+
+class Response < Flydata::QueryBasedSync::Response
+end
+
+end
+end
+end

data/lib/flydata/source_postgresql/query_based_sync/response_handler.rb

@@ -0,0 +1,12 @@
+require 'flydata/query_based_sync/response_handler'
+
+module Flydata
+module SourcePostgresql
+module QueryBasedSync
+
+  class ResponseHandler < Flydata::QueryBasedSync::ResponseHandler
+  end
+
+end
+end
+end

data/lib/flydata/source_postgresql/sync_generate_table_ddl.rb

@@ -1,6 +1,5 @@
 require 'flydata/source/sync_generate_table_ddl'
-require 'flydata-core/table_def/postgresql_table_def'
-require 'flydata/source_postgresql/pg_client'
+require 'flydata/source_postgresql/table_meta'
 
 module Flydata
 module SourcePostgresql
@@ -21,96 +20,43 @@ class SyncGenerateTableDdl < Source::SyncGenerateTableDdl
   def each_source_tabledef(tables, options, &block)
     # PostgreSQL options.
     tables = tables.clone
-    missing_tables = []
-    begin
-      if tables.to_s == '' || tables.to_s == '[]'
-        raise ArgumentError, "tables is nil or empty"
-      end
-      _each_tabledef(tables, options, &block)
-    rescue TableMissingError => e
-      tables.delete e.table
-      missing_tables << e.table
-      return missing_tables if tables.empty?
-      retry
+    if tables.to_s == '' || tables.to_s == '[]'
+      raise ArgumentError, "tables is nil or empty"
     end
-    missing_tables
+    _each_tabledef(tables, options, &block)
   end
 
   private
 
-  COLUMNS_QUERY = <<EOS
-SELECT c.table_name, c.column_name, c.data_type, c.character_octet_length,
-  c.numeric_precision, c.numeric_scale, c.is_nullable, c.column_default, i.indisprimary AS is_primary
-FROM pg_index i
-JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY(i.indkey)
-RIGHT JOIN
-(SELECT (table_catalog ||'.'|| table_schema ||'.'|| table_name)::regclass AS regid, *
-FROM information_schema.columns) c
- ON i.indrelid = c.regid AND a.attname = c.column_name
-WHERE c.table_schema = $1 AND c.table_name IN (%s)
-ORDER BY c.table_name, c.ordinal_position;
-EOS
-  TABLE_PLACEHOLDER_START_NUM = 2 # because $1 is used by table_schema
+  def table_meta(tables, options)
+    Flydata::SourcePostgresql::TableMeta.new(options, tables)
+  end
 
   def _each_tabledef(tables, options, &block)
-    cli = PGClient.new(options)
+    all_table_meta = options[:table_meta]
+    unless all_table_meta
+      all_table_meta = table_meta(tables, options)
+      all_table_meta.reload
+    end
 
-    # TODO call the query for every 50 tables
-    res = cli.query(COLUMNS_QUERY, [options['schema']] + tables,
-                    placeholder_start_num: TABLE_PLACEHOLDER_START_NUM,
-                    placeholder_size: tables.size)
+    missing_tables = []
 
-    create_opt = {}
-    if options.has_key?(:skip_primary_key_check)
-      create_opt[:skip_primary_key_check] = options[:skip_primary_key_check]
-    end
-    current_table = nil
-    columns = []
-    completed_tables = []
-    res.each do |row|
-      table_name = row["table_name"]
-      unless table_name == current_table
-        unless columns.empty?
-          tabledef = create_tabledef_and_yield(columns, create_opt, &block)
-          completed_tables << current_table
-          columns = []
-          break unless tabledef
-        end
-        current_table = table_name
+    tables.each do |table|
+      t_meta = all_table_meta[table]
+      unless t_meta && !t_meta.empty?
+        missing_tables << table
+        next
       end
-      columns << row
-    end
-    unless columns.empty?
-      create_tabledef_and_yield(columns, create_opt, &block)
-      completed_tables << current_table
-    end
-    missing_tables = tables - completed_tables
-    unless missing_tables.empty?
-      raise TableMissingError.new("Table is missing", missing_tables.first)
-    end
-  end
-
-  def placeholder_string(num_items, start_num)
-    num_items.times.collect{|i| "$#{i + start_num}"}.join(",")
-  end
 
-  def create_tabledef_and_yield(columns, create_opt, &block)
-    pg_tabledef = nil
-    begin
-      pg_tabledef = FlydataCore::TableDef::PostgresqlTableDef.create(columns, create_opt)
-      yield(pg_tabledef, nil) if pg_tabledef
-    rescue FlydataCore::TableDefError => e
-      yield(nil, e)
+      table_def = t_meta[:table_def]
+      if table_def
+        yield(table_def, nil)
+      else
+        yield(nil, t_meta[:table_def_err])
+      end
     end
-    pg_tabledef
-  end
 
-  class TableMissingError < RuntimeError
-    def initialize(message, table)
-      super(message)
-      @table = table
-    end
-     attr_reader :table
+    missing_tables
   end
 end
 

data/lib/flydata/source_postgresql/table_meta.rb

@@ -0,0 +1,168 @@
+require 'flydata/table_meta'
+require 'flydata-core/postgresql/pg_client'
+require 'flydata-core/postgresql/snapshot'
+require 'flydata-core/table_def/postgresql_table_def'
+
+
+module Flydata
+module SourcePostgresql
+
+  # Fetch and keep table meta information
+  #
+  #  <table-name(Symbol)>:
+  #    primary_keys: <Array of String>    # Set primary key names. ex: ['group_id', 'category_id']
+  #    pk_positions: <Array of Integer>   # Set the ordinal position of primary keys. ex: [1, 3]
+  #    max_row_size: <Integer>            # byte, calculated based on column size
+  #    max_num_rows_per_query>: <Integer> # max number of rows per query
+  #    raw_columns: <Hash>                # raw information schema data
+  #    columns: table_def.columns
+  #    table_def:
+  #      columns:
+  #        table:
+  #        column:
+  #        type:
+  #        not_null:
+  #        primary_key:
+  #        default:
+  #        column_size: (new)  # Set in `PostgresqlTableDef.parse_one_column_def`
+  #
+  class TableMeta < Flydata::TableMeta
+
+    GET_TABLE_META_SQL = <<EOT
+SELECT
+  c.table_catalog,
+  c.table_schema,
+  c.table_name,
+  c.column_name,
+  c.ordinal_position,
+  c.column_default,
+  c.is_nullable,
+  c.data_type,
+  c.character_maximum_length,
+  c.character_octet_length,
+  CASE WHEN c.data_type='money' THEN 19
+   ELSE c.numeric_precision END AS numeric_precision,
+  c.numeric_precision_radix,
+  CASE WHEN c.data_type='money' THEN (
+    SELECT CASE WHEN scale IS NULL THEN 0 ELSE scale END
+      FROM (SELECT length(substring(999999::money::char varying, '\.([0-9]+)$')) AS scale) s)
+   ELSE c.numeric_scale END AS numeric_scale,
+  c.datetime_precision,
+  i.indisprimary AS is_primary
+FROM
+  pg_index i
+  JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY(i.indkey)
+  RIGHT JOIN
+    (SELECT
+      (table_catalog ||'.'|| table_schema ||'.'|| table_name)::regclass AS regid,
+      *
+     FROM information_schema.columns
+    ) c
+  ON i.indrelid = c.regid AND a.attname = c.column_name
+WHERE
+  c.table_catalog = '%{database}'
+  AND c.table_schema IN (%{schema})
+  AND c.table_name IN (%{tables})
+ORDER BY
+  c.table_name, c.ordinal_position;
+EOT
+
+    GET_CURRENT_SNAPSHOT_SQL = "SELECT txid_current_snapshot();"
+
+    DEFAULT_MAX_FETCH_RECORD_SIZE = 50000
+    #DEFAULT_MAX_FETCH_RECORD_SIZE = 8
+
+    def initialize(dbconf, tables, schema = nil) # 64mb
+      @dbconf = dbconf
+      @database = dbconf[:dbname] || dbconf[:database] || dbconf['database']
+      @tables = tables
+      @schema = schema || dbconf[:schema] || dbconf['schema']
+    end
+
+    attr_reader :current_snapshot
+
+    def reload(pg_client = nil)
+      schema = @schema ? "'#{@schema}'" : "select current_schema"
+      sql = GET_TABLE_META_SQL % {
+        database: @database,
+        schema: schema,
+        tables: @tables.collect{|t| "'#{t}'"}.join(','),
+      }
+
+      conn = pg_client
+      if conn.nil?
+        local_conn = conn = FlydataCore::Postgresql::PGClient.new(@dbconf)
+      end
+
+      # Set table_meta
+      columns = conn.query(sql)
+      @table_meta = build_table_meta(columns)
+
+      # Set current snapshot
+      current_snapshot_str = conn.
+        query(GET_CURRENT_SNAPSHOT_SQL).first['txid_current_snapshot']
+      @current_snapshot = FlydataCore::Postgresql::Snapshot.new(current_snapshot_str)
+
+      self
+    ensure
+      local_conn.close rescue nil if local_conn
+    end
+
+    def build_table_meta(columns)
+      ret = Hash.new{|h,k| h[k]={} }
+
+      # Put ret[<table-name-sym>][:raw_columns]
+      columns.each do |col|
+        column_name = col['column_name'].to_sym
+        table_name = col['table_name'].to_sym
+        t_meta = ret[table_name]
+        t_meta[:raw_columns] ||= {}
+        t_meta[:raw_columns][column_name] = col
+      end
+
+      ret.each do |table_name, t_meta|
+        begin
+          table_def = FlydataCore::TableDef::PostgresqlTableDef.create(
+                      t_meta[:raw_columns].values, :skip_primary_key_check)
+        rescue FlydataCore::TableDefError => e
+          t_meta.merge!(
+            table_name: table_name,
+            table_def_err: e,
+          )
+          # Skip when getting an error when parsing the columns
+          next
+        end
+
+        primary_keys = []
+        pk_positions = []
+        table_def.columns.each.with_index(1) do |col, index|
+          col_name = col[:column]
+          if col[:primary_key]
+            primary_keys << col_name
+            pk_positions << index.to_s
+          end
+        end
+
+        t_meta.merge!(
+          table_name: table_name.to_s,
+          primary_keys: primary_keys,
+          pk_positions: pk_positions,
+          #max_row_size: max_row_size,                  #TODO: calculation
+          #max_num_rows_per_query: max_row_size + 128,  #TODO: calculation
+          max_num_rows_per_query: DEFAULT_MAX_FETCH_RECORD_SIZE,
+          columns: table_def.columns,
+          table_def: table_def,
+        )
+      end
+
+      ret
+    end
+
+    def calc_column_size(column)
+      #TODO: Implement the check logic based on column type
+      124
+    end
+  end
+
+end
+end

data/lib/flydata/sync_file_manager.rb

@@ -9,7 +9,7 @@ module Flydata
   class SyncFileManager
     DUMP_DIR = ENV['FLYDATA_DUMP'] || File.join(FLYDATA_HOME, 'dump')
     BACKUP_DIR = ENV['FLYDATA_BACKUP'] || File.join(FLYDATA_HOME, 'backup')
-    TABLE_POSITIONS_DIR = ENV['FLYDATA_TABLE_POSITIONS'] || File.join(FLYDATA_HOME, 'positions')
+    TABLE_POSITIONS_DIR = FLYDATA_TABLE_POSITIONS_DIR
 
     def initialize(data_entry, source = nil)
       @data_entry = data_entry
@@ -65,7 +65,6 @@ module Flydata
 
     def save_generated_ddl(tables, contents = "1")
       tables = [ tables ] unless tables.kind_of?(Array)
-      table_positions_dir_path = ENV['FLYDATA_TABLE_POSITIONS'] || File.join(FLYDATA_HOME, 'positions')
       #Create positions if dir does not exist
       unless File.directory?(table_positions_dir_path)
         FileUtils.mkdir_p(table_positions_dir_path)
@@ -76,7 +75,6 @@ module Flydata
     end
 
     def get_new_table_list(tables, file_type)
-      table_positions_dir_path = ENV['FLYDATA_TABLE_POSITIONS'] || File.join(FLYDATA_HOME, 'positions')
       new_tables = []
       tables.each do |table|
         new_tables << table unless File.exists?(File.join(table_positions_dir_path, "#{table}.#{file_type}"))
@@ -408,13 +406,15 @@ module Flydata
     end
 
     def get_table_source_raw_pos(table_name) #returns String. interface for fluentd
-      file = File.join(table_positions_dir_path, table_name + ".binlog.pos")
+      file = table_source_pos_file_path(table_name)
       return nil unless File.exists?(file)
 
       File.open(file, 'r').readline
     end
 
-
+    def table_source_pos_file_path(table_name)
+      File.join(table_positions_dir_path, table_name + ".binlog.pos")
+    end
 
     def install_table_source_pos_files(tables)
       FileUtils.mkdir_p(table_positions_dir_path) unless Dir.exists?(table_positions_dir_path)