coopy 0.6.4

data/lib/coopy/scraperwiki_utils.rb

@@ -0,0 +1,23 @@
+def link_tables(watch_scraper,watch_tables)
+  sql = ScraperwikiSqlWrapper.new(ScraperWiki)
+  watch_tables.each { |tbl| sql.copy_table_structure(watch_scraper,tbl) }
+end
+
+class CoopyResult
+  attr_accessor :html
+end
+
+def sync_table(watch_scraper,tbl,keys)
+  sql = ScraperwikiSqlWrapper.new(ScraperWiki)
+  sql.set_primary_key(keys) unless keys.nil? 
+  cmp = SqlCompare.new(sql,"main.#{tbl}","#{watch_scraper}.#{tbl}")
+  sinks = DiffOutputGroup.new
+  render = DiffRenderHtml.new
+  sinks << render
+  sinks << DiffApplySql.new(sql,"main.#{tbl}")
+  cmp.set_output(sinks)
+  cmp.apply
+  result = CoopyResult.new
+  result.html = render.to_string
+  result
+end

data/lib/coopy/sequel_sql_wrapper.rb

@@ -0,0 +1,73 @@
+require 'sql_wrapper'
+require 'sequel'
+
+class SequelSqlBare < SqlWrapper
+  def initialize(db)
+    @db = db
+    @tname = nil
+    @t = nil
+  end
+
+  def sync_table(tbl)
+    tbl = @tname if tbl.nil?
+    tbl = @db.tables[0] if tbl.nil?
+    return @t if tbl==@tname
+    @tname = tbl
+    @t = @db[tbl]
+  end
+
+  def enhash(cols,vals)
+    Hash[*cols.map{|c| c.to_sym}.zip(vals).flatten]
+  end
+
+  def insert(tbl,cols,vals)
+    sync_table(tbl)
+    @t.insert(enhash(cols,vals))
+  end
+
+  def delete(tbl,cols,vals)
+    sync_table(tbl)
+    @t.filter(enhash(cols,vals)).delete
+  end
+  
+  def update(tbl,set_cols,set_vals,cond_cols,cond_vals)
+    sync_table(tbl)
+    @t.filter(enhash(cond_cols,cond_vals)).update(enhash(set_cols,set_vals))
+  end
+
+  def transaction(&block)
+    @db.transaction(&block)
+  end
+
+  def columns(tbl)
+    sync_table(tbl)
+    @db.schema(@tname)
+  end
+
+  def column_names(tbl)
+    columns(tbl).map{|x| x[0]}
+  end
+  
+  def primary_key(tbl)
+    cols = columns(tbl)
+    cols.select{|x| x[1][:primary_key]}.map{|x| x[0]}
+  end
+
+  def index(tbl)
+    key = primary_key(tbl)
+    @t.select(*key)
+  end
+
+  def fetch(sql,names)
+    @db.fetch(sql) do |row|
+      yield names.map{|n| row[n]}
+    end
+  end
+end
+
+
+class SequelSqlWrapper < SequelSqlBare
+  def initialize(*params)
+    super(Sequel.connect(*params))
+  end
+end

data/lib/coopy/sql_compare.rb

@@ -0,0 +1,222 @@
+require 'coopy/diff_columns'
+require 'coopy/row_change'
+
+class SqlCompare
+  def initialize(db1,db2)
+    @db1 = db
+    @db2 = db2
+    @table1 = nil
+    @table2 = nil
+    @single_db = false
+    raise "not implemented yet"
+  end
+
+  def initialize(db,table1,table2)
+    @db1 = db
+    @db2 = db.clone
+    @table1 = table1
+    @table2 = table2
+    @single_db = true
+  end
+
+  def set_output(patch)
+    @patch = patch
+  end
+
+  def apply
+    apply_single
+  end
+
+  # We are not implementing full comparison, just an adequate subset
+  # for easy cases (a table with a trustworthy primary key, and constant
+  # columns).  Make sure we are not trying to do something we're not ready 
+  # for.
+  def validate_schema
+    all_cols1 = @db1.column_names(@table1)
+    all_cols2 = @db2.column_names(@table2)
+    if all_cols1 != all_cols2
+      raise "Columns do not match, please use full coopy toolbox"
+    end
+
+    key_cols1 = @db1.primary_key(@table1)
+    key_cols2 = @db2.primary_key(@table2)
+    if key_cols1 != key_cols2
+      raise "Primary keys do not match, please use full coopy toolbox"
+    end
+  end
+
+  def keyify(lst)
+    lst.map{|x| x.to_s}.join("___")
+  end
+
+  # When working within a single database, we can delegate more work to SQL.
+  # So we specialize this case.
+  def apply_single
+    validate_schema
+
+    # Prepare some lists of columns.
+    key_cols = @db1.primary_key(@table1)
+    data_cols = @db1.except_primary_key(@table1)
+    all_cols = @db1.column_names(@table1)
+
+    # Let our public know we are beginning.
+    @patch.begin_diff
+
+    # Advertise column names.
+    @rc_columns = DiffColumns.new
+    @rc_columns.title_row = all_cols
+    @rc_columns.update(0)
+    cells = all_cols.map{|v| { :txt => v, :value => v, :cell_mode => "" }}
+    rc = RowChange.new("@@",cells)
+    @patch.apply_row(rc)
+
+    # If requested, we will be providing context rows around changed rows.
+    # This is not a natural thing to do with SQL, so we do it only on request.
+    # When requested, we need to buffer row changes.
+    @pending_rcs = []
+
+    # Prepare some useful SQL fragments to assemble later.
+    sql_table1 = @db1.quote_table(@table1)
+    sql_table2 = @db1.quote_table(@table2)
+    sql_key_cols = key_cols.map{|c| @db1.quote_column(c)}.join(",")
+    sql_all_cols = all_cols.map{|c| @db1.quote_column(c)}.join(",")
+    sql_key_match = key_cols.map{|c| @db1.quote_column(c)}.map{|c| "#{sql_table1}.#{c} IS #{sql_table2}.#{c}"}.join(" AND ")
+    sql_data_mismatch = data_cols.map{|c| @db1.quote_column(c)}.map{|c| "#{sql_table1}.#{c} IS NOT #{sql_table2}.#{c}"}.join(" OR ")
+
+    # For one query we will need to interleave columns from two tables.  For
+    # portability we need to give these columns distinct names.
+    weave = all_cols.map{|c| [[sql_table1,@db1.quote_column(c)],
+                              [sql_table2,@db2.quote_column(c)]]}.flatten(1)
+    dbl_cols = weave.map{|c| "#{c[0]}.#{c[1]}"}
+    sql_dbl_cols = weave.map{|c| "#{c[0]}.#{c[1]} AS #{c[0].gsub(/[^a-zA-Z0-9]/,'_')}_#{c[1].gsub(/[^a-zA-Z0-9]/,'_')}"}.join(",")
+
+    # Prepare a map of primary key offsets.
+    keys_in_all_cols = key_cols.each.map{|c| all_cols.index(c)}
+    keys_in_dbl_cols = keys_in_all_cols.map{|x| 2*x}
+
+    # Find rows in table2 that are not in table1.
+    sql = "SELECT #{sql_all_cols} FROM #{sql_table2} WHERE NOT EXISTS (SELECT 1 FROM #{sql_table1} WHERE #{sql_key_match})"
+    apply_inserts(sql,all_cols,keys_in_all_cols)
+
+    # Find rows in table1 and table2 that differ while having the same primary
+    # key.
+    sql = "SELECT #{sql_dbl_cols} FROM #{sql_table1} INNER JOIN #{sql_table2} ON #{sql_key_match} WHERE #{sql_data_mismatch}"
+    apply_updates(sql,dbl_cols,keys_in_dbl_cols)
+
+    # Find rows that are in table1 but not table2
+    sql = "SELECT #{sql_all_cols} FROM #{sql_table1} WHERE NOT EXISTS (SELECT 1 FROM #{sql_table2} WHERE #{sql_key_match})"
+    apply_deletes(sql,all_cols,keys_in_all_cols)
+
+    # If we are supposed to provide context, we need to deal with row order.
+    if @patch.want_context
+      sql = "SELECT #{sql_all_cols}, 0 AS __coopy_tag__ FROM #{sql_table1} UNION SELECT #{sql_all_cols}, 1 AS __coopy_tag__ FROM #{sql_table2} ORDER BY #{sql_key_cols}, __coopy_tag__"
+      apply_with_context(sql,all_cols,keys_in_all_cols)
+    end
+
+    # Done!
+    @patch.end_diff
+  end
+
+
+  def apply_inserts(sql,all_cols,keys_in_all_cols)
+    @db1.fetch(sql,all_cols) do |row|
+      cells = row.map{|v| { :txt => v, :value => v, :cell_mode => "" }}
+      rc = RowChange.new("+++",cells)
+      apply_rc(rc,row,keys_in_all_cols)
+    end
+  end
+
+
+  def apply_updates(sql,dbl_cols,keys_in_dbl_cols)
+    @db1.fetch(sql,dbl_cols) do |row|
+      pairs = row.enum_for(:each_slice,2).to_a
+      cells = pairs.map do |v| 
+        if v[0]==v[1] 
+          { :txt => v[0], :value => v[0], :cell_mode => "" }
+        else
+          { :txt => v[0], :value => v[0], :new_value => v[1], :cell_mode => "->" }
+        end
+      end
+      rc = RowChange.new("->",cells)
+      apply_rc(rc,row,keys_in_dbl_cols)
+    end
+  end
+
+
+  def apply_deletes(sql,all_cols,keys_in_all_cols)
+    @db1.fetch(sql,all_cols) do |row|
+      cells = row.map{|v| { :txt => v, :value => v, :cell_mode => "" }}
+      rc = RowChange.new("---",cells)
+      apply_rc(rc,row,keys_in_all_cols)
+    end
+  end
+
+  def apply_rc(rc,row,keys_in_cols)
+    rc.columns = @rc_columns
+    if @patch.want_context
+      rc.key = keyify(row.values_at(*keys_in_cols))
+      @pending_rcs << rc
+    else
+      @patch.apply_row(rc)
+    end
+  end
+
+  def emit_skip(row)
+    cells = row.map{|v| { :txt => "...", :value => "...", :cell_mode => "" }}
+    rc = RowChange.new("...",cells)
+    rc.columns = @rc_columns
+    @patch.apply_row(rc)
+  end
+
+  # Do the context dance.
+  def apply_with_context(sql,all_cols,keys_in_all_cols)
+    hits = {}
+    @pending_rcs.each do |rc|
+      hits[rc.key] = rc
+    end 
+    hist = []
+    n = 2
+    pending = 0
+    skipped = false
+    noted = false
+    last_row = nil
+    @db1.fetch(sql,all_cols + ["__coopy_tag__"]) do |row|
+      tag = row.pop.to_i
+      k = keyify(row.values_at(*keys_in_all_cols))
+      if hits[k]
+        emit_skip(row) if skipped
+        hist.each do |row0|
+          cells = row0.map{|v| { :txt => v, :value => v, :cell_mode => "" }}
+          rc = RowChange.new("",cells)
+          rc.columns = @rc_columns
+          @patch.apply_row(rc)
+        end
+        hist.clear
+        pending = n
+        @patch.apply_row(hits[k])
+        hits.delete(k)
+        skipped = false
+        noted =  true
+      elsif tag == 1
+        # ignore redundant row
+      elsif pending>0
+        emit_skip(row) if skipped
+        cells = row.map{|v| { :txt => v, :value => v, :cell_mode => "" }}
+        rc = RowChange.new("",cells)
+        rc.columns = @rc_columns
+        @patch.apply_row(rc)
+        pending = pending-1
+        skipped = false
+      else
+        hist << row
+        if hist.length>n
+          skipped = true
+          last_row = row
+          hist.shift
+        end
+      end
+    end
+    emit_skip(last_row) if skipped and noted
+  end
+end
+

data/lib/coopy/sql_wrapper.rb

@@ -0,0 +1,34 @@
+class SqlWrapper
+  def insert(tbl,cols,vals)
+  end
+
+  def delete(tbl,cols,vals)
+  end
+
+  def update(tbl,set_cols,set_vals,cond_cols,cond_vals)
+  end
+
+  def column_names(tbl)
+    []
+  end
+
+  def primary_key(tbl)
+    []
+  end
+
+  def except_primary_key(tbl)
+    column_names(tbl)-primary_key(tbl)
+  end
+
+  def fetch(sql)
+    []
+  end
+
+  def quote_column(c)
+    c.to_s
+  end
+
+  def quote_table(t)
+    t.to_s
+  end
+end

data/lib/coopy/sqlite_sql_wrapper.rb

@@ -0,0 +1,143 @@
+require 'coopy/sql_wrapper'
+
+class SqliteSqlWrapper < SqlWrapper
+  def initialize(db)
+    @db = db
+    @t = nil
+    @qt = nil
+    @pk = nil
+    @info = {}
+  end
+
+  def set_primary_key(lst)
+    @pk = lst
+  end
+
+  def sqlite_execute(template,vals)
+    return @db.execute(template,*vals)
+  end
+
+  def get_table_names
+    sqlite_execute("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name",[]).flatten
+  end
+
+  def complete_table(tbl)
+    @t = tbl unless tbl.nil?
+    @t = get_table_names[0] if @t.nil?
+    @t
+  end
+
+  def quote_with_dots(x)
+    return x if x.match(/^[a-zA-Z0-9_]+$/)
+    x.split('.').map{|p| "`#{p}`"}.join('.')
+  end
+
+  def quote_table(tbl)
+    complete_table(tbl)
+    return @t if @t.match(/^[a-zA-Z0-9_]+$/)
+    quote_with_dots(@t)
+  end
+
+  def quote_column(col)
+    return col if col.match(/^[a-zA-Z0-9_]+$/)
+    quote_with_dots(col)
+  end
+
+  def insert(tbl,cols,vals)
+    tbl = quote_table(tbl)
+    template = cols.map{|x| '?'}.join(",")
+    template = "INSERT INTO #{tbl} VALUES(#{template})"
+    sqlite_execute(template,vals)
+  end
+
+  def delete(tbl,cols,vals)
+    tbl = quote_table(tbl)
+    template = cols.map{|c| quote_column(c) + ' IS ?'}.join(" AND ")
+    template = "DELETE FROM #{tbl} WHERE #{template}"
+    sqlite_execute(template,vals)
+  end
+  
+  def update(tbl,set_cols,set_vals,cond_cols,cond_vals)
+    tbl = quote_table(tbl)
+    conds = cond_cols.map{|c| quote_column(c) + ' IS ?'}.join(" AND ")
+    sets = set_cols.map{|c| quote_column(c) + ' = ?'}.join(", ")
+    template = "UPDATE #{tbl} SET #{sets} WHERE #{conds}"
+    v = set_vals + cond_vals
+    sqlite_execute(template,v)
+  end
+
+  def transaction(&block)
+    # not yet mapped, not yet used
+    block.call
+  end
+
+  def pragma(tbl,info)
+    if tbl.include? '.'
+      dbname, tbname, *ignore = tbl.split('.')
+      dbname = quote_with_dots(dbname)
+      tbname = quote_with_dots(tbname)
+      query = "PRAGMA #{dbname}.#{info}(#{tbname})"
+    else
+      tbl = quote_with_dots(tbl)
+      query = "PRAGMA #{info}(#{tbl})"
+    end
+    result = sqlite_execute(query,[])
+    result
+  end
+
+  def part(row,n,name)
+    row[n]
+  end
+
+  def columns(tbl)
+    tbl = complete_table(tbl)
+    @info[tbl] = pragma(tbl,"table_info") unless @info.has_key? tbl
+    @info[tbl]
+  end
+
+  def column_names(tbl)
+    columns(tbl).map{|c| part(c,1,"name")}
+  end
+
+  def fetch(sql,names)
+    sqlite_execute(sql,[]).each do |row|
+      yield row
+    end
+  end
+
+  def primary_key(tbl)
+    return @pk unless @pk.nil?
+    cols = columns(tbl)
+    cols = cols.select{|c| part(c,5,"pk").to_s=="1"}.map{|c| part(c,1,"name")}
+    if cols.length == 0
+      cols = pk_from_unique_index(tbl)
+    end
+    @pk = cols if cols.length>0
+    cols
+  end
+
+  def pk_from_unique_index(tbl)
+    pragma(tbl,"index_list").each do |row|
+      if part(row,2,"unique").to_s == "1"
+        idx = part(row,1,"name")
+        return pragma(idx,"index_info").map{|r| part(r,2,"name")}
+      end
+    end
+    nil
+  end
+
+  # copy the structure of an attached table, along with any indexes
+  def copy_table_structure(rdb,tbl)
+    template = "SELECT sql, type from X.sqlite_master WHERE tbl_name = ? ORDER BY type DESC"
+    lsql = template.gsub('X',"main")
+    rsql = template.gsub('X',quote_with_dots(rdb))
+    args = [quote_with_dots(tbl)]
+    lschema = sqlite_execute(lsql,args)
+    rschema = sqlite_execute(rsql,args)
+    if lschema.length>0
+      return false
+    end
+    rschema.each{ |row| sqlite_execute(row[0],[]) }
+    true
+  end
+end