coopy 0.6.4.1 → 1.0.0

data/lib/coopy/row_change.rb

@@ -1,25 +0,0 @@
-class RowChange
-  attr_accessor :row_mode
-  attr_accessor :cells
-  attr_accessor :columns
-  attr_accessor :key
-
-  def initialize(row_mode,cells)
-    @row_mode = row_mode
-    @cells = cells
-    @key = nil
-  end
-
-  def active_columns
-    return [] if @columns.nil?
-    @columns.column_by_offset
-  end
-
-  def value_at(column)
-    @cells[column[:in_offset]][:value]
-  end
-
-  def new_value_at(column)
-    @cells[column[:in_offset]][:new_value]
-  end
-end

data/lib/coopy/scraperwiki_sql_wrapper.rb

@@ -1,8 +0,0 @@
-require 'sqlite_sql_wrapper'
-
-# Tweak sqlite wrapper slightly to match ScraperWiki's API
-class ScraperwikiSqlWrapper < SqliteSqlWrapper
-  def sqlite_execute(template,vals)
-    @db.sqliteexecute(template,vals)["data"]
-  end
-end

data/lib/coopy/scraperwiki_utils.rb

@@ -1,23 +0,0 @@
-def link_tables(watch_scraper,watch_tables)
-  sql = ScraperwikiSqlWrapper.new(ScraperWiki)
-  watch_tables.each { |tbl| sql.copy_table_structure(watch_scraper,tbl) }
-end
-
-class CoopyResult
-  attr_accessor :html
-end
-
-def sync_table(watch_scraper,tbl,keys)
-  sql = ScraperwikiSqlWrapper.new(ScraperWiki)
-  sql.set_primary_key(keys) unless keys.nil? 
-  cmp = SqlCompare.new(sql,"main.#{tbl}","#{watch_scraper}.#{tbl}")
-  sinks = DiffOutputGroup.new
-  render = DiffRenderHtml.new
-  sinks << render
-  sinks << DiffApplySql.new(sql,"main.#{tbl}")
-  cmp.set_output(sinks)
-  cmp.apply
-  result = CoopyResult.new
-  result.html = render.to_string
-  result
-end

data/lib/coopy/sequel_sql_wrapper.rb

@@ -1,73 +0,0 @@
-require 'sql_wrapper'
-require 'sequel'
-
-class SequelSqlBare < SqlWrapper
-  def initialize(db)
-    @db = db
-    @tname = nil
-    @t = nil
-  end
-
-  def sync_table(tbl)
-    tbl = @tname if tbl.nil?
-    tbl = @db.tables[0] if tbl.nil?
-    return @t if tbl==@tname
-    @tname = tbl
-    @t = @db[tbl]
-  end
-
-  def enhash(cols,vals)
-    Hash[*cols.map{|c| c.to_sym}.zip(vals).flatten]
-  end
-
-  def insert(tbl,cols,vals)
-    sync_table(tbl)
-    @t.insert(enhash(cols,vals))
-  end
-
-  def delete(tbl,cols,vals)
-    sync_table(tbl)
-    @t.filter(enhash(cols,vals)).delete
-  end
-  
-  def update(tbl,set_cols,set_vals,cond_cols,cond_vals)
-    sync_table(tbl)
-    @t.filter(enhash(cond_cols,cond_vals)).update(enhash(set_cols,set_vals))
-  end
-
-  def transaction(&block)
-    @db.transaction(&block)
-  end
-
-  def columns(tbl)
-    sync_table(tbl)
-    @db.schema(@tname)
-  end
-
-  def column_names(tbl)
-    columns(tbl).map{|x| x[0]}
-  end
-  
-  def primary_key(tbl)
-    cols = columns(tbl)
-    cols.select{|x| x[1][:primary_key]}.map{|x| x[0]}
-  end
-
-  def index(tbl)
-    key = primary_key(tbl)
-    @t.select(*key)
-  end
-
-  def fetch(sql,names)
-    @db.fetch(sql) do |row|
-      yield names.map{|n| row[n]}
-    end
-  end
-end
-
-
-class SequelSqlWrapper < SequelSqlBare
-  def initialize(*params)
-    super(Sequel.connect(*params))
-  end
-end

data/lib/coopy/sql_compare.rb

@@ -1,222 +0,0 @@
-require 'coopy/diff_columns'
-require 'coopy/row_change'
-
-class SqlCompare
-  def initialize(db1,db2)
-    @db1 = db
-    @db2 = db2
-    @table1 = nil
-    @table2 = nil
-    @single_db = false
-    raise "not implemented yet"
-  end
-
-  def initialize(db,table1,table2)
-    @db1 = db
-    @db2 = db.clone
-    @table1 = table1
-    @table2 = table2
-    @single_db = true
-  end
-
-  def set_output(patch)
-    @patch = patch
-  end
-
-  def apply
-    apply_single
-  end
-
-  # We are not implementing full comparison, just an adequate subset
-  # for easy cases (a table with a trustworthy primary key, and constant
-  # columns).  Make sure we are not trying to do something we're not ready 
-  # for.
-  def validate_schema
-    all_cols1 = @db1.column_names(@table1)
-    all_cols2 = @db2.column_names(@table2)
-    if all_cols1 != all_cols2
-      raise "Columns do not match, please use full coopy toolbox"
-    end
-
-    key_cols1 = @db1.primary_key(@table1)
-    key_cols2 = @db2.primary_key(@table2)
-    if key_cols1 != key_cols2
-      raise "Primary keys do not match, please use full coopy toolbox"
-    end
-  end
-
-  def keyify(lst)
-    lst.map{|x| x.to_s}.join("___")
-  end
-
-  # When working within a single database, we can delegate more work to SQL.
-  # So we specialize this case.
-  def apply_single
-    validate_schema
-
-    # Prepare some lists of columns.
-    key_cols = @db1.primary_key(@table1)
-    data_cols = @db1.except_primary_key(@table1)
-    all_cols = @db1.column_names(@table1)
-
-    # Let our public know we are beginning.
-    @patch.begin_diff
-
-    # Advertise column names.
-    @rc_columns = DiffColumns.new
-    @rc_columns.title_row = all_cols
-    @rc_columns.update(0)
-    cells = all_cols.map{|v| { :txt => v, :value => v, :cell_mode => "" }}
-    rc = RowChange.new("@@",cells)
-    @patch.apply_row(rc)
-
-    # If requested, we will be providing context rows around changed rows.
-    # This is not a natural thing to do with SQL, so we do it only on request.
-    # When requested, we need to buffer row changes.
-    @pending_rcs = []
-
-    # Prepare some useful SQL fragments to assemble later.
-    sql_table1 = @db1.quote_table(@table1)
-    sql_table2 = @db1.quote_table(@table2)
-    sql_key_cols = key_cols.map{|c| @db1.quote_column(c)}.join(",")
-    sql_all_cols = all_cols.map{|c| @db1.quote_column(c)}.join(",")
-    sql_key_match = key_cols.map{|c| @db1.quote_column(c)}.map{|c| "#{sql_table1}.#{c} IS #{sql_table2}.#{c}"}.join(" AND ")
-    sql_data_mismatch = data_cols.map{|c| @db1.quote_column(c)}.map{|c| "#{sql_table1}.#{c} IS NOT #{sql_table2}.#{c}"}.join(" OR ")
-
-    # For one query we will need to interleave columns from two tables.  For
-    # portability we need to give these columns distinct names.
-    weave = all_cols.map{|c| [[sql_table1,@db1.quote_column(c)],
-                              [sql_table2,@db2.quote_column(c)]]}.flatten(1)
-    dbl_cols = weave.map{|c| "#{c[0]}.#{c[1]}"}
-    sql_dbl_cols = weave.map{|c| "#{c[0]}.#{c[1]} AS #{c[0].gsub(/[^a-zA-Z0-9]/,'_')}_#{c[1].gsub(/[^a-zA-Z0-9]/,'_')}"}.join(",")
-
-    # Prepare a map of primary key offsets.
-    keys_in_all_cols = key_cols.each.map{|c| all_cols.index(c)}
-    keys_in_dbl_cols = keys_in_all_cols.map{|x| 2*x}
-
-    # Find rows in table2 that are not in table1.
-    sql = "SELECT #{sql_all_cols} FROM #{sql_table2} WHERE NOT EXISTS (SELECT 1 FROM #{sql_table1} WHERE #{sql_key_match})"
-    apply_inserts(sql,all_cols,keys_in_all_cols)
-
-    # Find rows in table1 and table2 that differ while having the same primary
-    # key.
-    sql = "SELECT #{sql_dbl_cols} FROM #{sql_table1} INNER JOIN #{sql_table2} ON #{sql_key_match} WHERE #{sql_data_mismatch}"
-    apply_updates(sql,dbl_cols,keys_in_dbl_cols)
-
-    # Find rows that are in table1 but not table2
-    sql = "SELECT #{sql_all_cols} FROM #{sql_table1} WHERE NOT EXISTS (SELECT 1 FROM #{sql_table2} WHERE #{sql_key_match})"
-    apply_deletes(sql,all_cols,keys_in_all_cols)
-
-    # If we are supposed to provide context, we need to deal with row order.
-    if @patch.want_context
-      sql = "SELECT #{sql_all_cols}, 0 AS __coopy_tag__ FROM #{sql_table1} UNION SELECT #{sql_all_cols}, 1 AS __coopy_tag__ FROM #{sql_table2} ORDER BY #{sql_key_cols}, __coopy_tag__"
-      apply_with_context(sql,all_cols,keys_in_all_cols)
-    end
-
-    # Done!
-    @patch.end_diff
-  end
-
-
-  def apply_inserts(sql,all_cols,keys_in_all_cols)
-    @db1.fetch(sql,all_cols) do |row|
-      cells = row.map{|v| { :txt => v, :value => v, :cell_mode => "" }}
-      rc = RowChange.new("+++",cells)
-      apply_rc(rc,row,keys_in_all_cols)
-    end
-  end
-
-
-  def apply_updates(sql,dbl_cols,keys_in_dbl_cols)
-    @db1.fetch(sql,dbl_cols) do |row|
-      pairs = row.enum_for(:each_slice,2).to_a
-      cells = pairs.map do |v| 
-        if v[0]==v[1] 
-          { :txt => v[0], :value => v[0], :cell_mode => "" }
-        else
-          { :txt => v[0], :value => v[0], :new_value => v[1], :cell_mode => "->" }
-        end
-      end
-      rc = RowChange.new("->",cells)
-      apply_rc(rc,row,keys_in_dbl_cols)
-    end
-  end
-
-
-  def apply_deletes(sql,all_cols,keys_in_all_cols)
-    @db1.fetch(sql,all_cols) do |row|
-      cells = row.map{|v| { :txt => v, :value => v, :cell_mode => "" }}
-      rc = RowChange.new("---",cells)
-      apply_rc(rc,row,keys_in_all_cols)
-    end
-  end
-
-  def apply_rc(rc,row,keys_in_cols)
-    rc.columns = @rc_columns
-    if @patch.want_context
-      rc.key = keyify(row.values_at(*keys_in_cols))
-      @pending_rcs << rc
-    else
-      @patch.apply_row(rc)
-    end
-  end
-
-  def emit_skip(row)
-    cells = row.map{|v| { :txt => "...", :value => "...", :cell_mode => "" }}
-    rc = RowChange.new("...",cells)
-    rc.columns = @rc_columns
-    @patch.apply_row(rc)
-  end
-
-  # Do the context dance.
-  def apply_with_context(sql,all_cols,keys_in_all_cols)
-    hits = {}
-    @pending_rcs.each do |rc|
-      hits[rc.key] = rc
-    end 
-    hist = []
-    n = 2
-    pending = 0
-    skipped = false
-    noted = false
-    last_row = nil
-    @db1.fetch(sql,all_cols + ["__coopy_tag__"]) do |row|
-      tag = row.pop.to_i
-      k = keyify(row.values_at(*keys_in_all_cols))
-      if hits[k]
-        emit_skip(row) if skipped
-        hist.each do |row0|
-          cells = row0.map{|v| { :txt => v, :value => v, :cell_mode => "" }}
-          rc = RowChange.new("",cells)
-          rc.columns = @rc_columns
-          @patch.apply_row(rc)
-        end
-        hist.clear
-        pending = n
-        @patch.apply_row(hits[k])
-        hits.delete(k)
-        skipped = false
-        noted =  true
-      elsif tag == 1
-        # ignore redundant row
-      elsif pending>0
-        emit_skip(row) if skipped
-        cells = row.map{|v| { :txt => v, :value => v, :cell_mode => "" }}
-        rc = RowChange.new("",cells)
-        rc.columns = @rc_columns
-        @patch.apply_row(rc)
-        pending = pending-1
-        skipped = false
-      else
-        hist << row
-        if hist.length>n
-          skipped = true
-          last_row = row
-          hist.shift
-        end
-      end
-    end
-    emit_skip(last_row) if skipped and noted
-  end
-end
-

data/lib/coopy/sql_wrapper.rb

@@ -1,34 +0,0 @@
-class SqlWrapper
-  def insert(tbl,cols,vals)
-  end
-
-  def delete(tbl,cols,vals)
-  end
-
-  def update(tbl,set_cols,set_vals,cond_cols,cond_vals)
-  end
-
-  def column_names(tbl)
-    []
-  end
-
-  def primary_key(tbl)
-    []
-  end
-
-  def except_primary_key(tbl)
-    column_names(tbl)-primary_key(tbl)
-  end
-
-  def fetch(sql)
-    []
-  end
-
-  def quote_column(c)
-    c.to_s
-  end
-
-  def quote_table(t)
-    t.to_s
-  end
-end

data/lib/coopy/sqlite_sql_wrapper.rb

@@ -1,143 +0,0 @@
-require 'coopy/sql_wrapper'
-
-class SqliteSqlWrapper < SqlWrapper
-  def initialize(db)
-    @db = db
-    @t = nil
-    @qt = nil
-    @pk = nil
-    @info = {}
-  end
-
-  def set_primary_key(lst)
-    @pk = lst
-  end
-
-  def sqlite_execute(template,vals)
-    return @db.execute(template,*vals)
-  end
-
-  def get_table_names
-    sqlite_execute("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name",[]).flatten
-  end
-
-  def complete_table(tbl)
-    @t = tbl unless tbl.nil?
-    @t = get_table_names[0] if @t.nil?
-    @t
-  end
-
-  def quote_with_dots(x)
-    return x if x.match(/^[a-zA-Z0-9_]+$/)
-    x.split('.').map{|p| "`#{p}`"}.join('.')
-  end
-
-  def quote_table(tbl)
-    complete_table(tbl)
-    return @t if @t.match(/^[a-zA-Z0-9_]+$/)
-    quote_with_dots(@t)
-  end
-
-  def quote_column(col)
-    return col if col.match(/^[a-zA-Z0-9_]+$/)
-    quote_with_dots(col)
-  end
-
-  def insert(tbl,cols,vals)
-    tbl = quote_table(tbl)
-    template = cols.map{|x| '?'}.join(",")
-    template = "INSERT INTO #{tbl} VALUES(#{template})"
-    sqlite_execute(template,vals)
-  end
-
-  def delete(tbl,cols,vals)
-    tbl = quote_table(tbl)
-    template = cols.map{|c| quote_column(c) + ' IS ?'}.join(" AND ")
-    template = "DELETE FROM #{tbl} WHERE #{template}"
-    sqlite_execute(template,vals)
-  end
-  
-  def update(tbl,set_cols,set_vals,cond_cols,cond_vals)
-    tbl = quote_table(tbl)
-    conds = cond_cols.map{|c| quote_column(c) + ' IS ?'}.join(" AND ")
-    sets = set_cols.map{|c| quote_column(c) + ' = ?'}.join(", ")
-    template = "UPDATE #{tbl} SET #{sets} WHERE #{conds}"
-    v = set_vals + cond_vals
-    sqlite_execute(template,v)
-  end
-
-  def transaction(&block)
-    # not yet mapped, not yet used
-    block.call
-  end
-
-  def pragma(tbl,info)
-    if tbl.include? '.'
-      dbname, tbname, *ignore = tbl.split('.')
-      dbname = quote_with_dots(dbname)
-      tbname = quote_with_dots(tbname)
-      query = "PRAGMA #{dbname}.#{info}(#{tbname})"
-    else
-      tbl = quote_with_dots(tbl)
-      query = "PRAGMA #{info}(#{tbl})"
-    end
-    result = sqlite_execute(query,[])
-    result
-  end
-
-  def part(row,n,name)
-    row[n]
-  end
-
-  def columns(tbl)
-    tbl = complete_table(tbl)
-    @info[tbl] = pragma(tbl,"table_info") unless @info.has_key? tbl
-    @info[tbl]
-  end
-
-  def column_names(tbl)
-    columns(tbl).map{|c| part(c,1,"name")}
-  end
-
-  def fetch(sql,names)
-    sqlite_execute(sql,[]).each do |row|
-      yield row
-    end
-  end
-
-  def primary_key(tbl)
-    return @pk unless @pk.nil?
-    cols = columns(tbl)
-    cols = cols.select{|c| part(c,5,"pk").to_s=="1"}.map{|c| part(c,1,"name")}
-    if cols.length == 0
-      cols = pk_from_unique_index(tbl)
-    end
-    @pk = cols if cols.length>0
-    cols
-  end
-
-  def pk_from_unique_index(tbl)
-    pragma(tbl,"index_list").each do |row|
-      if part(row,2,"unique").to_s == "1"
-        idx = part(row,1,"name")
-        return pragma(idx,"index_info").map{|r| part(r,2,"name")}
-      end
-    end
-    nil
-  end
-
-  # copy the structure of an attached table, along with any indexes
-  def copy_table_structure(rdb,tbl)
-    template = "SELECT sql, type from X.sqlite_master WHERE tbl_name = ? ORDER BY type DESC"
-    lsql = template.gsub('X',"main")
-    rsql = template.gsub('X',quote_with_dots(rdb))
-    args = [quote_with_dots(tbl)]
-    lschema = sqlite_execute(lsql,args)
-    rschema = sqlite_execute(rsql,args)
-    if lschema.length>0
-      return false
-    end
-    rschema.each{ |row| sqlite_execute(row[0],[]) }
-    true
-  end
-end