nbadw-util 0.1.0

data/LICENSE

@@ -0,0 +1,3 @@
+== nbadw
+
+Put appropriate LICENSE for your project here.

data/README

@@ -0,0 +1,3 @@
+== nbadw
+
+You should document your project here.

data/Rakefile

@@ -0,0 +1,48 @@
+# encoding: utf-8
+require 'rubygems'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |s|
+    s.name = "nbadw-util"
+    s.summary = %Q{NB Aquatic Data Warehouse - Models and Database Utilities}
+    s.email = "casey.colin@gmail.com"
+    s.homepage = "http://github.com/colincasey/nbadw"
+    s.description = "Database models, migrations, and utilities for the New Brunswick Aquatic Data Warehouse"
+    s.authors = ["Colin Casey"]
+    s.add_dependency 'sequel', '>= 3.5.0'
+    s.rubygems_version = '1.3.1'
+    s.files = FileList['lib/**/*.rb'] + ['README.rdoc', 'LICENSE', 'VERSION.yml', 'Rakefile']
+    s.files.exclude('main.rb')
+  end
+rescue LoadError => e
+  if e.message =~ /jeweler/
+    puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
+  else
+    puts e.message + ' -- while loading jeweler.'
+  end
+end
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = 'NB Aquatic Data Warehouse'
+  rdoc.options << '--line-numbers' << '--inline-source'
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |t|
+    t.libs << 'spec'
+    t.test_files = FileList['spec/*_spec.rb']
+    t.verbose = true
+  end
+rescue LoadError
+  if RUBY_PLATFORM =~ /java/
+    puts "RCov is not available. In order to run rcov, you must: sudo gem install jruby-rcov"
+  else
+    puts "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+  end
+end

data/lib/nbadw/util/copy_database_task.rb

@@ -0,0 +1,311 @@
+require 'sequel'
+require 'sequel/extensions/schema_dumper'
+require 'sequel/extensions/migration'
+require 'nbadw/util/progress_bar'
+require 'sequel/jdbc_access_adapter'
+
+module NBADW
+  module Util
+    class CopyDatabaseTask
+      attr_reader :source, :destination, :page_size, :except
+      
+      def initialize(src, dest, options = {})
+        @source = Sequel.connect(src, :single_threaded => true)
+        @destination = Sequel.connect(dest, :single_threaded => true)
+        @page_size = options[:page_size] || :unlimited
+        @verify_data = !!options[:verify_data]
+        @except = options[:except] || []
+      end
+
+      def self.start(src, dest, options = {})
+        print "Initializing copy operation"
+        task = new(src, dest, options)
+        begin
+          task.copy
+        rescue Exception => e
+          puts "...fail!!!"
+          puts "Reason: #{e.message}"
+          puts e.backtrace.join("\n")
+        end
+      end
+      
+      def copy
+        puts "..."
+        puts "#{source.tables.length} tables, #{format_number(total_records(source))} records"
+        copy_schema
+        copy_data
+        copy_indexes
+        verify_data if verify_data?
+        puts "...copy completed"
+      end
+
+      def copy_schema
+        begin
+          run_callback :before_copy_schema
+        
+          tables = source.tables
+          progress = ProgressBar.new("Schema copy", tables.length)
+
+          tables.each do |t|
+            next if except.include?(t.to_s)
+            args = { :table => t, :schema => source.dump_table_schema(t.to_sym, :indexes => false) }
+            run_callback :before_create_table, args
+            migration = "Class.new(Sequel::Migration) do \n def up \n #{args[:schema]} \n end \n end"
+            eval(migration).apply(destination, :up)
+            run_callback :after_create_table, args
+            progress.inc(1)
+          end
+
+          run_callback :after_copy_schema
+        ensure
+          progress.finish if progress
+        end
+      end
+
+      def copy_data
+        run_callback :before_copy_data
+
+        progress = ProgressBar.new("Data copy", source.tables.size)
+        begin
+          source.tables.each do |table_name|
+            next if except.include?(table_name.to_s)
+            src_table = source[table_name.to_sym]
+            dst_table = destination[table_name.to_sym]
+            args = { :table => table_name }
+            page_size == :unlimited ? copy_table_without_limit(src_table, dst_table, args) : copy_table_with_limit(src_table, dst_table, args)
+            progress.inc(1)
+          end
+        ensure
+          progress.finish
+        end
+
+        run_callback :after_copy_data
+      end
+
+      def copy_table_without_limit(src_table, dst_table, args = {})
+        src_table.each do |row|
+          args.merge!({ :row => row })
+          run_callback :before_copy_row, args
+          dst_table.insert(row)
+          run_callback :after_copy_row, args
+        end
+      end
+
+      def copy_table_with_limit(src_table, dst_table, args = {})
+        count = src_table.count
+        offset = 0
+        while(offset < count) do
+          rows = src_table.limit(page_size, offset).all
+          rows.each_with_index do |row, i|
+            args.merge!({ :row => row, :index => i, :offset => offset })
+            run_callback :before_copy_row, args
+            dst_table.insert(row)
+            run_callback :after_copy_row, args
+          end
+          offset += rows.size
+        end
+      end
+
+      def copy_indexes
+        begin
+          run_callback :before_copy_indexes
+
+          tables = source.tables
+          progress = ProgressBar.new("Index copy", tables.length)
+
+          tables.each do |t|
+            next if except.include?(t.to_s)
+            args = { :table => t, :indexes => source.send(:dump_table_indexes, t.to_sym, :add_index) }
+            run_callback :before_add_indexes, args
+            migration = "Class.new(Sequel::Migration) do \n def up \n #{args[:indexes]} \n end \n end"
+            eval(migration).apply(destination, :up)
+            run_callback :after_add_indexes, args
+            progress.inc(1)
+          end
+
+          run_callback :after_copy_indexes
+        ensure
+          progress.finish if progress
+        end
+      end
+
+      def verify_data
+        tables = source.tables
+        progress = ProgressBar.new("Verify data", tables.length)
+        begin
+          tables.each do |table_name|
+            next if except.include?(table_name.to_s)
+            src_table = source[table_name.to_sym]
+            dst_table = destination[table_name.to_sym]
+            page_size == :unlimited ? verify_table_without_limit(table_name, src_table, dst_table) : verify_table_with_limit(table_name, src_table, dst_table)
+            progress.inc(1)
+          end
+        ensure
+          progress.finish if progress
+        end
+      end
+
+      def verify_table_without_limit(table_name, src_table, dst_table)
+        src_table.each do |row|
+          row_found = dst_table.filter(row).first
+          raise "no matching row found in #{table_name} for #{row.inspect}" unless row_found
+          verify_row(table_name, row, row_found)
+        end
+      end
+
+      def verify_table_with_limit(table_name, src_table, dst_table)
+        count = src_table.count
+        offset = 0
+        while(offset < count) do
+          rows = src_table.limit(page_size, offset).all
+          rows.each do |row|
+            row_found = dst_table.filter(row).first
+            raise "no matching row found in #{table_name} for #{row.inspect}" unless row_found
+            verify_row(table_name, row, row_found)
+          end
+          offset += rows.length
+        end
+      end
+
+      def verify_row(table_name, row1, row2)
+        diff = {}
+        row1.each do |col, val|
+          eql = case val
+          when Time then (val - row1[col]).abs < 1  # time fields are sometimes off by very miniscule fractions
+          else           val == row1[col]
+          end
+          diff[col] = "#{val}, #{row2[col]}" unless eql
+        end
+        raise "row does not match exactly - expected #{row1.inspect}, but was #{row2.inspect} - in table #{table_name}, diff #{diff.inspect}" unless diff.empty?
+      end
+
+      def verify_data?
+        @verify_data
+      end
+
+      def total_records(db)
+        db.tables.inject(0) { |total, table_name| total += db[table_name.to_sym].count }
+      end
+
+      def format_number(num)
+        num.to_s.gsub(/(\d)(?=(\d\d\d)+(?!\d))/, "\\1,")
+      end
+
+      # the following is a callback system that helps to handle slight
+      # differences when copying between database types
+      class << self
+        def callbacks
+          @callbacks ||= []
+        end
+
+        def before(callback, opts = {}, &block)
+          add_callback(:before, callback, opts, &block)
+        end
+
+        def after(callback, opts = {}, &block)
+          add_callback(:after, callback, opts, &block)
+        end
+
+        def add_callback(type, callback, opts, &block)
+          callback_config = {
+            :type     => type,
+            :callback => callback,
+            :adapter  => opts[:adapter] || :all,
+            :for      => opts[:for],
+            :logic    => block
+          }
+          callbacks << callback_config
+        end
+      end
+
+      # prevent MySQL from changing '0' values on insert since we'd like an exact copy
+      before :copy_schema, :adapter => :mysql, :for => :destination do |src, dst, args|
+        dst.run("SET sql_mode = 'NO_AUTO_VALUE_ON_ZERO';")
+      end
+
+      # fix to catch schema dumps for PostgreSQL which set an invalid boolean default
+      before :create_table, :adapter => :postgres, :for => :destination do |src, dst, args|
+        schema = args[:schema]
+        schema = schema.split("\n").collect do |line|
+          if line.match(/TrueClass/)
+            line = line.sub(/:default=>(\d)/) { |match| ":default=>#{$1 == '0' ? 'true' : 'false'}"  }
+          end
+          line
+        end.join("\n")
+        args[:schema] = schema
+      end
+
+      # this fixes the string as primary keys
+      before :create_table, :adapter => :access, :for => :source do |src, dst, args|
+        table =  args[:table].to_s
+        pks = src.schema(args[:table]).collect do |col_schema|
+          col, opts = col_schema
+          opts[:primary_key] ? col_schema : nil
+        end.compact
+
+        if pks.size == 1 && pks[0][1][:type] == :string
+          col, opts = pks[0]
+          schema = args[:schema]
+          schema = schema.split("\n").collect do |line|
+            line = "  String :#{col}, :size=>#{opts[:column_size]}, :null=>false" if line.match(/primary_key/)
+            line = "  primary_key [:#{col}]\nend" if line.match(/^end/)
+            line
+          end.join("\n")
+          args[:schema] = schema
+        end
+      end
+
+      # When copying from access, convert all BigDecimal columns to Float or lose precision!
+      before :create_table, :adapter => :access, :for => :source do |src, dst, args|
+        args[:schema] = args[:schema].gsub(/BigDecimal/, 'Float')
+      end
+
+      STRING_TO_INT_FIXES = [
+        { :table => "auxuserdbselectedsites", :column => "aquaticsiteuseid" },
+        { :table => "auxuserdbselectedsiteuse", :column => "aquaticsiteuseid" },
+        { :table => "cdtranslation - dfo stock mating", :column => "mating code" },
+        { :table => "del-missing age class in tblfishmeasurement", :column => "fishsampleid" },
+        { :table => "del-missing age class in tblfishmeasurement-robin", :column => "fishsampleid" },
+        { :table => "selections", :column => "selectionid" },
+        { :table => "tblelectrofishingmethoddetail", :column => "aquaticactivitydetailid" },
+        { :table => "tbloldhabitatsurvey", :column => "habitatsurveyid" }
+      ]
+      # not sure what's up here...
+      before :create_table, :adapter => :postgres, :for => :destination do |src, dst, args|
+        table = args[:table].to_s.downcase
+        if fix = STRING_TO_INT_FIXES.detect { |fix| fix[:table] == table }
+          schema = args[:schema]
+          schema = schema.split("\n").collect do |line|
+            line = "  Integer :\"#{fix[:column]}\"" if line.match(/#{fix[:column]}/)
+            line
+          end.join("\n")
+          args[:schema] = schema
+        end
+      end
+
+      # determines which callbacks to run (is this needlessly complex?)
+      def run_callback(full_callback, args = {})
+        full_callback.to_s.match(/(before|after)_(.*)/)
+        type, callback = $1.to_sym, $2.to_sym
+        CopyDatabaseTask.callbacks.each do |callback_config|
+          if callback_config[:type] == type && callback_config[:callback] == callback # callback matches
+            # which adapters should we check against?
+            adapters = [:all] # always check for all...
+            if callback_config[:for] == :destination # only destination?
+              adapters << destination.database_type.to_sym
+            elsif callback_config[:for] == :source   # only source?
+              adapters << source.database_type.to_sym
+            else                                     # or both?
+              adapters << destination.database_type.to_sym
+              adapters << source.database_type.to_sym
+            end
+            # if the adapter matches, run the callback
+            if adapters.include?(callback_config[:adapter])
+              callback_config[:logic].call(source, destination, args)
+            end
+          end
+        end
+      end
+    end # CopyDatabaseTask
+  end # Util
+end # NBADW

data/lib/nbadw/util/progress_bar.rb

@@ -0,0 +1,236 @@
+#
+# Ruby/ProgressBar - a text progress bar library
+#
+# Copyright (C) 2001-2005 Satoru Takabayashi <satoru@namazu.org>
+#     All rights reserved.
+#     This is free software with ABSOLUTELY NO WARRANTY.
+#
+# You can redistribute it and/or modify it under the terms
+# of Ruby's license.
+#
+
+class ProgressBar
+  VERSION = "0.9"
+
+  def initialize (title, total, out = STDERR)
+    @title = title
+    @total = total
+    @out = out
+    @terminal_width = 80
+    @bar_mark = "="
+    @current = 0
+    @previous = 0
+    @finished_p = false
+    @start_time = Time.now
+    @previous_time = @start_time
+    @title_width = 14
+    @format = "%-#{@title_width}s %3d%% %s %s"
+    @format_arguments = [:title, :percentage, :bar, :stat]
+    clear
+    show
+  end
+  attr_reader   :title
+  attr_reader   :current
+  attr_reader   :total
+  attr_accessor :start_time
+
+  private
+  def fmt_bar
+    bar_width = do_percentage * @terminal_width / 100
+    sprintf("|%s%s|", 
+            @bar_mark * bar_width, 
+            " " *  (@terminal_width - bar_width))
+  end
+
+  def fmt_percentage
+    do_percentage
+  end
+
+  def fmt_stat
+    if @finished_p then elapsed else eta end
+  end
+
+  def fmt_stat_for_file_transfer
+    if @finished_p then 
+      sprintf("%s %s %s", bytes, transfer_rate, elapsed)
+    else 
+      sprintf("%s %s %s", bytes, transfer_rate, eta)
+    end
+  end
+
+  def fmt_title
+    @title[0,(@title_width - 1)] + ":"
+  end
+
+  def convert_bytes (bytes)
+    if bytes < 1024
+      sprintf("%6dB", bytes)
+    elsif bytes < 1024 * 1000 # 1000kb
+      sprintf("%5.1fKB", bytes.to_f / 1024)
+    elsif bytes < 1024 * 1024 * 1000  # 1000mb
+      sprintf("%5.1fMB", bytes.to_f / 1024 / 1024)
+    else
+      sprintf("%5.1fGB", bytes.to_f / 1024 / 1024 / 1024)
+    end
+  end
+
+  def transfer_rate
+    bytes_per_second = @current.to_f / (Time.now - @start_time)
+    sprintf("%s/s", convert_bytes(bytes_per_second))
+  end
+
+  def bytes
+    convert_bytes(@current)
+  end
+
+  def format_time (t)
+    t = t.to_i
+    sec = t % 60
+    min  = (t / 60) % 60
+    hour = t / 3600
+    sprintf("%02d:%02d:%02d", hour, min, sec);
+  end
+
+  # ETA stands for Estimated Time of Arrival.
+  def eta
+    if @current == 0
+      "ETA:  --:--:--"
+    else
+      elapsed = Time.now - @start_time
+      eta = elapsed * @total / @current - elapsed;
+      sprintf("ETA:  %s", format_time(eta))
+    end
+  end
+
+  def elapsed
+    elapsed = Time.now - @start_time
+    sprintf("Time: %s", format_time(elapsed))
+  end
+  
+  def eol
+    if @finished_p then "\n" else "\r" end
+  end
+
+  def do_percentage
+    if @total.zero?
+      100
+    else
+      @current  * 100 / @total
+    end
+  end
+
+  def get_width
+    # FIXME: I don't know how portable it is.
+    default_width = 80
+    begin
+      tiocgwinsz = 0x5413
+      data = [0, 0, 0, 0].pack("SSSS")
+      if @out.ioctl(tiocgwinsz, data) >= 0 then
+        rows, cols, xpixels, ypixels = data.unpack("SSSS")
+        if cols > 0 then cols else default_width end
+      else
+        default_width
+      end
+    rescue Exception
+      default_width
+    end
+  end
+
+  def show
+    arguments = @format_arguments.map {|method| 
+      method = sprintf("fmt_%s", method)
+      send(method)
+    }
+    line = sprintf(@format, *arguments)
+
+    width = get_width
+    if line.length == width - 1 
+      @out.print(line + eol)
+      @out.flush
+    elsif line.length >= width
+      @terminal_width = [@terminal_width - (line.length - width + 1), 0].max
+      if @terminal_width == 0 then @out.print(line + eol) else show end
+    else # line.length < width - 1
+      @terminal_width += width - line.length + 1
+      show
+    end
+    @previous_time = Time.now
+  end
+
+  def show_if_needed
+    if @total.zero?
+      cur_percentage = 100
+      prev_percentage = 0
+    else
+      cur_percentage  = (@current  * 100 / @total).to_i
+      prev_percentage = (@previous * 100 / @total).to_i
+    end
+
+    # Use "!=" instead of ">" to support negative changes
+    if cur_percentage != prev_percentage || 
+        Time.now - @previous_time >= 1 || @finished_p
+      show
+    end
+  end
+
+  public
+  def clear
+    @out.print "\r"
+    @out.print(" " * (get_width - 1))
+    @out.print "\r"
+  end
+
+  def finish
+    @current = @total
+    @finished_p = true
+    show
+  end
+
+  def finished?
+    @finished_p
+  end
+
+  def file_transfer_mode
+    @format_arguments = [:title, :percentage, :bar, :stat_for_file_transfer]
+  end
+
+  def format= (format)
+    @format = format
+  end
+
+  def format_arguments= (arguments)
+    @format_arguments = arguments
+  end
+
+  def halt
+    @finished_p = true
+    show
+  end
+
+  def inc (step = 1)
+    @current += step
+    @current = @total if @current > @total
+    show_if_needed
+    @previous = @current
+  end
+
+  def set (count)
+    if count < 0 || count > @total
+      raise "invalid count: #{count} (total: #{@total})"
+    end
+    @current = count
+    show_if_needed
+    @previous = @current
+  end
+
+  def inspect
+    "#<ProgressBar:#{@current}/#{@total}>"
+  end
+end
+
+class ReversedProgressBar < ProgressBar
+  def do_percentage
+    100 - super
+  end
+end
+

data/lib/sequel/adapters/jdbc/access.rb

@@ -0,0 +1,44 @@
+require 'sequel/adapters/shared/access'
+
+module Sequel
+  module JDBC
+    class Database
+      # Alias the generic JDBC version so it can be called directly later
+      alias jdbc_schema_parse_table schema_parse_table
+    end
+    
+    # Database and Dataset instance methods for MSSQL specific
+    # support via JDBC.
+    module Access
+      # Database instance methods for MSSQL databases accessed via JDBC.
+      module DatabaseMethods
+        PRIMARY_KEY_INDEX_RE = /\Apk__/i.freeze
+        
+        include Sequel::Access::DatabaseMethods
+        
+        # Return instance of Sequel::JDBC::MSSQL::Dataset with the given opts.
+        def dataset(opts=nil)
+          Sequel::JDBC::Access::Dataset.new(self, opts)
+        end
+        
+        private
+                
+        # Call the generic JDBC version instead of MSSQL version,
+        # since the JDBC version handles primary keys.
+        def schema_parse_table(table, opts={})
+          jdbc_schema_parse_table(table, opts)
+        end
+        
+        # Primary key indexes appear to start with pk__ on MSSQL
+        def primary_key_index_re
+          PRIMARY_KEY_INDEX_RE
+        end
+      end
+      
+      # Dataset class for MSSQL datasets accessed via JDBC.
+      class Dataset < JDBC::Dataset
+        include Sequel::Access::DatasetMethods
+      end
+    end
+  end
+end

data/lib/sequel/adapters/shared/access.rb

@@ -0,0 +1,416 @@
+module Sequel
+  module Access
+    module DatabaseMethods
+      AUTO_INCREMENT = 'COUNTER(1,1)'.freeze
+      SERVER_VERSION_RE = /^(\d+)\.(\d+)\.(\d+)/.freeze
+      SQL_BEGIN = "BEGIN TRANSACTION".freeze
+      SQL_COMMIT = "COMMIT TRANSACTION".freeze
+      SQL_ROLLBACK = "ROLLBACK TRANSACTION".freeze
+      SQL_ROLLBACK_TO_SAVEPOINT = 'ROLLBACK TRANSACTION autopoint_%d'.freeze
+      SQL_SAVEPOINT = 'SAVE TRANSACTION autopoint_%d'.freeze
+      TEMPORARY = "#".freeze
+      
+      def database_type
+        :access
+      end
+        
+      def supports_savepoints?
+        false
+      end
+
+      private
+      
+      # MSSQL uses the IDENTITY(1,1) column for autoincrementing columns.
+      def auto_increment_sql
+        AUTO_INCREMENT
+      end
+      
+      # MSSQL specific syntax for altering tables.
+      def alter_table_sql(table, op)
+        case op[:op]
+        when :add_column
+          "ALTER TABLE #{quote_schema_table(table)} ADD #{column_definition_sql(op)}"
+        when :rename_column
+          "SP_RENAME #{literal("#{quote_schema_table(table)}.#{quote_identifier(op[:name])}")}, #{literal(op[:new_name].to_s)}, 'COLUMN'"
+        when :set_column_type
+          "ALTER TABLE #{quote_schema_table(table)} ALTER COLUMN #{quote_identifier(op[:name])} #{type_literal(op)}"
+        when :set_column_null
+          sch = schema(table).find{|k,v| k.to_s == op[:name].to_s}.last
+          type = {:type=>sch[:db_type]}
+          type[:size] = sch[:max_chars] if sch[:max_chars]
+          "ALTER TABLE #{quote_schema_table(table)} ALTER COLUMN #{quote_identifier(op[:name])} #{type_literal(type)} #{'NOT ' unless op[:null]}NULL"
+        when :set_column_default
+          "ALTER TABLE #{quote_schema_table(table)} ADD CONSTRAINT #{quote_identifier("sequel_#{table}_#{op[:name]}_def")} DEFAULT #{literal(op[:default])} FOR #{quote_identifier(op[:name])}"
+        else
+          super(table, op)
+        end
+      end
+      
+      # SQL to start a new savepoint
+      def begin_savepoint_sql(depth)
+        SQL_SAVEPOINT % depth
+      end
+
+      # SQL to BEGIN a transaction.
+      def begin_transaction_sql
+        SQL_BEGIN
+      end
+      
+      # Commit the active transaction on the connection, does not commit/release
+      # savepoints.
+      def commit_transaction(conn)
+        log_connection_execute(conn, commit_transaction_sql) unless Thread.current[:sequel_transaction_depth] > 1
+      end
+
+      # SQL to COMMIT a transaction.
+      def commit_transaction_sql
+        SQL_COMMIT
+      end
+      
+      # The SQL to drop an index for the table.
+      def drop_index_sql(table, op)
+        "DROP INDEX #{quote_identifier(op[:name] || default_index_name(table, op[:columns]))} ON #{quote_schema_table(table)}"
+      end
+      
+      # Always quote identifiers in the metadata_dataset, so schema parsing works.
+      def metadata_dataset
+        ds = super
+        ds.quote_identifiers = true
+        ds
+      end
+      
+      # SQL to rollback to a savepoint
+      def rollback_savepoint_sql(depth)
+        SQL_ROLLBACK_TO_SAVEPOINT % depth
+      end
+      
+      # SQL to ROLLBACK a transaction.
+      def rollback_transaction_sql
+        SQL_ROLLBACK
+      end
+      
+      # MSSQL uses the INFORMATION_SCHEMA to hold column information.  This method does
+      # not support the parsing of primary key information.
+      def schema_parse_table(table_name, opts)
+        m = output_identifier_meth
+        m2 = input_identifier_meth
+        ds = metadata_dataset.from(:information_schema__tables___t).
+         join(:information_schema__columns___c, :table_catalog=>:table_catalog,
+              :table_schema => :table_schema, :table_name => :table_name).
+         select(:column_name___column, :data_type___db_type, :character_maximum_length___max_chars, :column_default___default, :is_nullable___allow_null).
+         filter(:c__table_name=>m2.call(table_name.to_s))
+        if schema = opts[:schema] || default_schema
+          ds.filter!(:table_schema=>schema)
+        end
+        ds.map do |row|
+          row[:allow_null] = row[:allow_null] == 'YES' ? true : false
+          row[:default] = nil if blank_object?(row[:default])
+          row[:type] = schema_column_type(row[:db_type])
+          [m.call(row.delete(:column)), row]
+        end
+      end
+      
+      # SQL fragment for marking a table as temporary
+      def temporary_table_sql
+        TEMPORARY
+      end
+      
+      # MSSQL has both datetime and timestamp classes, most people are going
+      # to want datetime
+      def type_literal_generic_datetime(column)
+        :datetime
+      end
+
+      # MSSQL has both datetime and timestamp classes, most people are going
+      # to want datetime
+      def type_literal_generic_time(column)
+        column[:only_time] ? :time : :datetime
+      end
+      
+      # MSSQL doesn't have a true boolean class, so it uses bit
+      def type_literal_generic_trueclass(column)
+        :bit
+      end
+      
+      # MSSQL uses image type for blobs
+      def type_literal_generic_file(column)
+        :image
+      end
+    end
+  
+    module DatasetMethods
+      BOOL_TRUE = '1'.freeze
+      BOOL_FALSE = '0'.freeze
+      COMMA_SEPARATOR = ', '.freeze
+      DELETE_CLAUSE_METHODS = Dataset.clause_methods(:delete, %w'with from output from2 where')
+      INSERT_CLAUSE_METHODS = Dataset.clause_methods(:insert, %w'with into columns output values')
+      SELECT_CLAUSE_METHODS = Dataset.clause_methods(:select, %w'with limit distinct columns from table_options join where group order having compounds')
+      UPDATE_CLAUSE_METHODS = Dataset.clause_methods(:update, %w'with table set output from where')
+      WILDCARD = LiteralString.new('*').freeze
+      CONSTANT_MAP = {:CURRENT_DATE=>'CAST(CURRENT_TIMESTAMP AS DATE)'.freeze, :CURRENT_TIME=>'CAST(CURRENT_TIMESTAMP AS TIME)'.freeze}
+
+      # Split out from fetch rows to allow processing of JDBC result sets
+      # that don't come from issuing an SQL string.
+      def process_result_set(result)
+        # get column names
+        meta = result.getMetaData
+        cols = []
+        i = 0
+        meta.getColumnCount.times{cols << [output_identifier(meta.getColumnLabel(i+=1)), i]}
+        @columns = cols.map{|c| c.at(0)}
+        row = {}
+        blk = if @convert_types
+          lambda{ |n, i|
+            begin
+              row[n] = convert_type(result.getObject(i))
+            rescue
+              # XXX: this is because HXTT driver throws an error here
+              if n == :column_def && row[:type_name] == 'TIMESTAMP'
+                row[:column_def] = nil
+              end
+            end
+          }
+        else
+          lambda{|n, i| row[n] = result.getObject(i)}
+        end
+        # get rows
+        rsmd = result.get_meta_data
+        num_cols = rsmd.get_column_count
+
+        while result.next
+          row = {}
+          cols.each(&blk)
+          yield row
+        end
+      end
+
+      # MSSQL uses + for string concatenation
+      def complex_expression_sql(op, args)
+        case op
+        when :'||'
+          super(:+, args)
+        else
+          super(op, args)
+        end
+      end
+      
+      # MSSQL doesn't support the SQL standard CURRENT_DATE or CURRENT_TIME
+      def constant_sql(constant)
+        CONSTANT_MAP[constant] || super
+      end
+      
+      # When returning all rows, if an offset is used, delete the row_number column
+      # before yielding the row.
+      def fetch_rows(sql, &block)
+        @opts[:offset] ? super(sql) {|r| r.delete(:"recno()"); yield r} : super(sql, &block)
+      end
+      
+      # MSSQL uses the CONTAINS keyword for full text search
+      def full_text_search(cols, terms, opts = {})
+        filter("CONTAINS (#{literal(cols)}, #{literal(terms)})")
+      end
+      
+      # MSSQL uses a UNION ALL statement to insert multiple values at once.
+      def multi_insert_sql(columns, values)
+        [insert_sql(columns, LiteralString.new(values.map {|r| "SELECT #{expression_list(r)}" }.join(" UNION ALL ")))]
+      end
+
+      # Allows you to do .nolock on a query
+      def nolock
+        clone(:table_options => "(NOLOCK)")
+      end
+
+      # Include an OUTPUT clause in the eventual INSERT, UPDATE, or DELETE query.
+      #
+      # The first argument is the table to output into, and the second argument
+      # is either an Array of column values to select, or a Hash which maps output
+      # column names to selected values, in the style of #insert or #update.
+      #
+      # Output into a returned result set is not currently supported.
+      #
+      # Examples:
+      #
+      #   dataset.output(:output_table, [:deleted__id, :deleted__name])
+      #   dataset.output(:output_table, :id => :inserted__id, :name => :inserted__name)
+      def output(into, values)
+        output = {}
+        case values
+          when Hash:
+            output[:column_list], output[:select_list] = values.keys, values.values
+          when Array:
+            output[:select_list] = values
+        end
+        output[:into] = into
+        clone({:output => output})
+      end
+
+      # An output method that modifies the receiver.
+      def output!(into, values)
+        mutation_method(:output, into, values)
+      end
+
+      # MSSQL uses [] to quote identifiers
+      def quoted_identifier(name)
+        "[#{name}]"
+      end
+      
+      # Pagination queries (i.e., limit with offset) are supported HXTT
+      # with the help of the recno() function which returns the
+      # row number of each record
+      def select_sql
+        return super unless offset = @opts[:offset]        
+        if @opts[:select]
+          @opts[:select] << :recno.sql_function
+        else
+          @opts[:select] = [WILDCARD, :recno.sql_function]
+        end
+        s = unlimited.where("BETWEEN (recno(), #{@opts[:offset] + 1}, #{@opts[:limit] + @opts[:offset]})")
+        s.select_sql
+      end
+#      def select_sql
+#        return super unless offset = @opts[:offset]
+#        raise(Error, 'Access requires an order be provided if using an offset') unless order = @opts[:order]
+#
+#        total_rows = unlimited.count
+#        if @opts[:limit] + @opts[:offset] > total_rows
+#          correction = @opts[:limit] + @opts[:offset] - total_rows
+#          @opts[:limit] = @opts[:limit] - correction
+#        end
+#
+#        s0 = unlimited.limit(@opts[:limit] + @opts[:offset]).order(order)
+#        s1 = unlimited.from(s0.as('s1')).limit(@opts[:limit]).reverse_order(order)
+#        s2 = unlimited.from(s1.as('s2')).order(order)
+#        s2.select_sql
+#      end
+
+      # The version of the database server.
+      def server_version
+        db.server_version(@opts[:server])
+      end
+
+      # Microsoft SQL Server does not support INTERSECT or EXCEPT
+      def supports_intersect_except?
+        false
+      end
+      
+      # MSSQL does not support IS TRUE
+      def supports_is_true?
+        false
+      end
+
+      # MSSQL 2005+ supports window functions
+      def supports_window_functions?
+        true
+      end
+
+      private
+
+      # MSSQL can modify joined datasets
+      def check_modification_allowed!
+        raise(InvalidOperation, "Grouped datasets cannot be modified") if opts[:group]
+      end
+
+      # MSSQL supports the OUTPUT clause for DELETE statements.
+      # It also allows prepending a WITH clause.
+      def delete_clause_methods
+        DELETE_CLAUSE_METHODS
+      end
+
+      # Handle the with clause for delete, insert, and update statements
+      # to be the same as the insert statement.
+      def delete_with_sql(sql)
+        select_with_sql(sql)
+      end
+      alias insert_with_sql delete_with_sql
+      alias update_with_sql delete_with_sql
+      
+      # MSSQL raises an error if you try to provide more than 3 decimal places
+      # for a fractional timestamp.  This probably doesn't work for smalldatetime
+      # fields.
+      def format_timestamp_usec(usec)
+        sprintf(".%03d", usec/1000)
+      end
+
+      # MSSQL supports FROM clauses in DELETE and UPDATE statements.
+      def from_sql(sql)
+        if (opts[:from].is_a?(Array) && opts[:from].size > 1) || opts[:join]
+          select_from_sql(sql)
+          select_join_sql(sql)
+        end
+      end
+      alias delete_from2_sql from_sql
+      alias update_from_sql from_sql
+      
+      # MSSQL supports the OUTPUT clause for INSERT statements.
+      # It also allows prepending a WITH clause.
+      def insert_clause_methods
+        INSERT_CLAUSE_METHODS
+      end
+
+      # MSSQL uses a literal hexidecimal number for blob strings
+      def literal_blob(v)
+        blob = '0x'
+        v.each_byte{|x| blob << sprintf('%02x', x)}
+        blob
+      end
+      
+      # Use unicode string syntax for all strings
+      def literal_string(v)
+        "N#{super}"
+      end
+      
+      # Use 0 for false on MSSQL
+      def literal_false
+        BOOL_FALSE
+      end
+
+      # Use 1 for true on MSSQL
+      def literal_true
+        BOOL_TRUE
+      end
+      
+      # The alias to use for the row_number column when emulating OFFSET
+      def row_number_column
+        :x_sequel_row_number_x
+      end
+
+      # MSSQL adds the limit before the columns
+      def select_clause_methods
+        SELECT_CLAUSE_METHODS
+      end
+
+      # MSSQL uses TOP for limit
+      def select_limit_sql(sql)
+        sql << " TOP #{@opts[:limit]}" if @opts[:limit]
+      end
+
+      # MSSQL uses the WITH statement to lock tables
+      def select_table_options_sql(sql)
+        sql << " WITH #{@opts[:table_options]}" if @opts[:table_options]
+      end
+
+      # SQL fragment for MSSQL's OUTPUT clause.
+      def output_sql(sql)
+        return unless output = @opts[:output]
+        sql << " OUTPUT #{column_list(output[:select_list])}"
+        if into = output[:into]
+          sql << " INTO #{table_ref(into)}"
+          if column_list = output[:column_list]
+            cl = []
+            column_list.each { |k, v| cl << literal(String === k ? k.to_sym : k) }
+            sql << " (#{cl.join(COMMA_SEPARATOR)})"
+          end
+        end
+      end
+      alias delete_output_sql output_sql
+      alias update_output_sql output_sql
+      alias insert_output_sql output_sql
+
+      # MSSQL supports the OUTPUT clause for UPDATE statements.
+      # It also allows prepending a WITH clause.
+      def update_clause_methods
+        UPDATE_CLAUSE_METHODS
+      end
+    end
+  end
+end

data/lib/sequel/jdbc_access_adapter.rb

@@ -0,0 +1,8 @@
+# encoding: utf-8
+require 'sequel/adapters/jdbc'
+
+Sequel::JDBC::DATABASE_SETUP[:access] = proc do |db|
+  require 'sequel/adapters/jdbc/access'
+  db.extend(Sequel::JDBC::Access::DatabaseMethods)
+  com.hxtt.sql.access.AccessDriver
+end

metadata

@@ -0,0 +1,72 @@
+--- !ruby/object:Gem::Specification 
+name: nbadw-util
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+platform: ruby
+authors: 
+- Colin Casey
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-10-22 00:00:00 -03:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: sequel
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 3.5.0
+    version: 
+description: Database models, migrations, and utilities for the New Brunswick Aquatic Data Warehouse
+email: casey.colin@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README
+files: 
+- LICENSE
+- Rakefile
+- lib/nbadw/util/copy_database_task.rb
+- lib/nbadw/util/progress_bar.rb
+- lib/sequel/adapters/jdbc/access.rb
+- lib/sequel/adapters/shared/access.rb
+- lib/sequel/jdbc_access_adapter.rb
+- README
+has_rdoc: true
+homepage: http://github.com/colincasey/nbadw
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: NB Aquatic Data Warehouse - Models and Database Utilities
+test_files: []
+