data_task 0.0.2

data/lib/data_task/adapters/support/transactions.rb

@@ -0,0 +1,28 @@
+require_relative '../../db'
+
+module Rake
+  module DataTask
+
+    class Db
+      module StandardTransactions
+
+        def with_transaction do_commit, &block
+          execute "begin;"
+          yield
+          close_command = do_commit ? "commit;" : "rollback;"
+          execute close_command
+        end
+
+        def with_transaction_commit &block
+          with_transaction true, &block
+        end
+
+        def with_transaction_rollback &block
+          with_transaction false, &block
+        end
+
+      end
+    end
+
+  end
+end

data/lib/data_task/data.rb

@@ -0,0 +1,34 @@
+require_relative './db'
+
+module Rake
+  module DataTask
+
+    # Represents a table accessed via a database, roughly analogous to the File class.
+    class Data
+
+      attr_accessor :name
+      attr_accessor :adapter
+
+      def initialize data_name, adapter
+        @name = data_name
+        @adapter = adapter
+      end
+
+      def exists? options={}
+        @adapter.data_exists?(@name, options)
+      end
+
+      alias_method :exist?, :exists?
+
+      def mtime
+        @adapter.data_mtime(@name)
+      end
+
+      def to_s
+        @name
+      end
+
+    end
+
+  end
+end

data/lib/data_task/db.rb

@@ -0,0 +1,57 @@
+require 'yaml'
+require 'logger'
+require_relative './sql'
+require_relative './util'
+
+module Rake
+  module DataTask
+
+    class Db
+      # This is the base class for SQL-compliant relational databases. It contains utility methods
+      # that probably don't vary across databases, and it shouldn't be instantiated.
+
+      LOG = Logger.new(STDOUT)
+      LOG.level = Logger::WARN
+
+      TABLE_TRACKER_NAME = 'last_operations'
+
+      # enumerate case-sensitive, DBMS-specific values that we store in tracking tables
+      # this can be overridden in child classes for specific databases
+      TABLE_TRACKER_COLUMNS = { 
+        :relation_name => { :data_type => :text },
+        :relation_type => {
+          :data_type => :text,
+          :values => {
+            :table => 'table', 
+            :view => 'view'
+          }
+        },
+        :operation => {
+          :data_type => :text,
+          :values => {
+            :create => 'create', 
+            :insert => 'insert',
+            :update => 'update',
+            :truncate => 'truncate',
+            :delete => 'delete'
+          }
+        },
+        :time => { :data_type => :timestamp }
+      }
+
+      def table_tracker_columns
+        TABLE_TRACKER_COLUMNS
+      end
+
+      def operation_values
+        table_tracker_columns[:operation][:values]
+      end
+
+      def relation_type_values
+        table_tracker_columns[:relation_type][:values]
+      end
+
+    end
+
+  end
+end

data/lib/data_task/sql.rb

@@ -0,0 +1,49 @@
+require 'time'
+
+module Rake
+  module DataTask
+
+    class Sql
+
+      # Parse a single string value into an object using the supplied type logic.
+      #
+      # @param r [Array] an array (table) of arrays (rows), usually resulting from a database query
+      # @param &type_logic [Block] code that takes the first 
+      # @raise [TypeError] if r contains more than one row or column
+      # @returns [Object] the return value of &type_logic on the value in r
+      def self.parse_single_value r, &type_logic
+        if r.nil? || r.empty? || r == [[]] || r == [[nil]]
+          return nil
+        elsif r.length > 1
+          raise TypeError, 'Query must result in a single row'
+        elsif r.first.length > 1
+          raise TypeError, 'Query must result in a single column'
+        end
+        yield(r)
+      end
+
+      # Get a single integer via SQL.
+      #
+      # @param r [Array] an array containing a single result from a query
+      # @returns [Integer] the single result converted to an Integer
+      def self.get_single_int r
+        parse_single_value r do
+          Integer(r.first.first)
+        end
+      end
+
+      # Get a single time via SQL.
+      #
+      # @param r [Array] an array containing a single result from a query
+      # @returns [Time] the single result converted to Ruby's local time
+      def self.get_single_time r
+        parse_single_value r do
+          t = Time.parse(r.first.first)
+          DateTime.parse(t.to_s)
+        end
+      end
+
+    end
+
+  end
+end

data/lib/data_task/tasks/examples.rake

@@ -0,0 +1,35 @@
+require 'data_task/adapters/postgres'
+require 'data_task/adapters/sqlite'
+
+# set up adapters to two databases
+postgres = Rake::DataTask::Postgres.new(
+  'host' => 'localhost', 'port' => 5432, 'database' => 'example', 'username' => 'postgres')
+sqlite = Rake::DataTask::Sqlite.new('database' => 'example')
+
+
+desc "Build a data file."
+file 'raw.txt' do
+  File.open('raw.txt', 'w') { |file| file.write("v1") }
+end
+
+desc "Load a data file into PostgreSQL for analysis."
+data postgres['raw'] => 'raw.txt' do
+  postgres.create_table "raw", nil, "(var1 text)"
+  postgres.execute "copy raw from '#{File.expand_path('raw.txt',Dir.pwd)}'"
+end
+
+desc "Perform analysis in PostgreSQL."
+data postgres['analyzed'] => postgres['raw'] do
+  # perform analysis ...
+  postgres.create_table "analyzed", "select * from raw", nil
+end
+
+desc "Archive analysis results in SQLite."
+data sqlite['analyzed_archive'] => postgres['analyzed'] do
+  sqlite.create_table "analyzed_archive", nil, "(var1 text)"
+  r = postgres.execute "select var1 from analyzed"
+  sqlite.execute <<-EOSQL
+    insert into analyzed_archive values 
+    ('#{ r.flatten.join("'),('") }')
+  EOSQL
+end

data/lib/data_task/util.rb

@@ -0,0 +1,6 @@
+class Array
+  def to_quoted_s
+    commad = self.join("','")
+    "'#{commad}'"
+  end
+end

data/lib/data_task/version.rb

@@ -0,0 +1,5 @@
+module Rake
+  module DataTask
+    VERSION = "0.0.2"
+  end
+end

data/lib/data_task.rb

@@ -0,0 +1,76 @@
+require 'rake'
+require 'data_task/version'
+require 'data_task/data'
+require 'data_task/adapters/postgres'
+require 'data_task/adapters/sqlite'
+
+module Rake
+  # #########################################################################
+  # A DataTask is a task that includes time based dependencies. If any of a
+  # DataTask's prerequisites has a timestamp that is later than the data
+  # represented by this task, then the data must be rebuilt (using the
+  # supplied actions).
+  module DataTask
+
+    class DataTask < ::Rake::Task
+
+      # Instantiate a new DataTask.
+      #
+      # @param [Data] data the Data object that keeps track of existence and modification
+      # @param [Rake::Application] app required by the parent class's constructor 
+      def initialize(data, app)
+        super
+        @data = data
+      end
+
+      # Is this table task needed? Yes if it doesn't exist, or if its time stamp
+      # is out of date.
+      def needed?
+        !@data.exist? || out_of_date?(timestamp)
+      end
+
+      # Time stamp for data task.
+      def timestamp
+        if @data.exist?
+          mtime = @data.mtime.to_time
+          raise "Table #{name} exists but modified time is unavailable." if mtime.nil?
+          mtime
+        else
+          Rake::EARLY
+        end
+      end
+
+      private
+
+      # Are there any prerequisites with a later time than the given time stamp?
+      def out_of_date?(stamp)
+        @prerequisites.any? do |n| 
+          prereq_time = application[n, @scope].timestamp
+          return false if prereq_time == Rake::EARLY
+
+          prereq_time > stamp
+        end
+      end
+
+      # ----------------------------------------------------------------
+      # Task class methods.
+      #
+      class << self
+        # Apply the scope to the task name according to the rules for this kind
+        # of task. Table based tasks ignore the scope when creating the name.
+        def scope_name(scope, task_name)
+          task_name
+        end
+      end
+
+    end
+
+  end
+end
+
+def data(*args, &block)
+  # The task name in *args here is a Data returned by the adapter. Rake will key this task by 
+  # Data.to_s in @tasks [Array]. All task recording and lookup in Rake is already done via to_s 
+  # already to accomdate tasks named by symbols.
+  Rake::DataTask::DataTask.define_task(*args, &block)
+end

data/test/config/database.yml

@@ -0,0 +1,10 @@
+sqlite_test:
+  adapter: sqlite
+  database: ci_test
+
+postgres_test:
+  adapter: postgres
+  database: ci_test
+  host: localhost
+  port: 5432
+  username: postgres

data/test/data_spec.rb

@@ -0,0 +1,81 @@
+require_relative './helper.rb'
+
+module Rake
+  module DataTask
+
+    describe Data do
+
+      test_data_name = "test"
+
+      def mtime_updated? data, operation
+        original_mtime = data.mtime
+        sleep(1)
+        operation.call
+        data.mtime > original_mtime
+      end
+
+      around do |test|
+        @adapter = get_adapter
+        @adapter.with_transaction_rollback do
+          test.call
+        end
+      end
+
+      it "has a modified time after creation" do
+        @adapter.with_tracking do
+          @adapter.create_data test_data_name, nil, "(var1 integer)"
+          t = Data.new(test_data_name, @adapter)
+          t.mtime.to_time.must_be :>, Time.new(0)
+        end
+      end
+
+      it "has an updated modified time after insert" do
+        @adapter.with_tracking do
+          @adapter.create_data test_data_name, nil, "(var1 integer)"
+          t = Data.new(test_data_name, @adapter)
+          operation = lambda do
+            @adapter.execute "insert into #{test_data_name} values (1)"
+          end
+          mtime_updated?(t, operation).must_equal true
+        end
+      end
+
+      it "has an updated modified time after update" do
+        @adapter.with_tracking do
+          @adapter.create_data test_data_name, nil, "(var1 integer, var2 integer)"
+          t = Data.new(test_data_name, @adapter)
+          @adapter.execute "insert into #{test_data_name} values (1, 1)"
+          operation = lambda do 
+            @adapter.execute "update #{test_data_name} set var2 = 2 where var1 = 1"
+          end
+          mtime_updated?(t, operation).must_equal true
+        end
+      end
+
+      it "has an updated modified time after delete" do
+        @adapter.with_tracking do
+          @adapter.create_data test_data_name, nil, "(var1 integer)"
+          t = Data.new(test_data_name, @adapter)
+          @adapter.execute "insert into #{test_data_name} values (1)"
+          operation = lambda do
+            @adapter.execute "delete from #{test_data_name}"
+          end
+          mtime_updated?(t, operation).must_equal true
+        end
+      end
+
+      it "has an updated modified time after truncate" do
+        @adapter.with_tracking do
+          @adapter.create_data test_data_name, nil, "(var1 integer)"
+          t = Data.new(test_data_name, @adapter)
+          @adapter.execute "insert into #{test_data_name} values (1)"
+          operation = lambda do
+            @adapter.truncate_data test_data_name
+          end
+          mtime_updated?(t, operation).must_equal true
+        end
+      end
+
+    end
+  end
+end

data/test/db_spec.rb

@@ -0,0 +1,202 @@
+require_relative './helper.rb'
+
+module Rake
+  module DataTask
+
+    describe Db do
+
+      around do |test|
+        @adapter = get_adapter
+        @adapter.with_transaction_rollback do
+          test.call
+        end
+      end
+
+      test_table = "test_table"
+      test_view = "test_view"
+
+      it "finds a table when it exists" do
+        if !@adapter.table_exists?(test_table)
+          @adapter.execute <<-EOSQL
+            create table #{test_table} (var1 text)
+          EOSQL
+        end
+        @adapter.table_exists?(test_table).must_equal true
+      end
+
+      it "does not find a table when it does not exist" do
+        if @adapter.table_exists?(test_table)
+          @adapter.execute <<-EOSQL
+            drop table #{test_table}
+          EOSQL
+        end
+        @adapter.table_exists?(test_table).must_equal false
+      end
+
+      it "creates a table when called to" do
+        @adapter.with_tracking do
+
+          @adapter.create_table test_table, nil, '(var1 text)'
+          @adapter.table_exists?(test_table).must_equal true
+
+        end
+      end
+
+      it "drops a table when called to" do
+        @adapter.with_tracking do
+
+          @adapter.create_table test_table, nil, '(var1 text)'
+          @adapter.drop_table test_table
+          @adapter.table_exists?(test_table).must_equal false
+
+        end
+      end
+
+      it "creates a view when called to" do
+        @adapter.with_tracking do
+
+          @adapter.create_table test_table, nil, '(var1 text)'
+          @adapter.create_view test_view, "select * from #{test_table}"
+          @adapter.view_exists?(test_view).must_equal true
+
+        end
+      end
+
+      it "drops a view when called to" do
+        @adapter.with_tracking do
+
+          @adapter.create_table test_table, nil, '(var1 text)'
+          @adapter.create_view test_view, "select * from #{test_table}"
+          @adapter.drop_view test_view
+          @adapter.view_exists?(test_view).must_equal false
+
+        end
+      end
+
+      it "drops a view when the underlying table is dropped" do
+        @adapter.with_tracking do
+
+          @adapter.create_table test_table, nil, '(var1 text)'
+          @adapter.create_view test_view, "select * from #{test_table}"
+          @adapter.drop_table test_table
+          @adapter.view_exists?(test_view).must_equal false
+
+        end
+      end
+
+      it "updates the tracking table when it creates a table" do
+        @adapter.with_tracking do
+
+          @adapter.create_table test_table, nil, '(var1 text)'
+           tracked_create = Sql.get_single_int( 
+            @adapter.execute <<-EOSQL
+            select 1 from #{Db::TABLE_TRACKER_NAME} 
+            where 
+              relation_name = '#{test_table}' and
+              relation_type = '#{@adapter.relation_type_values[:table]}' and
+              operation = '#{@adapter.operation_values[:create]}'
+            EOSQL
+          )
+          tracked_create.must_equal 1
+
+        end
+      end
+
+      it "updates the tracking table when it drops a table" do
+        @adapter.with_tracking do
+
+          @adapter.create_table test_table, nil, '(var1 text)'
+          @adapter.drop_table test_table
+          still_tracking_table = Sql.get_single_int(
+            @adapter.execute <<-EOSQL
+            select 1 from #{Db::TABLE_TRACKER_NAME} 
+            where 
+              relation_name = '#{test_table}' and
+              relation_type = '#{@adapter.relation_type_values[:table]}'
+            EOSQL
+          )
+          still_tracking_table.must_be_nil
+
+        end
+      end
+
+      it "updates the tracking table on insert to a tracked table" do
+        @adapter.with_tracking do
+
+          @adapter.create_table test_table, nil, '(var1 text)'
+          @adapter.execute <<-EOSQL
+            insert into #{test_table} values ('a')
+          EOSQL
+          tracked_insert = Sql.get_single_int( 
+            @adapter.execute <<-EOSQL
+            select 1 from #{Db::TABLE_TRACKER_NAME} 
+            where 
+              relation_name = '#{test_table}' and
+              relation_type = '#{@adapter.relation_type_values[:table]}' and
+              operation = '#{@adapter.operation_values[:insert]}'
+            EOSQL
+          )
+          tracked_insert.must_equal 1
+
+        end
+      end
+
+      it "updates the tracking table on update on a tracked table" do
+        @adapter.with_tracking do
+
+          @adapter.create_table test_table, nil, '(var1 text, var2 text)'
+          @adapter.execute <<-EOSQL
+            insert into #{test_table} values ('a', 'a')
+          EOSQL
+          @adapter.execute <<-EOSQL
+            update #{test_table} set var2 = 'b' where var1 = 'a'
+          EOSQL
+
+          tracked_insert = Sql.get_single_int( 
+            @adapter.execute <<-EOSQL
+            select 1 from #{Db::TABLE_TRACKER_NAME} 
+            where 
+              relation_name = '#{test_table}' and
+              relation_type = '#{@adapter.relation_type_values[:table]}' and
+              operation = '#{@adapter.operation_values[:update]}'
+            EOSQL
+          )
+          tracked_insert.must_equal 1
+
+        end
+      end
+
+      it "updates the tracking table on truncate of a tracked table" do
+        @adapter.with_tracking do
+
+          @adapter.create_table test_table, nil, '(var1 text)'
+          @adapter.truncate_table test_table
+          tracked_truncate = Sql.get_single_int( 
+            @adapter.execute <<-EOSQL
+            select 1 from #{Db::TABLE_TRACKER_NAME} 
+            where 
+              relation_name = '#{test_table}' and
+              relation_type = '#{@adapter.relation_type_values[:table]}' and
+              operation = '#{@adapter.operation_values[:truncate]}'
+            EOSQL
+          )
+          tracked_truncate.must_equal 1
+
+        end
+      end
+
+      it "says it is tracking tables after tracking is set up" do
+        @adapter.tear_down_tracking
+        @adapter.set_up_tracking
+        @adapter.tracking_tables?.must_equal true
+      end
+
+      it "says it is not tracking tables after tracking is torn down" do
+        @adapter.tear_down_tracking
+        (@adapter.tracking_tables?).must_equal false
+      end
+
+    end
+
+  end
+end

data/test/helper.rb

@@ -0,0 +1,37 @@
+# https://coveralls.io
+require 'coveralls'
+Coveralls.wear!
+
+require 'minitest/autorun'
+require 'minitest/around/spec'
+require 'minitest-spec-context'
+
+require 'logger'
+
+require 'data_task/sql'
+require 'data_task/db'
+require 'data_task/data'
+require 'data_task'
+
+require 'data_task/adapters/sqlite'
+require 'data_task/adapters/postgres'
+
+def get_adapter
+  # connect an adapter to the configured database for testing
+  config = YAML.load_file('test/config/database.yml')[ENV['DATATASK_ENV'] || 'sqlite_test']
+  klass = "Rake::DataTask::#{config['adapter'].capitalize}".split('::').inject(Object) {|memo, name| memo = memo.const_get(name); memo}
+  adapter = klass.new(config)
+
+  # extend the adapter to enable clean tracking setup/teardown within each test
+  adapter.extend(TrackingSetupTeardownHelper)
+
+  adapter
+end
+
+module TrackingSetupTeardownHelper
+  def with_tracking &ops
+    set_up_tracking
+    ops.call
+    tear_down_tracking
+  end
+end