data_task 0.0.2

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    ZDk5OWQzMDU3YzNkZGUyZTczODUxNTdhODk4MTg4MjU0MGU3NDYwOQ==
+  data.tar.gz: !binary |-
+    NzRmMWMzNjM4MGJjYmZjYTkwYTIwOTFiNjQ0YTVjZWZjNmU1NmQwNg==
+SHA512:
+  metadata.gz: !binary |-
+    NmNmMjc3MzFmMzBmOGI2MmFjNDBkODAyNDQ5YmZiNzBjMjFmY2JhNDFhMTk5
+    M2NmZjk5MzU4ZWRiNzFkMDYxYTgxN2I1MTM0ZjI0OGUzMjRjOTM4YmVmNmVm
+    ZjlkYzhhM2Y0YmU5YTdiNTY2NTMyMTVlMzhiNDE2ODU5YzQ2M2I=
+  data.tar.gz: !binary |-
+    MzdiNGYyN2IyOWU4YWNkYjgwNzE1MzRmYThlNGNlODU1NmQ2N2ZhZDU1YTcz
+    YWEwZjQyM2Q4NWJkYmFlZGI4YjVhNWIxY2M4ODZjMzc3OGM4YmFlZWQ3YTcx
+    ZTcwNDMyZTQ3YThmYjY2MjIyNGQ1OGMyMzNlMmEzOTdjY2Y2NTQ=

data/.gitignore

@@ -0,0 +1,39 @@
+*.gem
+*.rbc
+.config
+Gemfile.lock
+InstalledFiles
+coverage
+doc/
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalisation:
+/.bundle/
+/lib/bundler/man/
+
+# vim
+[._]*.s[a-w][a-z]
+[._]s[a-w][a-z]
+*.un~
+Session.vim
+.netrwhist
+*~
+
+# host-specific configuration
+/config/database.yml

data/.travis.yml

@@ -0,0 +1,17 @@
+language: ruby
+cache: bundler
+rvm:
+  - 1.9.3
+services: postgresql
+before_script:
+  - psql -c 'create database ci_test;' -U postgres
+  - sqlite3 ci_test ''
+
+env:
+  - DATATASK_ENV=postgres_test
+  - DATATASK_ENV=sqlite_test
+
+notifications:
+  email:
+    on_success: never
+    on_failure: change

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Shahin Saneinejad
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,93 @@
+[![Build Status](https://travis-ci.org/shahin/data_task.svg?branch=master)](https://travis-ci.org/shahin/data_task)
+[![Coverage Status](https://img.shields.io/coveralls/shahin/data_task.svg)](https://coveralls.io/r/shahin/data_task?branch=master)
+
+# DataTask
+
+DataTask extends Rake's dependency-based programming language to databases. This gem provides the `data` task, analogous to Rake's built-in `file` task but extended to work with pluggable backends beyond the local filesystem.
+
+Adapters are included for Sqlite3, PostgreSQL, and Greenplum.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'data_task'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install data_task
+
+## Usage
+
+To write your first data task, connect to your database by instantiating an adapter:
+
+```
+postgres = Rake::DataTask::Postgres.new(
+  'host' => 'localhost', 
+  'port' => 5432, 
+  'database' => 'example', 
+  'username' => 'postgres'
+  )
+```
+
+Then use this adapter instance as the target for a data task:
+
+```
+desc "Load a data file into PostgreSQL for analysis."
+data postgres['raw'] => 'raw.txt' do
+  # Add loading logic here
+end
+```
+
+Rake will run this task if and only if (a) the table 'raw' is does not exist yet, or (b) the table 'raw' exists but has a timestamp earlier than the file 'raw.txt'. Since database tables now have timestamps associated with them, they can serve as targets or as dependencies in data tasks.
+
+Here's a runnable example Rakefile:
+
+```
+require 'rake'
+require 'data_task'
+
+# connect to the database
+postgres = Rake::DataTask::Postgres.new(
+  'host' => 'localhost', 
+  'port' => 5432, 
+  'database' => 'example', 
+  'username' => 'postgres'
+  )
+
+# mark raw.txt as a potential dependency
+file 'raw.txt'
+
+# define a loader for the postgres table 'raw', dependent on raw.txt
+desc "Load a data file into PostgreSQL for analysis."
+data postgres['raw'] => 'raw.txt' do
+  postgres.create_table 'raw', nil, '(var1 text)'
+  postgres.execute "copy raw.txt to raw"
+end
+```
+
+To run it: 
+
+1. paste the example into a file named 'Rakefile',
+2. make sure the PostgreSQL configuration matches your server,
+3. open a terminal and run the commands below:
+
+```
+$ echo "v1" > raw.txt
+$ rake 'raw'
+```
+
+The contents of raw.txt should be in your table 'raw' on PostgreSQL. Running the rake command a second time will result in no operations as long as raw.txt hasn't changed. With big data files, this can be a big time-saver.
+
+
+## Contributing
+
+1. Fork it ( https://github.com/shahin/data_task/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,17 @@
+require 'data_task'
+
+desc "Run tests"
+task :default => :'data_task:test'
+
+namespace :data_task do
+
+  require 'bundler/gem_tasks'
+  require 'rake/testtask'
+
+  Rake::TestTask.new do |t|
+    t.libs << "spec"
+    t.test_files = FileList['test/**/*_spec.rb', 'test/test_*.rb']
+    t.verbose
+  end
+
+end

data/data_task.gemspec

@@ -0,0 +1,31 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'data_task/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "data_task"
+  spec.version       = Rake::DataTask::VERSION
+  spec.authors       = ["Shahin Saneinejad"]
+  spec.email         = ["shahin.saneinejad@gmail.com"]
+  spec.summary       = %q{A Rake task for managing data across multiple datastores.}
+  spec.description   = %q{DataTask provides dependency-based programming for data workflows on top of the Rake build tool.}
+  spec.homepage      = "https://github.com/shahin/data_task"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.required_ruby_version  = '>= 1.9.3'
+
+  spec.add_runtime_dependency 'rake', '~> 10.0.4'
+  spec.add_runtime_dependency 'pg', '~> 0.17.1'
+  spec.add_runtime_dependency 'sqlite3'
+
+  spec.add_development_dependency 'bundler', '~> 1.6'
+  spec.add_development_dependency 'minitest-around', '~> 0.2'
+  spec.add_development_dependency 'minitest-spec-context', '~> 0.0.3'
+  spec.add_development_dependency 'coveralls'
+end

data/lib/data_task/adapters/greenplum.rb

@@ -0,0 +1,159 @@
+require 'pg'
+require_relative 'support/transactions'
+require_relative 'support/booleans'
+require_relative './postgres'
+
+module Rake
+  module DataTask
+
+    class Greenplum < Postgres
+
+      TABLE_TRACKER_HELPER_NAME = "operations"
+
+      def self.set_up_tracking options
+        tear_down_tracking options
+        super
+
+        execute "alter table #{TABLE_TRACKER_NAME} rename to #{TABLE_TRACKER_HELPER_NAME}"
+
+        # Greenplum tracks CREATE and TRUNCATE operations in its pg_stat_operations system view.
+        # Join this view with the tracking table so that we can track CREATE and TRUNCATE from within
+        # the database instead of from application code.
+
+        execute <<-EOSQL
+          create view fixed_pg_stat_operations as
+          -- GP's pg_stat_operations enum values like 'TABLE' are inconsistent so fix them here
+          select
+            pso.classname, 
+            pso.objname,
+            pso.objid,
+            pso.schemaname,
+            pso.usestatus,
+            pso.usename,
+            pso.actionname,
+            case 
+              when pso.actionname = 'TRUNCATE' then '#{relation_type_values[:table]}'
+              when pso.subtype = 'TABLE' then '#{relation_type_values[:table]}'
+              else pso.subtype
+            end as subtype,
+            pso.statime
+          from pg_stat_operations pso
+        EOSQL
+
+        execute <<-EOSQL
+          create view #{TABLE_TRACKER_NAME} as 
+          select
+            relation_name,
+            relation_type,
+            operation,
+            time
+          from (
+
+            select
+              a.*,
+              rank() over (partition by relation_name, relation_type order by time desc)
+            from (
+
+              -- select all CREATE and TRUNCATE operations tracked by Greenplum
+              select
+                pso.objname as relation_name,
+                pso.subtype as relation_type,
+                pso.actionname as operation,
+                pso.statime as time
+              from fixed_pg_stat_operations pso
+              where pso.actionname not in ('ANALYZE', 'VACUUM')
+
+              union all
+
+              -- select all operations tracked by Greenplum (PostgreSQL) table rules 
+              select
+                ttb.relation_name,
+                ttb.relation_type,
+                ttb.operation,
+                ttb.time
+              from 
+                #{TABLE_TRACKER_HELPER_NAME} ttb
+                -- return only operations for tables that exist in system tables
+                join fixed_pg_stat_operations pso on (
+                  ttb.relation_name = pso.objname and
+                  ttb.relation_type = pso.subtype and
+                  pso.actionname = 'CREATE'
+                )
+
+              ) a
+            ) b 
+          -- take only the latest operation per table
+          where rank = 1
+        EOSQL
+
+        # make sure we do deletes and inserts on the helper table, not the view
+        execute <<-EOSQL
+          create rule delete_operation_record as on delete to #{TABLE_TRACKER_NAME} 
+            do instead
+            delete from #{TABLE_TRACKER_HELPER_NAME} 
+            where
+              relation_name = OLD.relation_name and
+              relation_type = OLD.relation_type and
+              operation = OLD.operation
+          ;
+
+          create rule insert_operation_record as on insert to #{TABLE_TRACKER_NAME} 
+            do instead
+            insert into #{TABLE_TRACKER_HELPER_NAME} values (
+              NEW.relation_name,
+              NEW.relation_type,
+              NEW.operation,
+              NEW.time
+            )
+          ;
+        EOSQL
+      end
+
+      def self.tear_down_tracking options
+        drop_view "fixed_pg_stat_operations"
+        drop_view TABLE_TRACKER_NAME
+        drop_table TABLE_TRACKER_HELPER_NAME
+      end
+
+      def tracking_tables?
+        view_exists?(TABLE_TRACKER_NAME)
+      end
+
+      def drop_table table_name
+        execute "drop table if exists #{table_name} cascade"
+        return if table_name.casecmp(TABLE_TRACKER_HELPER_NAME) == 0 || 
+          table_name.casecmp(TABLE_TRACKER_NAME) == 0
+        track_drop table_name
+      end
+
+      def track_drop table_name
+        execute <<-EOSQL
+          delete from #{TABLE_TRACKER_HELPER_NAME} 
+          where 
+            relation_name = '#{table_name}' and 
+            relation_type = '#{relation_type_values[:table]}'
+        EOSQL
+      end
+
+
+
+      private
+
+        def operations_supported_by_db
+          operations_supported_by_rules & [:create, :truncate]
+        end
+
+        def track_creation table_name, n_tuples
+          # nothing to do; Greenplum tracks this operation in system tables already
+          return nil
+        end
+
+        def track_truncate table_name
+          # nothing to do; Greenplum tracks this operation in system tables already
+          return nil
+        end
+
+    end
+
+  end
+end