purview 1.0.0.alpha

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 4753e561eece41d8838e916c7914ae3b3d155b3b
+  data.tar.gz: 8430637788951088580baa1f533540968974f28c
+SHA512:
+  metadata.gz: 742e3ab26e898924c9054b461d41c700144859910b96ccfd2df6ac6e9d51bb2945a61f73282445f7c239c8adfeb4cc53bdf518aad195b91cc931cdeca24639bb
+  data.tar.gz: 43d8b1a2b5f6cf401a33e254a8b63169b1764fe3a88b83391a91b6068203b0bd02df339746982580864aba2a3c3f9ae0ba079fd938f31fe687d37e38b7aff987

data/.gitignore

@@ -0,0 +1,33 @@
+*.gem
+*.rbc
+/.config
+/InstalledFiles
+/coverage/
+/pkg/
+/script/
+/spec/reports/
+/test/tmp/
+/test/version_tmp/
+/tmp/
+tags
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalisation:
+/.bundle/
+/lib/bundler/man/
+
+## Ruby artifacts:
+.ruby-gemset
+.ruby-version
+.rvmrc
+Gemfile.lock

data/.travis.yml

@@ -0,0 +1,18 @@
+language: ruby
+
+matrix:
+  allow_failures:
+    - rvm: jruby-19mode
+
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - 2.1.0
+  - 2.1.1
+  - 2.1.2
+  - 2.1.3
+  - 2.1.4
+  - 2.1.5
+  - 2.2.0
+  - jruby-19mode
+  - rbx-2

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 Jonathan W. Zaleski
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,143 @@
+# Purview
+
+[![Build Status](https://secure.travis-ci.org/jzaleski/purview.png?branch=master)](http://travis-ci.org/jzaleski/purview)
+[![Dependency Status](https://gemnasium.com/jzaleski/purview.png)](https://gemnasium.com/jzaleski/purview)
+
+A framework designed to simplify data warehousing
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'purview'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install purview
+
+## Usage
+
+Load the `MySQL` client (for `PostgreSQL` simply change 'mysql2' to 'pg')
+```ruby
+require 'mysql2'
+```
+
+Set the database-name (this can be anything, but it must exist)
+```ruby
+database_name = :data_warehouse
+```
+
+Combine all the configuration options and instantiate the `Database` (for
+`PostgreSQL` simply change `MySQL` to `PostgreSQL`)
+```ruby
+database_opts = {}
+
+database = Purview::Databases::MySQL.new(
+  database_name,
+  database_opts
+)
+```
+
+Set the table-name (this can be anything, but it must exist)
+```ruby
+table_name = :users
+```
+
+Define the `Column(s)` (available column-types: `Boolean`, `CreatedTimestamp`,
+`Date`, `Float`, `Id`, `Integer`, `Money`, `String`, `Text`, `Time`, `Timestamp`,
+`UpdatedTimestamp` & `UUID` -- the `Id`, `CreatedTimestamp` & `UpdatedTimestamp`
+columns are required for all tables)
+```ruby
+columns = [
+  Purview::Columns::Id.new(:id),
+  Purview::Columns::String.new(:name, :nullable => false),
+  Purview::Columns::String.new(:email, :nullable => false, :limit => 100),
+  Purview::Columns::CreatedTimestamp.new(:created_at),
+  Purview::Columns::UpdatedTimestamp.new(:updated_at),
+]
+```
+
+Configure the `Puller` (available puller-types: `URI`)
+```ruby
+puller_opts = {
+  :type => Purview::Pullers::URI,
+  :uri => 'http://feed.test.com/users',
+}
+```
+
+Configure the `Parser` (available parser-types: `CSV` & `TSV`)
+```ruby
+parser_opts = {
+  :type => Purview::Parsers::TSV,
+}
+```
+
+Configure the `Loader` (for `PostgreSQL` simply change `MySQL` to `PostgreSQL`)
+```ruby
+loader_opts = {
+  :type => Purview::Loaders::MySQL,
+}
+```
+
+Configure the `starting_timestamp` (this is the min-date to pull and can vary
+between `Table(s)`)
+```ruby
+starting_timestamp = Time.parse('2012-01-01 00:00:00Z')
+```
+
+Combine all the configuration options and instantiate the `Table`
+```ruby
+table_opts = {
+  :columns => columns,
+  :database => database,
+  :loader => loader_opts,
+  :parser => parser_opts,
+  :puller => puller_opts,
+  :starting_timestamp => starting_timestamp,
+}
+
+table = Purview::Tables::Raw.new(
+  table_name,
+  table_opts
+)
+```
+
+Add the `Table` to the `Database` (schema). In order for [the] `Table` to be
+`sync[ed]` it *must* be added to [the] `Database`
+```ruby
+database.add_table(table)
+```
+
+Create the `Table`. Recommended for testing purposes *only*. For production
+environments you will likely want an external process to manage the schema (for
+`PostgreSQL` simply change `Mysql2::Error` to `PG::DuplicateTable`)
+```ruby
+begin
+  database.create_table(
+    database.connect,
+    table
+  )
+rescue Mysql2::Error; end
+```
+
+Sync the `Database`. This process will select a [candidate] `Table`, pull data
+from its [remote-]source and reconcile the new data against the main-table (e.g.
+perform `INSERTs`, `UPDATEs` and `DELETEs`). When multiple `Table(s)` are
+configured the least recently pulled and available (`enabled` and not `locked`)
+`Table` will be selected (you will likely want to configure some process to load
+the schema run the `sync` at regularly scheduled intervals)
+```ruby
+database.sync
+```
+
+## Contributing
+
+1. Fork it ( http://github.com/jzaleski/purview/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,11 @@
+require 'bundler/gem_tasks'
+
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec) do |task|
+  task.rspec_opts = %w[--format progress]
+end
+
+task :test => :spec
+
+task :default => :test

data/TODO

@@ -0,0 +1,81 @@
+... CLOSED TASKS ....
+
+* Handle blank values a bit more intelligently (all blanks -> nil)
+  * Add support for INSERTs, UPDATEs and DELETEs (of table-data)
+* Deal w/ NULL values
+* Deal w/ BOOL values
+* Build out MVP for PostgreSQL loader
+  * Create temporary table from table-metadata and load all of the parsed rows
+    into it (build create, update and delete logic atop the temporary table)
+  * Map [object-]types to database types (in the loader?)
+* Add windowing to pulls (simulate w/ 2 different data-files if necessary)
+  * Store max-timestamp pulled (NULL or previous window + window size)
+  * Add configuration to `Table` for window-size (in the future this can scale
+    dynamically)
+* Include [window] timestamps in query-string for pull[er]
+* Figure out an intelligent way to solve the chicken <-> egg problem for
+  `table_metadata`
+* Fail fast if there are no candidate tables (`next_table`)
+* Ensure that the puller does not try to go into the future (`next_window`)
+* Add in a primitive logging facility (can be temporary)
+* Enhance `next_table_sql`
+* Lock tables (feeds) during pull, parse and load
+* Verify table-locking mechanism behaves as expected
+* Ability to drop a table
+* Reorder args: "table, table_name" to: "table_name, table"
+* Reorder args: "table, index_name, *columns" to "index_name, table, *columns"
+* Ability to create an index
+* Ability to drop an index
+* Indices on `created_at` and `updated_at` columns
+* Gracefully handle table {,un}locking in case of error
+* Add debugging code
+* Raise no-window & no-table exceptions (making them catchable upstream)
+* Minimize race-condition around table-locking (use where on UPDATE, raise if
+  zero rows are updated)
+* Log number of delete, inserts and updates in `loader`
+* `load_temporary_table` should bomb if there is data outside the window
+* Delete from outside window where id in temporary table
+* Rename `data` to `rows` in `execute`
+* Rename `Postgres` to `PostgreSQL`
+* Finish PostgreSQL support (requires `pg` gem)
+* Finish MySQL support (requires `mysql2` gem)
+* Usage/configuration examples in README.md
+
+... ONGOING TASKS ...
+
+* Add tests
+* Add comments where appropriate
+* DRY/Refactor
+
+... OPEN TASKS ....
+
+* Create table class to encapulate `table_metadata` logic
+* Build out change-log tables
+* Build out canonical, fact and aggregate tables (and related transforms)
+* Configurable re-pull window (do this automatically once up to current?)
+* Add schema management capabilities (detect schema-deltas and suggestion
+  modifications)
+
+... DEFERRED TASKS ...
+
+* Fix JRuby bundle/build
+
+... CLOSED QUESTIONS ...
+
+* Where is the best place to create the connection?
+  * Immediately? Then it's open during the pull, etc.
+  * Lazily? Makes for connection logic all over the place
+    * Perhaps it can be centralized entirely into the `Database` class?
+* Consider using ISO-8601 timestamps in query-string (not sure on this one, it
+  is slightly more fragile because of URL-encoding, etc.)
+
+... OPEN QUESTIONS ...
+
+* Parallel pulls from the same "source"? Can this be done in one request?
+* Rename columns during pull/parse `source_name` & `target_name` perhaps? Maybe
+  create a new type of column?
+
+... INTEGRATION CONSIDERATIONS ...
+
+* Deployment
+* Scheduling (Daemon? CRON?)

data/lib/purview/columns/base.rb

@@ -0,0 +1,65 @@
+module Purview
+  module Columns
+    class Base
+      attr_reader :name
+
+      def initialize(name, opts={})
+        @name = name.to_sym
+        @opts = default_opts.merge(opts)
+      end
+
+      def default
+        opts[:default]
+      end
+
+      def default?
+        !!default
+      end
+
+      def limit
+        opts[:limit]
+      end
+
+      def limit?
+        !!limit
+      end
+
+      def nullable
+        coalesce(opts[:nullable], true)
+      end
+
+      def nullable?
+        !!nullable
+      end
+
+      def parse(value)
+        blank = blank?(value)
+        return nil if blank && nullable?
+        raise %{Unexpected blank value for column: "#{name}"} if blank
+        type.parse(value)
+      end
+
+      def primary_key
+        opts[:primary_key]
+      end
+
+      def primary_key?
+        !!primary_key
+      end
+
+      def type
+        coalesce(opts[:type], Purview::Types::String)
+      end
+
+      private
+
+      include Purview::Mixins::Helpers
+
+      attr_reader :opts
+
+      def default_opts
+        {}
+      end
+    end
+  end
+end

data/lib/purview/columns/boolean.rb

@@ -0,0 +1,11 @@
+module Purview
+  module Columns
+    class Boolean < Base
+      private
+
+      def default_opts
+        super.merge(:type => Purview::Types::Boolean)
+      end
+    end
+  end
+end

data/lib/purview/columns/created_timestamp.rb

@@ -0,0 +1,11 @@
+module Purview
+  module Columns
+    class CreatedTimestamp < Timestamp
+      private
+
+      def default_opts
+        super.merge(:nullable => false)
+      end
+    end
+  end
+end

data/lib/purview/columns/date.rb

@@ -0,0 +1,11 @@
+module Purview
+  module Columns
+    class Date < Base
+      private
+
+      def default_opts
+        super.merge(:type => Purview::Types::Date)
+      end
+    end
+  end
+end

data/lib/purview/columns/float.rb

@@ -0,0 +1,11 @@
+module Purview
+  module Columns
+    class Float < Base
+      private
+
+      def default_opts
+        super.merge(:type => Purview::Types::Float)
+      end
+    end
+  end
+end

data/lib/purview/columns/id.rb

@@ -0,0 +1,11 @@
+module Purview
+  module Columns
+    class Id < Integer
+      private
+
+      def default_opts
+        super.merge(:nullable => false, :primary_key => true)
+      end
+    end
+  end
+end

data/lib/purview/columns/integer.rb

@@ -0,0 +1,11 @@
+module Purview
+  module Columns
+    class Integer < Base
+      private
+
+      def default_opts
+        super.merge(:type => Purview::Types::Integer)
+      end
+    end
+  end
+end

data/lib/purview/columns/money.rb

@@ -0,0 +1,11 @@
+module Purview
+  module Columns
+    class Money < Base
+      private
+
+      def default_opts
+        super.merge(:type => Purview::Types::Float)
+      end
+    end
+  end
+end

data/lib/purview/columns/string.rb

@@ -0,0 +1,11 @@
+module Purview
+  module Columns
+    class String < Base
+      private
+
+      def default_opts
+        super.merge(:type => Purview::Types::String, :limit => 255)
+      end
+    end
+  end
+end

data/lib/purview/columns/text.rb

@@ -0,0 +1,11 @@
+module Purview
+  module Columns
+    class Text < Base
+      private
+
+      def default_opts
+        super.merge(:type => Purview::Types::Text)
+      end
+    end
+  end
+end

data/lib/purview/columns/time.rb

@@ -0,0 +1,11 @@
+module Purview
+  module Columns
+    class Time < Base
+      private
+
+      def default_opts
+        super.merge(:type => Purview::Types::Time)
+      end
+    end
+  end
+end

data/lib/purview/columns/timestamp.rb

@@ -0,0 +1,11 @@
+module Purview
+  module Columns
+    class Timestamp < Base
+      private
+
+      def default_opts
+        super.merge(:type => Purview::Types::Timestamp)
+      end
+    end
+  end
+end

data/lib/purview/columns/updated_timestamp.rb

@@ -0,0 +1,11 @@
+module Purview
+  module Columns
+    class UpdatedTimestamp < Timestamp
+      private
+
+      def default_opts
+        super.merge(:nullable => false)
+      end
+    end
+  end
+end

data/lib/purview/columns/uuid.rb

@@ -0,0 +1,11 @@
+module Purview
+  module Columns
+    class UUID < Base
+      private
+
+      def default_opts
+        super.merge(:type => Purview::Types::UUID, :limit => 36)
+      end
+    end
+  end
+end

data/lib/purview/columns.rb

@@ -0,0 +1,14 @@
+require 'purview/columns/base'
+require 'purview/columns/boolean'
+require 'purview/columns/date'
+require 'purview/columns/float'
+require 'purview/columns/integer'
+require 'purview/columns/id'
+require 'purview/columns/money'
+require 'purview/columns/string'
+require 'purview/columns/text'
+require 'purview/columns/time'
+require 'purview/columns/timestamp'
+require 'purview/columns/created_timestamp'
+require 'purview/columns/updated_timestamp'
+require 'purview/columns/uuid'

data/lib/purview/connections/base.rb

@@ -0,0 +1,55 @@
+module Purview
+  module Connections
+    class Base
+      def initialize(opts={})
+        @opts = opts
+      end
+
+      def connect
+        @connection ||= new_connection
+        self
+      end
+
+      def disconnect
+        connection.close
+        @connection = nil
+        self
+      end
+
+      def execute(sql)
+        logger.debug("Executing: #{sql}")
+        result = execute_sql(sql)
+        Purview::Structs::Result.new(
+          :rows => extract_rows(result),
+          :rows_affected => extract_rows_affected(result)
+        )
+      end
+
+      def with_transaction
+        raise %{All "#{Base}(s)" must override the "with_transaction" method}
+      end
+
+      private
+
+      include Purview::Mixins::Logger
+
+      attr_reader :opts, :connection
+
+      def execute_sql(sql)
+        raise %{All "#{Base}(s)" must override the "execute_sql" method}
+      end
+
+      def extract_rows(result)
+        raise %{All "#{Base}(s)" must override the "extract_rows" method}
+      end
+
+      def extract_rows_affected(result)
+        raise %{All "#{Base}(s)" must override the "extract_rows_affected" method}
+      end
+
+      def new_connection
+        raise %{All "#{Base}(s)" must override the "new_connection" method}
+      end
+    end
+  end
+end

data/lib/purview/connections/mysql.rb

@@ -0,0 +1,39 @@
+module Purview
+  module Connections
+    class MySQL < Base
+      def with_transaction
+        connection.query(BEGIN_TRANSACTION)
+        yield.tap { |result| connection.query(COMMIT_TRANSACTION) }
+      rescue Mysql2::Error
+        connection.query(ROLLBACK_TRANSACTION)
+        raise
+      end
+
+      private
+
+      BEGIN_TRANSACTION = 'BEGIN'
+      COMMIT_TRANSACTION = 'COMMIT'
+      ROLLBACK_TRANSACTION = 'ROLLBACK'
+
+      def execute_sql(sql)
+        connection.query(sql, query_opts)
+      end
+
+      def extract_rows(result)
+        result && result.to_a
+      end
+
+      def extract_rows_affected(result)
+        connection.affected_rows
+      end
+
+      def new_connection
+        Mysql2::Client.new(opts)
+      end
+
+      def query_opts
+        { :cast => false }
+      end
+    end
+  end
+end