purview 1.0.0.alpha

data/lib/purview/tables/base.rb

@@ -0,0 +1,140 @@
+module Purview
+  module Tables
+    class Base
+      attr_reader :name
+
+      def initialize(name, opts={})
+        @name = name
+        @opts = opts
+      end
+
+      def columns
+        opts[:columns]
+      end
+
+      def column_names
+        columns.map(&:name)
+      end
+
+      def columns_by_name
+        {}.tap do |result|
+          columns.each do |column|
+            result[column.name] = column
+          end
+        end
+      end
+
+      def columns_of_type(type)
+        columns.select { |column| column.is_a?(type) }
+      end
+
+      def created_timestamp_column
+        columns_of_type(Purview::Columns::CreatedTimestamp).first
+      end
+
+      def data_columns
+        columns - [
+          created_timestamp_column,
+          id_column,
+          updated_timestamp_column,
+        ]
+      end
+
+      def database
+        opts[:database]
+      end
+
+      def id_column
+        columns_of_type(Purview::Columns::Id).first
+      end
+
+      def indexed_columns
+        (opts[:indexed_columns] || []).tap do |indexed_columns|
+          indexed_columns << [created_timestamp_column]
+          indexed_columns << [updated_timestamp_column]
+        end
+      end
+
+      def starting_timestamp
+        opts[:starting_timestamp]
+      end
+
+      def sync(connection, window)
+        raw_data = puller.pull(window)
+        parser.validate(raw_data)
+        parsed_data = parser.parse(raw_data)
+        loader.load(
+          connection,
+          parsed_data,
+          window
+        )
+      end
+
+      def temporary_name
+        "#{name}_#{Time.now.utc.to_i}"
+      end
+
+      def updated_timestamp_column
+        columns_of_type(Purview::Columns::UpdatedTimestamp).first
+      end
+
+      def window_size
+        opts[:window_size] || (60 * 60)
+      end
+
+      private
+
+      include Purview::Mixins::Logger
+
+      attr_reader :opts
+
+      def extract_type_option(opts)
+        opts[:type]
+      end
+
+      def filter_type_option(opts)
+        opts.select { |key| key != :type }
+      end
+
+      def loader
+        loader_type.new(loader_opts)
+      end
+
+      def loader_opts
+        merge_table_option(filter_type_option(opts[:loader]))
+      end
+
+      def loader_type
+        extract_type_option(opts[:loader])
+      end
+
+      def merge_table_option(opts)
+        { :table => self }.merge(opts)
+      end
+
+      def parser
+        parser_type.new(parser_opts)
+      end
+
+      def parser_opts
+        merge_table_option(filter_type_option(opts[:parser]))
+      end
+
+      def parser_type
+        extract_type_option(opts[:parser])
+      end
+
+      def puller
+        puller_type.new(puller_opts)
+      end
+
+      def puller_opts
+        filter_type_option(opts[:puller])
+      end
+
+      def puller_type
+        extract_type_option(opts[:puller])
+      end
+    end
+  end
+end

data/lib/purview/tables/raw.rb

@@ -0,0 +1,13 @@
+module Purview
+  module Tables
+    class Raw < Base
+      def name
+        "#{super}_raw"
+      end
+
+      def window_size
+        opts[:window_size] || (24 * 60 * 60)
+      end
+    end
+  end
+end

data/lib/purview/tables.rb

@@ -0,0 +1,2 @@
+require 'purview/tables/base'
+require 'purview/tables/raw'

data/lib/purview/types/base.rb

@@ -0,0 +1,9 @@
+module Purview
+  module Types
+    class Base
+      def self.parse(value)
+        raise %{All "#{Base}(s)" must override the "parse" method}
+      end
+    end
+  end
+end

data/lib/purview/types/boolean.rb

@@ -0,0 +1,9 @@
+module Purview
+  module Types
+    class Boolean < Base
+      def self.parse(value)
+        !!(value =~ /\A(true|t|yes|y|1)\z/i)
+      end
+    end
+  end
+end

data/lib/purview/types/date.rb

@@ -0,0 +1,9 @@
+module Purview
+  module Types
+    class Date < Base
+      def self.parse(value)
+        ::Date.parse(value)
+      end
+    end
+  end
+end

data/lib/purview/types/float.rb

@@ -0,0 +1,9 @@
+module Purview
+  module Types
+    class Float < Base
+      def self.parse(value)
+        Float(value)
+      end
+    end
+  end
+end

data/lib/purview/types/integer.rb

@@ -0,0 +1,9 @@
+module Purview
+  module Types
+    class Integer < Base
+      def self.parse(value)
+        Integer(value)
+      end
+    end
+  end
+end

data/lib/purview/types/money.rb

@@ -0,0 +1,9 @@
+module Purview
+  module Types
+    class Money < Base
+      def self.parse(value)
+        Float(value)
+      end
+    end
+  end
+end

data/lib/purview/types/string.rb

@@ -0,0 +1,9 @@
+module Purview
+  module Types
+    class String < Base
+      def self.parse(value)
+        String(value)
+      end
+    end
+  end
+end

data/lib/purview/types/text.rb

@@ -0,0 +1,9 @@
+module Purview
+  module Types
+    class Text < Base
+      def self.parse(value)
+        String(value)
+      end
+    end
+  end
+end

data/lib/purview/types/time.rb

@@ -0,0 +1,9 @@
+module Purview
+  module Types
+    class Time < Base
+      def self.parse(value)
+        ::Time.parse(value)
+      end
+    end
+  end
+end

data/lib/purview/types/timestamp.rb

@@ -0,0 +1,9 @@
+module Purview
+  module Types
+    class Timestamp < Base
+      def self.parse(value)
+        ::Time.parse(value)
+      end
+    end
+  end
+end

data/lib/purview/types/uuid.rb

@@ -0,0 +1,9 @@
+module Purview
+  module Types
+    class UUID < Base
+      def self.parse(value)
+        String(value)
+      end
+    end
+  end
+end

data/lib/purview/types.rb

@@ -0,0 +1,11 @@
+require 'purview/types/base'
+require 'purview/types/boolean'
+require 'purview/types/date'
+require 'purview/types/float'
+require 'purview/types/integer'
+require 'purview/types/money'
+require 'purview/types/string'
+require 'purview/types/text'
+require 'purview/types/time'
+require 'purview/types/timestamp'
+require 'purview/types/uuid'

data/lib/purview/version.rb

@@ -0,0 +1,3 @@
+module Purview
+  VERSION = '1.0.0.alpha'
+end

data/lib/purview.rb

@@ -0,0 +1,27 @@
+require 'csv'
+require 'date'
+require 'net/http'
+require 'openssl'
+require 'ostruct'
+require 'set'
+require 'time'
+require 'uri'
+
+%w[mysql2 pg].each { |gem| begin; require gem; rescue LoadError; end }
+abort 'Could not load the `mysql2` or `pg` gem' unless defined?(Mysql2) || defined?(PG)
+
+require 'purview/mixins'
+require 'purview/refinements'
+
+require 'purview/columns'
+require 'purview/connections'
+require 'purview/databases'
+require 'purview/exceptions'
+require 'purview/loaders'
+require 'purview/loggers'
+require 'purview/parsers'
+require 'purview/pullers'
+require 'purview/structs'
+require 'purview/tables'
+require 'purview/types'
+require 'purview/version'

data/purview.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'purview/version'
+
+Gem::Specification.new do |gem|
+  gem.name = 'purview'
+  gem.version = Purview::VERSION
+  gem.authors = ['Jonathan W. Zaleski']
+  gem.email = ['JonathanZaleski@gmail.com']
+  gem.summary = 'A framework designed to simplify data warehousing'
+  gem.description = 'Coming soon!'
+  gem.homepage = 'https://github.com/jzaleski/purview'
+  gem.license = 'MIT'
+
+  gem.files = `git ls-files`.split($/)
+  gem.executables = gem.files.grep(%r{^bin/}) { |file| File.basename(file) }
+  gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ['lib']
+
+  gem.add_development_dependency 'mysql2', '~> 0.3'
+  gem.add_development_dependency 'pg', '~> 0.18'
+
+  gem.add_development_dependency 'bundler', '~> 1.0'
+  gem.add_development_dependency 'pry', '~> 0.10'
+  gem.add_development_dependency 'rake', '~> 10.4'
+  gem.add_development_dependency 'rspec', '~> 3.2'
+end

data/spec/spec_helper.rb

@@ -0,0 +1,5 @@
+require 'rspec'
+
+RSpec.configure do |config|
+  config.color_enabled = true if config.respond_to?(:color_enabled)
+end

metadata

@@ -0,0 +1,210 @@
+--- !ruby/object:Gem::Specification
+name: purview
+version: !ruby/object:Gem::Version
+  version: 1.0.0.alpha
+platform: ruby
+authors:
+- Jonathan W. Zaleski
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-04-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.18'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.18'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.4'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+description: Coming soon!
+email:
+- JonathanZaleski@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- TODO
+- lib/purview.rb
+- lib/purview/columns.rb
+- lib/purview/columns/base.rb
+- lib/purview/columns/boolean.rb
+- lib/purview/columns/created_timestamp.rb
+- lib/purview/columns/date.rb
+- lib/purview/columns/float.rb
+- lib/purview/columns/id.rb
+- lib/purview/columns/integer.rb
+- lib/purview/columns/money.rb
+- lib/purview/columns/string.rb
+- lib/purview/columns/text.rb
+- lib/purview/columns/time.rb
+- lib/purview/columns/timestamp.rb
+- lib/purview/columns/updated_timestamp.rb
+- lib/purview/columns/uuid.rb
+- lib/purview/connections.rb
+- lib/purview/connections/base.rb
+- lib/purview/connections/mysql.rb
+- lib/purview/connections/postgresql.rb
+- lib/purview/databases.rb
+- lib/purview/databases/base.rb
+- lib/purview/databases/mysql.rb
+- lib/purview/databases/postgresql.rb
+- lib/purview/exceptions.rb
+- lib/purview/exceptions/base.rb
+- lib/purview/exceptions/could_not_acquire_lock.rb
+- lib/purview/exceptions/lock_already_released.rb
+- lib/purview/exceptions/no_table.rb
+- lib/purview/exceptions/no_window.rb
+- lib/purview/exceptions/rows_outside_window.rb
+- lib/purview/exceptions/table.rb
+- lib/purview/loaders.rb
+- lib/purview/loaders/base.rb
+- lib/purview/loaders/mysql.rb
+- lib/purview/loaders/postgresql.rb
+- lib/purview/loggers.rb
+- lib/purview/loggers/base.rb
+- lib/purview/loggers/console.rb
+- lib/purview/mixins.rb
+- lib/purview/mixins/helpers.rb
+- lib/purview/mixins/logger.rb
+- lib/purview/parsers.rb
+- lib/purview/parsers/base.rb
+- lib/purview/parsers/csv.rb
+- lib/purview/parsers/tsv.rb
+- lib/purview/pullers.rb
+- lib/purview/pullers/base.rb
+- lib/purview/pullers/uri.rb
+- lib/purview/refinements.rb
+- lib/purview/refinements/object.rb
+- lib/purview/refinements/time.rb
+- lib/purview/structs.rb
+- lib/purview/structs/base.rb
+- lib/purview/structs/result.rb
+- lib/purview/structs/window.rb
+- lib/purview/tables.rb
+- lib/purview/tables/base.rb
+- lib/purview/tables/raw.rb
+- lib/purview/types.rb
+- lib/purview/types/base.rb
+- lib/purview/types/boolean.rb
+- lib/purview/types/date.rb
+- lib/purview/types/float.rb
+- lib/purview/types/integer.rb
+- lib/purview/types/money.rb
+- lib/purview/types/string.rb
+- lib/purview/types/text.rb
+- lib/purview/types/time.rb
+- lib/purview/types/timestamp.rb
+- lib/purview/types/uuid.rb
+- lib/purview/version.rb
+- purview.gemspec
+- spec/spec_helper.rb
+homepage: https://github.com/jzaleski/purview
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">"
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.3
+signing_key: 
+specification_version: 4
+summary: A framework designed to simplify data warehousing
+test_files:
+- spec/spec_helper.rb