axiom-optimizer 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 36b3b6f4025783f94a149341825fa522193892e3
+  data.tar.gz: 3bf6368e2943f1d4fb0a8640a71a6a857b48fe67
+SHA512:
+  metadata.gz: 2d78a4f99334b80b65fff5cfef9ae4a050b6156a6d81d4508bc0b6ae211423f759eaecd7dd78c7f47eb2d2369f4c65af2e7ab0e94d2002a3a7bef7f9719159b7
+  data.tar.gz: b131979736952d33d94e473730e7066e33711837922342f7124a20128560dc58e05cd56ef2432d2706bba594be7514f8cc7b1a2fe1e2a5e76bf0fcdda7f505c0

data/.gitignore

@@ -0,0 +1,37 @@
+## MAC OS
+.DS_Store
+
+## TEXTMATE
+*.tmproj
+tmtags
+
+## EMACS
+*~
+\#*
+.\#*
+
+## VIM
+*.swp
+
+## Rubinius
+*.rbc
+.rbx
+
+## PROJECT::GENERAL
+*.gem
+coverage
+profiling
+turbulence
+rdoc
+pkg
+tmp
+doc
+log
+.yardoc
+measurements
+
+## BUNDLER
+.bundle
+Gemfile.lock
+
+## PROJECT::SPECIFIC

data/.rspec

@@ -0,0 +1,4 @@
+--color
+--format progress
+--profile
+--order random

data/.rvmrc

@@ -0,0 +1 @@
+rvm use @$(basename `pwd`) --create

data/.travis.yml

@@ -0,0 +1,35 @@
+language: ruby
+before_install: gem install bundler
+bundler_args: --without yard guard benchmarks
+script: "bundle exec rake ci"
+rvm:
+  - ree
+  - 1.8.7
+  - 1.9.2
+  - 1.9.3
+  - 2.0.0
+  - ruby-head
+  - rbx-18mode
+  - rbx-19mode
+  - jruby-18mode
+notifications:
+  irc:
+    channels:
+      - "irc.freenode.org#datamapper"
+    on_success: never
+    on_failure: change
+  email:
+    recipients:
+      - dan.kubb@gmail.com
+    on_success: never
+    on_failure: change
+matrix:
+  include:
+    - rvm: jruby-19mode
+      env: JRUBY_OPTS="$JRUBY_OPTS --debug"
+    - rvm: jruby-head
+      env: JRUBY_OPTS="$JRUBY_OPTS --debug"
+  allow_failures:
+    # mutant fails
+    - rvm: 1.9.3
+    - rvm: rbx-19mode

data/CONTRIBUTING.md

@@ -0,0 +1,11 @@
+Contributing
+------------
+
+* If you want your code merged into the mainline, please discuss the proposed changes with me before doing any work on it. This library is still in early development, and the direction it is going may not always be clear. Some features may not be appropriate yet, may need to be deferred until later when the foundation for them is laid, or may be more applicable in a plugin.
+* Fork the project.
+* Make your feature addition or bug fix.
+  * Follow this [style guide](https://github.com/dkubb/styleguide).
+* Add specs for it. This is important so I don't break it in a future version unintentionally. Tests must cover all branches within the code, and code must be fully covered.
+* Commit, do not mess with Rakefile, version, or history. (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
+* Run "rake ci". This must pass and not show any regressions in the metrics for the code to be merged.
+* Send me a pull request. Bonus points for topic branches.

data/Gemfile

@@ -0,0 +1,10 @@
+# encoding: utf-8
+
+source 'https://rubygems.org'
+
+gemspec
+
+gem 'axiom', '~> 0.1.0', :git => 'https://github.com/dkubb/axiom.git'
+
+gem 'devtools', :git => 'https://github.com/datamapper/devtools.git'
+eval File.read('Gemfile.devtools')

data/Gemfile.devtools

@@ -0,0 +1,57 @@
+# encoding: utf-8
+
+group :development do
+  gem 'rake',  '~> 10.0.4'
+  gem 'rspec', '~> 2.13.0'
+  gem 'yard',  '~> 0.8.5.2'
+end
+
+group :yard do
+  gem 'kramdown', '~> 1.0.1'
+end
+
+group :guard do
+  gem 'guard',         '~> 1.7.0'
+  gem 'guard-bundler', '~> 1.0.0'
+  gem 'guard-rspec',   '~> 2.5.2'
+
+  # file system change event handling
+  gem 'listen',     '~> 0.7.3'
+  gem 'rb-fchange', '~> 0.0.6', :require => false
+  gem 'rb-fsevent', '~> 0.9.3', :require => false
+  gem 'rb-inotify', '~> 0.9.0', :require => false
+
+  # notification handling
+  gem 'libnotify',               '~> 0.8.0', :require => false
+  gem 'rb-notifu',               '~> 0.0.4', :require => false
+  gem 'terminal-notifier-guard', '~> 1.5.3', :require => false
+end
+
+group :metrics do
+  gem 'backports',       '~> 3.3', '>= 3.3.0'
+  gem 'coveralls',       '~> 0.6.4'
+  gem 'flay',            '~> 2.1.0'
+  gem 'flog',            '~> 3.2.3'
+  gem 'reek',            '~> 1.3.1', :git => 'https://github.com/troessner/reek.git'
+  gem 'simplecov',       '~> 0.7.1'
+  gem 'yardstick',       '~> 0.9.5'
+  gem 'yard-spellcheck', '~> 0.1.5'
+
+  platforms :mri_19, :rbx do
+    gem 'mutant', '~> 0.2.20'
+  end
+
+  platforms :rbx do
+    gem 'pelusa', '~> 0.2.2'
+  end
+end
+
+group :benchmarks do
+  gem 'rbench', '~> 0.2.3'
+end
+
+platform :jruby do
+  group :jruby do
+    gem 'jruby-openssl', '~> 0.8.5'
+  end
+end

data/Guardfile

@@ -0,0 +1,23 @@
+# encoding: utf-8
+
+guard :bundler do
+  watch('Gemfile')
+end
+
+guard :rspec, :cli => File.read('.rspec').split.join(' '), :keep_failed => false do
+  # run all specs if configuration is modified
+  watch('Guardfile')           { 'spec' }
+  watch('Gemfile.lock')        { 'spec' }
+  watch('spec/spec_helper.rb') { 'spec' }
+
+  # run all specs if supporting files files are modified
+  watch(%r{\Aspec/(?:lib|support|shared)/.+\.rb\z}) { 'spec' }
+
+  # run unit specs if associated lib code is modified
+  watch(%r{\Alib/(.+)\.rb\z})                                         { |m| Dir["spec/unit/#{m[1]}"]         }
+  watch(%r{\Alib/(.+)/support/(.+)\.rb\z})                            { |m| Dir["spec/unit/#{m[1]}/#{m[2]}"] }
+  watch("lib/#{File.basename(File.expand_path('../', __FILE__))}.rb") { 'spec'                               }
+
+  # run a spec if it is modified
+  watch(%r{\Aspec/(?:unit|integration)/.+_spec\.rb\z})
+end

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2011-2013 Dan Kubb
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,111 @@
+axiom-optimizer
+===============
+
+Relational algebra optimizer
+
+[![Gem Version](https://badge.fury.io/rb/axiom-optimizer.png)][gem]
+[![Build Status](https://secure.travis-ci.org/dkubb/axiom-optimizer.png?branch=master)][travis]
+[![Dependency Status](https://gemnasium.com/dkubb/axiom-optimizer.png)][gemnasium]
+[![Code Climate](https://codeclimate.com/github/dkubb/axiom-optimizer.png)][codeclimate]
+[![Coverage Status](https://coveralls.io/repos/dkubb/axiom-optimizer/badge.png?branch=master)][coveralls]
+
+[gem]: https://rubygems.org/gems/axiom-optimizer
+[travis]: https://travis-ci.org/dkubb/axiom-optimizer
+[gemnasium]: https://gemnasium.com/dkubb/axiom-optimizer
+[codeclimate]: https://codeclimate.com/github/dkubb/axiom-optimizer
+[coveralls]: https://coveralls.io/r/dkubb/axiom-optimizer
+
+Installation
+------------
+
+With Rubygems:
+
+```bash
+$ gem install axiom-optimizer
+$ irb -rubygems
+>> require 'axiom-optimizer'
+=> true
+```
+
+With git and local working copy:
+
+```bash
+$ git clone git://github.com/dkubb/axiom-optimizer.git
+$ cd axiom-optimizer
+$ rake install
+$ irb -rubygems
+>> require 'axiom-optimizer'
+=> true
+```
+
+Usage
+-----
+
+```ruby
+# optimize a relation
+new_relation = relation.optimize
+new_relation = relation.optimize(optimizer)
+
+# optimize a scalar function
+new_function = function.optimize
+new_function = function.optimize(optimizer)
+
+# optimize an aggregate function
+new_aggregate = function.aggregate
+new_aggregate = function.aggregate(optimizer)
+```
+
+Description
+-----------
+
+The purpose of this gem is to provide a simple API that can be used to optimize a [axiom](https://github.com/dkubb/axiom) relation, scalar or aggregate function. An optional optimizer can be passed in to the #optimize method and return an equivalent but simplified version of the object.
+
+One of the primary benefits of Relational Algebra is that it's based on logic, and the rules for simplifying logic are well known and studied. An optimizer can pass through user-generated objects and typically find ways to simplify or organize them in a way that will be more efficient when the operation is executed.
+
+The goal is not to replace the advanced optimizers that are inside most databases and datastores, but to augment it with some simple optimizations that make the user provided query easier for the datastore to accept. On the ruby side we have knowledge about intent and can perform semantic optimization that the datastore otherwise would not be able to perform. In many cases we have richer constraints and data than many datastores and we can use that information to simplify and possibly short-circuit queries that could otherwise never return valid results.
+
+With the ability to provide custom optimizers we can even target output to a structure optimized for specific datastores. All operations in relational algebra can be transformed into other equivalent operations, ones that are more efficient for the target datastore to execute. The built-in optimizers included in this gem are only a starting point; the intention is to expand them as well as help others create custom optimizers that are optimized for each datastore.
+
+Design
+------
+
+The contract for an optimizer instance is simple:
+
+1. it must respond to #call, and accept an optimizable object as it's only argument
+2. it must return an equivalent object
+3. it must return the exact object when it cannot perform further optimizations
+
+The optimizer can perform whatever logic it wishes on the object or any of it's contained objects as far down the tree as it likes as long as the requirements for (2) and (3) are met.
+
+Inside this gem we have the concept of an optimizer chain. It's a chain of responsibility, which means it's a set of objects chained together to form a pipeline. The object is passed into the head of the pipeline and is either matched and returned by one of the optimizers, or it is already fully optimized and passes through to the end of the chain and is returned as-is. This chain organization has proven to be extremely effective, and it is trivial to re-order or add new optimizers into the middle of the chain as needed. Further work will be made to provide APIs to make this even simpler.
+
+Here is an example of an optimizer chain for the restriction operator (think WHERE clause in SQL):
+
+```ruby
+Axiom::Algebra::Restriction.optimizer = chain(
+  Tautology,            # does the predicate match everything?
+  Contradiction,        # does the predicate match nothing?
+  RestrictionOperand,   # does the restriction contain another restriction?
+  SetOperand,           # does the restriction contain a set operation?
+  OrderOperand,         # does the restriction contain an order?
+  EmptyOperand,         # does the restriction contain an empty relation?
+  MaterializedOperand,  # does the restriction contain a materialized relation?
+  UnoptimizedOperand    # does the restriction contain an unoptimized relation?
+)
+```
+
+The restriction operator enters this pipeline, and tests are made at each stage. If the test returns true, then an optimization is performed. Usually the goal is to eliminate work performed by the system or collapse the tree of operations down into something simpler. More aggressive optimizations are usually checked first because we would like to prune as much of the tree as possible up-front. In this case, the first test checks to see if the restriction matches everything, in which case it's pretty much a no-op, and we can drop it altogether. If that's not the case, we then test to see if it matches nothing, if that's the case then we can return an empty relation. Then we test if it's another restriction, in which case we can "AND" the predicates for both restrictions together and return a single restriction operation. And so on down the list with the first match winning.
+
+We always perform at least two optimization passes on each object, because once a tree has been simplified there could be further optimizations possible. Essentially we keep passing the objects into their corresponding optimizer chains until it passes through unchanged. This may seem rather expensive, and I guess it is, but optimization is very fast. Also it doesn't appear to affect performance much in practice due to our convention of testing for the most aggressive optimizations first; often it results in something that is completely optimized on the first try.
+
+Once the optimization passes are finished, and no further optimization is possible, the result of an #optimize call is memoized. Further calls to #optimize will always return the same object.
+
+Contributing
+------------
+
+See [CONTRIBUTING.md](CONTRIBUTING.md) for details.
+
+Copyright
+---------
+
+Copyright © 2011-2013 Dan Kubb. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,5 @@
+# encoding: utf-8
+
+require 'devtools'
+
+Devtools.init_rake_tasks

data/TODO

@@ -0,0 +1,144 @@
+* Add optimizer for empty right operand to an Insertion
+  * Should factor out the Insertion
+
+* Add optimizer for empty left operand to a Deletion
+  * Should return an Empty relation, since deleting anything from an empty
+    relation always results in an empty Relation
+* Add optimizer for empty right operand to a Deletion
+  * Should factor out the Deletion
+
+* Add an assertion to every #optimize spec that if #optimizable? is true, then
+  the object returned by #optimize must *not* be equivalent. This will ensure
+  that the guard clause in #optimizable? only evaluates to true if the
+  optimization is actually going to return a new object; no point in going
+  through the work of optimizing if it won't result in any change, and it should
+  be possible to test for this before carrying out the work.
+  * Add a shared spec for #optimize and #optimizable?. Note that the current
+    shared spec for #optimize is for the Relation#optimize integration specs
+    (it should probably be renamed to avoid confusion)
+    * Should ensure that when #optimize is being tested #optimizable? should
+      only return true.
+    * Should ensure #optimizable? only returns true if #optimize returns an
+      object that is different (should return equivalent tuples, but have
+      different internal state, otherwise the #optimize step was totally
+      unecessary)
+
+* Rename the Util.constant? method as Util.value?
+
+* Pull up Rename rather than pushing it down
+  * Currently a Rename applies to every tuple in a result-set, and
+    doing the rename prematurely means that lots of tuples that would
+    otherwise be filtered out are going to be processed. A better
+    approach should be to push the rename up, dropping renames that
+    are projected away. It should not distribute over Binary ops
+    unless the other side of the binary op has he same aliases.
+
+* For Binary operations, when the left or right is materialized then
+  change the other relation to use a restriction to filter the records.
+  * This will help optimize the fetching of the underlying records.
+
+* More optimizations:
+  * Union
+    * A Union of relations with the same base, header, and restrictions should
+      try to combine into a single relation with the restrictions using OR.
+
+  * Intersection
+    * An Intersection of relations with the same base, header, and restrictions
+      should try to combine into a single relation with the restrictions using
+      AND.
+    * Use the Join::RightMaterializedOperand and Join::LeftMaterializedOperand
+      optimizers to simplify Intersection operations
+
+  * Difference
+    * A Difference of relations with the same base and restrictions should
+      try to combine into a single relation with the restrictions using NOT.
+
+  * Summarization
+    * Add OrderSummarizePer for factoring out Order objects inside a Summarization
+    * When there are no aggregate functions, drop the Summarization and
+      return the summarize_per (?)
+      * Use the UnchangedHeader optimizer as a base class
+    * When summarize_per is an Order, the Order can be dropped.
+
+  * Projection
+    * When it contains a Restriction, if the removed attributes are *not*
+      used in the predicate, then move the Restriction to contain the
+      Projection.
+    * When a Projection contains a Restriction, wrap the Projection
+      in the Restriction, projecting away any attributes not used in
+      the restriction. If there are any remaining attributes, then
+      wrap the operation in a Projection removing those attributes.
+      * If all attributes are being used in the Restriction do nothing
+    * When an attribute is renamed, then projected away, the alias should be
+      removed from the rename.
+    * When an attribute is added by a summarization, then projected away, the
+      attribute should be removed from the summarization.
+    * It does not distribute over Intersection or Difference, but see if
+      perhaps an exception can be made if there is a functional dependency
+      between the columns projected away and the one remaining. Then I *think*
+      it might still work, but more research will be needed.
+    * When a Projection contains a Join, wrap the Join with a Projection
+      of all the headers, minus those used in the Join. If there were
+      any used, then wrap the whole operation in a Projection with
+      the remaining attributes.
+      * If all the attributes are used in the Join, do nothing
+      * Try to use the same approach for Product
+      * Test if it's possible to fully distribute projections over
+        joins rather than splitting it up like this.
+
+  * Rename
+    * When wrapping a Summarization or Extension, and renaming the new attribute,
+      it should change the new attribute name, and remove it from the rename.
+
+  * Restriction
+    * Figure out how to reorganize the Restriction predicates so that all
+      similar operations are closer together to allow more efficient
+      optimizations. This would allow optimizations of stuff like this:
+
+        "attr1 = ? OR attr2 = ? OR attr1 = ?"
+
+      Into:
+
+        "attr1 IN(..) OR attr2 = ?"
+
+    * When it has an equality on a unique attribute:
+      * Limit with a limit >= 1 can be factored out
+      * Offset with an offset > 0 can be transformed into an empty
+        relation, since at most there can be only one match.
+
+   * When wrapping a Product and the predicate only contains
+     equality/conjunction between attributes from the left and right
+     operands, then transform into a Join.
+
+  * Order
+    * When the operation is an instance of Order, and the operand is sorted in
+      reversed order, change to a Reverse operation. This should occur after the
+      Order optimization that collapses two Order objects into one; it would be
+      expected that the operand is a Limit or Offset, but it probably isn't
+      necessary to test that.
+
+  * Connective
+    * "attr > ? OR attr > ?" -> "attr > ?", with the least restrictive value
+      * Do the same for >=, <, <=
+    * "attr > ? AND attr > ?" -> "attr > ?", with the most restrictive value
+      * Do the same for >=, <, <=
+    * "attr > 5 OR attr == 5" -> "attr >= 5"
+    * "attr < 5 OR attr == 5" -> "attr <= 5"
+    * "attr" = "string" AND "attr" =~ /string/ -> "attr" = "string"
+      * If the regexp matches the constant, then it should be
+        optimized down to a constant match. If it does not match
+        then it should be optimized to a Contradiction.
+    * Constant folding, eg:
+      "attr1 > attr2 AND attr1 = 5" -> "5 > attr2 AND attr1 = 5"
+      * This will probably only work across Conjunctions.
+    * "attr > 5 AND attr = 6" -> "attr = 6", because attr must be
+      equal to 6. this will probably be related to constant folding;
+      the first expression will become 6 > 5, which evaluates to a
+      Tautology, then the expression is a Tautology AND attr = 6,
+      which simplifies down to attr = 6.
+    * "attr < 5 AND attr = 6" -> "Contradiction", because attr must be equal to
+      6, and 6 < 5 evaluates to a Contradiction.  A Contradiction AND attr = 6
+      simplifies down to a Contradiction.
+
+  * Inclusion/Exclusion
+    * When the enumerable is a contiguous sequence transform it into a Range