mongery 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 8360d8979a97ea0236f28a3aa49bcc35cd9f3861
+  data.tar.gz: 6d63c13b93d91147b1e18956dc7e53ebd9555368
+SHA512:
+  metadata.gz: 394339097c78801844b502505add3cb95881871b51eca88edda565d56333034893d7e092a28e31a286e1f1c5f59b66cea80db55a51fe850df3aa28464e1ffa99
+  data.tar.gz: f9a1ee7321cf2e71f0bc308f0caaa5755016a68ad4be1409b789c5712289331cf36df4df2c097978872def591d91782f4a2b3bfd41e0b8c9e355bed281981e99

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in mongery.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Tatsuhiko Miyagawa
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,42 @@
+# Mongery
+
+Mongery helps you to migrate off of MongoDB object storage to PostgreSQL with JSON column by translating Mongo query into Arel object, ready to use with ActiveRecord connection.
+
+## Limitation
+
+MOngery currently supports the limited set of queries to use with PostgreSQL 9.3. Most query on JSON data will end up with the full table scan. Index using hstore columns and/or 9.4 `jsonb` types are planned but not implemented.
+
+See the spec file in `spec` directory for the supported conversion.
+
+## Usage
+
+Create a table with the following structure. The name of the table could be anything.
+
+```
+CREATE TABLE objects (
+  id varchar(32) not null primary key,
+  data json not null,
+  updated_at timestamp DEFAULT current_timestamp
+)
+```
+
+Currently Mongery assumes the `id` values are stored as `_id` key duplicated in the json data as well. This can be customized in the future updates.
+
+```ruby
+builder = Mongery::Builder.new(:objects)
+
+builder.find({ _id: 'abcd' }).limit(1).to_sql
+# => SELECT data FROM objects WHERE id = 'abcd' LIMIT 1
+
+builder.find({ age: {"$gte" => 21 } }).sort({ name: -1 }).to_sql
+# => SELECT data FROM objects WHERE (data->>'age')::integer >= 21 ORDER BY data->>'name' DESC
+```
+
+## License
+
+MIT
+
+## Copyright
+
+Tatsuhiko Miyagawa
+

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/lib/mongery.rb

@@ -0,0 +1,177 @@
+require "mongery/version"
+require "arel"
+
+module Mongery
+  # Translate Mongo query to Arel AST
+
+  class Builder
+    attr_reader :model, :table
+
+    def initialize(model, engine = ActiveRecord::Base)
+      @model = model
+      @table = Arel::Table.new(model, engine)
+    end
+
+    def find(args)
+       Query.new(table).where(args)
+    end
+  end
+
+  class Query
+    attr_reader :table
+
+    def initialize(table)
+      @table = table
+    end
+
+    def where(args)
+      condition = translate(args)
+      arel.where(condition) if condition
+      self
+    end
+
+    def arel
+      @arel ||= table.project(table[:data])
+    end
+
+    def to_arel
+      arel
+    end
+
+    def to_sql
+      to_arel.to_sql
+    end
+
+    def limit(number)
+      arel.take(number)
+      self
+    end
+
+    def skip(number)
+      arel.skip(number)
+      self
+    end
+
+    def sort(params)
+      params.each do |col, val|
+        order = val > 0 ? :asc : :desc
+        case col.to_s
+        when "_id"
+          arel.order(table[:id].send(order))
+        else
+          arel.order(sql_json_path(col).send(order))
+        end
+      end
+      self
+    end
+
+    def translate(query)
+      chain(:and, query.map { |col, value| translate_cv(col, value) })
+    end
+
+    def translate_cv(col, value)
+      case col.to_s
+      when "_id"
+        translate_value(table[:id], value)
+      when "$or"
+        chain(:or, value.map {|q| translate(q) })
+      when "$and"
+        chain(:and, value.map { |q| translate(q) })
+      when /^\$/
+        raise UnsupportedQuery, "Unsupported operator #{col}"
+      else
+        translate_value_json(sql_json_path(col), value)
+      end
+    end
+
+    def translate_value(col, value, json = false)
+      case value
+      when Hash
+        ops = value.keys
+        if ops.size > 1
+          raise UnsupportedQuery, "Multiple operator supported: #{ops.join(", ")}"
+        end
+
+        val = value[ops.first]
+        case ops.first
+        when "$in"
+          col.in(val)
+        when "$gt", "$gte", "$lt", "$lte"
+          col.send(COMPARE_MAPS[ops.first], val)
+        when "$eq"
+          col.eq(val)
+        when "$ne"
+          col.not_eq(val)
+        when /^\$/
+          raise UnsupportedQuery, "Unknown operator #{ops.first}"
+        end
+      else
+        col.eq(value)
+      end
+    end
+
+    COMPARE_MAPS = { "$gt" => :gt, "$gte" => :gteq, "$lt" => :lt, "$lte" => :lteq }
+
+    def translate_value_json(col, value, json = false)
+      case value
+      when String, Numeric, TrueClass, FalseClass
+        # in Postgres 9.3, you can't compare numeric
+        col.eq(value.to_s)
+      when NilClass
+        # You can't use IS NULL
+        col.eq('')
+      when Hash
+        ops = value.keys
+        if ops.size > 1
+          raise UnsupportedQuery, "Multiple operator supported: #{ops.join(", ")}"
+        end
+
+        val = value[ops.first]
+        case ops.first
+        when "$in"
+          chain(:or, val.map { |val| col.matches(%Q[%"#{val}"%]) })
+        when "$gt", "$gte", "$lt", "$lte"
+          wrap_numeric(col, val).send(COMPARE_MAPS[ops.first], val)
+        when "$eq"
+          wrap_numeric(col, val).eq(val)
+        when "$ne"
+          wrap_numeric(col, val).not_eq(val)
+        when /^\$/
+          raise UnsupportedQuery, "Unknown operator #{ops.first}"
+        end
+      end
+    end
+
+    def wrap_numeric(col, val)
+      case val
+      when Float
+        Arel.sql("(#{col})::float")
+      when Integer
+        Arel.sql("(#{col})::integer")
+      else
+        col
+      end
+    end
+
+    def sql_json_path(col)
+      path = "data"
+      parts = col.to_s.split('.')
+      parts.each_with_index do |part, index|
+        sep = index == parts.size - 1 ? "->>" : "->"
+        path += "#{sep}'#{part}'"
+      end
+      Arel.sql(path)
+    end
+
+    def chain(op, conditions)
+      result = nil
+      conditions.each do |cond|
+        result = result ? result.send(op, cond) : cond
+      end
+      result
+    end
+  end
+
+  class UnsupportedQuery < StandardError
+  end
+end

data/lib/mongery/version.rb

@@ -0,0 +1,3 @@
+module Mongery
+  VERSION = "0.0.1"
+end

data/mongery.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'mongery/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "mongery"
+  spec.version       = Mongery::VERSION
+  spec.authors       = ["Tatsuhiko Miyagawa"]
+  spec.email         = ["miyagawa@bulknews.net"]
+  spec.summary       = %q{Convert MongoDB query to Arel for PostgreSQL + JSON}
+  spec.description   = %q{}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "arel", ">= 4.0.2"
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "activerecord"
+  spec.add_development_dependency "pg"
+end

data/spec/mongery/builder_spec.rb

@@ -0,0 +1,73 @@
+require 'spec_helper'
+
+describe Mongery::Builder do
+  tests = [
+    [ { }, { },
+      /^SELECT "test"\."data" FROM "test"$/ ],
+    [ { _id: "foo" }, { },
+      /WHERE "test"\."id" = 'foo'$/ ],
+    [ { _id: { "$in" => ["foo", "bar"] } }, { },
+      /WHERE "test"\."id" IN \('foo', 'bar'\)$/ ],
+    [ { _id: "foo" }, { limit: 1 },
+      /WHERE "test"\."id" = 'foo' LIMIT 1$/ ],
+    [ { _id: "foo" }, { limit: 1, skip: 10 },
+      /WHERE "test"\."id" = 'foo' LIMIT 1 OFFSET 10$/ ],
+    [ { _id: "foo" }, { skip: 10, sort: { _id: 1 } },
+      /WHERE "test"\."id" = 'foo'  ORDER BY "test"\."id" ASC OFFSET 10$/ ],
+    [ { _id: "foo" }, { sort: { name: -1, email: 1 } },
+      /WHERE "test"\."id" = 'foo'  ORDER BY data->>'name' DESC, data->>'email' ASC$/ ],
+    [ { "_id" => "foo" }, { },
+      /WHERE "test"\."id" = 'foo'$/ ],
+    [ { name: "foo" }, { },
+      /WHERE data->>'name' = 'foo'$/ ],
+    [ { name: nil }, { },
+      /WHERE data->>'name' = ''$/ ],
+    [ { name: "foo", other: "bar" }, { },
+      /WHERE data->>'name' = 'foo' AND data->>'other' = 'bar'/ ],
+    [ { weight: 66 }, { },
+      /WHERE data->>'weight' = '66'$/ ],
+    [ { weight: { "$gt" => 66 } }, { },
+      /WHERE \(data->>'weight'\)::integer > 66$/ ],
+    [ { weight: { "$gt" => 66.0 } }, { },
+      /WHERE \(data->>'weight'\)::float > 66\.0$/ ],
+    [ { weight: { "$lte" => 66 } }, { },
+      /WHERE \(data->>'weight'\)::integer <= 66$/ ],
+    [ { age: { '$eq' => 10 }}, { },
+      /WHERE \(data->>'age'\)::integer = 10/ ],
+    [ { age: { '$ne' => 10 }}, { },
+      /WHERE \(data->>'age'\)::integer != 10/ ],
+    [ { bool: true }, { },
+      /WHERE data->>'bool' = 'true'$/ ],
+    [ { 'email.address' => 'john@example.com' }, { },
+      /WHERE data->'email'->>'address' = 'john@example.com'$/ ],
+    [ { type: "food", "$or" => [{name: "miso"}, {name: "tofu"}]}, { },
+      /WHERE data->>'type' = 'food' AND \(data->>'name' = 'miso' OR data->>'name' = 'tofu'\)$/ ],
+    [ { "$or" => [{ _id: "foo" }, { _id: "bar" }] }, { },
+      /WHERE \("test"\."id" = 'foo' OR "test"\."id" = 'bar'\)$/ ],
+    [ { "$or" => [{ name: "John" }, { weight: 120 }] }, { },
+      /WHERE \(data->>'name' = 'John' OR data->>'weight' = '120'\)$/ ],
+    [ { "$and" => [{ _id: "foo" }, { name: "bar" }] }, { },
+      /WHERE "test"\."id" = 'foo' AND data->>'name' = 'bar'$/ ],
+    [ { "$and" => [{ name: "John" }, { weight: 120 }] }, { },
+      /WHERE data->>'name' = 'John' AND data->>'weight' = '120'$/ ],
+    [ { "$and" => [{ "$or" => [{name: "John"}, {email: "john"}] }, {_id: "Bob"}] }, { },
+      /WHERE \(data->>'name' = 'John' OR data->>'email' = 'john'\) AND "test"\."id" = 'Bob'$/ ],
+    [ { ids: {"$in" => [ "foo" ]} }, { },
+      /WHERE data->>'ids' ILIKE '%"foo"%'$/ ],
+    [ { ids: {"$in" => [ "foo", "bar" ]} }, { },
+      /WHERE \(data->>'ids' ILIKE '%"foo"%' OR data->>'ids' ILIKE '%"bar"%'\)$/ ],
+  ]
+
+  tests.each do |query, condition, sql|
+    context "with query #{query}" do
+      subject do
+        Mongery::Builder.new(:test).find(query).tap { |q|
+          condition.each do |method, value|
+            q.send(method, value)
+          end
+        }.to_sql
+      end
+      it { should match sql }
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,13 @@
+require 'mongery'
+require 'rspec'
+require 'active_record'
+
+RSpec.configure do |config|
+  ActiveRecord::Base.establish_connection(
+    adapter: 'postgresql',
+    encoding: 'unicode',
+    database: ENV['PG_DATABASE'],
+    username: ENV['USER'],
+    password: nil,
+  )
+end

metadata

@@ -0,0 +1,141 @@
+--- !ruby/object:Gem::Specification
+name: mongery
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Tatsuhiko Miyagawa
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-10-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: arel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.0.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.0.2
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: ''
+email:
+- miyagawa@bulknews.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/mongery.rb
+- lib/mongery/version.rb
+- mongery.gemspec
+- spec/mongery/builder_spec.rb
+- spec/spec_helper.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Convert MongoDB query to Arel for PostgreSQL + JSON
+test_files:
+- spec/mongery/builder_spec.rb
+- spec/spec_helper.rb
+has_rdoc: