bmg 0.17.8 → 0.18.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 600739b827185e0d02252824b7b5616662a378b1c907a632ba695536e76d631e
-  data.tar.gz: f327e31860ef19e0ab8177ebd0e3d0c927c003cb25b8e48af202145cdf1ceece
+  metadata.gz: f1ce59d00b630f644e5716eaff17a83116c1342ea0b6ba7d9174b1f5f4eadd6e
+  data.tar.gz: 2156d1a8eb88999e749f434a8e94d2b4e70c636fadb10470493f19bb805a19a3
 SHA512:
-  metadata.gz: 2cd09c0006211c6db5d220ee2371669bd3e29bf32c71eae9fb9ba2c4698392ed172412bbd71c9eded5df34172ab469aa6d20a4117ef263955ed73e8a128cf856
-  data.tar.gz: 8777307569c78001741a597b3ecd9974aa1ee94780495d86f97af38313dcd185c9516d43b9c1635cae5e57606c7422d3e6b3f4889be0b2adcdfeb9f2c30dee24
+  metadata.gz: 8e3e714d698ff7c47c2f61b4056c73acaa39983306402eeb54d1957b90959af51fcf20a543c5c17242d260fe835b33c19dd960e3cd4920a66d49ea81a22ee4b0
+  data.tar.gz: 89fe1cc4c3157adf7373fb755e0cf7eb5dd8d93f7061558454d27649b44aca558f936f006e3238dce410ffdfe29821d59feac3b31b5d64241ffc76192bba149f

data/Gemfile

@@ -1,5 +1,2 @@
 source "https://rubygems.org"
 gemspec
-
-# gem "predicate", github: "enspirit/predicate", branch: "placeholders"
-# gem "predicate", path: "../predicate"

data/README.md

@@ -1,16 +1,30 @@
 # Bmg, a relational algebra (Alf's successor)!
 
+[![Build Status](https://travis-ci.com/enspirit/bmg.svg?branch=master)](https://travis-ci.com/enspirit/bmg)
+
 Bmg is a relational algebra implemented as a ruby library. It implements the
 [Relation as First-Class Citizen](http://www.try-alf.org/blog/2013-10-21-relations-as-first-class-citizen)
-paradigm contributed with Alf a few years ago.
-
-Like Alf, Bmg can be used to query relations in memory, from various files,
-SQL databases, and any data sources that can be seen as serving relations.
-Cross data-sources joins are supported, as with Alf.
-
-Unlike Alf, Bmg does not make any core ruby extension and exposes the
-object-oriented syntax only (not Alf's functional one). Bmg implementation is
-also much simpler, and make its easier to implement user-defined relations.
+paradigm contributed with [Alf](http://www.try-alf.org/) a few years ago.
+
+Bmg can be used to query relations in memory, from various files, SQL databases,
+and any data source that can be seen as serving relations. Cross data-sources
+joins are supported, as with Alf. For differences with Alf, see a section
+further down this README.
+
+## Outline
+
+* [Example](#example)
+* [Where are base relations coming from?](#where-are-base-relations-coming-from)
+  * [Memory relations](#memory-relations)
+  * [Connecting to SQL databases](#connecting-to-sql-databases)
+  * [Reading files (csv, excel, text)](#reading-files-csv-excel-text)
+  * [Your own relations](#your-own-relations)
+* [List of supported operators](#supported-operators)
+* [How is this different?](#how-is-this-different)
+  * [... from similar libraries](#-from-similar-libraries)
+  * [... from Alf](#-from-alf)
+* [Contribute](#contribute)
+* [License](#license)
 
 ## Example
 
@@ -27,7 +41,7 @@ suppliers = Bmg::Relation.new([
 ])
 
 by_city = suppliers
-  .restrict(Predicate.neq(status: 30))
+  .exclude(status: 30)
   .extend(upname: ->(t){ t[:name].upcase })
   .group([:sid, :name, :status], :suppliers_in)
 
@@ -35,76 +49,158 @@ puts JSON.pretty_generate(by_city)
 # [{...},...]
 ```
 
-## Connecting to a SQL database
+## Where are base relations coming from?
+
+Bmg sees relations as sets/enumerable of symbolized Ruby hashes. The following
+sections show you how to get them in the first place, to enter Relationland.
+
+### Memory relations
+
+If you have an Array of Hashes -- in fact any Enumerable -- you can easily get
+a Relation using either `Bmg::Relation.new` or `Bmg.in_memory`.
+
+```ruby
+# this...
+r = Bmg::Relation.new [{id: 1}, {id: 2}]
+
+# is the same as this...
+r = Bmg.in_memory [{id: 1}, {id: 2}]
+
+# entire algebra is available on `r`
+```
+
+### Connecting to SQL databases
 
-Bmg requires `sequel >= 3.0` to connect to SQL databases.
+Bmg currently requires `sequel >= 3.0` to connect to SQL databases. You also
+need to require `bmg/sequel`.
 
 ```ruby
 require 'sqlite3'
 require 'bmg'
 require 'bmg/sequel'
+```
 
-DB = Sequel.connect("sqlite://suppliers-and-parts.db")
+Then `Bmg.sequel` serves relations for tables of your SQL database:
 
+```ruby
+DB = Sequel.connect("sqlite://suppliers-and-parts.db")
 suppliers = Bmg.sequel(:suppliers, DB)
+```
+
+The entire algebra is available on those relations. As long as you keep using
+operators that can be translated to SQL, results remain SQL-able:
 
+```ruby
 big_suppliers = suppliers
-  .restrict(Predicate.neq(status: 30))
+  .exclude(status: 30)
+  .project([:sid, :name])
 
 puts big_suppliers.to_sql
-# SELECT `t1`.`sid`, `t1`.`name`, `t1`.`status`, `t1`.`city` FROM `suppliers` AS 't1' WHERE (`t1`.`status` != 30)
+# SELECT `t1`.`sid`, `t1`.`name` FROM `suppliers` AS 't1' WHERE (`t1`.`status` != 30)
+```
 
-puts JSON.pretty_generate(big_suppliers)
-# [{...},...]
+Operators not translatable to SQL are available too (such as `group` below).
+Bmg fallbacks to memory operators for them, but remains capable of pushing some
+operators down the tree as illustrated below (the restriction on `:city` is
+pushed to the SQL server):
+
+```ruby
+Bmg.sequel(:suppliers, sequel_db)
+  .project([:sid, :name, :city])
+  .group([:sid, :name], :suppliers_in)
+  .restrict(city: ["Paris", "London"])
+  .debug
+
+# (group
+#   (sequel SELECT `t1`.`sid`, `t1`.`name`, `t1`.`city` FROM `suppliers` AS 't1' WHERE (`t1`.`city` IN ('Paris', 'London')))
+#   [:sid, :name, :status]
+#   :suppliers_in
+#   {:array=>false})
 ```
 
-## How is this different from similar libraries?
+### Reading files (csv, excel, text)
 
-1. The libraries you probably know (Sequel, Arel, SQLAlchemy, Korma, jOOQ,
-   etc.) do not implement a genuine relational algebra: their support for
-   chaining relational operators is limited (yielding errors or wrong SQL
-   queries). Bmg **always** allows chaining operators. If it does not, it's
-   a bug. In other words, the following query is 100% valid:
+Bmg provides simple adapters to read files and reach Relationland as soon as
+possible.
 
-       relation
-         .restrict(...)    # aka where
-         .union(...)
-         .summarize(...)   # aka group by
-         .restrict(...)
+#### CSV files
 
-2. Bmg supports in memory relations, json relations, csv relations, SQL
-   relations and so on. It's not tight to SQL generation, and supports
-   queries accross multiple data sources.
+```ruby
+csv_options = { col_sep: ",", quote_char: '"' }
+r = Bmg.csv("path/to/a/file.csv", csv_options)
+```
 
-3. Bmg makes a best effort to optimize queries, simplifying both generated
-   SQL code (low-level accesses to datasources) and in-memory operations.
+Options are directly transmitted to `::CSV.new`, check ruby's standard
+library.
 
-4. Bmg supports various *structuring* operators (group, image, autowrap,
-   autosummarize, etc.) and allows building 'non flat' relations.
+#### Excel files
 
-## How is this different from Alf?
+You will need to add [`roo`](https://github.com/roo-rb/roo) to your Gemfile to
+read `.xls` and `.xlsx` files with Bmg.
 
-1. Bmg's implementation is much simpler than Alf, and uses no ruby core
-   extention.
+```ruby
+roo_options = { skip: 1 }
+r = Bmg.excel("path/to/a/file.xls", roo_options)
+```
 
-2. We are confident using Bmg in production. Systematic inspection of query
-   plans is suggested though. Alf was a bit too experimental to be used on
-   (critical) production systems.
+Options are directly transmitted to `Roo::Spreadsheet.open`, check roo's
+documentation.
 
-2. Alf exposes a functional syntax, command line tool, restful tools and
-   many more. Bmg is limited to the core algebra, main Relation abstraction
-   and SQL generation.
+#### Text files
 
-3. Bmg is less strict regarding conformance to relational theory, and
-   may actually expose non relational features (such as support for null,
-   left_join operator, etc.). Sharp tools hurt, use them with great care.
+There is also a straightforward way to read text files and convert lines to
+tuples.
 
-4. Bmg does not yet implement all operators documented on try-alf.org, even
-   if we plan to eventually support them all.
+```ruby
+r = Bmg.text_file("path/to/a/file.txt")
+r.type.attrlist
+# => [:line, :text]
+```
 
-5. Bmg has a few additional operators that prove very useful on real
-   production use cases: prefix, suffix, autowrap, autosummarize, left_join,
-   rxmatch, etc.
+Without options tuples will have `:line` and `:text` attributes, the former
+being the line number (starting at 1) and the latter being the line itself
+(stripped).
+
+The are a couple of options (see `Bmg::Reader::Textfile`). The most useful one
+is the use a of a Regexp with named captures to automatically extract
+attributes:
+
+```ruby
+r = Bmg.text_file("path/to/a/file.txt", parse: /GET (?<url>([^\s]+))/)
+r.type.attrlist
+# => [:line, :url]
+```
+
+In this scenario, non matching lines are skipped. The `:line` attribute keeps
+being used to have at least one candidate key (so to speak).
+
+### Your own relations
+
+As noted earlier, Bmg has a simple relation interface where you only have to
+provide an iteration of symbolized tuples.
+
+```ruby
+class MyRelation
+  include Bmg::Relation
+
+  def each
+    yield(id: 1, name: "Alf", year: 2014)
+    yield(id: 2, name: "Bmg", year: 2018)
+  end
+end
+
+MyRelation.new
+  .restrict(Predicate.gt(:year, 2015))
+  .allbut([:year])
+```
+
+As shown, creating adapters on top of various data source is straighforward.
+Adapters can also participate to query optimization (such as pushing
+restrictions down the tree) by overriding the underscored version of operators
+(e.g. `_restrict`).
+
+Have a look at `Bmg::Algebra` for the protocol and `Bmg::Sql::Relation` for an
+example. Keep in touch with the team if you need some help.
 
 ## Supported operators
 
@@ -114,6 +210,7 @@ r.autowrap(split: '_')                       # structure a flat relation, split:
 r.autosummarize([:a, :b, ...], x: :sum)      # (experimental) usual summarizers supported
 r.constants(x: 12, ...)                      # add constant attributes (sometimes useful in unions)
 r.extend(x: ->(t){ ... }, ...)               # add computed attributes
+r.exclude(predicate)                         # shortcut for restrict(!predicate)
 r.group([:a, :b, ...], :x)                   # relation-valued attribute from attributes
 r.image(right, :x, [:a, :b, ...])            # relation-valued attribute from another relation
 r.join(right, [:a, :b, ...])                 # natural join on a join key
@@ -137,14 +234,95 @@ t.transform(&:to_s)                          # similar, but Proc-driven
 t.transform(:foo => :upcase, ...)            # specific-attrs tranformation
 t.transform([:to_s, :upcase])                # chain-transformation
 r.union(right)                               # relational union
+r.where(predicate)                           # alias for restrict(predicate)
 ```
 
-## Who is behind Bmg?
+## How is this different?
+
+### ... from similar libraries?
+
+1. The libraries you probably know (Sequel, Arel, SQLAlchemy, Korma, jOOQ,
+   etc.) do not implement a genuine relational algebra. Their support for
+   chaining relational operators is thus limited (restricting your expression
+   power and/or raising errors and/or outputting wrong or counterintuitive
+   SQL code). Bmg **always** allows chaining operators. If it does not, it's
+   a bug.
+
+   For instance the expression below is 100% valid in Bmg. The last where
+   clause applies to the result of the summarize (while SQL requires a `HAVING`
+   clause, or a `SELECT ... FROM (SELECT ...) r`).
+
+      ```ruby
+      relation
+        .where(...)
+        .union(...)
+        .summarize(...)   # aka group by
+        .where(...)
+      ```
+
+2. Bmg supports in memory relations, json relations, csv relations, SQL
+   relations and so on. It's not tight to SQL generation, and supports
+   queries accross multiple data sources.
+
+3. Bmg makes a best effort to optimize queries, simplifying both generated
+   SQL code (low-level accesses to datasources) and in-memory operations.
+
+4. Bmg supports various *structuring* operators (group, image, autowrap,
+   autosummarize, etc.) and allows building 'non flat' relations.
+
+5. Bmg can use full ruby power when that helps (e.g. regular expressions in
+   WHERE clauses or ruby code in EXTEND clauses). This may prevent Bmg from
+   delegating work to underlying data sources (e.g. SQL server) and should
+   therefore be used with care though.
+
+### ... from Alf?
+
+If you use Alf (or used it in the past), below are the main differences between
+Bmg and Alf. Bmg has NOT been written to be API-compatible with Alf and will
+probably never be.
+
+1. Bmg's implementation is much simpler than Alf and uses no ruby core
+   extention.
+
+2. We are confident using Bmg in production. Systematic inspection of query
+   plans is advised though. Alf was a bit too experimental to be used on
+   (critical) production systems.
+
+3. Alf exposes a functional syntax, command line tool, restful tools and
+   many more. Bmg is limited to the core algebra, main Relation abstraction
+   and SQL generation.
 
-Bernard Lambeau (bernard@klaro.cards) is Alf & Bmg main engineer & maintainer.
+4. Bmg is less strict regarding conformance to relational theory, and
+   may actually expose non relational features (such as support for null,
+   left_join operator, etc.). Sharp tools hurt, use them with care.
+
+5. Unlike Alf::Relation instances of Bmg::Relation capture query-trees, not
+   values. Currently two instances `r1` and `r2` are not equal even if they
+   define the same mathematical relation. As a consequence joining on
+   relation-valued attributes does not work as expected in Bmg until further
+   notice.
+
+6. Bmg does not implement all operators documented on try-alf.org, even if
+   we plan to eventually support most of them.
+
+7. Bmg has a few additional operators that prove very useful on real
+   production use cases: prefix, suffix, autowrap, autosummarize, left_join,
+   rxmatch, etc.
+
+8. Bmg optimizes queries and compiles them to SQL on the fly, while Alf was
+   building an AST internally first. Strictly speaking this makes Bmg less
+   powerful than Alf since optimizations cannot be turned off for now.
+
+## Contribute
+
+Please use github issues and pull requests for all questions, bug reports,
+and contributions. Don't hesitate to get in touch with us with an early code
+spike if you plan to add non trivial features.
+
+## Licence
+
+This software is distributed by Enspirit SRL under a MIT Licence. Please
+contact Bernard Lambeau (blambeau@gmail.com) with any question.
 
 Enspirit (https://enspirit.be) and Klaro App (https://klaro.cards) are both
 actively using and contributing to the library.
-
-Feel free to contact us for help, ideas and/or contributions. Please use github
-issues and pull requests if possible if code is involved.

data/lib/bmg.rb

@@ -1,6 +1,7 @@
 require 'path'
 require 'predicate'
 require 'forwardable'
+require 'set'
 module Bmg
 
   def in_memory(enumerable, type = Type::ANY)
@@ -8,6 +9,11 @@ module Bmg
   end
   module_function :in_memory
 
+  def text_file(path, options = {}, type = Type::ANY)
+    Reader::TextFile.new(type, path, options).spied(main_spy)
+  end
+  module_function :text_file
+
   def csv(path, options = {}, type = Type::ANY)
     Reader::Csv.new(type, path, options).spied(main_spy)
   end

data/lib/bmg/algebra/shortcuts.rb

@@ -2,6 +2,14 @@ module Bmg
   module Algebra
     module Shortcuts
 
+      def where(predicate)
+        restrict(predicate)
+      end
+
+      def exclude(predicate)
+        restrict(!Predicate.coerce(predicate))
+      end
+
       def rxmatch(attrs, matcher, options = {})
         predicate = attrs.inject(Predicate.contradiction){|p,a|
           p | Predicate.match(a, matcher, options)

data/lib/bmg/reader.rb

@@ -9,3 +9,4 @@ module Bmg
 end
 require_relative "reader/csv"
 require_relative "reader/excel"
+require_relative "reader/text_file"

data/lib/bmg/reader/text_file.rb

@@ -0,0 +1,56 @@
+module Bmg
+  module Reader
+    class TextFile
+      include Reader
+
+      DEFAULT_OPTIONS = {
+        strip: true,
+        parse: nil
+      }
+
+      def initialize(type, path, options = {})
+        options = { parse: options } if options.is_a?(Regexp)
+        @path = path
+        @options = DEFAULT_OPTIONS.merge(options)
+        @type = infer_type(type)
+      end
+      attr_reader :path, :options
+
+    public # Relation
+
+      def each
+        path.each_line.each_with_index do |text, line|
+          text = text.strip if strip?
+          parsed = parse(text)
+          yield({line: 1+line}.merge(parsed)) if parsed
+        end
+      end
+
+    private
+
+      def infer_type(base)
+        return base unless base == Bmg::Type::ANY
+        attr_list = if rx = options[:parse]
+          [:line] + rx.names.map(&:to_sym)
+        else
+          [:line, :text]
+        end
+        base
+          .with_attrlist(attr_list)
+          .with_keys([[:line]])
+      end
+
+      def strip?
+        options[:strip]
+      end
+
+      def parse(text)
+        return { text: text } unless rx = options[:parse]
+        if match = rx.match(text)
+          TupleAlgebra.symbolize_keys(match.named_captures)
+        end
+      end
+
+    end # class TextFile
+  end # module Reader
+end # module Bmg

data/lib/bmg/relation.rb

@@ -17,6 +17,16 @@ module Bmg
       self
     end
 
+    def type
+      Bmg::Type::ANY
+    end
+
+    def with_type(type)
+      dup.tap{|r|
+        r.type = type
+      }
+    end
+
     def with_typecheck
       dup.tap{|r|
         r.type = r.type.with_typecheck

data/lib/bmg/relation/spied.rb

@@ -24,7 +24,7 @@ module Bmg
       protected :type=
 
       def each(&bl)
-        spy.call(self)
+        spy.call(self) if bl
         operand.each(&bl)
       end
 

data/lib/bmg/support/tuple_algebra.rb

@@ -19,5 +19,11 @@ module Bmg
     end
     module_function :rename
 
+    def symbolize_keys(h)
+      return h if h.empty?
+      h.each_with_object({}){|(k,v),h| h[k.to_sym] = v }
+    end
+    module_function :symbolize_keys
+
   end # module TupleAlgebra
 end # module Bmg

data/lib/bmg/support/tuple_transformer.rb

@@ -26,11 +26,7 @@ module Bmg
 
       def transform_tuple(tuple, with)
         case with
-        when Symbol
-          tuple.each_with_object({}){|(k,v),dup|
-            dup[k] = transform_attr(v, with)
-          }
-        when Proc
+        when Symbol, Proc, Regexp
           tuple.each_with_object({}){|(k,v),dup|
             dup[k] = transform_attr(v, with)
           }
@@ -51,6 +47,9 @@ module Bmg
         case with
         when Symbol
           value.send(with)
+        when Regexp
+          m = with.match(value.to_s)
+          m.nil? ? m : m.to_s
         when Proc
           with.call(value)
         when Hash

data/lib/bmg/version.rb

@@ -1,8 +1,8 @@
 module Bmg
   module Version
     MAJOR = 0
-    MINOR = 17
-    TINY  = 8
+    MINOR = 18
+    TINY  = 0
   end
   VERSION = "#{Version::MAJOR}.#{Version::MINOR}.#{Version::TINY}"
 end

data/tasks/test.rake

@@ -6,17 +6,24 @@ namespace :test do
   desc "Runs unit tests"
   RSpec::Core::RakeTask.new(:unit) do |t|
     t.pattern = "spec/unit/**/test_*.rb"
-    t.rspec_opts = ["-Ilib", "-Ispec/unit", "--fail-fast", "--color", "--backtrace", "--format=progress"]
+    t.rspec_opts = ["-Ilib", "-Ispec/unit", "--color", "--backtrace", "--format=progress"]
   end
   tests << :unit
 
   desc "Runs integration tests"
   RSpec::Core::RakeTask.new(:integration) do |t|
     t.pattern = "spec/integration/**/test_*.rb"
-    t.rspec_opts = ["-Ilib", "-Ispec/integration", "--fail-fast", "--color", "--backtrace", "--format=progress"]
+    t.rspec_opts = ["-Ilib", "-Ispec/integration", "--color", "--backtrace", "--format=progress"]
   end
   tests << :integration
 
+  desc "Runs github regression tests"
+  RSpec::Core::RakeTask.new(:regression) do |t|
+    t.pattern = "spec/regression/**/test_*.rb"
+    t.rspec_opts = ["-Ilib", "-Ispec/regression", "--color", "--backtrace", "--format=progress"]
+  end
+  tests << :regression
+
   task :all => tests
 end
 

metadata

@@ -1,49 +1,49 @@
 --- !ruby/object:Gem::Specification
 name: bmg
 version: !ruby/object:Gem::Version
-  version: 0.17.8
+  version: 0.18.0
 platform: ruby
 authors:
 - Bernard Lambeau
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-09-10 00:00:00.000000000 Z
+date: 2021-03-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: predicate
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.4'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.4.0
+        version: 2.5.0
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.4'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.4.0
+        version: 2.5.0
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
 - !ruby/object:Gem::Dependency
   name: path
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -78,14 +78,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '2.7'
+        version: '2.8'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '2.7'
+        version: '2.8'
 - !ruby/object:Gem::Dependency
   name: sequel
   requirement: !ruby/object:Gem::Requirement
@@ -154,6 +154,7 @@ files:
 - lib/bmg/reader.rb
 - lib/bmg/reader/csv.rb
 - lib/bmg/reader/excel.rb
+- lib/bmg/reader/text_file.rb
 - lib/bmg/relation.rb
 - lib/bmg/relation/empty.rb
 - lib/bmg/relation/in_memory.rb
@@ -288,7 +289,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.0.8
 signing_key: 
 specification_version: 4
 summary: Bmg is Alf's successor.