bmg 0.17.6 → 0.18.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e67d11656619195bfaa20a9d55fff49d593ccaa3ad558d0876c82534977d4e04
-  data.tar.gz: 7ecd9b926d4289feb0fec3efcafcf7b14c483b70f8c4e47b87e3ba038db64774
+  metadata.gz: 2c9b490328160c9816102f7dd34b3ee46207ab4362cd2af6d8623c9fc5d4b87a
+  data.tar.gz: d113dc14ae1811f2b5565af9e84a0ffbc82b7842c46284af7fcad3bc9f3b7470
 SHA512:
-  metadata.gz: 4d99c96391c73c096e3e981155cbe53e637c909854a0e3fe92bae991eb7da1be7cf78188c87f9fb53cdbb90e5dd0d92b4fec0501f8d1fcf7f1cdb1900828d575
-  data.tar.gz: f8ff33a830852b1e0c5648596ef5f11cae7fc5f7bd817c1ed30a73ce3da81a92bfb42871eecd928a3ce7b717499fb9a975ee8f174a1b013054af506ed056b825
+  metadata.gz: fd42b8787cc093c6452386760744d7480c64c8be186ab73b87a548a1b853643b46a6ad48cf019da9d77fddbb38c824eb0e0bef3ef5fa2f333f1d1240f5a0bd84
+  data.tar.gz: 7bd0cd32a28963123b03b59152fa03d6470bbdf1a3c6cfe0b17a6667a80d97e65d5309def4e31ec9cbb47353d409ad9ad2a6d1fbc9c2ec10797460852836ce35

data/Gemfile

@@ -1,5 +1,2 @@
 source "https://rubygems.org"
 gemspec
-
-# gem "predicate", github: "enspirit/predicate", branch: "placeholders"
-# gem "predicate", path: "../predicate"

data/README.md

@@ -1,16 +1,30 @@
 # Bmg, a relational algebra (Alf's successor)!
 
+[![Build Status](https://travis-ci.com/enspirit/bmg.svg?branch=master)](https://travis-ci.com/enspirit/bmg)
+
 Bmg is a relational algebra implemented as a ruby library. It implements the
 [Relation as First-Class Citizen](http://www.try-alf.org/blog/2013-10-21-relations-as-first-class-citizen)
-paradigm contributed with Alf a few years ago.
-
-Like Alf, Bmg can be used to query relations in memory, from various files,
-SQL databases, and any data sources that can be seen as serving relations.
-Cross data-sources joins are supported, as with Alf.
-
-Unlike Alf, Bmg does not make any core ruby extension and exposes the
-object-oriented syntax only (not Alf's functional one). Bmg implementation is
-also much simpler, and make its easier to implement user-defined relations.
+paradigm contributed with [Alf](http://www.try-alf.org/) a few years ago.
+
+Bmg can be used to query relations in memory, from various files, SQL databases,
+and any data source that can be seen as serving relations. Cross data-sources
+joins are supported, as with Alf. For differences with Alf, see a section
+further down this README.
+
+## Outline
+
+* [Example](#example)
+* [Where are base relations coming from?](#where-are-base-relations-coming-from)
+  * [Memory relations](#memory-relations)
+  * [Connecting to SQL databases](#connecting-to-sql-databases)
+  * [Reading files (csv, excel, text)](#reading-files-csv-excel-text)
+  * [Your own relations](#your-own-relations)
+* [List of supported operators](#supported-operators)
+* [How is this different?](#how-is-this-different)
+  * [... from similar libraries](#-from-similar-libraries)
+  * [... from Alf](#-from-alf)
+* [Contribute](#contribute)
+* [License](#license)
 
 ## Example
 
@@ -27,7 +41,7 @@ suppliers = Bmg::Relation.new([
 ])
 
 by_city = suppliers
-  .restrict(Predicate.neq(status: 30))
+  .exclude(status: 30)
   .extend(upname: ->(t){ t[:name].upcase })
   .group([:sid, :name, :status], :suppliers_in)
 
@@ -35,76 +49,158 @@ puts JSON.pretty_generate(by_city)
 # [{...},...]
 ```
 
-## Connecting to a SQL database
+## Where are base relations coming from?
+
+Bmg sees relations as sets/enumerable of symbolized Ruby hashes. The following
+sections show you how to get them in the first place, to enter Relationland.
+
+### Memory relations
+
+If you have an Array of Hashes -- in fact any Enumerable -- you can easily get
+a Relation using either `Bmg::Relation.new` or `Bmg.in_memory`.
+
+```ruby
+# this...
+r = Bmg::Relation.new [{id: 1}, {id: 2}]
+
+# is the same as this...
+r = Bmg.in_memory [{id: 1}, {id: 2}]
+
+# entire algebra is available on `r`
+```
+
+### Connecting to SQL databases
 
-Bmg requires `sequel >= 3.0` to connect to SQL databases.
+Bmg currently requires `sequel >= 3.0` to connect to SQL databases. You also
+need to require `bmg/sequel`.
 
 ```ruby
 require 'sqlite3'
 require 'bmg'
 require 'bmg/sequel'
+```
 
-DB = Sequel.connect("sqlite://suppliers-and-parts.db")
+Then `Bmg.sequel` serves relations for tables of your SQL database:
 
+```ruby
+DB = Sequel.connect("sqlite://suppliers-and-parts.db")
 suppliers = Bmg.sequel(:suppliers, DB)
+```
+
+The entire algebra is available on those relations. As long as you keep using
+operators that can be translated to SQL, results remain SQL-able:
 
+```ruby
 big_suppliers = suppliers
-  .restrict(Predicate.neq(status: 30))
+  .exclude(status: 30)
+  .project([:sid, :name])
 
 puts big_suppliers.to_sql
-# SELECT `t1`.`sid`, `t1`.`name`, `t1`.`status`, `t1`.`city` FROM `suppliers` AS 't1' WHERE (`t1`.`status` != 30)
+# SELECT `t1`.`sid`, `t1`.`name` FROM `suppliers` AS 't1' WHERE (`t1`.`status` != 30)
+```
 
-puts JSON.pretty_generate(big_suppliers)
-# [{...},...]
+Operators not translatable to SQL are available too (such as `group` below).
+Bmg fallbacks to memory operators for them, but remains capable of pushing some
+operators down the tree as illustrated below (the restriction on `:city` is
+pushed to the SQL server):
+
+```ruby
+Bmg.sequel(:suppliers, sequel_db)
+  .project([:sid, :name, :city])
+  .group([:sid, :name], :suppliers_in)
+  .restrict(city: ["Paris", "London"])
+  .debug
+
+# (group
+#   (sequel SELECT `t1`.`sid`, `t1`.`name`, `t1`.`city` FROM `suppliers` AS 't1' WHERE (`t1`.`city` IN ('Paris', 'London')))
+#   [:sid, :name, :status]
+#   :suppliers_in
+#   {:array=>false})
 ```
 
-## How is this different from similar libraries?
+### Reading files (csv, excel, text)
 
-1. The libraries you probably know (Sequel, Arel, SQLAlchemy, Korma, jOOQ,
-   etc.) do not implement a genuine relational algebra: their support for
-   chaining relational operators is limited (yielding errors or wrong SQL
-   queries). Bmg **always** allows chaining operators. If it does not, it's
-   a bug. In other words, the following query is 100% valid:
+Bmg provides simple adapters to read files and reach Relationland as soon as
+possible.
 
-       relation
-         .restrict(...)    # aka where
-         .union(...)
-         .summarize(...)   # aka group by
-         .restrict(...)
+#### CSV files
 
-2. Bmg supports in memory relations, json relations, csv relations, SQL
-   relations and so on. It's not tight to SQL generation, and supports
-   queries accross multiple data sources.
+```ruby
+csv_options = { col_sep: ",", quote_char: '"' }
+r = Bmg.csv("path/to/a/file.csv", csv_options)
+```
 
-3. Bmg makes a best effort to optimize queries, simplifying both generated
-   SQL code (low-level accesses to datasources) and in-memory operations.
+Options are directly transmitted to `::CSV.new`, check ruby's standard
+library.
 
-4. Bmg supports various *structuring* operators (group, image, autowrap,
-   autosummarize, etc.) and allows building 'non flat' relations.
+#### Excel files
 
-## How is this different from Alf?
+You will need to add [`roo`](https://github.com/roo-rb/roo) to your Gemfile to
+read `.xls` and `.xlsx` files with Bmg.
 
-1. Bmg's implementation is much simpler than Alf, and uses no ruby core
-   extention.
+```ruby
+roo_options = { skip: 1 }
+r = Bmg.excel("path/to/a/file.xls", roo_options)
+```
 
-2. We are confident using Bmg in production. Systematic inspection of query
-   plans is suggested though. Alf was a bit too experimental to be used on
-   (critical) production systems.
+Options are directly transmitted to `Roo::Spreadsheet.open`, check roo's
+documentation.
 
-2. Alf exposes a functional syntax, command line tool, restful tools and
-   many more. Bmg is limited to the core algebra, main Relation abstraction
-   and SQL generation.
+#### Text files
 
-3. Bmg is less strict regarding conformance to relational theory, and
-   may actually expose non relational features (such as support for null,
-   left_join operator, etc.). Sharp tools hurt, use them with great care.
+There is also a straightforward way to read text files and convert lines to
+tuples.
 
-4. Bmg does not yet implement all operators documented on try-alf.org, even
-   if we plan to eventually support them all.
+```ruby
+r = Bmg.text_file("path/to/a/file.txt")
+r.type.attrlist
+# => [:line, :text]
+```
 
-5. Bmg has a few additional operators that prove very useful on real
-   production use cases: prefix, suffix, autowrap, autosummarize, left_join,
-   rxmatch, etc.
+Without options tuples will have `:line` and `:text` attributes, the former
+being the line number (starting at 1) and the latter being the line itself
+(stripped).
+
+The are a couple of options (see `Bmg::Reader::Textfile`). The most useful one
+is the use a of a Regexp with named captures to automatically extract
+attributes:
+
+```ruby
+r = Bmg.text_file("path/to/a/file.txt", parse: /GET (?<url>([^\s]+))/)
+r.type.attrlist
+# => [:line, :url]
+```
+
+In this scenario, non matching lines are skipped. The `:line` attribute keeps
+being used to have at least one candidate key (so to speak).
+
+### Your own relations
+
+As noted earlier, Bmg has a simple relation interface where you only have to
+provide an iteration of symbolized tuples.
+
+```ruby
+class MyRelation
+  include Bmg::Relation
+
+  def each
+    yield(id: 1, name: "Alf", year: 2014)
+    yield(id: 2, name: "Bmg", year: 2018)
+  end
+end
+
+MyRelation.new
+  .restrict(Predicate.gt(:year, 2015))
+  .allbut([:year])
+```
+
+As shown, creating adapters on top of various data source is straighforward.
+Adapters can also participate to query optimization (such as pushing
+restrictions down the tree) by overriding the underscored version of operators
+(e.g. `_restrict`).
+
+Have a look at `Bmg::Algebra` for the protocol and `Bmg::Sql::Relation` for an
+example. Keep in touch with the team if you need some help.
 
 ## Supported operators
 
@@ -114,8 +210,10 @@ r.autowrap(split: '_')                       # structure a flat relation, split:
 r.autosummarize([:a, :b, ...], x: :sum)      # (experimental) usual summarizers supported
 r.constants(x: 12, ...)                      # add constant attributes (sometimes useful in unions)
 r.extend(x: ->(t){ ... }, ...)               # add computed attributes
+r.exclude(predicate)                         # shortcut for restrict(!predicate)
 r.group([:a, :b, ...], :x)                   # relation-valued attribute from attributes
 r.image(right, :x, [:a, :b, ...])            # relation-valued attribute from another relation
+r.images({:x => r1, :y => r2}, [:a, ...])    # shortcut over image(r1, :x, ...).image(r2, :y, ...)
 r.join(right, [:a, :b, ...])                 # natural join on a join key
 r.join(right, :a => :x, :b => :y, ...)       # natural join after right reversed renaming
 r.left_join(right, [:a, :b, ...], {...})     # left join with optional default right tuple
@@ -137,14 +235,95 @@ t.transform(&:to_s)                          # similar, but Proc-driven
 t.transform(:foo => :upcase, ...)            # specific-attrs tranformation
 t.transform([:to_s, :upcase])                # chain-transformation
 r.union(right)                               # relational union
+r.where(predicate)                           # alias for restrict(predicate)
 ```
 
-## Who is behind Bmg?
+## How is this different?
+
+### ... from similar libraries?
+
+1. The libraries you probably know (Sequel, Arel, SQLAlchemy, Korma, jOOQ,
+   etc.) do not implement a genuine relational algebra. Their support for
+   chaining relational operators is thus limited (restricting your expression
+   power and/or raising errors and/or outputting wrong or counterintuitive
+   SQL code). Bmg **always** allows chaining operators. If it does not, it's
+   a bug.
+
+   For instance the expression below is 100% valid in Bmg. The last where
+   clause applies to the result of the summarize (while SQL requires a `HAVING`
+   clause, or a `SELECT ... FROM (SELECT ...) r`).
+
+      ```ruby
+      relation
+        .where(...)
+        .union(...)
+        .summarize(...)   # aka group by
+        .where(...)
+      ```
+
+2. Bmg supports in memory relations, json relations, csv relations, SQL
+   relations and so on. It's not tight to SQL generation, and supports
+   queries accross multiple data sources.
+
+3. Bmg makes a best effort to optimize queries, simplifying both generated
+   SQL code (low-level accesses to datasources) and in-memory operations.
+
+4. Bmg supports various *structuring* operators (group, image, autowrap,
+   autosummarize, etc.) and allows building 'non flat' relations.
+
+5. Bmg can use full ruby power when that helps (e.g. regular expressions in
+   WHERE clauses or ruby code in EXTEND clauses). This may prevent Bmg from
+   delegating work to underlying data sources (e.g. SQL server) and should
+   therefore be used with care though.
+
+### ... from Alf?
+
+If you use Alf (or used it in the past), below are the main differences between
+Bmg and Alf. Bmg has NOT been written to be API-compatible with Alf and will
+probably never be.
+
+1. Bmg's implementation is much simpler than Alf and uses no ruby core
+   extention.
+
+2. We are confident using Bmg in production. Systematic inspection of query
+   plans is advised though. Alf was a bit too experimental to be used on
+   (critical) production systems.
+
+3. Alf exposes a functional syntax, command line tool, restful tools and
+   many more. Bmg is limited to the core algebra, main Relation abstraction
+   and SQL generation.
 
-Bernard Lambeau (bernard@klaro.cards) is Alf & Bmg main engineer & maintainer.
+4. Bmg is less strict regarding conformance to relational theory, and
+   may actually expose non relational features (such as support for null,
+   left_join operator, etc.). Sharp tools hurt, use them with care.
+
+5. Unlike Alf::Relation instances of Bmg::Relation capture query-trees, not
+   values. Currently two instances `r1` and `r2` are not equal even if they
+   define the same mathematical relation. As a consequence joining on
+   relation-valued attributes does not work as expected in Bmg until further
+   notice.
+
+6. Bmg does not implement all operators documented on try-alf.org, even if
+   we plan to eventually support most of them.
+
+7. Bmg has a few additional operators that prove very useful on real
+   production use cases: prefix, suffix, autowrap, autosummarize, left_join,
+   rxmatch, etc.
+
+8. Bmg optimizes queries and compiles them to SQL on the fly, while Alf was
+   building an AST internally first. Strictly speaking this makes Bmg less
+   powerful than Alf since optimizations cannot be turned off for now.
+
+## Contribute
+
+Please use github issues and pull requests for all questions, bug reports,
+and contributions. Don't hesitate to get in touch with us with an early code
+spike if you plan to add non trivial features.
+
+## Licence
+
+This software is distributed by Enspirit SRL under a MIT Licence. Please
+contact Bernard Lambeau (blambeau@gmail.com) with any question.
 
 Enspirit (https://enspirit.be) and Klaro App (https://klaro.cards) are both
 actively using and contributing to the library.
-
-Feel free to contact us for help, ideas and/or contributions. Please use github
-issues and pull requests if possible if code is involved.

data/lib/bmg.rb

@@ -1,6 +1,7 @@
 require 'path'
 require 'predicate'
 require 'forwardable'
+require 'set'
 module Bmg
 
   def in_memory(enumerable, type = Type::ANY)
@@ -8,6 +9,11 @@ module Bmg
   end
   module_function :in_memory
 
+  def text_file(path, options = {}, type = Type::ANY)
+    Reader::TextFile.new(type, path, options).spied(main_spy)
+  end
+  module_function :text_file
+
   def csv(path, options = {}, type = Type::ANY)
     Reader::Csv.new(type, path, options).spied(main_spy)
   end

data/lib/bmg/algebra.rb

@@ -174,6 +174,7 @@ module Bmg
 
     def transform(transformation = nil, options = {}, &proc)
       transformation, options = proc, (transformation || {}) unless proc.nil?
+      return self if transformation.is_a?(Hash) && transformation.empty?
       _transform(self.type.transform(transformation, options), transformation, options)
     end
 

data/lib/bmg/algebra/shortcuts.rb

@@ -2,6 +2,14 @@ module Bmg
   module Algebra
     module Shortcuts
 
+      def where(predicate)
+        restrict(predicate)
+      end
+
+      def exclude(predicate)
+        restrict(!Predicate.coerce(predicate))
+      end
+
       def rxmatch(attrs, matcher, options = {})
         predicate = attrs.inject(Predicate.contradiction){|p,a|
           p | Predicate.match(a, matcher, options)
@@ -31,6 +39,12 @@ module Bmg
         self.image(right.rename(renaming), as, on.keys, options)
       end
 
+      def images(rights, on = [], options = {})
+        rights.each_pair.inject(self){|memo,(as,right)|
+          memo.image(right, as, on, options)
+        }
+      end
+
       def join(right, on = [])
         return super unless on.is_a?(Hash)
         renaming = Hash[on.map{|k,v| [v,k] }]

data/lib/bmg/operator/allbut.rb

@@ -63,12 +63,38 @@ module Bmg
 
     protected ### optimization
 
+      def _allbut(type, butlist)
+        operand.allbut(self.butlist|butlist)
+      end
+
+      def _matching(type, right, on)
+        # Always possible to push the matching, since by construction
+        # `on` can only use attributes that have not been trown away,
+        # hence they exist on `operand` too.
+        operand.matching(right, on).allbut(butlist)
+      end
+
+      def _page(type, ordering, page_index, options)
+        return super unless self.preserving_key?
+        operand.page(ordering, page_index, options).allbut(butlist)
+      end
+
+      def _project(type, attrlist)
+        operand.project(attrlist)
+      end
+
       def _restrict(type, predicate)
         operand.restrict(predicate).allbut(butlist)
       end
 
     protected ### inspect
 
+      def preserving_key?
+        operand.type.knows_keys? && operand.type.keys.find{|k|
+          (k & butlist).empty?
+        }
+      end
+
       def args
         [ butlist ]
       end

data/lib/bmg/operator/autosummarize.rb

@@ -24,6 +24,22 @@ module Bmg
 
     public
 
+      def self.same(*args)
+        Same.new(*args)
+      end
+
+      def self.group(*args)
+        Group.new(*args)
+      end
+
+      def self.y_by_x(*args)
+        YByX.new(*args)
+      end
+
+      def self.ys_by_x(*args)
+        YsByX.new(*args)
+      end
+
       def each(&bl)
         h = {}
         @operand.each do |tuple|
@@ -41,6 +57,12 @@ module Bmg
         [:autosummarize, operand.to_ast, by.dup, sums.dup]
       end
 
+    public ### for internal reasons
+
+      def _count
+        operand._count
+      end
+
     protected
 
       def _restrict(type, predicate)
@@ -175,11 +197,11 @@ module Bmg
         end
 
         def init(v)
-          [v]
+          v.nil? ? [] : [v]
         end
 
         def sum(v1, v2)
-          v1 << v2
+          v2.nil? ? v1 : (v1 << v2)
         end
 
         def term(v)
@@ -211,11 +233,11 @@ module Bmg
         end
 
         def init(v)
-          [v]
+          v.nil? ? [] : [v]
         end
 
         def sum(v1, v2)
-          v1 << v2
+          v2.nil? ? v1 : (v1 << v2)
         end
 
         def term(v)

data/lib/bmg/operator/autowrap.rb

@@ -52,6 +52,12 @@ module Bmg
         [ :autowrap, operand.to_ast, @original_options.dup ]
       end
 
+    public ### for internal reasons
+
+      def _count
+        operand._count
+      end
+
     protected ### optimization
 
       def _autowrap(type, opts)
@@ -86,6 +92,16 @@ module Bmg
         false
       end
 
+      def _matching(type, right, on)
+        if (wrapped_roots! & on).empty?
+          operand.matching(right, on).autowrap(options)
+        else
+          super
+        end
+      rescue UnknownAttributesError
+        super
+      end
+
       def _page(type, ordering, page_index, opts)
         attrs = ordering.map{|(a,d)| a }
         if (wrapped_roots! & attrs).empty?
@@ -97,6 +113,16 @@ module Bmg
         super
       end
 
+      def _project(type, attrlist)
+        if (wrapped_roots! & attrlist).empty?
+          operand.project(attrlist).autowrap(options)
+        else
+          super
+        end
+      rescue UnknownAttributesError
+        super
+      end
+
       def _rename(type, renaming)
         # 1. Can't optimize if renaming applies to a wrapped one
         return super unless (wrapped_roots! & renaming.keys).empty?

data/lib/bmg/operator/constants.rb

@@ -54,6 +54,12 @@ module Bmg
         [ :constants, operand.to_ast, constants.dup ]
       end
 
+    public ### for internal reasons
+
+      def _count
+        operand._count
+      end
+
     protected ### optimization
 
       def _page(type, ordering, page_index, options)

data/lib/bmg/operator/extend.rb

@@ -53,6 +53,12 @@ module Bmg
         [ :extend, operand.to_ast, extension.dup ]
       end
 
+    public ### for internal reasons
+
+      def _count
+        operand._count
+      end
+
     protected ### optimization
 
       def _allbut(type, butlist)

data/lib/bmg/operator/image.rb

@@ -99,9 +99,10 @@ module Bmg
           key = tuple_project(t, on)
           index[key].operand << tuple_image(t, on)
         end
-        if options[:array]
+        if opt = options[:array]
+          sorter = to_sorter(opt)
           index = index.each_with_object({}) do |(k,v),ix|
-            ix[k] = v.to_a
+            ix[k] = sorter ? v.to_a.sort(&sorter) : v.to_a
           end
         end
         index
@@ -154,8 +155,32 @@ module Bmg
         end
       end
 
+    public ### for internal reasons
+
+      def _count
+        left._count
+      end
+
     protected ### optimization
 
+      def _allbut(type, butlist)
+        if butlist.include?(as)
+          left.allbut(butlist - [as])
+        elsif (butlist & on).empty?
+          left.allbut(butlist).image(right, as, on, options)
+        else
+          super
+        end
+      end
+
+      def _matching(type, m_right, m_on)
+        if m_on.include?(as)
+          super
+        else
+          left.matching(m_right, m_on).image(right, as, on, options)
+        end
+      end
+
       def _page(type, ordering, page_index, opts)
         if ordering.map{|(k,v)| k}.include?(as)
           super
@@ -166,6 +191,14 @@ module Bmg
         end
       end
 
+      def _project(type, attrlist)
+        if attrlist.include?(as)
+          super
+        else
+          left.project(attrlist)
+        end
+      end
+
       def _restrict(type, predicate)
         on_as, rest = predicate.and_split([as])
         if rest.tautology?
@@ -227,6 +260,11 @@ module Bmg
         Relation::InMemory.new(image_type, Set.new)
       end
 
+      def to_sorter(opt)
+        return nil unless opt.is_a?(Array)
+        Ordering.new(opt).comparator
+      end
+
     public
 
       def to_s

data/lib/bmg/operator/page.rb

@@ -45,13 +45,7 @@ module Bmg
     protected ### inspect
 
       def comparator
-        ->(t1, t2) {
-          ordering.each do |(attr,direction)|
-            c = t1[attr] <=> t2[attr]
-            return (direction == :desc ? -c : c) unless c==0
-          end
-          0
-        }
+        Ordering.new(@ordering).comparator
       end
 
       def args

data/lib/bmg/operator/project.rb

@@ -31,7 +31,7 @@ module Bmg
       def each
         seen = {}
         @operand.each do |tuple|
-          projected = project(tuple)
+          projected = tuple_project(tuple)
           unless seen.has_key?(projected)
             yield(projected)
             seen[projected] = true
@@ -74,7 +74,7 @@ module Bmg
 
     private
 
-      def project(tuple)
+      def tuple_project(tuple)
         tuple.dup.delete_if{|k,_| !@attrlist.include?(k) }
       end
 

data/lib/bmg/operator/rename.rb

@@ -60,6 +60,12 @@ module Bmg
         [ :rename, operand.to_ast, renaming.dup ]
       end
 
+    public ### for internal reasons
+
+      def _count
+        operand._count
+      end
+
     protected ### optimization
 
       def _page(type, ordering, page_index, options)

data/lib/bmg/operator/transform.rb

@@ -23,7 +23,7 @@ module Bmg
 
     protected
 
-      attr_reader :transformation
+      attr_reader :transformation, :options
 
     public
 
@@ -40,6 +40,43 @@ module Bmg
 
     protected ### optimization
 
+      def _allbut(type, butlist)
+        # `allbut` can always be pushed down the tree. unlike
+        # `extend` the Proc that might be used cannot use attributes
+        # in butlist, so it's safe to strip them away.
+        if transformer.knows_attrlist?
+          # We just need to clean the transformation
+          attrlist = transformer.to_attrlist
+          thrown = attrlist & butlist
+          t = transformation.dup.reject{|k,v| thrown.include?(k) }
+          operand.allbut(butlist).transform(t, options)
+        else
+          operand.allbut(butlist).transform(transformation, options)
+        end
+      end
+
+      def _project(type, attrlist)
+        if transformer.knows_attrlist?
+          t = transformation.dup.select{|k,v| attrlist.include?(k) }
+          operand.project(attrlist).transform(t, options)
+        else
+          operand.project(attrlist).transform(transformation, options)
+        end
+      end
+
+      def _restrict(type, predicate)
+        return super unless transformer.knows_attrlist?
+        top, bottom = predicate.and_split(transformer.to_attrlist)
+        if top == predicate
+          super
+        else
+          operand
+            .restrict(bottom)
+            .transform(transformation, options)
+            .restrict(top)
+        end
+      end
+
     protected ### inspect
 
       def args

data/lib/bmg/reader.rb

@@ -9,3 +9,4 @@ module Bmg
 end
 require_relative "reader/csv"
 require_relative "reader/excel"
+require_relative "reader/text_file"

data/lib/bmg/reader/text_file.rb

@@ -0,0 +1,56 @@
+module Bmg
+  module Reader
+    class TextFile
+      include Reader
+
+      DEFAULT_OPTIONS = {
+        strip: true,
+        parse: nil
+      }
+
+      def initialize(type, path, options = {})
+        options = { parse: options } if options.is_a?(Regexp)
+        @path = path
+        @options = DEFAULT_OPTIONS.merge(options)
+        @type = infer_type(type)
+      end
+      attr_reader :path, :options
+
+    public # Relation
+
+      def each
+        path.each_line.each_with_index do |text, line|
+          text = text.strip if strip?
+          parsed = parse(text)
+          yield({line: 1+line}.merge(parsed)) if parsed
+        end
+      end
+
+    private
+
+      def infer_type(base)
+        return base unless base == Bmg::Type::ANY
+        attr_list = if rx = options[:parse]
+          [:line] + rx.names.map(&:to_sym)
+        else
+          [:line, :text]
+        end
+        base
+          .with_attrlist(attr_list)
+          .with_keys([[:line]])
+      end
+
+      def strip?
+        options[:strip]
+      end
+
+      def parse(text)
+        return { text: text } unless rx = options[:parse]
+        if match = rx.match(text)
+          TupleAlgebra.symbolize_keys(match.named_captures)
+        end
+      end
+
+    end # class TextFile
+  end # module Reader
+end # module Bmg

data/lib/bmg/relation.rb

@@ -17,6 +17,16 @@ module Bmg
       self
     end
 
+    def type
+      Bmg::Type::ANY
+    end
+
+    def with_type(type)
+      dup.tap{|r|
+        r.type = type
+      }
+    end
+
     def with_typecheck
       dup.tap{|r|
         r.type = r.type.with_typecheck
@@ -100,6 +110,18 @@ module Bmg
       end
     end
 
+    def count
+      if type.knows_keys?
+        project(type.keys.first)._count
+      else
+        self._count
+      end
+    end
+
+    def _count
+      to_a.size
+    end
+
     # Returns a json representation
     def to_json(*args, &bl)
       to_a.to_json(*args, &bl)
@@ -113,9 +135,10 @@ module Bmg
     # When no string_or_io is used, the method uses a string.
     #
     # The method always returns the string_or_io.
-    def to_csv(options = {}, string_or_io = nil)
+    def to_csv(options = {}, string_or_io = nil, preferences = nil)
       options, string_or_io = {}, options unless options.is_a?(Hash)
-      Writer::Csv.new(options).call(self, string_or_io)
+      string_or_io, preferences = nil, string_or_io if string_or_io.is_a?(Hash)
+      Writer::Csv.new(options, preferences).call(self, string_or_io)
     end
 
     # Converts to an sexpr expression.

data/lib/bmg/relation/empty.rb

@@ -19,6 +19,10 @@ module Bmg
       def each(&bl)
       end
 
+      def _count
+        0
+      end
+
       def to_ast
         [ :empty ]
       end

data/lib/bmg/relation/in_memory.rb

@@ -17,6 +17,16 @@ module Bmg
         @operand.each(&bl)
       end
 
+      def _count
+        if operand.respond_to?(:count)
+          operand.count
+        elsif operand.respond_to?(:size)
+          operand.size
+        else
+          super
+        end
+      end
+
       def to_ast
         [ :in_memory, operand ]
       end

data/lib/bmg/relation/materialized.rb

@@ -16,6 +16,12 @@ module Bmg
       end
       protected :type=
 
+    public
+
+      def _count
+        operand._count
+      end
+
     public
 
       def each(&bl)

data/lib/bmg/relation/spied.rb

@@ -24,10 +24,15 @@ module Bmg
       protected :type=
 
       def each(&bl)
-        spy.call(self)
+        spy.call(self) if bl
         operand.each(&bl)
       end
 
+      def count
+        spy.call(self) if bl
+        operand.count
+      end
+
       def to_ast
         [ :spied, operand.to_ast, spy ]
       end

data/lib/bmg/sequel/relation.rb

@@ -33,6 +33,10 @@ module Bmg
         base_table.update(arg)
       end
 
+      def _count
+        dataset.count
+      end
+
       def to_ast
         [:sequel, dataset.sql]
       end

data/lib/bmg/sql/relation.rb

@@ -10,7 +10,6 @@ module Bmg
       end
 
       attr_accessor :type
-      protected :type=
 
     protected
 

data/lib/bmg/support.rb

@@ -1,3 +1,5 @@
 require_relative 'support/tuple_algebra'
 require_relative 'support/tuple_transformer'
 require_relative 'support/keys'
+require_relative 'support/ordering'
+require_relative 'support/output_preferences'

data/lib/bmg/support/ordering.rb

@@ -0,0 +1,20 @@
+module Bmg
+  class Ordering
+
+    def initialize(attrs)
+      @attrs = attrs
+    end
+    attr_reader :attrs
+
+    def comparator
+      ->(t1, t2) {
+        attrs.each do |(attr,direction)|
+          c = t1[attr] <=> t2[attr]
+          return (direction == :desc ? -c : c) unless c==0
+        end
+        0
+      }
+    end
+
+  end # class Ordering
+end # module Bmg

data/lib/bmg/support/output_preferences.rb

@@ -0,0 +1,44 @@
+module Bmg
+  class OutputPreferences
+
+    DEFAULT_PREFS = {
+      attributes_ordering: nil,
+      extra_attributes: :after
+    }
+
+    def initialize(options)
+      @options = DEFAULT_PREFS.merge(options)
+    end
+    attr_reader :options
+
+    def self.dress(arg)
+      return arg if arg.is_a?(OutputPreferences)
+      arg = {} if arg.nil?
+      new(arg)
+    end
+
+    def attributes_ordering
+      options[:attributes_ordering]
+    end
+
+    def extra_attributes
+      options[:extra_attributes]
+    end
+
+    def order_attrlist(attrlist)
+      return attrlist if attributes_ordering.nil?
+      index = Hash[attributes_ordering.each_with_index.to_a]
+      attrlist.sort{|a,b|
+        ai, bi = index[a], index[b]
+        if ai && bi
+          ai <=> bi
+        elsif ai
+          extra_attributes == :after ? -1 : 1
+        else
+          extra_attributes == :after ? 1 : -1
+        end
+      }
+    end
+
+  end # class OutputPreferences
+end # module Bmg

data/lib/bmg/support/tuple_algebra.rb

@@ -19,5 +19,11 @@ module Bmg
     end
     module_function :rename
 
+    def symbolize_keys(h)
+      return h if h.empty?
+      h.each_with_object({}){|(k,v),h| h[k.to_sym] = v }
+    end
+    module_function :symbolize_keys
+
   end # module TupleAlgebra
 end # module Bmg

data/lib/bmg/support/tuple_transformer.rb

@@ -26,11 +26,7 @@ module Bmg
 
       def transform_tuple(tuple, with)
         case with
-        when Symbol
-          tuple.each_with_object({}){|(k,v),dup|
-            dup[k] = transform_attr(v, with)
-          }
-        when Proc
+        when Symbol, Proc, Regexp
           tuple.each_with_object({}){|(k,v),dup|
             dup[k] = transform_attr(v, with)
           }
@@ -51,8 +47,13 @@ module Bmg
         case with
         when Symbol
           value.send(with)
+        when Regexp
+          m = with.match(value.to_s)
+          m.nil? ? m : m.to_s
         when Proc
           with.call(value)
+        when Hash
+          with[value]
         else
           raise ArgumentError, "Unexpected transformation `#{with.inspect}`"
         end

data/lib/bmg/version.rb

@@ -1,8 +1,8 @@
 module Bmg
   module Version
     MAJOR = 0
-    MINOR = 17
-    TINY  = 6
+    MINOR = 18
+    TINY  = 2
   end
   VERSION = "#{Version::MAJOR}.#{Version::MINOR}.#{Version::TINY}"
 end

data/lib/bmg/writer/csv.rb

@@ -6,26 +6,37 @@ module Bmg
       DEFAULT_OPTIONS = {
       }
 
-      def initialize(options)
-        @options = DEFAULT_OPTIONS.merge(options)
+      def initialize(csv_options, output_preferences = nil)
+        @csv_options = DEFAULT_OPTIONS.merge(csv_options)
+        @output_preferences = OutputPreferences.dress(output_preferences)
       end
-      attr_reader :options
+      attr_reader :csv_options, :output_preferences
 
       def call(relation, string_or_io = nil)
         require 'csv'
         string_or_io, to_s = string_or_io.nil? ? [StringIO.new, true] : [string_or_io, false]
-        headers = relation.type.to_attrlist if relation.type.knows_attrlist?
-        csv = nil
+        headers, csv = infer_headers(relation.type), nil
         relation.each do |tuple|
           if csv.nil?
-            headers = tuple.keys if headers.nil?
-            csv = CSV.new(string_or_io, options.merge(headers: headers))
+            headers = infer_headers(tuple) if headers.nil?
+            csv = CSV.new(string_or_io, csv_options.merge(headers: headers))
           end
           csv << headers.map{|h| tuple[h] }
         end
         to_s ? string_or_io.string : string_or_io
       end
 
+    private
+
+      def infer_headers(from)
+        attrlist = if from.is_a?(Type) && from.knows_attrlist?
+          from.to_attrlist
+        elsif from.is_a?(Hash)
+          from.keys
+        end
+        attrlist ? output_preferences.order_attrlist(attrlist) : nil
+      end
+
     end # class Csv
   end # module Writer
 end # module Bmg

data/tasks/test.rake

@@ -6,17 +6,24 @@ namespace :test do
   desc "Runs unit tests"
   RSpec::Core::RakeTask.new(:unit) do |t|
     t.pattern = "spec/unit/**/test_*.rb"
-    t.rspec_opts = ["-Ilib", "-Ispec/unit", "--fail-fast", "--color", "--backtrace", "--format=progress"]
+    t.rspec_opts = ["-Ilib", "-Ispec/unit", "--color", "--backtrace", "--format=progress"]
   end
   tests << :unit
 
   desc "Runs integration tests"
   RSpec::Core::RakeTask.new(:integration) do |t|
     t.pattern = "spec/integration/**/test_*.rb"
-    t.rspec_opts = ["-Ilib", "-Ispec/integration", "--fail-fast", "--color", "--backtrace", "--format=progress"]
+    t.rspec_opts = ["-Ilib", "-Ispec/integration", "--color", "--backtrace", "--format=progress"]
   end
   tests << :integration
 
+  desc "Runs github regression tests"
+  RSpec::Core::RakeTask.new(:regression) do |t|
+    t.pattern = "spec/regression/**/test_*.rb"
+    t.rspec_opts = ["-Ilib", "-Ispec/regression", "--color", "--backtrace", "--format=progress"]
+  end
+  tests << :regression
+
   task :all => tests
 end
 

metadata

@@ -1,49 +1,49 @@
 --- !ruby/object:Gem::Specification
 name: bmg
 version: !ruby/object:Gem::Version
-  version: 0.17.6
+  version: 0.18.2
 platform: ruby
 authors:
 - Bernard Lambeau
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-08-28 00:00:00.000000000 Z
+date: 2021-04-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: predicate
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.4'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.4.0
+        version: 2.5.0
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.4'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.4.0
+        version: 2.5.0
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
 - !ruby/object:Gem::Dependency
   name: path
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -78,14 +78,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '2.7'
+        version: '2.8'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '2.7'
+        version: '2.8'
 - !ruby/object:Gem::Dependency
   name: sequel
   requirement: !ruby/object:Gem::Requirement
@@ -154,6 +154,7 @@ files:
 - lib/bmg/reader.rb
 - lib/bmg/reader/csv.rb
 - lib/bmg/reader/excel.rb
+- lib/bmg/reader/text_file.rb
 - lib/bmg/relation.rb
 - lib/bmg/relation/empty.rb
 - lib/bmg/relation/in_memory.rb
@@ -260,6 +261,8 @@ files:
 - lib/bmg/summarizer/variance.rb
 - lib/bmg/support.rb
 - lib/bmg/support/keys.rb
+- lib/bmg/support/ordering.rb
+- lib/bmg/support/output_preferences.rb
 - lib/bmg/support/tuple_algebra.rb
 - lib/bmg/support/tuple_transformer.rb
 - lib/bmg/type.rb
@@ -287,7 +290,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.0.8
 signing_key: 
 specification_version: 4
 summary: Bmg is Alf's successor.