bmg 0.17.4 → 0.18.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1e48d4a2d3c4bea8271e40ad4db52f43438de7f754079cc4c5bcd5e98ce4865d
-  data.tar.gz: 3fff9f251a2b766ab3de631748ccc6fe6f6b96ccef88507cb3ed76766fea3992
+  metadata.gz: f1ce59d00b630f644e5716eaff17a83116c1342ea0b6ba7d9174b1f5f4eadd6e
+  data.tar.gz: 2156d1a8eb88999e749f434a8e94d2b4e70c636fadb10470493f19bb805a19a3
 SHA512:
-  metadata.gz: facfbb798258cfdec3803cc7b93999d23c8a0776ead53c074266f4f658a27feee1a3e1dcc1145e7ad45bb03579d22f5d65108152516ac0af0d79aa59805ee016
-  data.tar.gz: 10271e608276bc2e00ccee3d6a94287cb2211d945898cb9cfa54764a05027c9e6a6c34a11e9725f9ed47df4df537fe054d8c3f17b24f58da9713761121523e73
+  metadata.gz: 8e3e714d698ff7c47c2f61b4056c73acaa39983306402eeb54d1957b90959af51fcf20a543c5c17242d260fe835b33c19dd960e3cd4920a66d49ea81a22ee4b0
+  data.tar.gz: 89fe1cc4c3157adf7373fb755e0cf7eb5dd8d93f7061558454d27649b44aca558f936f006e3238dce410ffdfe29821d59feac3b31b5d64241ffc76192bba149f

data/Gemfile

@@ -1,5 +1,2 @@
 source "https://rubygems.org"
 gemspec
-
-# gem "predicate", github: "enspirit/predicate", branch: "placeholders"
-# gem "predicate", path: "../predicate"

data/README.md

@@ -1,16 +1,30 @@
 # Bmg, a relational algebra (Alf's successor)!
 
+[![Build Status](https://travis-ci.com/enspirit/bmg.svg?branch=master)](https://travis-ci.com/enspirit/bmg)
+
 Bmg is a relational algebra implemented as a ruby library. It implements the
 [Relation as First-Class Citizen](http://www.try-alf.org/blog/2013-10-21-relations-as-first-class-citizen)
-paradigm contributed with Alf a few years ago.
-
-Like Alf, Bmg can be used to query relations in memory, from various files,
-SQL databases, and any data sources that can be seen as serving relations.
-Cross data-sources joins are supported, as with Alf.
-
-Unlike Alf, Bmg does not make any core ruby extension and exposes the
-object-oriented syntax only (not Alf's functional one). Bmg implementation is
-also much simpler, and make its easier to implement user-defined relations.
+paradigm contributed with [Alf](http://www.try-alf.org/) a few years ago.
+
+Bmg can be used to query relations in memory, from various files, SQL databases,
+and any data source that can be seen as serving relations. Cross data-sources
+joins are supported, as with Alf. For differences with Alf, see a section
+further down this README.
+
+## Outline
+
+* [Example](#example)
+* [Where are base relations coming from?](#where-are-base-relations-coming-from)
+  * [Memory relations](#memory-relations)
+  * [Connecting to SQL databases](#connecting-to-sql-databases)
+  * [Reading files (csv, excel, text)](#reading-files-csv-excel-text)
+  * [Your own relations](#your-own-relations)
+* [List of supported operators](#supported-operators)
+* [How is this different?](#how-is-this-different)
+  * [... from similar libraries](#-from-similar-libraries)
+  * [... from Alf](#-from-alf)
+* [Contribute](#contribute)
+* [License](#license)
 
 ## Example
 
@@ -27,7 +41,7 @@ suppliers = Bmg::Relation.new([
 ])
 
 by_city = suppliers
-  .restrict(Predicate.neq(status: 30))
+  .exclude(status: 30)
   .extend(upname: ->(t){ t[:name].upcase })
   .group([:sid, :name, :status], :suppliers_in)
 
@@ -35,76 +49,158 @@ puts JSON.pretty_generate(by_city)
 # [{...},...]
 ```
 
-## Connecting to a SQL database
+## Where are base relations coming from?
+
+Bmg sees relations as sets/enumerable of symbolized Ruby hashes. The following
+sections show you how to get them in the first place, to enter Relationland.
+
+### Memory relations
+
+If you have an Array of Hashes -- in fact any Enumerable -- you can easily get
+a Relation using either `Bmg::Relation.new` or `Bmg.in_memory`.
+
+```ruby
+# this...
+r = Bmg::Relation.new [{id: 1}, {id: 2}]
+
+# is the same as this...
+r = Bmg.in_memory [{id: 1}, {id: 2}]
+
+# entire algebra is available on `r`
+```
+
+### Connecting to SQL databases
 
-Bmg requires `sequel >= 3.0` to connect to SQL databases.
+Bmg currently requires `sequel >= 3.0` to connect to SQL databases. You also
+need to require `bmg/sequel`.
 
 ```ruby
 require 'sqlite3'
 require 'bmg'
 require 'bmg/sequel'
+```
 
-DB = Sequel.connect("sqlite://suppliers-and-parts.db")
+Then `Bmg.sequel` serves relations for tables of your SQL database:
 
+```ruby
+DB = Sequel.connect("sqlite://suppliers-and-parts.db")
 suppliers = Bmg.sequel(:suppliers, DB)
+```
+
+The entire algebra is available on those relations. As long as you keep using
+operators that can be translated to SQL, results remain SQL-able:
 
+```ruby
 big_suppliers = suppliers
-  .restrict(Predicate.neq(status: 30))
+  .exclude(status: 30)
+  .project([:sid, :name])
 
 puts big_suppliers.to_sql
-# SELECT `t1`.`sid`, `t1`.`name`, `t1`.`status`, `t1`.`city` FROM `suppliers` AS 't1' WHERE (`t1`.`status` != 30)
+# SELECT `t1`.`sid`, `t1`.`name` FROM `suppliers` AS 't1' WHERE (`t1`.`status` != 30)
+```
 
-puts JSON.pretty_generate(big_suppliers)
-# [{...},...]
+Operators not translatable to SQL are available too (such as `group` below).
+Bmg fallbacks to memory operators for them, but remains capable of pushing some
+operators down the tree as illustrated below (the restriction on `:city` is
+pushed to the SQL server):
+
+```ruby
+Bmg.sequel(:suppliers, sequel_db)
+  .project([:sid, :name, :city])
+  .group([:sid, :name], :suppliers_in)
+  .restrict(city: ["Paris", "London"])
+  .debug
+
+# (group
+#   (sequel SELECT `t1`.`sid`, `t1`.`name`, `t1`.`city` FROM `suppliers` AS 't1' WHERE (`t1`.`city` IN ('Paris', 'London')))
+#   [:sid, :name, :status]
+#   :suppliers_in
+#   {:array=>false})
 ```
 
-## How is this different from similar libraries?
+### Reading files (csv, excel, text)
 
-1. The libraries you probably know (Sequel, Arel, SQLAlchemy, Korma, jOOQ,
-   etc.) do not implement a genuine relational algebra: their support for
-   chaining relational operators is limited (yielding errors or wrong SQL
-   queries). Bmg **always** allows chaining operators. If it does not, it's
-   a bug. In other words, the following query is 100% valid:
+Bmg provides simple adapters to read files and reach Relationland as soon as
+possible.
 
-       relation
-         .restrict(...)    # aka where
-         .union(...)
-         .summarize(...)   # aka group by
-         .restrict(...)
+#### CSV files
 
-2. Bmg supports in memory relations, json relations, csv relations, SQL
-   relations and so on. It's not tight to SQL generation, and supports
-   queries accross multiple data sources.
+```ruby
+csv_options = { col_sep: ",", quote_char: '"' }
+r = Bmg.csv("path/to/a/file.csv", csv_options)
+```
 
-3. Bmg makes a best effort to optimize queries, simplifying both generated
-   SQL code (low-level accesses to datasources) and in-memory operations.
+Options are directly transmitted to `::CSV.new`, check ruby's standard
+library.
 
-4. Bmg supports various *structuring* operators (group, image, autowrap,
-   autosummarize, etc.) and allows building 'non flat' relations.
+#### Excel files
 
-## How is this different from Alf?
+You will need to add [`roo`](https://github.com/roo-rb/roo) to your Gemfile to
+read `.xls` and `.xlsx` files with Bmg.
 
-1. Bmg's implementation is much simpler than Alf, and uses no ruby core
-   extention.
+```ruby
+roo_options = { skip: 1 }
+r = Bmg.excel("path/to/a/file.xls", roo_options)
+```
 
-2. We are confident using Bmg in production. Systematic inspection of query
-   plans is suggested though. Alf was a bit too experimental to be used on
-   (critical) production systems.
+Options are directly transmitted to `Roo::Spreadsheet.open`, check roo's
+documentation.
 
-2. Alf exposes a functional syntax, command line tool, restful tools and
-   many more. Bmg is limited to the core algebra, main Relation abstraction
-   and SQL generation.
+#### Text files
 
-3. Bmg is less strict regarding conformance to relational theory, and
-   may actually expose non relational features (such as support for null,
-   left_join operator, etc.). Sharp tools hurt, use them with great care.
+There is also a straightforward way to read text files and convert lines to
+tuples.
 
-4. Bmg does not yet implement all operators documented on try-alf.org, even
-   if we plan to eventually support them all.
+```ruby
+r = Bmg.text_file("path/to/a/file.txt")
+r.type.attrlist
+# => [:line, :text]
+```
 
-5. Bmg has a few additional operators that prove very useful on real
-   production use cases: prefix, suffix, autowrap, autosummarize, left_join,
-   rxmatch, etc.
+Without options tuples will have `:line` and `:text` attributes, the former
+being the line number (starting at 1) and the latter being the line itself
+(stripped).
+
+The are a couple of options (see `Bmg::Reader::Textfile`). The most useful one
+is the use a of a Regexp with named captures to automatically extract
+attributes:
+
+```ruby
+r = Bmg.text_file("path/to/a/file.txt", parse: /GET (?<url>([^\s]+))/)
+r.type.attrlist
+# => [:line, :url]
+```
+
+In this scenario, non matching lines are skipped. The `:line` attribute keeps
+being used to have at least one candidate key (so to speak).
+
+### Your own relations
+
+As noted earlier, Bmg has a simple relation interface where you only have to
+provide an iteration of symbolized tuples.
+
+```ruby
+class MyRelation
+  include Bmg::Relation
+
+  def each
+    yield(id: 1, name: "Alf", year: 2014)
+    yield(id: 2, name: "Bmg", year: 2018)
+  end
+end
+
+MyRelation.new
+  .restrict(Predicate.gt(:year, 2015))
+  .allbut([:year])
+```
+
+As shown, creating adapters on top of various data source is straighforward.
+Adapters can also participate to query optimization (such as pushing
+restrictions down the tree) by overriding the underscored version of operators
+(e.g. `_restrict`).
+
+Have a look at `Bmg::Algebra` for the protocol and `Bmg::Sql::Relation` for an
+example. Keep in touch with the team if you need some help.
 
 ## Supported operators
 
@@ -114,6 +210,7 @@ r.autowrap(split: '_')                       # structure a flat relation, split:
 r.autosummarize([:a, :b, ...], x: :sum)      # (experimental) usual summarizers supported
 r.constants(x: 12, ...)                      # add constant attributes (sometimes useful in unions)
 r.extend(x: ->(t){ ... }, ...)               # add computed attributes
+r.exclude(predicate)                         # shortcut for restrict(!predicate)
 r.group([:a, :b, ...], :x)                   # relation-valued attribute from attributes
 r.image(right, :x, [:a, :b, ...])            # relation-valued attribute from another relation
 r.join(right, [:a, :b, ...])                 # natural join on a join key
@@ -132,15 +229,100 @@ r.restrict(a: "foo", b: "bar", ...)          # relational restriction, aka where
 r.rxmatch([:a, :b, ...], /xxx/)              # regex match kind of restriction
 r.summarize([:a, :b, ...], x: :sum)          # relational summarization
 r.suffix(:_foo, but: [:a, ...])              # suffix kind of renaming
+t.transform(:to_s)                           # all-attrs transformation
+t.transform(&:to_s)                          # similar, but Proc-driven
+t.transform(:foo => :upcase, ...)            # specific-attrs tranformation
+t.transform([:to_s, :upcase])                # chain-transformation
 r.union(right)                               # relational union
+r.where(predicate)                           # alias for restrict(predicate)
 ```
 
-## Who is behind Bmg?
+## How is this different?
+
+### ... from similar libraries?
+
+1. The libraries you probably know (Sequel, Arel, SQLAlchemy, Korma, jOOQ,
+   etc.) do not implement a genuine relational algebra. Their support for
+   chaining relational operators is thus limited (restricting your expression
+   power and/or raising errors and/or outputting wrong or counterintuitive
+   SQL code). Bmg **always** allows chaining operators. If it does not, it's
+   a bug.
+
+   For instance the expression below is 100% valid in Bmg. The last where
+   clause applies to the result of the summarize (while SQL requires a `HAVING`
+   clause, or a `SELECT ... FROM (SELECT ...) r`).
+
+      ```ruby
+      relation
+        .where(...)
+        .union(...)
+        .summarize(...)   # aka group by
+        .where(...)
+      ```
+
+2. Bmg supports in memory relations, json relations, csv relations, SQL
+   relations and so on. It's not tight to SQL generation, and supports
+   queries accross multiple data sources.
+
+3. Bmg makes a best effort to optimize queries, simplifying both generated
+   SQL code (low-level accesses to datasources) and in-memory operations.
+
+4. Bmg supports various *structuring* operators (group, image, autowrap,
+   autosummarize, etc.) and allows building 'non flat' relations.
+
+5. Bmg can use full ruby power when that helps (e.g. regular expressions in
+   WHERE clauses or ruby code in EXTEND clauses). This may prevent Bmg from
+   delegating work to underlying data sources (e.g. SQL server) and should
+   therefore be used with care though.
+
+### ... from Alf?
+
+If you use Alf (or used it in the past), below are the main differences between
+Bmg and Alf. Bmg has NOT been written to be API-compatible with Alf and will
+probably never be.
+
+1. Bmg's implementation is much simpler than Alf and uses no ruby core
+   extention.
+
+2. We are confident using Bmg in production. Systematic inspection of query
+   plans is advised though. Alf was a bit too experimental to be used on
+   (critical) production systems.
+
+3. Alf exposes a functional syntax, command line tool, restful tools and
+   many more. Bmg is limited to the core algebra, main Relation abstraction
+   and SQL generation.
 
-Bernard Lambeau (bernard@klaro.cards) is Alf & Bmg main engineer & maintainer.
+4. Bmg is less strict regarding conformance to relational theory, and
+   may actually expose non relational features (such as support for null,
+   left_join operator, etc.). Sharp tools hurt, use them with care.
+
+5. Unlike Alf::Relation instances of Bmg::Relation capture query-trees, not
+   values. Currently two instances `r1` and `r2` are not equal even if they
+   define the same mathematical relation. As a consequence joining on
+   relation-valued attributes does not work as expected in Bmg until further
+   notice.
+
+6. Bmg does not implement all operators documented on try-alf.org, even if
+   we plan to eventually support most of them.
+
+7. Bmg has a few additional operators that prove very useful on real
+   production use cases: prefix, suffix, autowrap, autosummarize, left_join,
+   rxmatch, etc.
+
+8. Bmg optimizes queries and compiles them to SQL on the fly, while Alf was
+   building an AST internally first. Strictly speaking this makes Bmg less
+   powerful than Alf since optimizations cannot be turned off for now.
+
+## Contribute
+
+Please use github issues and pull requests for all questions, bug reports,
+and contributions. Don't hesitate to get in touch with us with an early code
+spike if you plan to add non trivial features.
+
+## Licence
+
+This software is distributed by Enspirit SRL under a MIT Licence. Please
+contact Bernard Lambeau (blambeau@gmail.com) with any question.
 
 Enspirit (https://enspirit.be) and Klaro App (https://klaro.cards) are both
 actively using and contributing to the library.
-
-Feel free to contact us for help, ideas and/or contributions. Please use github
-issues and pull requests if possible if code is involved.

data/lib/bmg.rb

@@ -1,6 +1,7 @@
 require 'path'
 require 'predicate'
 require 'forwardable'
+require 'set'
 module Bmg
 
   def in_memory(enumerable, type = Type::ANY)
@@ -8,6 +9,11 @@ module Bmg
   end
   module_function :in_memory
 
+  def text_file(path, options = {}, type = Type::ANY)
+    Reader::TextFile.new(type, path, options).spied(main_spy)
+  end
+  module_function :text_file
+
   def csv(path, options = {}, type = Type::ANY)
     Reader::Csv.new(type, path, options).spied(main_spy)
   end
@@ -38,11 +44,13 @@ module Bmg
   require_relative 'bmg/operator'
 
   require_relative 'bmg/reader'
+  require_relative 'bmg/writer'
 
   require_relative 'bmg/relation/empty'
   require_relative 'bmg/relation/in_memory'
   require_relative 'bmg/relation/spied'
   require_relative 'bmg/relation/materialized'
+  require_relative 'bmg/relation/proxy'
 
   # Deprecated
   Leaf = Relation::InMemory

data/lib/bmg/algebra.rb

@@ -172,6 +172,16 @@ module Bmg
     end
     protected :_summarize
 
+    def transform(transformation = nil, options = {}, &proc)
+      transformation, options = proc, (transformation || {}) unless proc.nil?
+      _transform(self.type.transform(transformation, options), transformation, options)
+    end
+
+    def _transform(type, transformation, options)
+      Operator::Transform.new(type, self, transformation, options)
+    end
+    protected :_transform
+
     def union(other, options = {})
       return self if other.is_a?(Relation::Empty)
       _union self.type.union(other.type), other, options

data/lib/bmg/algebra/shortcuts.rb

@@ -2,6 +2,14 @@ module Bmg
   module Algebra
     module Shortcuts
 
+      def where(predicate)
+        restrict(predicate)
+      end
+
+      def exclude(predicate)
+        restrict(!Predicate.coerce(predicate))
+      end
+
       def rxmatch(attrs, matcher, options = {})
         predicate = attrs.inject(Predicate.contradiction){|p,a|
           p | Predicate.match(a, matcher, options)

data/lib/bmg/operator.rb

@@ -46,4 +46,5 @@ require_relative 'operator/rename'
 require_relative 'operator/restrict'
 require_relative 'operator/rxmatch'
 require_relative 'operator/summarize'
+require_relative 'operator/transform'
 require_relative 'operator/union'

data/lib/bmg/operator/transform.rb

@@ -0,0 +1,57 @@
+module Bmg
+  module Operator
+    #
+    # Transform operator.
+    #
+    # Transforms existing attributes through computations
+    #
+    # Example:
+    #
+    #   [{ a: 1 }] transform { a: ->(t){ t[:a]*2 } } => [{ a: 4 }]
+    #
+    class Transform
+      include Operator::Unary
+
+      DEFAULT_OPTIONS = {}
+
+      def initialize(type, operand, transformation, options = {})
+        @type = type
+        @operand = operand
+        @transformation = transformation
+        @options = DEFAULT_OPTIONS.merge(options)
+      end
+
+    protected
+
+      attr_reader :transformation
+
+    public
+
+      def each
+        t = transformer
+        @operand.each do |tuple|
+          yield t.call(tuple)
+        end
+      end
+
+      def to_ast
+        [ :transform, operand.to_ast, transformation.dup ]
+      end
+
+    protected ### optimization
+
+    protected ### inspect
+
+      def args
+        [ transformation ]
+      end
+
+    private
+
+      def transformer
+        @transformer ||= TupleTransformer.new(transformation)
+      end
+
+    end # class Transform
+  end # module Operator
+end # module Bmg

data/lib/bmg/reader.rb

@@ -9,3 +9,4 @@ module Bmg
 end
 require_relative "reader/csv"
 require_relative "reader/excel"
+require_relative "reader/text_file"

data/lib/bmg/reader/text_file.rb

@@ -0,0 +1,56 @@
+module Bmg
+  module Reader
+    class TextFile
+      include Reader
+
+      DEFAULT_OPTIONS = {
+        strip: true,
+        parse: nil
+      }
+
+      def initialize(type, path, options = {})
+        options = { parse: options } if options.is_a?(Regexp)
+        @path = path
+        @options = DEFAULT_OPTIONS.merge(options)
+        @type = infer_type(type)
+      end
+      attr_reader :path, :options
+
+    public # Relation
+
+      def each
+        path.each_line.each_with_index do |text, line|
+          text = text.strip if strip?
+          parsed = parse(text)
+          yield({line: 1+line}.merge(parsed)) if parsed
+        end
+      end
+
+    private
+
+      def infer_type(base)
+        return base unless base == Bmg::Type::ANY
+        attr_list = if rx = options[:parse]
+          [:line] + rx.names.map(&:to_sym)
+        else
+          [:line, :text]
+        end
+        base
+          .with_attrlist(attr_list)
+          .with_keys([[:line]])
+      end
+
+      def strip?
+        options[:strip]
+      end
+
+      def parse(text)
+        return { text: text } unless rx = options[:parse]
+        if match = rx.match(text)
+          TupleAlgebra.symbolize_keys(match.named_captures)
+        end
+      end
+
+    end # class TextFile
+  end # module Reader
+end # module Bmg

data/lib/bmg/relation.rb

@@ -17,6 +17,16 @@ module Bmg
       self
     end
 
+    def type
+      Bmg::Type::ANY
+    end
+
+    def with_type(type)
+      dup.tap{|r|
+        r.type = type
+      }
+    end
+
     def with_typecheck
       dup.tap{|r|
         r.type = r.type.with_typecheck
@@ -105,6 +115,20 @@ module Bmg
       to_a.to_json(*args, &bl)
     end
 
+    # Writes the relation data to CSV.
+    #
+    # `string_or_io` and `options` are what CSV::new itself
+    # recognizes, default options are CSV's.
+    #
+    # When no string_or_io is used, the method uses a string.
+    #
+    # The method always returns the string_or_io.
+    def to_csv(options = {}, string_or_io = nil, preferences = nil)
+      options, string_or_io = {}, options unless options.is_a?(Hash)
+      string_or_io, preferences = nil, string_or_io if string_or_io.is_a?(Hash)
+      Writer::Csv.new(options, preferences).call(self, string_or_io)
+    end
+
     # Converts to an sexpr expression.
     def to_ast
       raise "Bmg is missing a feature!"

data/lib/bmg/relation/proxy.rb

@@ -0,0 +1,63 @@
+module Bmg
+  module Relation
+    #
+    # This module can be used to create typed collection on top
+    # of Bmg relations. Algebra methods will be delegated to the
+    # decorated relation, and results wrapped in a new instance
+    # of the class.
+    #
+    module Proxy
+
+      def initialize(relation)
+        @relation = relation
+      end
+
+      def method_missing(name, *args, &bl)
+        if @relation.respond_to?(name)
+          res = @relation.send(name, *args, &bl)
+          res.is_a?(Relation) ? _proxy(res) : res
+        else
+          super
+        end
+      end
+
+      def respond_to?(name, *args)
+        @relation.respond_to?(name) || super
+      end
+
+      [
+        :extend
+      ].each do |name|
+        define_method(name) do |*args, &bl|
+          res = @relation.send(name, *args, &bl)
+          res.is_a?(Relation) ? _proxy(res) : res
+        end
+      end
+
+      [
+        :one,
+        :one_or_nil
+      ].each do |meth|
+        define_method(meth) do |*args, &bl|
+          res = @relation.send(meth, *args, &bl)
+          res.nil? ? nil : _proxy_tuple(res)
+        end
+      end
+
+      def to_json(*args, &bl)
+        @relation.to_json(*args, &bl)
+      end
+
+    protected
+
+      def _proxy(relation)
+        self.class.new(relation)
+      end
+
+      def _proxy_tuple(tuple)
+        tuple
+      end
+
+    end # module Proxy
+  end # class Relation
+end # module Bmg

data/lib/bmg/relation/spied.rb

@@ -24,7 +24,7 @@ module Bmg
       protected :type=
 
       def each(&bl)
-        spy.call(self)
+        spy.call(self) if bl
         operand.each(&bl)
       end
 

data/lib/bmg/support.rb

@@ -1,2 +1,4 @@
 require_relative 'support/tuple_algebra'
+require_relative 'support/tuple_transformer'
 require_relative 'support/keys'
+require_relative 'support/output_preferences'

data/lib/bmg/support/keys.rb

@@ -7,6 +7,10 @@ module Bmg
 
   public ## tools
 
+    def select(&bl)
+      Keys.new(@keys.select(&bl), false)
+    end
+
   public ## algebra
 
     def allbut(oldtype, newtype, butlist)

data/lib/bmg/support/output_preferences.rb

@@ -0,0 +1,44 @@
+module Bmg
+  class OutputPreferences
+
+    DEFAULT_PREFS = {
+      attributes_ordering: nil,
+      extra_attributes: :after
+    }
+
+    def initialize(options)
+      @options = DEFAULT_PREFS.merge(options)
+    end
+    attr_reader :options
+
+    def self.dress(arg)
+      return arg if arg.is_a?(OutputPreferences)
+      arg = {} if arg.nil?
+      new(arg)
+    end
+
+    def attributes_ordering
+      options[:attributes_ordering]
+    end
+
+    def extra_attributes
+      options[:extra_attributes]
+    end
+
+    def order_attrlist(attrlist)
+      return attrlist if attributes_ordering.nil?
+      index = Hash[attributes_ordering.each_with_index.to_a]
+      attrlist.sort{|a,b|
+        ai, bi = index[a], index[b]
+        if ai && bi
+          ai <=> bi
+        elsif ai
+          extra_attributes == :after ? -1 : 1
+        else
+          extra_attributes == :after ? 1 : -1
+        end
+      }
+    end
+
+  end # class OutputPreferences
+end # module Bmg

data/lib/bmg/support/tuple_algebra.rb

@@ -19,5 +19,11 @@ module Bmg
     end
     module_function :rename
 
+    def symbolize_keys(h)
+      return h if h.empty?
+      h.each_with_object({}){|(k,v),h| h[k.to_sym] = v }
+    end
+    module_function :symbolize_keys
+
   end # module TupleAlgebra
 end # module Bmg

data/lib/bmg/support/tuple_transformer.rb

@@ -0,0 +1,63 @@
+module Bmg
+  class TupleTransformer
+
+    def initialize(transformation)
+      @transformation = transformation
+    end
+
+    def self.new(arg)
+      return arg if arg.is_a?(TupleTransformer)
+      super
+    end
+
+    def call(tuple)
+      transform_tuple(tuple, @transformation)
+    end
+
+    def knows_attrlist?
+      @transformation.is_a?(Hash)
+    end
+
+    def to_attrlist
+      @transformation.keys
+    end
+
+    private
+
+      def transform_tuple(tuple, with)
+        case with
+        when Symbol, Proc, Regexp
+          tuple.each_with_object({}){|(k,v),dup|
+            dup[k] = transform_attr(v, with)
+          }
+        when Hash
+          with.each_with_object(tuple.dup){|(k,v),dup|
+            dup[k] = transform_attr(dup[k], v)
+          }
+        when Array
+          with.inject(tuple){|dup,on|
+            transform_tuple(dup, on)
+          }
+        else
+          raise ArgumentError, "Unexpected transformation `#{with.inspect}`"
+        end
+      end
+
+      def transform_attr(value, with)
+        case with
+        when Symbol
+          value.send(with)
+        when Regexp
+          m = with.match(value.to_s)
+          m.nil? ? m : m.to_s
+        when Proc
+          with.call(value)
+        when Hash
+          with[value]
+        else
+          raise ArgumentError, "Unexpected transformation `#{with.inspect}`"
+        end
+      end
+
+  end # module TupleTransformer
+end # module Bmg

data/lib/bmg/type.rb

@@ -241,6 +241,31 @@ module Bmg
       }
     end
 
+    def transform(transformation, options = {})
+      transformer = TupleTransformer.new(transformation)
+      if typechecked? && knows_attrlist? && transformer.knows_attrlist?
+        known_attributes!(transformer.to_attrlist)
+      end
+      keys = if options[:key_preserving]
+        self._keys
+      elsif transformer.knows_attrlist? && knows_keys?
+        touched_attrs = transformer.to_attrlist
+        keys = self._keys.select{|k| (k & touched_attrs).empty? }
+      else
+        nil
+      end
+      pred = if transformer.knows_attrlist?
+        attr_list = transformer.to_attrlist
+        predicate.and_split(attr_list).last
+      else
+        Predicate.tautology
+      end
+      dup.tap{|x|
+        x.keys = keys
+        x.predicate = pred
+      }
+    end
+
     def union(other)
       if typechecked? && knows_attrlist? && other.knows_attrlist?
         missing = self.attrlist - other.attrlist

data/lib/bmg/version.rb

@@ -1,8 +1,8 @@
 module Bmg
   module Version
     MAJOR = 0
-    MINOR = 17
-    TINY  = 4
+    MINOR = 18
+    TINY  = 0
   end
   VERSION = "#{Version::MAJOR}.#{Version::MINOR}.#{Version::TINY}"
 end

data/lib/bmg/writer.rb

@@ -0,0 +1 @@
+require_relative 'writer/csv'

data/lib/bmg/writer/csv.rb

@@ -0,0 +1,42 @@
+module Bmg
+  module Writer
+    class Csv
+      include Writer
+
+      DEFAULT_OPTIONS = {
+      }
+
+      def initialize(csv_options, output_preferences = nil)
+        @csv_options = DEFAULT_OPTIONS.merge(csv_options)
+        @output_preferences = OutputPreferences.dress(output_preferences)
+      end
+      attr_reader :csv_options, :output_preferences
+
+      def call(relation, string_or_io = nil)
+        require 'csv'
+        string_or_io, to_s = string_or_io.nil? ? [StringIO.new, true] : [string_or_io, false]
+        headers, csv = infer_headers(relation.type), nil
+        relation.each do |tuple|
+          if csv.nil?
+            headers = infer_headers(tuple) if headers.nil?
+            csv = CSV.new(string_or_io, csv_options.merge(headers: headers))
+          end
+          csv << headers.map{|h| tuple[h] }
+        end
+        to_s ? string_or_io.string : string_or_io
+      end
+
+    private
+
+      def infer_headers(from)
+        attrlist = if from.is_a?(Type) && from.knows_attrlist?
+          from.to_attrlist
+        elsif from.is_a?(Hash)
+          from.keys
+        end
+        attrlist ? output_preferences.order_attrlist(attrlist) : nil
+      end
+
+    end # class Csv
+  end # module Writer
+end # module Bmg

data/tasks/test.rake

@@ -6,17 +6,24 @@ namespace :test do
   desc "Runs unit tests"
   RSpec::Core::RakeTask.new(:unit) do |t|
     t.pattern = "spec/unit/**/test_*.rb"
-    t.rspec_opts = ["-Ilib", "-Ispec/unit", "--fail-fast", "--color", "--backtrace", "--format=progress"]
+    t.rspec_opts = ["-Ilib", "-Ispec/unit", "--color", "--backtrace", "--format=progress"]
   end
   tests << :unit
 
   desc "Runs integration tests"
   RSpec::Core::RakeTask.new(:integration) do |t|
     t.pattern = "spec/integration/**/test_*.rb"
-    t.rspec_opts = ["-Ilib", "-Ispec/integration", "--fail-fast", "--color", "--backtrace", "--format=progress"]
+    t.rspec_opts = ["-Ilib", "-Ispec/integration", "--color", "--backtrace", "--format=progress"]
   end
   tests << :integration
 
+  desc "Runs github regression tests"
+  RSpec::Core::RakeTask.new(:regression) do |t|
+    t.pattern = "spec/regression/**/test_*.rb"
+    t.rspec_opts = ["-Ilib", "-Ispec/regression", "--color", "--backtrace", "--format=progress"]
+  end
+  tests << :regression
+
   task :all => tests
 end
 

metadata

@@ -1,91 +1,91 @@
 --- !ruby/object:Gem::Specification
 name: bmg
 version: !ruby/object:Gem::Version
-  version: 0.17.4
+  version: 0.18.0
 platform: ruby
 authors:
 - Bernard Lambeau
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-23 00:00:00.000000000 Z
+date: 2021-03-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: predicate
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.4'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.4.0
+        version: 2.5.0
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.4'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.4.0
+        version: 2.5.0
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: path
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10'
-  type: :development
+        version: '2.0'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
-  name: rspec
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.6'
+        version: '13'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.6'
+        version: '13'
 - !ruby/object:Gem::Dependency
-  name: path
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '3.6'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '3.6'
 - !ruby/object:Gem::Dependency
   name: roo
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '2.7'
+        version: '2.8'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '2.7'
+        version: '2.8'
 - !ruby/object:Gem::Dependency
   name: sequel
   requirement: !ruby/object:Gem::Requirement
@@ -149,14 +149,17 @@ files:
 - lib/bmg/operator/shared/nary.rb
 - lib/bmg/operator/shared/unary.rb
 - lib/bmg/operator/summarize.rb
+- lib/bmg/operator/transform.rb
 - lib/bmg/operator/union.rb
 - lib/bmg/reader.rb
 - lib/bmg/reader/csv.rb
 - lib/bmg/reader/excel.rb
+- lib/bmg/reader/text_file.rb
 - lib/bmg/relation.rb
 - lib/bmg/relation/empty.rb
 - lib/bmg/relation/in_memory.rb
 - lib/bmg/relation/materialized.rb
+- lib/bmg/relation/proxy.rb
 - lib/bmg/relation/spied.rb
 - lib/bmg/sequel.rb
 - lib/bmg/sequel/ext.rb
@@ -258,9 +261,13 @@ files:
 - lib/bmg/summarizer/variance.rb
 - lib/bmg/support.rb
 - lib/bmg/support/keys.rb
+- lib/bmg/support/output_preferences.rb
 - lib/bmg/support/tuple_algebra.rb
+- lib/bmg/support/tuple_transformer.rb
 - lib/bmg/type.rb
 - lib/bmg/version.rb
+- lib/bmg/writer.rb
+- lib/bmg/writer/csv.rb
 - tasks/gem.rake
 - tasks/test.rake
 homepage: http://github.com/enspirit/bmg
@@ -282,7 +289,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.0.8
 signing_key: 
 specification_version: 4
 summary: Bmg is Alf's successor.