monolens 0.5.3 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bef48815edf1f5c767fd7e41ad3720046b4d6a7c
-  data.tar.gz: 366cfea423fca4d3a7aa7471a67fd19571016b50
+  metadata.gz: 836c097ee70c9517d08ce544cbd1cd9b600a846b
+  data.tar.gz: 1af2f7a32e00bef2e3d69a0451f94c367e83d5d6
 SHA512:
-  metadata.gz: 1e0ae29c6ab1c18b22d3a96c7455983116f5433ba537ddf8a4c6a8c5ef289eb562a3d2d955da6218eb7ccf27a34488405f3fb66798524b03943deb61498093b3
-  data.tar.gz: 8513b04a89c60ded94f11a7296b613bf23358515b4f5e035b933afa9259e1f4ad7c0cc282d1d0c96841feefcaf46743aad325ed877c7e4b476663cb079f281f3
+  metadata.gz: dd6e356d995c5e1d28b9f46491abec5a95ccaa8f1efaa463600dbe4b146a97ad5ef56ecb19b69be000c24f91a4bb74afde5cbf0ce15ebcda8fd9b7b262e72e53
+  data.tar.gz: e78d2ce7b2cfc47d83bd53ae179b1ae29d24c9edc78a131b1d2b085ab6a1d35cf2cdb3edccca256c58a937510b7d0de4440847a15b56e2d288ff2e5e9b0140a9

data/README.md

@@ -1,68 +1,60 @@
-# Monolens - Declarative data transformation scripts
+# Monolens - Declarative data transformations
 
-Monolens implements declarative data transformation
-pipelines. It is inspired by [Project Cambria](https://www.inkandswitch.com/cambria/)
-but is not as ambitious, and is not currently compatible with it.
+Declarative data transformations expressed as simple .yaml or
+.json files. They are great to
+
+- clean an Excel file
+- transform a .json file
+- transform data from a .csv file
+- upgrade a .yaml configuration
+- etc.
+
+Monolens let's you tackle those tasks with small programs that are
+simple, declarative, robust, secure, reusable and sharable.
 
 ## Features / Limitations
 
-* Allows defining common data transformations on scalars
-  (e.g. string, dates), objects and arrays.
 * Declarative & language agnostic
-* Safe (no path to code injection)
+* Allows transforming scalars (e.g. string, dates), objects and arrays.
+* Support for (simplified) jsonpath interpolation when defining objects
+* Support for macros (monolens is an homoiconic language)
+* Secure: not Turing Complete, no code injection, no RegExp DDoS
 
 * Requires ruby >= 2.6
-* There is no validation of lens files for now
-
-## Example
-
-Given the following input file, say `input.json`:
-
-```json
-[
-  {
-    "status": "open",
-    "body": " Hello world"
-  },
-  {
-    "status": "closed",
-    "body": " Foo bar baz"
-  }
-]
-```
+* Not reached 1.0 yet, still experimental
+
+## Documentation & Examples
+
+Please refer to the `documentation/` folder for a longer introduction,
+documentation of the stdlib, and documented use-cases:
+
+- [Introduction](./documentation/1-introduction.md)
+- [Standard library](./documentation/stdlib)
+- [Use cases](./documentation/use-cases)
+  - [Kubernetes data templates](./documentation/use-cases/data-templates/)
+  - [Migrating database seeds](./documentation/use-cases/data-transformation/)
+
+## Getting started
+
+### In shell
 
-The following monolens file, say `lens.yml`
-
-```yaml
----
-version: 1.0
-lenses:
-- array.map:
-  - object.transform:
-      status:
-      - str.upcase
-      body:
-      - str.strip
-  - object.rename:
-      body: description
 ```
+gem install monolens
+```
+
+Then:
 
-will generate the following result:
-
-```json
-[
-  {
-    "status": "OPEN",
-    "description": "Hello world"
-  },
-  {
-    "status": "CLOSED",
-    "description": "Foo bar baz"
-  }
-]
+```shell
+monolens --help
+monolens lens.yaml input.json
 ```
 
-In ruby:
+### In ruby
+
+```ruby
+# Gemfile
+gem 'monolens', '< 1.0'
+```
 
 ```ruby
 require 'monolens'
@@ -73,39 +65,13 @@ input  = JSON.parse(File.read('input.json'))
 result = lens.call(input)
 ```
 
-## Available lenses
+## Credits
 
-```
-core.dig          - Extract from the input value (object or array) using a path.
-core.chain        - Applies a chain of lenses to an input value
-core.mapping      - Converts the input value via a key:value mapping
-core.literal      - Returns a constant value takens as lens definition
-
-str.strip         - Remove leading and trailing spaces of an input string
-str.split         - Splits the input string as an array
-str.downcase      - Converts the input string to lowercase
-str.upcase        - Converts the input string to uppercase
-
-skip.null         - Aborts the current lens transformation if nil
-
-object.extend     - Adds key/value(s) to the input object
-object.rename     - Rename some keys of the input object
-object.transform  - Applies specific lenses to specific values of the input object
-object.keys       - Applies a lens to all keys of the input object
-object.values     - Applies a lens to all values of the input object
-object.select     - Builds an object by selecting key/values from the input object
-
-coerce.date       - Coerces the input value to a date
-coerce.datetime   - Coerces the input value to a datetime
-coerce.string     - Coerces the input value to a string (aka to_s)
-coerce.integer    - Coerces the input value to an integer
-
-array.compact     - Removes null from the input array
-array.join        - Joins values of the input array as a string
-array.map         - Apply a lens to each member of an Array
-
-check.notEmpty    - Throws an error if the input is null or empty
-```
+* Monolens is inspired by [Project Cambria](https://www.inkandswitch.com/cambria/)
+but is not as ambitious, and is not currently compatible with it.
+
+* The name of some lenses mimic Tutorial D / relational algebra (Date & Darwen).
+  See also [Bmg](https://github.com/enspirit/bmg)
 
 ## Public API
 

data/lib/monolens/command.rb

@@ -11,41 +11,51 @@ module Monolens
       @stdout = stdout
       @stderr = stderr
       @pretty = false
+      @enclose = []
+      @output_format = :json
+      @stream = false
+      @fail_strategy = 'fail'
+      @override = false
+      #
+      @input_file = nil
+      @use_stdin = false
     end
     attr_reader :argv, :stdin, :stdout, :stderr
-    attr_reader :pretty
+    attr_reader :pretty, :stream, :override
+    attr_reader :enclose_map, :fail_strategy
+    attr_reader :input_file, :use_stdin
 
     def self.call(argv, stdin = $stdin, stdout = $stdout, stderr = $stderr)
       new(argv, stdin, stdout, stderr).call
     end
 
     def call
-      lens, input = options.parse!(argv)
-      show_help_and_exit if lens.nil? || input.nil?
+      lens, @input_file = options.parse!(argv)
+      show_help_and_exit if lens.nil? || (@input_file.nil? && !use_stdin)
 
-      lens, input = read_file(lens), read_file(input)
+      lens_data, input = read_file(lens), read_input
+      lens = build_lens(lens_data)
       error_handler = ErrorHandler.new
-      lens = Monolens.lens(lens)
       result = lens.call(input, error_handler: error_handler)
 
       unless error_handler.empty?
         stderr.puts(error_handler.report)
       end
 
-      if result
-        output = if pretty
-          JSON.pretty_generate(result)
-        else
-          result.to_json
-        end
-
-        stdout.puts output
-      end
+      output_result(result) if result
     rescue Monolens::LensError => ex
       stderr.puts("[#{ex.location.join('/')}] #{ex.message}")
       do_exit(-2)
     end
 
+    def read_input
+      if use_stdin
+        JSON.parse(stdin.read)
+      else
+        read_file(@input_file)
+      end
+    end
+
     def read_file(file)
       case ::File.extname(file)
       when /json$/
@@ -87,10 +97,97 @@ module Monolens
           stdout.puts "Monolens v#{VERSION} - (c) Enspirit #{Date.today.year}"
           do_exit(0)
         end
+        opts.on('-m', '--map', 'Enclose the loaded lens inside an array.map') do
+          @enclose << :map
+        end
+        opts.on('-l', '--literal', 'Enclose the loaded lens inside core.literal') do
+          @enclose << :literal
+        end
+        opts.on('--on-error=STRATEGY', 'Apply a specific strategy on error') do |strategy|
+          @fail_strategy = strategy
+        end
+        opts.on('-ILIB', 'Add a folder to ruby load path') do |lib|
+          $LOAD_PATH.unshift(lib)
+        end
+        opts.on('-rLIB', 'Add a ruby require of a lib') do |lib|
+          require(lib)
+        end
+        opts.on( '--stdin', 'Takes input data from STDIN') do
+          @use_stdin = true
+        end
         opts.on('-p', '--[no-]pretty', 'Show version and exit') do |pretty|
           @pretty = pretty
         end
+        opts.on('-y', '--yaml', 'Print output in YAML') do
+          @output_format = :yaml
+        end
+        opts.on('-s', '--stream', 'Stream mode: output each result item separately') do
+          @stream = true
+        end
+        opts.on('-j', '--json', 'Print output in JSON') do
+          @output_format = :json
+        end
+        opts.on('--override', 'Write output back to the input file') do
+          @override = true
+        end
+      end
+    end
+
+    def build_lens(lens_data)
+      lens_data = @enclose.inject(lens_data) do |memo, lens_name|
+        case lens_name
+        when :map
+          {
+            'array.map' => {
+              'on_error' => ['handler', fail_strategy].compact,
+              'lenses' => memo,
+            }
+          }
+        when :literal
+          {
+            'core.literal' => {
+              'defn' => memo,
+            }
+          }
+        end
+      end
+      Monolens.lens(lens_data)
+    end
+
+    def output_result(result)
+      with_output_io do |io|
+        output = case @output_format
+        when :json
+          output_json(result, io)
+        when :yaml
+          output_yaml(result, io)
+        end
+      end
+    end
+
+    def with_output_io(&block)
+      if override
+        ::File.open(@input_file, 'w', &block)
+      else
+        block.call(stdout)
+      end
+    end
+
+    def output_json(result, io)
+      method = pretty ? :pretty_generate : :generate
+      if stream
+        fail!("Stream mode only works with an output Array") unless result.is_a?(::Enumerable)
+        result.each do |item|
+          io.puts JSON.send(method, item)
+        end
+      else
+        io.puts JSON.send(method, result)
       end
     end
+
+    def output_yaml(result, io)
+      output = stream ? YAML.dump_stream(*result) : YAML.dump(result)
+      io.puts output
+    end
   end
 end

data/lib/monolens/error.rb

@@ -1,6 +1,8 @@
 module Monolens
   class Error < StandardError
   end
+  class TypeError < Error
+  end
   class LensError < Error
     def initialize(message, location = [])
       super(message)

data/lib/monolens/file.rb

@@ -2,6 +2,12 @@ module Monolens
   class File
     include Lens
 
+    signature(Type::Any, Type::Any, {
+      version: [Type::Any, true],
+      macros: [Type::Map.of(Type::Name, Type::Any), false],
+      lenses: [Type::Lenses, true],
+    })
+
     def call(arg, world = {})
       option(:lenses).call(arg, world)
     end

data/lib/monolens/jsonpath.rb

@@ -0,0 +1,76 @@
+module Monolens
+  class Jsonpath
+    def self.one_detect_rx(symbol)
+      symbol = "\\" + symbol if symbol == '$'
+      %r{^#{symbol}([.\[][^\s]+)?$}
+    end
+
+    def self.interpolate_detect_rx(symbol)
+      symbol = "\\" + symbol if symbol == '$'
+      %r{#{symbol}[.(]}
+    end
+
+    def self.interpolate_rx(symbol)
+      %r{
+        #{symbol}
+        (
+          (\.([a-zA-Z0-9.-_\[\]])+)
+        |
+          (\([^)]+\))
+        )
+      }x.freeze
+    end
+
+    INTERPOLATE_RXS = {
+      '$' => interpolate_rx("\\" + '$'),
+      '<' => interpolate_rx('<'),
+    }
+
+    DEFAULT_OPTIONS = {
+      root_symbol: '$',
+      use_symbols: true,
+    }
+
+    def initialize(path, options = {})
+      @path = path
+      @options = DEFAULT_OPTIONS.merge(options)
+      @interpolate_rx = INTERPOLATE_RXS[@options[:root_symbol]]
+      raise ArgumentError, "Unknown root symbol #{@options.inspect}" unless @interpolate_rx
+    end
+
+    def self.one(path, input, options = {})
+      Jsonpath.new(path, options).one(input)
+    end
+
+    def self.interpolate(str, input, options = {})
+      Jsonpath.new('', options).interpolate(str, input)
+    end
+
+    def one(input)
+      use_symbols = @options[:use_symbols]
+
+      parts = @path
+        .gsub(/[.\[\]\(\)]/, ';')
+        .split(';')
+        .reject{|p| p.nil? || p.empty? || p == '$' || p == '<' }
+        .map{|p|
+          case p
+          when /^'[^']+'$/
+            use_symbols ? p[1...-1].to_sym : p[1...-1]
+          when /^\d+$/
+            p.to_i
+          else
+            use_symbols ? p.to_sym : p
+          end
+        }
+
+      parts.empty? ? input : input.dig(*parts)
+    end
+
+    def interpolate(str, input)
+      str.gsub(@interpolate_rx) do |path|
+        Jsonpath.one(path, input, @options)
+      end
+    end
+  end
+end

data/lib/monolens/lens/options.rb

@@ -3,22 +3,24 @@ module Monolens
     class Options
       include FetchSupport
 
-      def initialize(options)
-        @options = case options
-        when Hash
-          options.dup
-        else
-          { lenses: options }
-        end
-        actual, lenses = fetch_on(:lenses, @options)
-        @options[actual] = Monolens.lens(lenses) if actual && lenses
-        @options.freeze
+      def initialize(options, registry, signature)
+        raise ArgumentError if options.nil?
+        raise ArgumentError if registry.nil?
+        raise ArgumentError if signature.nil?
+
+        @signature = signature
+        @registry =  compile_macros(options, registry)
+        @options = @signature.dress_options(options, @registry)
       end
-      attr_reader :options
-      private :options
+      attr_reader :options, :registry
+      private :options, :registry
 
       NO_DEFAULT = Object.new.freeze
 
+      def lens(arg, registry = @registry)
+        registry.lens(arg)
+      end
+
       def fetch(key, default = NO_DEFAULT, on = @options)
         if on.key?(key)
           on[key]
@@ -36,6 +38,18 @@ module Monolens
       def to_h
         @options.dup
       end
+
+    private
+
+      def compile_macros(options, registry)
+        return registry unless options.is_a?(Hash)
+        return registry unless macros = options[:macros] || options['macros']
+
+        registry.fork('self').tap{|r|
+          r.define_namespace 'self', Macros.new(macros, registry)
+        }
+      end
+
     end
   end
 end

data/lib/monolens/lens/signature/missing.rb

@@ -0,0 +1,11 @@
+module Monolens
+  module Lens
+    class Signature
+      class Missing < Signature
+        def _dress_options(options, registry)
+          options
+        end
+      end
+    end
+  end
+end

data/lib/monolens/lens/signature.rb

@@ -0,0 +1,60 @@
+require_relative 'signature/missing'
+
+module Monolens
+  module Lens
+    class Signature
+      MISSING = Missing.new
+
+      def initialize(input, output, options)
+        @input = input
+        @output = output
+        @options = symbolize(options)
+      end
+
+      def dress_options(options, registry)
+        case options
+        when ::Hash
+          _dress_options(options.dup, registry)
+        when ::Array
+          dress_options({lenses: options}, registry)
+        when ::String
+          dress_options({lenses: [options]}, registry)
+        else
+          raise Error, "Invalid options `#{options.to_json}`"
+        end
+      end
+
+    private
+
+      def _dress_options(options, registry)
+        options = symbolize(options)
+        ks, ls = @options.keys, options.keys
+
+        extra = ls - ks
+        fail!("Invalid option `#{extra.first}`") unless extra.empty?
+
+        missing = (ks - ls).select{|name|
+          @options[name].last
+        }
+        fail!("Missing option `#{missing.first}`") unless missing.empty?
+
+        ls.each_with_object({}) do |name,memo|
+          type = @options[name].first
+          memo[name] = type.dress(options[name], registry) do |err|
+            fail!("Invalid option `#{name}`: #{err}")
+          end
+        end
+      end
+
+      def symbolize(options)
+        options.each_with_object({}) do |(k,v),memo|
+          memo[k.to_sym] = v
+        end
+      end
+
+      def fail!(message)
+        raise TypeError, message
+      end
+    end
+  end
+end

data/lib/monolens/lens.rb

@@ -1,13 +1,30 @@
 require_relative 'lens/fetch_support'
 require_relative 'lens/options'
 require_relative 'lens/location'
+require_relative 'lens/signature'
 
 module Monolens
   module Lens
     include FetchSupport
 
-    def initialize(options = {})
-      @options = Options.new(options)
+    module ClassMethods
+      def signature(input = nil, output = nil, options = {})
+        return @signature if input.nil?
+
+        @signature = Signature.new(input, output, options)
+      end
+    end
+
+    def self.included(by)
+      by.extend(ClassMethods)
+    end
+
+    def initialize(options, registry)
+      raise ArgumentError if options.nil?
+      raise ArgumentError unless registry.is_a?(Registry)
+      raise ArgumentError, "Missing signature on #{self.class}" unless self.class.signature
+
+      @options = Options.new(options, registry, self.class.signature)
     end
     attr_reader :options
 
@@ -18,6 +35,10 @@ module Monolens
 
   protected
 
+    def lens(*args)
+      options.lens(*args)
+    end
+
     def option(name, default = nil)
       @options.fetch(name, default)
     end
@@ -52,8 +73,8 @@ module Monolens
       fail!("Hash expected, got #{arg.class}", world)
     end
 
-    def is_enumerable!(arg, world)
-      return if arg.is_a?(::Enumerable)
+    def is_array!(arg, world)
+      return if arg.is_a?(Type::Array)
 
       fail!("Enumerable expected, got #{arg.class}", world)
     end

data/lib/monolens/macros.rb

@@ -0,0 +1,28 @@
+module Monolens
+  class Macros
+    def initialize(macros, registry)
+      @macros = macros
+      @registry = registry
+    end
+
+    def factor_lens(namespace_name, lens_name, options, registry)
+      if defn = @macros[lens_name]
+        instantiate_macro(defn, options)
+      else
+        raise Error, "No such lens #{[namespace_name, lens_name].join('.')}"
+      end
+    end
+
+  private
+
+    def instantiate_macro(defn, options)
+      instantiated = Monolens::Core.literal({
+        defn: defn,
+        jsonpath: {
+          root_symbol: '<'
+        }
+      }, @registry).call(options)
+      @registry.lens(instantiated)
+    end
+  end
+end

data/lib/monolens/namespace.rb

@@ -0,0 +1,11 @@
+module Monolens
+  module Namespace
+    def factor_lens(namespace_name, lens_name, options, registry)
+      if private_method_defined?(lens_name, false)
+        send(lens_name, options, registry)
+      else
+        raise Error, "No such lens #{[namespace_name, lens_name].join('.')}"
+      end
+    end
+  end
+end