raka 0.2.3

data/lib/lang/r/io.R

@@ -0,0 +1,113 @@
+suppressPackageStartupMessages(library(RPostgreSQL))
+suppressPackageStartupMessages(library(ggplot2))
+suppressPackageStartupMessages(library(gridExtra))
+
+# input
+# -----
+file_input <- function(fileName, coltypes = NA) {
+    read.table(fileName, header = T, sep = ',', colClasses = coltypes)
+}
+
+join_file_input <- function(namesstr, key) {
+    files <- strsplit(namesstr, split = '\\+')[[1]]
+    dataList <- lapply(files, (function(f) fileInput(f)))
+    do.call(merge, c(dataList, by = key))
+}
+
+init_table_input <- function(conn_args, schema) {
+    function(name, cols = '*', where = 'true') {
+        colstr <- paste(cols, collapse = ',')
+
+        sql <- sprintf("SELECT %s FROM %s WHERE %s", colstr, name, where)
+        init_sql_input(conn_args, schema)(sql)
+    }
+}
+
+init_sql_input <- function(conn_args, schema) {
+    function(sql) {
+        conn <- do.call(dbConnect, c(dbDriver('PostgreSQL'), conn_args))
+        on.exit(dbDisconnect(conn))
+        dbGetQuery(conn, paste('SET search_path TO', paste(c(schema, 'public'), sep = ',')))
+        buffer <- dbSendQuery(conn, sql)
+        fetch(buffer, n=-1)
+    }
+}
+
+create_parents <- function(output) {
+    dir.create(dirname(output), showWarnings = FALSE, recursive = TRUE)
+}
+
+# Output
+# ------
+# TODO consider deprecating this, perhaps more specific ones like graph_output is still useful
+auto_output <- function(report, ...) {
+    actual <- switch(class(report)[1],
+                     gg = ggplot_output,
+                     data.frame = csv_output,
+                     print)
+    actual(report, ...)
+}
+
+rplot_output <- function(f, output, size = c(8, 8)) {
+    create_parents(output)
+    size <- ceiling(size / 2.54 * 300 / 96) # convert cm to inch
+    par(mar = c(0,0,0,0))
+    # use pdf since png can be very weird, e.g. radarplot
+    pdf(output, width = size[1], height = size[2], pointsize = 12)
+    f()
+    dev.off()
+}
+
+# use facet or latex subfigure, do not handle list anymore
+ggplot_output <- function(report, output, size = c(15, 15), fontScale = 1, compact = F) {
+    create_parents(output)
+    if (compact) {
+        margin <- 1
+    } else {
+        margin <- 5
+    }
+
+    report <- report + theme_bw() +
+        theme(axis.title = element_text(size = 13 * fontScale),
+              axis.text = element_text(size = 11 * fontScale),
+              legend.title = element_text(size = 13 * fontScale),
+              legend.text = element_text(size = 11 * fontScale),
+              legend.key.size = unit(13 * fontScale, 'pt'),
+              legend.margin = unit(0, "cm"),
+              legend.background = element_rect(fill = alpha('white', 0)),
+              plot.margin = unit(c(margin, margin, 0, 0),"mm"))
+
+    width <- size[1]
+    height <- size[2]
+
+    ggsave(report, filename = output,
+           dpi = 300, units = 'cm', width = width, height = height, limitsize = FALSE)
+}
+
+csv_output <- function(report, output) {
+    create_parents(output)
+    write.table(format(report, digits = 4), file = toString(output), row.names = FALSE,
+                sep = ',', quote = FALSE)
+}
+
+txt_output <- function(report, output) {
+    create_parents(output)
+    write.table(format(report, digits = 4), file = toString(output), row.names = FALSE,
+                sep = ',', quote = FALSE)
+}
+
+init_table_output <- function(conn_args, schema) {
+    function(report, output, placeholder = NULL) {
+        conn <- do.call(dbConnect, c(dbDriver('PostgreSQL'), conn_args))
+        on.exit(dbDisconnect(conn))
+        dbGetQuery(conn, paste('SET search_path TO', paste(c(schema, 'public'), sep = ',')))
+
+        if (dbExistsTable(conn, output)) {
+            dbRemoveTable(conn, output)
+        }
+        dbWriteTable(conn, output, report, row.names = F)
+        if (!is.null(placeholder)) {
+            write.table(data.frame(), file = placeholder, col.names = FALSE)
+        }
+    }
+}

data/lib/lang/shell/impl.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require_relative '../../protocol'
+
+# shell(bash) protocol
+class Shell
+  # @implements LanguageImpl
+  def build(code, _)
+    ['set -e', code].join "\n"
+  end
+
+  def run_script(env, fname, _)
+    run_cmd(env, "bash #{fname}")
+  end
+end
+
+creator :shell, Shell

data/lib/output_type.rb

@@ -0,0 +1,36 @@
+class OutputType
+	class Default
+		# File name is for checking the existence of outputs
+		def file_name(rule_name, scope = '')
+			scope.to_s + (scope.empty? ? '' : '/') + rule_name
+		end
+
+		# Real name is for further processing, like schema.table_name in database
+		# defaultly the file is the output, but when the file is merely a placeholder
+		# the two will differ
+		def real_name(rule_name, scope = '')
+			file_name
+		end
+
+		def ext
+			res = self.class.to_s
+			res[0] = res[0].downcase
+			res
+		end
+	end
+
+	class Table < Default
+		def real_name(file_name, scope = '')
+			stem = file_name.gsub(/\.table$/, '')
+			scope.to_s + (scope.empty? ? '' : '.') + stem
+		end
+	end
+
+	def create(sym)
+		self.class.const_get(sym.capitalize).new || Default.new
+	end
+
+	def self.parse_option(opt)
+		opt.map { |obj| obj.class == Symbol ? create(obj) : obj }
+	end
+end

data/lib/protocol.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+require 'open3'
+
+def remove_common_indent(code)
+  code.gsub(/^#{code.scan(/^[ \t]+(?=\S)/).min}/, '')
+end
+
+def create_tmp(content)
+  tmpfile = "/tmp/#{SecureRandom.uuid}"
+
+  File.open(tmpfile, 'w') do |f|
+    f.write content
+  end
+
+  tmpfile
+end
+
+# protocol conforms the interface:
+#
+# call(env, task) resolve
+#
+# call :: rake's main -> dsl task(see compiler) -> void
+# resolve :: str -> str
+
+# There are two methods to provide code to a language protocol, either a string literal
+# OR a ruby block. Cannot choose both.
+class LanguageProtocol
+  attr_writer :block
+
+  private
+
+  def wrap_template(code)
+    @script_template.gsub(/^\<code\>$/, code)
+  end
+
+  public
+
+  def initialize(language_impl, script_template: '<code>')
+    # contextual variables, will be passed later
+    @impl = language_impl
+    @script_template = script_template
+    @block = nil
+    @text = nil
+  end
+
+  # for syntax sugar like shell* <code text>
+  def *(text)
+    @text = text
+    [self]
+  end
+
+  # a block::str -> str should be given to resolve the bindings in code text
+  def call(env, task)
+    code = yield @text if @text
+    code = @block.call(task) if @block # do not resolve
+
+    env.logger.debug code
+    script_text = @impl.build(wrap_template(remove_common_indent(code)), task)
+    temp_script = create_tmp(script_text)
+    @impl.run_script env, temp_script, task
+    env.logger.debug script_text
+  end
+end
+
+# A special language protocol, just a wrapper for action, pass block instead of
+# string to execute
+# named RubyP to avoid name collision
+class RubyP
+  def initialize(&block)
+    @block = block
+  end
+
+  def call(_, task, &resolve)
+    @block.call(task, &resolve)
+    FileUtils.touch(task.name)
+  end
+end
+
+# use run instead of "ruby" to avoid name collision
+def run(&block)
+  [RubyP.new(&block)]
+end
+
+# helper functions to implement LanguageImpl
+def run_cmd(env, cmd)
+  env.logger.debug(cmd)
+  Open3.popen3(cmd) do |_stdin, stdout, stderr, _thread|
+    env.logger.debug(stdout.read)
+    err = stderr.read
+    env.logger.info(err) unless err.empty?
+  end
+end
+
+def pick_kwargs(klass, kwargs)
+  param_ref = klass.instance_method(:initialize).parameters
+    .select { |arg| arg.size == 2 && (arg[0] == :key || arg[0] == :keyreq) }
+    .map { |arg| arg[1] }
+  kwargs.select do |key, _value|
+    param_ref.include? key
+  end
+end
+
+def creator(name, klass, global_defaults = {})
+  global_config = global_defaults
+  define_singleton_method name do |*args, **kwargs, &block|
+    # pick keyword arguments for klass
+    kwargs = global_config.merge kwargs
+    impl = klass.new(*args, **pick_kwargs(klass, kwargs))
+    proto = LanguageProtocol.new(impl, **pick_kwargs(LanguageProtocol, kwargs))
+    if block
+      proto.block = block
+      [proto]
+    else
+      proto.define_singleton_method :config do |**config|
+        global_config = global_defaults.merge config
+      end
+      proto # if no block, allow configure or waiting for * to add code text
+    end
+  end
+end

data/lib/raka.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require 'logger'
+
+require_relative './compile'
+require_relative './protocol'
+require_relative './token'
+
+# initialize raka
+class Raka
+  Pattern = Pattern
+  attr_reader :logger
+
+  def create_logger(level)
+    @env.define_singleton_method :logger do
+      logger = Logger.new(STDOUT)
+      logger.level = level
+      logger
+    end
+  end
+
+  def define_token_creator(ext, ext_alias = nil)
+    # closures
+    env = @env
+    options = @options
+    scopes = @scopes
+    @env.define_singleton_method(ext_alias || ext) do |*args|
+      # Here the compiler are bound with @options so that when we change @options
+      # using methods like scope in Rakefile, the subsequent rules defined will honor
+      # the new settings
+      # clone to fix the scopes when defining rule
+      inline_scope_pattern = !args.empty? ? args[0] : nil
+      Token.new(
+        DSLCompiler.new(env, options), Context.new(ext, scopes.clone),
+        [], inline_scope_pattern
+      )
+    end
+  end
+
+  def initialize(env, options)
+    @env = env
+    defaults = {
+      output_types: [:csv], input_types: [],
+      type_aliases: {},
+      scopes: [],
+      lang: ['lang/shell'],
+      user_lang: []
+    }
+    @options = options = OpenStruct.new(defaults.merge(options))
+
+    create_logger options.log_level || (ENV['LOG_LEVEL'] || Logger::INFO).to_i
+
+    @options.input_types |= @options.output_types # any output can be used as intermediate
+    # specify root of scopes in options, scopes will append to each root
+    @scopes = options.scopes.empty? ? [] : [options.scopes]
+    @options.lang.each { |path| load File::join(File::dirname(__FILE__), "#{path}/impl.rb") }
+    @options.user_lang.each { |path| load path.to_s + '.rb' }
+
+    # These are where the dsl starts
+    @options.output_types.each do |ext|
+      define_token_creator(ext, @options.type_aliases[ext])
+    end
+  end
+
+  def scope(*names, &block)
+    @scopes.push(names)
+    block.call
+    @scopes.pop
+  end
+end