raka 0.3.18 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3f74a12c7ba0a07e2dd9b3f07fe4496b3829295151bfe29f5bdf6a88ae8a0ce9
-  data.tar.gz: 9f0d169ba48237872b1564059abb2906c8d2960209d9f6972ce7985d0c9f2e8e
+  metadata.gz: 788df7b5953f68a917e57a8508b51aa89e1ca404b91480bcbd132b3cd7666905
+  data.tar.gz: 00123c1ab2294b5e274942e65ec507561834b259b45e4ad636e3ffe4f269e996
 SHA512:
-  metadata.gz: 7d301a5e179ff2ca7f5182fe8b94a971c897b7893ea1fb3c12b77cfc81846ce310e5c9dd80b6407517b72a756a18ddbb409c73619c7358b9c71166655364cbba
-  data.tar.gz: 95b61083e26783b07a63f3cd3b61e8088379cabf338e00ddeb8287d13fa54964bf7faa5fae8b3010e622b138f13bfc1a06f9fa93c69e417dc2614abe2f81ce2a
+  metadata.gz: 68cac774b3aa673896b68201e6e3667075cf9fc53d943c9f48aac130c5bb801903ce15f5e68523d46c53dc98c2d9df1b23210f84a0e7c628653c40861418d2cb
+  data.tar.gz: 3216c9889ca95b84814ca69c85f42c1f972c0e3093baae37338b09c9fce325064df5ed60d2709334e0250e0eabdd357cbf2cf23088c2dadeaed50fe2ea416369

data/VERSION

@@ -1 +1 @@
-0.3.18
+0.4.0

data/lib/raka/compile.rb

@@ -34,7 +34,6 @@ class DSLCompiler
       name: name,
       deps: deps,
       deps_str: deps.join(','),
-      deps_stem_str: (deps.map { |d| stem(d) }).join(','),
       dep_scopes: deps.map { |d| File.dirname(d) },
       input: deps.first || '',
       task: task
@@ -78,7 +77,6 @@ class DSLCompiler
       .gsub('$^', task.deps_str)
       .gsub('$<', task.input || '')
       .gsub('$(deps)', task.deps_str)
-      .gsub('$(deps_stem)', task.deps_stem_str)
       .gsub('$(input)', task.input || '')
 
     protect_percent_symbol text do |safe_text|

data/lib/raka/lang/duckdb/impl.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require_relative '../../protocol'
+
+def bash(env, cmd)
+  code = remove_common_indent(
+    %(set -e
+      set -o pipefail
+
+      #{cmd}
+    )
+  )
+  env.send :sh, 'bash ' + create_tmp(code)
+end
+
+# DuckDB protocol with two modes:
+# 1. Persistent mode: operations on .db file with CREATE TABLE
+# 2. Ad-hoc mode: parquet in/out using COPY operations
+class Duckdb
+  def initialize(database: nil, params: {})
+    @params = params
+    @database = database
+    @mode = @database ? :persistent : :adhoc
+  end
+
+  def duckdb_cmd
+    case @mode
+    when :persistent
+      "duckdb #{@database}"
+    when :adhoc
+      'duckdb'
+    end
+  end
+
+  def build(code, _task)
+    # Replace parameter placeholders
+    processed_code = code
+    (@params || {}).each do |key, value|
+      processed_code = processed_code.gsub("$#{key}", "'#{value}'")
+    end
+
+    case @mode
+    when :persistent
+      "DROP TABLE IF EXISTS :_name_; CREATE TABLE :_name_ AS (#{processed_code});"
+    when :adhoc
+      "COPY (#{processed_code}) TO ':output:' (FORMAT PARQUET);"
+    end
+  end
+
+  def run_script(env, fname, task)
+    case @mode
+    when :persistent
+      bash env, %(
+      #{duckdb_cmd} -c "$(cat #{fname} | sed 's|:_name_|#{task.output_stem}|g')" | tee #{fname}.log
+      echo "#{@database}" > #{task.name}
+      )
+    when :adhoc
+      bash env, %(
+      cat #{fname} | sed 's|:output:|#{task.name}|g' | #{duckdb_cmd} | tee #{fname}.log
+      )
+    end
+  end
+end
+
+creator :duckdb, Duckdb

data/lib/raka/lang/psql/impl.rb

@@ -24,9 +24,10 @@ class Psql
   end
 
   # 1. do not add required argument here, so psql.config will work or we can only use psql(conn: xxx).config
-  def initialize(conn: nil, create: 'mview', params: {})
+  def initialize(conn: nil, create: 'mview', schema: '', params: {})
     @create = create
     @params = params
+    @schema = schema
     @conn = conn
   end
 
@@ -36,10 +37,10 @@ class Psql
 
     if @create.to_s == 'table'
       'DROP TABLE IF EXISTS :_schema_:_name_;' \
-        'CREATE TABLE :_schema_:_name_ AS (' + code + ');'
+        'CREATE TABLE :schema:_name_ AS (' + code + ');'
     elsif @create.to_s == 'mview'
       'DROP MATERIALIZED VIEW IF EXISTS :_schema_:_name_;' \
-        'CREATE MATERIALIZED VIEW :_schema_:_name_ AS (' + code + ');'
+        'CREATE MATERIALIZED VIEW :schema:_name_ AS (' + code + ');'
     else
       code
     end
@@ -47,12 +48,11 @@ class Psql
 
   def run_script(env, fname, task)
     param_str = (@params || {}).map { |k, v| "-v #{k}=\"#{v}\"" }.join(' ')
-    schema = task.rule_scopes.join('__')
-    out_schema = (task.rule_scopes + (task.target_scope.nil? ? [] : [task.target_scope])).join('__')
+    schema = @schema.empty? ? task.rule_scopes.join('__') : @schema
 
     bash env, %(
     #{sh_cmd(schema)} #{param_str} -v _name_=#{task.output_stem} \
-      -v _schema_=#{out_schema.empty? ? '' : out_schema + '.'} -f #{fname} | tee #{fname}.log
+      -v _schema_=#{schema.empty? ? '' : schema + '.'} -f #{fname} | tee #{fname}.log
     mv #{fname}.log #{task.name}
     )
   end

data/lib/raka.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'logger'
+require 'ostruct'
 
 require_relative './raka/compile'
 require_relative './raka/protocol'

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: raka
 version: !ruby/object:Gem::Version
-  version: 0.3.18
+  version: 0.4.0
 platform: ruby
 authors:
 - yarray
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-03-21 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -16,98 +15,98 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 13.0.0
+        version: 13.3.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 13.0.0
+        version: 13.3.0
 - !ruby/object:Gem::Dependency
-  name: rdoc
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 6.3.1
+        version: 2.1.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 6.3.1
+        version: 2.1.0
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: juwelier
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.1.0
+        version: 2.4.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.1.0
+        version: 2.4.0
 - !ruby/object:Gem::Dependency
-  name: juwelier
+  name: rdoc
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 2.1.0
+        version: 6.3.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 2.1.0
+        version: 6.3.1
 - !ruby/object:Gem::Dependency
-  name: test-unit
+  name: reek
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.3.0
+        version: '6.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.3.0
+        version: '6.0'
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.80.0
+        version: 1.79.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.80.0
+        version: 1.79.0
 - !ruby/object:Gem::Dependency
-  name: reek
+  name: test-unit
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '6.0'
+        version: 3.7.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '6.0'
+        version: 3.7.0
 description: An extensible, concise and light weight DSL on Rake to automate data
   processing tasks
 email: '08to09@gmail.com'
@@ -125,6 +124,7 @@ files:
 - lib/raka.rb
 - lib/raka/compile.rb
 - lib/raka/interface.rbs
+- lib/raka/lang/duckdb/impl.rb
 - lib/raka/lang/psql/impl.rb
 - lib/raka/lang/python/impl.rb
 - lib/raka/lang/r/impl.rb
@@ -136,7 +136,6 @@ homepage: http://github.com/yarray/raka
 licenses:
 - MIT
 metadata: {}
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -151,8 +150,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key:
+rubygems_version: 3.6.9
 specification_version: 4
 summary: Rake for data
 test_files: []