setl_tool 1.0.1

data/lib/setl_tool/extract.rb

@@ -0,0 +1,51 @@
+# Licensed to you under one or more contributor license agreements.
+# See the NOTICE file distributed with this work for additional
+# information regarding copyright ownership.  Ernesto Angel Celis de la
+# Fuente licenses this file to you under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "csv"
+
+# Extract data into a in-memory table
+class Extract
+
+  # In-memory table
+  attr_reader :table
+
+  # Currently src_type only 'csv' is supported
+  # Headers true if source has column names in first row
+  # src_path is an absolute path to the csv file
+  def initialize(src_type,
+                 headers,
+                 src_path)
+    @src_type = src_type
+    @headers = headers
+    @src_path = src_path
+  end
+
+  # Extract data from @src_type
+  def data
+    case @src_type
+    when "csv"
+      @table = CSV.read(@src_path,
+                        :headers => @headers)
+      # TODO FIX headers
+      #@col_headers = @table.headers
+      #if @headers
+      #  @table.by_col!
+      #@table.delete(0)
+      #end
+    else
+      puts "xlsl"
+    end
+  end
+end

data/lib/setl_tool/load.rb

@@ -0,0 +1,36 @@
+# Licensed to you under one or more contributor license agreements.
+# See the NOTICE file distributed with this work for additional
+# information regarding copyright ownership.  Ernesto Angel Celis de la
+# Fuente licenses this file to you under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'pg'
+
+class Load
+  attr_accessor :db_host
+  attr_accessor :db_name
+  attr_accessor :db_user
+  attr_accessor :db_passwd
+  attr_accessor :conn
+
+  def initialize(db_host, db_name, db_user, db_passwd)
+    @conn = PG.connect(host: db_host,
+                       dbname: db_name,
+                       user: db_user,
+                       password: db_passwd)
+    @conn.set_error_verbosity(PG::PQERRORS_VERBOSE)
+  end
+
+  def insert(stmt)
+    @conn.exec(stmt)
+  end
+end

data/lib/setl_tool/transform.rb

@@ -0,0 +1,70 @@
+# Licensed to you under one or more contributor license agreements.
+# See the NOTICE file distributed with this work for additional
+# information regarding copyright ownership.  Ernesto Angel Celis de la
+# Fuente licenses this file to you under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Transforms from an in memory table to a known SQL dialect statement
+class Transform
+
+  # Target Table in RDMS engine
+  attr_accessor :target_table
+  # In-Memory source data table
+  attr_accessor :table
+  # don't use, reserved
+  attr_reader :values
+  # Valid SQL dialect statement
+  attr_reader :stmt
+
+  # Currently only CSV to PostgreSQL is supported
+  def initialize(src_type = "csv",
+                target_db = "pg")
+    @src_type = src_type
+    @target_db = target_db
+    if @target_table == nil
+      @target_table = "#{@src_type}_import"
+    end
+    @values = Array.new
+  end
+
+  # Execute data transformation
+  def data
+    case @src_type
+      when "csv" then process_csv
+    end
+    case @target_db
+    when "pg" then to_pg
+    end
+  end
+
+  private
+  def process_csv
+    @table.each do |row|
+      values = row.map(&:inspect).join(', ')
+      stmt = "#{values}"
+      @values << stmt.gsub('"', "'")
+    end
+  end
+
+  private
+  def to_pg
+    values = ""
+    @values.each do |row|
+      values << "("
+      values << row
+      values << "),"
+    end
+    @stmt = "INSERT INTO #{@target_table} VALUES #{values.chop!};"
+  end
+end
+
+

data/lib/setl_tool.rb

@@ -0,0 +1,42 @@
+# Licensed to you under one or more contributor license agreements.
+# See the NOTICE file distributed with this work for additional
+# information regarding copyright ownership.  Ernesto Angel Celis de la
+# Fuente licenses this file to you under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'setl_tool/extract'
+require 'setl_tool/transform'
+require 'setl_tool/load'
+
+class Setl_tool
+
+  attr_accessor :target_db
+  attr_accessor :target_table
+  attr_reader :extract
+  attr_reader :transform
+  attr_accessor :load
+
+  # TODO Change headers to true
+  def initialize(src_type = "csv",
+                 headers = false,
+                src_path)
+    @src_type = src_type
+    @headers = headers
+    @src_path = src_path
+    @extract = Extract.new(@src_type,
+                           @headers,
+                           @src_path)
+    @transform = Transform.new
+    #@load = Load.new
+  end
+
+end

metadata

@@ -0,0 +1,49 @@
+--- !ruby/object:Gem::Specification
+name: setl_tool
+version: !ruby/object:Gem::Version
+  version: 1.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Ernesto Celis
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-08-05 00:00:00.000000000 Z
+dependencies: []
+description: Just what the summary and name implies
+email: ecelis@sdf.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/setl_tool.rb
+- lib/setl_tool/extract.rb
+- lib/setl_tool/load.rb
+- lib/setl_tool/transform.rb
+homepage: https://bitbucket.org/ecelis/seth-gem
+licenses:
+- Apache 2.0
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Simple ETL
+test_files: []