datapipe2 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 33537af6d0ecafd5f50fc7b4d1a9dd9687c68966
+  data.tar.gz: 138fd72d0c33d055fd3da2cc401ac39e44f8e765
+SHA512:
+  metadata.gz: f6f9739676a60053cbdc07a5aa2866553ae15f9cd9db8da9ad945226528b81b55226d579c236bbfa72e9ad00e09ed82dee2ca4490cb86805d9e7b4e0db9c21b9
+  data.tar.gz: 63046abefaeeacb871ff19d7a12abf7048005f39372a74d88fc5c23c6325e4a74829ef059590a452bc97de1608b871cee62dd208ed1375a1f0764682a0906aab

data/bin/datapipe2

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+#
+#
+# The application 'datapipe' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+# $LOAD_PATH.unshift '/guyirvine.com/datapipe2/lib'
+
+require 'rubygems'
+require 'datapipe2'
+
+if ARGV.length == 0
+  DataPipe2::Host.new.run
+else
+  method_name = ARGV.shift
+  send(method_name, *ARGV)
+end

data/lib/datapipe2.rb

@@ -0,0 +1,18 @@
+# Don't buffer stdout
+$stdout.sync = true
+
+# Don't buffer stdout
+module DataPipe2
+  require 'helper_functions'
+  require 'jobs'
+  require 'host'
+
+  require 'fns/db_to_csv'
+  require 'fns/db_to_json'
+  require 'fns/csv_to_db'
+
+  class DataPipe2lineError < StandardError
+  end
+  class EnvironmentVariableNotFoundError < DataPipe2lineError
+  end
+end

data/lib/fns/csv_to_db.rb

@@ -0,0 +1,16 @@
+require 'csv'
+
+def csv_to_db(file_name, table_name)
+  db_env_name = 'DB'
+  db = DataPipe2.get_fluid_db(db_env_name)
+  db.execute("TRUNCATE TABLE #{table_name}", [])
+
+  csv = CSV.read(file_name)
+  columns = csv.shift
+  sql = "INSERT INTO #{table_name} ( #{columns.join(',')} )
+          VALUES (#{Array.new(columns.length, '?').join(',')})"
+  csv.each do |row|
+    db.execute(sql, row)
+  end
+  DataPipe2.log "#{table_name}: #{csv.length}", true
+end

data/lib/fns/db_to_csv.rb

@@ -0,0 +1,16 @@
+require 'csv'
+
+def db_to_csv(sql, path, name)
+  db_env_name = 'DB'
+  db = DataPipe2.get_fluid_db(db_env_name)
+
+  file_path = "#{path}/#{name}"
+  File.delete(file_path) if File.exist?(file_path)
+
+  rst = db.queryForResultset(sql, [])
+
+  CSV.open(file_path, 'w') do |csv|
+    csv << rst[0].keys
+    rst.each { |r| csv << r.values }
+  end
+end

data/lib/fns/db_to_json.rb

@@ -0,0 +1,14 @@
+require 'FluidDb'
+require 'json'
+
+def db_to_json(sql, path, name)
+  # Dir.mkdir(path) unless Dir.exists?(path)
+  db_env_name = 'DB'
+  db = DataPipe2.get_fluid_db(db_env_name)
+
+  file_path = "#{path}/#{name}"
+  File.delete(file_path) if File.exist?(file_path)
+
+  rst = db.queryForResultset(sql, [])
+  File.write(file_path, rst.to_json)
+end

data/lib/helper_functions.rb

@@ -0,0 +1,27 @@
+require 'FluidDb'
+
+# DataPipe2 Helpers
+module DataPipe2
+  def self.log(string, verbose = false)
+    type = verbose ? 'VERB' : 'INFO'
+    if !ENV['VERBOSE'].nil? || !verbose
+      timestamp = Time.new.strftime('%Y-%m-%d %H:%M:%S')
+      puts "[#{type}] #{timestamp} :: #{string}"
+    end
+  end
+
+  def self.log_dsl(name, string, verbose = false)
+    log "name: #{name}, #{string}", verbose
+  end
+
+  def self.get_env_var(name)
+    fail EnvironmentVariableNotFoundError, name if ENV[name].nil?
+    ENV[name]
+  end
+
+  def self.get_fluid_db(env_name)
+    uri_string = get_env_var(env_name)
+    log "uri: #{uri_string}", true
+    FluidDb::Db(uri_string)
+  end
+end

data/lib/host.rb

@@ -0,0 +1,44 @@
+require 'rubygems'
+require 'rack'
+require 'thin'
+
+module DataPipe2
+  # Host
+  class Host
+    def run
+      libs = ENV['LIB'] ||= './lib'
+      libs.split(';').each do |path|
+        DataPipe2.log "Adding libdir: #{path}"
+        $LOAD_PATH.unshift path
+      end
+
+      @dsl_paths = ENV['DSL'] ||= './dsl'
+      DataPipe2.log "dsl_paths: #{@dsl_paths}"
+      @hash = {}
+
+      @hash['jobs'] = Jobs.new
+      long_run
+    end
+
+    def long_run
+      Kernel.loop do
+        begin
+          single_run
+
+          sleep 0.5
+        rescue SystemExit, Interrupt
+          puts 'Exiting on request ...'
+          break
+        end
+      end
+    end
+
+    def single_run
+      @dsl_paths.split(';').each do |dsl_dir|
+        Dir.glob("#{dsl_dir}/*.dsl").each do |dsl_path|
+          @hash['jobs'].call dsl_path
+        end
+      end
+    end
+  end
+end

data/lib/jobs.rb

@@ -0,0 +1,83 @@
+# Doc
+module DataPipe2
+  require 'parse-cron'
+
+  # Hold a single job definition
+  class Job
+    attr_reader :name, :next, :errorList
+
+    def initialize(path)
+      @path = path
+      @name = File.basename(path, '.dsl')
+      @cron_string = ''
+
+      @error_list = []
+      set_cron
+    end
+
+    # Job Error -> Time, Exception Class Name, nsg, backtrace
+    def add_error(e)
+      @error_list << "#{e.class.name}: #{e.message}\n#{e.backtrace.join("\n")}"
+    end
+
+    def clear_error
+      @error_list = []
+    end
+
+    def run_now
+      @next = Time.now - 1
+    end
+
+    def set_cron
+      tmp = ENV["#{@name}_CRON"] ||= '0 0 * * *'
+      return if tmp == @cron_string
+
+      @cron_string = tmp
+      @cron = CronParser.new(@cron_string)
+    end
+
+    def call
+      run if Time.now > @next
+    end
+
+    def run
+      begin
+        DataPipe2.log "path: #{@path}", true
+        DataPipe2.log "dsl: #{@name}"
+        load @path
+        clear_error
+      rescue SystemExit, Interrupt
+        raise
+      rescue StandardError => e
+        string = e.message
+        p e.backtrace
+        DataPipe2.log_dsl @name, string
+        add_error(e)
+      end
+
+      set_cron
+      @next = @cron.next(Time.now)
+    end
+  end
+
+  # Hold all jobs
+  class Jobs
+    attr_reader :hash, :byName
+
+    def initialize
+      @hash = {}
+      @by_name = {}
+    end
+
+    def call(path)
+      if @hash[path].nil?
+        j = Job.new(path)
+        @hash[path] = j
+        @by_name[j.name.downcase] = j
+        j.run
+      else
+        @hash[path].call
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,51 @@
+--- !ruby/object:Gem::Specification
+name: datapipe2
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Guy Irvine
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-05-02 00:00:00.000000000 Z
+dependencies: []
+description: Helping to move data around your system
+email: guy@guyirvine.com
+executables:
+- datapipe2
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/datapipe2.rb
+- lib/fns/csv_to_db.rb
+- lib/fns/db_to_csv.rb
+- lib/fns/db_to_json.rb
+- lib/helper_functions.rb
+- lib/host.rb
+- lib/jobs.rb
+- bin/datapipe2
+homepage: http://rubygems.org/gems/datapipe2
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14.1
+signing_key: 
+specification_version: 4
+summary: DataPipe2
+test_files: []