datapipe 0.0.2 → 0.0.3

data/bin/datapipe

@@ -10,9 +10,24 @@ $:.unshift "/guyirvine.com/Stream/DataPipe/lib"
 require 'rubygems'
 require 'DataPipe'
 
-def run_datapipe()
-        DataPipe::Host.new().run()
+@hash = Hash.new
+
+datapipe_thread = Thread.new do
+begin
+	h = DataPipe::Host.new
+	h.hash = @hash
+	h.run()
+rescue Exception=>e
+	puts e.message
+	puts e.backtrace
+end
 end
 
-run_datapipe
+
+s = DataPipe::WWW.new
+s.hash = @hash
+Rack::Handler::Thin.run s, :Port => 9292
+
+
+Thread.kill( datapipe_thread )
 

data/lib/DataPipe.rb

@@ -6,6 +6,7 @@ module DataPipe
     require 'helper_functions'
     require 'Jobs'
     require 'Host'
+    require 'WWW'
     
     
     class DataPipelineError<StandardError

data/lib/DbToDir.rb

@@ -1,15 +1,17 @@
-require 'FluidDb'
-require "json"
+require 'StreamBuilder'
 
 
 def DbToDir( db_env_name, sql, splitField, path, prefix )
-    Dir.mkdir( path ) unless Dir.exists?( path )
-
+#    Dir.mkdir( path ) unless Dir.exists?( path )
 
     db = DataPipe.getFluidDb( db_env_name )
 
     hash = Hash.new
-    db.queryForResultset( sql, [] ).each do |r|
+    rst = db.queryForResultset( sql, [] )
+    columns = rst[0].keys if rst.length > 0
+
+
+    rst.each do |r|
         hash[r[splitField]] = Array.new unless hash.has_key?(r[splitField])
 
         hash[r[splitField]] << r
@@ -19,9 +21,12 @@ def DbToDir( db_env_name, sql, splitField, path, prefix )
     Dir.glob( "#{basePath}*" ).each { |f| File.delete(f) }
 
     hash.each do |k,v|
-        File.write( "#{basePath}#{k}.js", v.to_json )
+        s = StreamBuilder.new
+                .f( columns )
+	v.each { |r| s.add *r.values }
+        File.write( "#{basePath}#{k}.js", s.serialize )
     end
-    
+
     return hash
 end
 

data/lib/DbToJson.rb

@@ -0,0 +1,29 @@
+require 'FluidDb'
+require "json"
+
+
+def DbToJson( db_env_name, sql, splitField, path, prefix )
+    Dir.mkdir( path ) unless Dir.exists?( path )
+
+
+    db = DataPipe.getFluidDb( db_env_name )
+
+    hash = Hash.new
+    rst = db.queryForResultset( sql, [] )
+    columns = rst[0].keys if rst.length > 0
+    rst.each do |r|
+        hash[r[splitField]] = Array.new unless hash.has_key?(r[splitField])
+
+        hash[r[splitField]] << r
+    end
+
+    basePath = "#{path}/#{prefix}-"
+    Dir.glob( "#{basePath}*" ).each { |f| File.delete(f) }
+
+    hash.each do |k,v|
+        File.write( "#{basePath}#{k}.js", v.to_json )
+    end
+    
+    return hash
+end
+

data/lib/DirToDb.rb

@@ -0,0 +1,27 @@
+require 'FluidDb'
+require "json"
+
+
+def DbToDir( db_env_name, sql, splitField, path, prefix )
+    Dir.mkdir( path ) unless Dir.exists?( path )
+
+
+    db = DataPipe.getFluidDb( db_env_name )
+
+    hash = Hash.new
+    db.queryForResultset( sql, [] ).each do |r|
+        hash[r[splitField]] = Array.new unless hash.has_key?(r[splitField])
+
+        hash[r[splitField]] << r
+    end
+
+    basePath = "#{path}/#{prefix}-"
+    Dir.glob( "#{basePath}*" ).each { |f| File.delete(f) }
+
+    hash.each do |k,v|
+        File.write( "#{basePath}#{k}.js", v.to_json )
+    end
+    
+    return hash
+end
+

data/lib/Host.rb

@@ -1,22 +1,29 @@
+require 'rubygems'
+require 'rack'
+require 'thin'
+
 
 module DataPipe
-    
+
     class Host
+
+	attr_accessor :hash
         
         def run
 
             libs = ENV["LIB"] ||= "./lib"
             dsl_paths = ENV["DSL"] ||= "./dsl"
-            
+puts "dsl_paths: #{dsl_paths}"            
             
             libs.split( ";" ).each do |path|
                 DataPipe.log "Adding libdir: #{path}"
                 $:.unshift path
-                
+            end                
                 loop = true
                 
                 
                 jobs = Jobs.new
+		@hash['jobs'] = jobs
                 while loop do
                     begin
                         dsl_paths.split( ";" ).each do |dsl_dir|
@@ -26,7 +33,6 @@ module DataPipe
                             end
                         end
                         
-                        
                         sleep 0.5
                         rescue SystemExit, Interrupt
                         puts "Exiting on request ..."
@@ -34,7 +40,6 @@ module DataPipe
                     end
                 end
                 
-            end
 
         end
         

data/lib/Jobs.rb

@@ -4,14 +4,32 @@ module DataPipe
     
 
 class Job
+
+	attr_reader :name, :next, :errorList
     
 	def initialize( path )
 		@path = path
 		@name = File.basename( path, ".dsl" )
 		@cronString = ""
+
+		@errorList = Array.new
 		self.setCron
 	end
 
+	def addError( e )
+#Job Error -> Time, Exception Class Name, nsg, backtrace
+		@errorList << "#{e.class.name}: #{e.message}\n#{e.backtrace.join( "\n" )}"
+	end
+
+	def clearError
+		@errorList = Array.new
+	end
+
+	def runNow
+		@next = Time.now - 1
+	end
+
+
 	def setCron
 		tmp = ENV["#{@name}_CRON"] ||= "0 0 * * *"
 		return if tmp == @cronString
@@ -29,12 +47,15 @@ class Job
 			DataPipe.log "path: #{@path}", true
 			DataPipe.log "dsl: #{@name}"
             load @path
+		self.clearError
             
             rescue SystemExit, Interrupt
 			raise
             rescue Exception => e
-            string = "#{e.class.name}: #{e.message}\n#{e.backtrace.join( "\n" )}"
+#            string = "#{e.class.name}: #{e.message}\n#{e.backtrace.join( "\n" )}"
+            string = e.message
             DataPipe.log_dsl @name, string
+		self.addError( e )
         end
         
 		self.setCron 
@@ -43,16 +64,21 @@ class Job
 end
 
 class Jobs
+
+	attr_reader :hash, :byName
     
 	def initialize
 		@hash = Hash.new
+		@byName = Hash.new
 	end
     
+
     
 	def call( path )
 		if @hash[path].nil? then
 			j = Job.new( path )
 			@hash[path] = j
+			@byName[j.name.downcase] = j
 			j.run
             else
 			@hash[path].call

data/lib/JsonToPgsql.rb

@@ -0,0 +1,25 @@
+require "FluidDb"
+require "json"
+
+
+def JsonToPgsql( source_env_name, destination_env_name, tableName, columns )
+    d = DataPipe.getFluidDb( destination_env_name )
+
+    d.execute( "TRUNCATE TABLE #{tableName}", [])
+    
+    results = s.connection.exec( sql )
+    
+    d.connection.exec( "COPY #{tableName} (#{columns.join( "," )}) FROM STDIN WITH DELIMITER AS '|' CSV;" )
+    
+    JSON.parse( IO.read( DataPipe.getEnvVar( source_env_name ) ) ).each do |row|
+        l = Array.new
+	columns.each do |name|
+            l << row[name]
+        end
+        d.connection.put_copy_data "#{l.join( '|' )}\n"
+    end
+    d.connection.put_copy_end
+    
+    DataPipe.log "#{tableName}: #{count}", true
+end
+

data/lib/PathFromRemote.rb

@@ -0,0 +1,27 @@
+require 'net/sftp'
+require 'uri'
+
+
+def PathFromRemote( remoteUri, localPath, prefix )
+
+	uri = URI.parse( remoteUri )
+        DataPipe.log "remoteUri: #{remoteUri}, localPath: #{localPath}, prefix: #{prefix}", true
+
+        Net::SFTP.start( uri.host, uri.user, :password => uri.password ) do |sftp|
+                Dir.glob( "#{localPath}/#{prefix}*" ).each do |path|
+                        File.rm( path )
+                end     
+
+		sftp.dir.foreach(uri.path) do |entry|
+			name = entry.name
+			if name[0,prefix.length] == prefix && entry.file? then
+				DataPipe.log "sftp.rm: #{uri.path}/#{name}"
+				sftp.download!( "#{uri.path}/#{name}" ) if name[0,prefix.length] == prefix && entry.file?
+				sftp.remove!( "#{uri.path}/#{name}" ) if name[0,prefix.length] == prefix && entry.file?
+			end
+		end
+
+        end
+
+end
+

data/lib/Stream.rb

@@ -0,0 +1,9 @@
+module Telemetry
+    
+
+    class StreamError<StandardError
+    end
+
+    
+end
+

data/lib/StreamBuilder.rb

@@ -0,0 +1,32 @@
+require 'json'
+
+
+class StreamBuilder
+
+	def initialize
+		@h = Hash['v',1,'d',{},'f',[],'l',[]]
+	end
+
+	def set( name, value )
+		@h['d'][name] = value
+		return self
+	end
+
+	def f( fieldList )
+		@h['f'] = fieldList
+
+		return self
+	end
+
+	def add( *args )
+		@h['l'].concat args
+
+		return self
+	end
+
+	def serialize
+		return @h.to_json
+	end
+
+end
+

data/lib/StreamParser.rb

@@ -0,0 +1,70 @@
+module Telemetry
+    
+    require 'json'
+    require 'date'
+
+    class StreamParserError<StandardError
+    end
+
+    class VersionNotSpecifiedError<StreamParserError
+    end
+    class VersionNotSupportedError<StreamParserError
+    end
+    class MissingFormatError<StreamParserError
+    end
+    class InvalidFormatError<StreamParserError
+    end
+    class MissingListError<StreamParserError
+    end
+    class MeasurementFieldNotSuppliedError<StreamParserError
+    end
+
+    class StreamParser
+
+        attr_reader :version, :list, :all_fields
+
+        def method_missing( meth, *args, &block )
+            raise VersionNotSupportedError.new if meth[0,5] == "parse"
+            
+            raise NoMethodError.new( "method: #{meth}" )
+        end
+
+        def parse1
+            raise MissingFormatError.new if @r['f'].nil?
+            raise InvalidFormatError.new if @r['f'].length == 0
+            raise MissingListError.new if @r['l'].nil?
+
+            @defaults = @r['d'] || {}
+
+            @list = Array.new
+            format = @r['f']
+
+            @all_fields = @defaults.keys + @r['f']
+
+            #Break list up into chunks, each chunk being the size of the format record
+            @r['l'].each_slice( format.length ).with_index do |el,idx|
+
+                obj = @defaults.clone
+                format.each_with_index do |name,idx|
+                    obj[name] = el[idx]
+                end
+                @list << obj
+            end
+
+        end
+
+        def initialize( payload )
+            @r = JSON.parse( payload )
+            
+            raise VersionNotSpecifiedError.new if @r['v'].nil?
+            @version = @r['v']
+            
+            self.send "parse#{version}"
+            
+            return self
+        end
+
+    end
+
+end
+

data/lib/WWW.rb

@@ -0,0 +1,68 @@
+require 'rubygems'
+require 'rack'
+require 'json'
+
+
+module DataPipe
+    
+    class WWW
+        
+        attr_accessor :hash
+        
+        def call(env)
+            @root = File.expand_path(File.dirname(__FILE__))
+            path = Rack::Utils.unescape(env['PATH_INFO'])
+#            path += 'index.html' if path == '/'
+            file = @root + "#{path}"
+            
+            params = Rack::Utils.parse_nested_query(env['QUERY_STRING'])
+
+            request = Rack::Request.new(env)
+            response = Rack::Response.new()
+	parts = request.path_info.downcase.split( "/" )
+	section = parts[1]
+	case true
+		when request.request_method == "GET" && request.path_info.downcase == "/" then
+			l = ['jobs', 'errors']
+            [ 200, {'Content-Type' => 'application/json'}, l.to_json ]
+			
+		when request.request_method == "GET" && section == "jobs" && parts.length == 2 then
+		l = Array.new
+		@hash['jobs'].hash.each do |k,job|
+			l << Hash['name',job.name,'next',job.next,'errors',job.errorList.length]
+			end
+            [ 200, {'Content-Type' => 'application/json'}, l.to_json ]
+
+
+
+                when request.request_method == "GET" && section == "jobs" && parts.length == 3 then
+			jobName = parts[2]
+			[404, {'Content-Type' => 'text/plain'}, "Not Found"] if @hash['jobs'].byName[jobName].nil?
+
+
+			job = @hash['jobs'].byName[jobName]
+                        h = Hash['name',job.name,'next',job.next,'errorList',job.errorList]
+            [ 200, {'Content-Type' => 'application/json'}, h.to_json ]
+
+
+                when request.request_method == "POST" && section == "jobs" && parts.length == 4 && parts[3] == 'run' then
+			jobName = parts[2]
+                        [404, {'Content-Type' => 'text/plain'}, "Not Found"] if @hash['jobs'].byName[jobName].nil?
+
+                @hash['jobs'].byName[ jobName ].runNow
+            [ 200, {'Content-Type' => 'text/plain'}, "Ok" ]
+
+
+		when request.request_method == "POST" && parts.length == 3 && parts[2] == 'run' then
+		@hash['jobs'].byName[ parts[1] ].runNow
+            [ 200, {'Content-Type' => 'text/plain'}, "Ok" ]
+			
+		else
+            [ 200, {'Content-Type' => 'text/plain'}, "Ok" ]
+	end
+
+            
+        end
+    end
+end
+

data/lib/helper_functions.rb

@@ -1,5 +1,8 @@
 module DataPipe
 
+require 'FluidDb'
+
+
 def DataPipe.log( string, verbose=false )
         type = verbose ? "VERB" : "INFO"
         if !ENV["VERBOSE"].nil? || !verbose then

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: datapipe
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-01 00:00:00.000000000 Z
+date: 2014-05-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &70106395416780 !ruby/object:Gem::Requirement
+  requirement: &70316502296420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70106395416780
+  version_requirements: *70316502296420
 - !ruby/object:Gem::Dependency
   name: fluiddb
-  requirement: &70106395416340 !ruby/object:Gem::Requirement
+  requirement: &70316502295960 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70106395416340
+  version_requirements: *70316502295960
 - !ruby/object:Gem::Dependency
   name: parse-cron
-  requirement: &70106395415920 !ruby/object:Gem::Requirement
+  requirement: &70316502295500 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,7 +43,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70106395415920
+  version_requirements: *70316502295500
 description: Helping to move data around your system
 email: guy@guyirvine.com
 executables:
@@ -53,12 +53,20 @@ extra_rdoc_files: []
 files:
 - lib/DataPipe.rb
 - lib/DbToDir.rb
+- lib/DbToJson.rb
+- lib/DirToDb.rb
 - lib/helper_functions.rb
 - lib/Host.rb
 - lib/Jobs.rb
+- lib/JsonToPgsql.rb
+- lib/PathFromRemote.rb
 - lib/PathToRemote.rb
 - lib/PgsqlToPgsql.rb
 - lib/SqlServerToPgsql.rb
+- lib/Stream.rb
+- lib/StreamBuilder.rb
+- lib/StreamParser.rb
+- lib/WWW.rb
 - bin/datapipe
 - LICENSE
 - README.md