datapipe2 0.0.11 → 0.0.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 50d3264229f3363592e873e03600b7830f205a46
-  data.tar.gz: d1226748e0b7892cb315b0082fe901ea397a14e0
+  metadata.gz: 9a4e42506c6c2a85e662dc284be5a740e4c81f39
+  data.tar.gz: 7f602f5afdd3d5821f1cb0ff50d4cb4cfe54e842
 SHA512:
-  metadata.gz: 6b444290a548ea22eefadb745da1fd8042f2d56bf020a894360cf04c29372ad71d893533fe22c965e16780948a9993fe6bcd11f3c3fa4e5bdb68c977c3b5577c
-  data.tar.gz: 8405a58c633c023750110419d10fd1bdd0cd2b9ffe86a769f2b48d075189ead6730663e5f040864766add4c2bb0f1d709c4fb5d2deb0ee7af41f35f2c80116f4
+  metadata.gz: be6cca3b89a584fc0052a3331e5a162dfd37b272907f115ed9c2a131070c8ecec148b62d59ed3968b7f52ad1f33069b314ea807dfa5b235f35ed810cb9fe08f1
+  data.tar.gz: dc4d0532c05ba9b587facb7e96eed7a404fa9086ed1d3d0c7eecc2ec8d0597317f3974d25e779b8ea82a1c60e39d8b391091bb0d8be86e334fcb0d22c6eb363b

data/lib/datapipe2.rb

@@ -12,6 +12,7 @@ module DataPipe2
   require 'fns/csv_to_db'
   require 'fns/mssql_to_pgsql'
   require 'fns/smb_to_local'
+  require 'fns/scp_to_local'
   require 'fns/db_to_db'
 
   class DataPipe2lineError < StandardError

data/lib/fns/db_to_db.rb

@@ -1,7 +1,7 @@
 
 def db_to_db(select_sql, table_name, columns)
-  s = FluidDb::Db(ENV['SOURCE_DB']) # source
-  d = FluidDb::Db(ENV['DEST_DB']) # destination
+  s = FluidDb::Db(ENV['SOURCE_DB']) if s.nil? # source
+  d = FluidDb::Db(ENV['DEST_DB']) if d.nil? # destination
 
   insert_cmd = %{
     INSERT INTO #{table_name} (#{columns})

data/lib/fns/scp_to_local.rb

@@ -0,0 +1,37 @@
+require 'net/sftp'
+require 'fileutils'
+require 'uri'
+
+def scp_to_local(scp_uri_string,
+                 remote_done_path,
+                 local_path,
+                 local_working_path = nil)
+  scp_uri = URI.parse(scp_uri_string)
+  remote_ready_path = "#{scp_uri.path}"
+
+  local_working_path = "/tmp/#{local_path}" if local_working_path.nil?
+  FileUtils.mkdir_p local_working_path
+  # Check we can write to local path
+
+  Net::SFTP.start(scp_uri.host, scp_uri.user) do |sftp|
+    # Check we can read and write from remote_ready_path
+    # Check we can write to remote_done_path
+    # puts "remote_ready_path: #{sftp.lstat!('remote_ready_path').permissions}"
+    # puts "remote_done_path: #{sftp.lstat!('remote_done_path').permissions}"
+    sftp.dir.foreach(remote_ready_path) do |entry|
+      if entry.file?
+        puts entry.name
+        sftp.download!(
+          "#{remote_ready_path}/#{entry.name}",
+          "#{local_working_path}/#{entry.name}")
+
+        FileUtils.mv("#{local_working_path}/#{entry.name}", local_path)
+
+        sftp.rename!("#{remote_ready_path}/#{entry.name}",
+                     "#{remote_done_path}/#{entry.name}")
+      end
+    end
+  end
+
+  FileUtils.rm_rf local_working_path
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datapipe2
 version: !ruby/object:Gem::Version
-  version: 0.0.11
+  version: 0.0.13
 platform: ruby
 authors:
 - Guy Irvine
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-07-07 00:00:00.000000000 Z
+date: 2016-08-03 00:00:00.000000000 Z
 dependencies: []
 description: zero to one for moving data
 email: guy@guyirvine.com
@@ -23,6 +23,7 @@ files:
 - lib/fns/db_to_db.rb
 - lib/fns/db_to_json.rb
 - lib/fns/mssql_to_pgsql.rb
+- lib/fns/scp_to_local.rb
 - lib/fns/smb_to_local.rb
 - lib/helper_functions.rb
 - lib/host.rb