data_imp 0.1.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 86e11f4b6817ad75532ca9af858c261295516645
-  data.tar.gz: 7cdd8b503afa39ce898eaf8a77a5034f86af448e
+SHA256:
+  metadata.gz: 4c6777102891d246c75858d7cb7b60bc85c880990e6564a81bddc871241ae9d1
+  data.tar.gz: fc0e1a5a3d6aa1ec738fbd3f8dfb9b51a58185a08f9831e25f359f60c5724a54
 SHA512:
-  metadata.gz: 84f36d210ff7c43709501f10eed5289e2f7d4d965c2cb1fcfb0edca5134e2af08e5a0821b307ef22ee20c414e43e73398e26879b773ab2cbef33d9f7af7dba91
-  data.tar.gz: 9735f74a69ccee71adb71b3b4bb28273525cbad80eb7158f337f7357a38bd83d413cc920b95ceea2fb2adf9157ee3d06f3af7a6e6aad98229b89cccc40fbbac0
+  metadata.gz: 21ef743537443b66646dbf1558f5cbbb22b712c131a1aecd7318282c31a3ecc2ebc8bc5db4d101651678387b757ae6ae699abd6f19fcd3aaaccf3ca646081bcc
+  data.tar.gz: efa9c8697c30157e4685a465371cae73c24c7d8ea60260bdf50c346559b6161ad4567ac6e64fa08979964ed49ce5163724186e31e3dfc746940d7ed8f6c28d23

data/data_imp.gemspec

@@ -22,9 +22,10 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency 'activesupport'
 
-  spec.add_development_dependency "bundler", "~> 1.14.6"
+  spec.add_development_dependency "bundler", "~> 2.3.7"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "roo" #, "~> 3.0"
   spec.add_development_dependency "roo-xls" #, "~> 3.0"
+  spec.add_development_dependency "tiny_tds"
 end

data/lib/data_imp/class_methods.rb

@@ -1,63 +1,8 @@
-require 'pathname'
 
 require_relative "parser"
 require_relative "porter"
+require_relative "importer"
 
 module DataImp::ClassMethods
-  def self.extended(mod)
-    mod.delegate :data_dir, :find_parser, :find_importer, to: :class
-  end
-
-  def root= dir
-    @root = dir
-  end
-
-  def root
-    @root ||= Pathname.pwd
-  end
-
-  def data_dir= dir
-    path = Pathname.new(dir)
-    @data_dir = 
-      if path.absolute?
-        path
-      else
-        root.join(path)
-      end
-  end
-
-  def data_dir
-    @data_dir ||= root.join('data')
-  end
-
-  def import(file=nil,*args,&block)
-    file.strip!
-    return if file =~ /^#/
-    new(file, *args,&block).import
-  end
-
-  def import_list *args
-    args.each do |arg| 
-      arg.each_line do |file|
-        import file
-      end
-    end
-  end
-
-  def find_parser type
-    return type if type.kind_of? DataImp::Parser
-    type = type.to_s.camelize
-    const_get "#{type}Parser"
-  rescue NameError => e
-    DataImp::Parser.find_parser type
-  end
-
-  def find_importer type
-    return type if type.kind_of? DataImp::Porter
-    type = type.to_s.camelize
-    const_get "#{type}Importer"
-  rescue NameError => e
-    DataImp::Porter.find_importer type
-  end
 end
 

data/lib/data_imp/dir.rb

@@ -0,0 +1,28 @@
+require 'pathname'
+module DataImp::Dir
+  def self.extended(mod)
+    mod.delegate :data_dir, to: :class
+  end
+
+  def root= dir
+    @root = dir
+  end
+
+  def root
+    @root ||= Pathname.pwd
+  end
+
+  def data_dir= dir
+    path = Pathname.new(dir)
+    @data_dir = 
+      if path.absolute?
+        path
+      else
+        root.join(path)
+      end
+  end
+
+  def data_dir
+    @data_dir ||= root.join('data')
+  end
+end

data/lib/data_imp/finders.rb

@@ -0,0 +1,29 @@
+require_relative "parser"
+require_relative "porter"
+require_relative "importer"
+
+module DataImp::Finders
+  def self.extended(mod)
+    mod.delegate :find_parser, :find_importer, to: :class
+  end
+
+  def self.included(mod)
+    mod.delegate :const_get, to: :class
+  end
+
+  def find_parser type
+    return type if type.kind_of? DataImp::Parser
+    type = type.to_s.camelize
+    const_get "#{type}Parser"
+  rescue NameError => e
+    DataImp::Parser.find_parser type
+  end
+
+  def find_importer type
+    return type if type.kind_of? DataImp::Porter
+    type = type.to_s.camelize
+    const_get "#{type}Importer"
+  rescue NameError => e
+    DataImp::Porter.find_importer type
+  end
+end

data/lib/data_imp/import_methods.rb

@@ -0,0 +1,19 @@
+
+require_relative "parser"
+require_relative "porter"
+require_relative "importer"
+
+module DataImp::ImportMethods
+  def import(file=nil, **args, &block)
+    file.strip!
+    return if file =~ /^#/
+    new(file, **args, &block).import
+  end
+
+  def import_list(list, **args, &block)
+    list.each_line do |file|
+      import file, **args, &block
+    end
+  end
+end
+

data/lib/data_imp/importer.rb

@@ -0,0 +1,5 @@
+
+require_relative 'porter'
+DataImp::Importer = DataImp::Porter
+# class alias because someone is going to mix these up
+

data/lib/data_imp/parser/csv.rb

@@ -9,7 +9,7 @@ class DataImp::Parser::Csv < DataImp::Parser
   end
 
   def process_file &block
-    CSV.open(filename, options) do |csv|
+    CSV.open(filename, **options) do |csv|
       process csv, &block
     end
   end

data/lib/data_imp/tds/table.rb

@@ -0,0 +1,24 @@
+class DataImp::Tds::Table
+  attr_reader :client, :table, :schema
+  def initialize client, table, schema:
+    @client = client
+    @table = table
+    @schema = schema
+  end
+
+  def sql
+    if schema
+      "select * from [#{schema}].[#{table}]"
+    else
+      "select * from [#{table}]"
+    end
+  end
+
+  def result
+    @result ||= client.execute(sql)
+  end
+
+  def each &block
+    result.each &block
+  end
+end

data/lib/data_imp/tds.rb

@@ -0,0 +1,53 @@
+require 'tiny_tds'
+require_relative "finders"
+class DataImp::Tds
+  extend DataImp::Finders
+
+  def initialize options={}
+    @options = options
+  end
+
+  def options
+    @options ||= {}
+  end
+
+  def client
+    @client ||= TinyTds::Client.new options
+  end
+
+  def import table
+    table.strip!
+    return if table =~ /^#/
+    parts = table.split('.')
+    table = parts.pop
+    schema = parts.shift
+    importer = find_importer(table)
+    tbl = Table.new(client, table, schema: schema)
+    puts "Importing #{table} with #{importer}"
+    importer.before_all_imports
+    tbl.each do |hash, index|
+      porter = importer.new(hash,index)
+      begin
+        porter.before_import
+        porter.import
+        porter.after_import
+        show_progress index
+      rescue StandardError => e
+        warn "#{table}:#{index}:#{e.class.name}"
+        porter.on_error e
+      end
+    end
+    importer.after_all_imports
+    puts
+  end
+
+  def import_list list, *args, &block
+    list.each_line do |table|
+      import table, *args, &block
+    end
+  end
+
+  def show_progress index
+  end
+end
+require_relative "tds/table"

data/lib/data_imp/version.rb

@@ -1,3 +1,3 @@
 class DataImp
-  VERSION = "0.1.0"
+  VERSION = "0.3.0"
 end

data/lib/data_imp.rb

@@ -1,10 +1,19 @@
 
 require 'active_support/core_ext/string'
-require_relative "data_imp/version"
+
+class DataImp
+end
+
+require_relative "data_imp/finders"
+require_relative "data_imp/dir"
+require_relative "data_imp/import_methods"
 require_relative "data_imp/class_methods"
 
 # Dir[File.dirname(__FILE__) + '/data_imp/*.rb'].each {|file| require file }
 class DataImp
+  extend DataImp::Finders
+  extend DataImp::Dir
+  extend DataImp::ImportMethods
   extend DataImp::ClassMethods
   attr_accessor :file, :parser, :importer, :extname, :basename
 
@@ -41,30 +50,5 @@ class DataImp
     print '.' if index % 100 == 0
   end
 
-#  def import_file file
-#
-#    parser_class_name = ext.classify
-#    importer_class_name = base.concat('_importer').classify
-#    file = data_dir.join(file)
-#
-#    begin
-#      parser = const_get(parser_class_name)
-#      importer = importer_class_name.constantize
-#    rescue NameError => e
-#      warn "No importer class: #{e}"
-#      return 
-#    end
-#    parser.new(file, importer).import
-#  end
-
-#  def reader= reader
-#    return unless reader
-#    @reader = 
-#      case reader
-#      when IO,DataImp::Reader then reader
-#      when String then open(reader)
-#      when Pathname then reader.open
-#      end
-#  end
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: data_imp
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Robert W. Ferney
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-09-13 00:00:00.000000000 Z
+date: 2022-08-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.14.6
+        version: 2.3.7
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.14.6
+        version: 2.3.7
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -94,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: tiny_tds
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email:
 - rob@ferney.org
@@ -122,6 +136,10 @@ files:
 - data_imp.gemspec
 - lib/data_imp.rb
 - lib/data_imp/class_methods.rb
+- lib/data_imp/dir.rb
+- lib/data_imp/finders.rb
+- lib/data_imp/import_methods.rb
+- lib/data_imp/importer.rb
 - lib/data_imp/no_importer.rb
 - lib/data_imp/no_parser.rb
 - lib/data_imp/parser.rb
@@ -135,6 +153,8 @@ files:
 - lib/data_imp/parser/yaml.rb
 - lib/data_imp/porter.rb
 - lib/data_imp/porter/sample.rb
+- lib/data_imp/tds.rb
+- lib/data_imp/tds/table.rb
 - lib/data_imp/version.rb
 homepage: https://github.com/capnregex/data_imp
 licenses:
@@ -155,8 +175,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 3.3.7
 signing_key: 
 specification_version: 4
 summary: Scaffolding to make importing data easy