scaffolding 0.3.1 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a3f515c0d5ff226567d6fb2bc8f9c3c3fe162049
-  data.tar.gz: 5daf3967066221b99c8de505b9c7bf40a5c9482c
+  metadata.gz: 8671e3427494760d1568c7846a1728f05bd7eec5
+  data.tar.gz: cfb03213051ffb7ccf930f4380ffa10be042985a
 SHA512:
-  metadata.gz: d60112225f41d16ac48fc9fd0de1d2df58924c2d1a322ee50a24ccd9b6763950a9d7b9e5bb2aba984395c5db2f4eb65d97e005414cf1374155a9049629026659
-  data.tar.gz: 34253e8fc6472899720662470e2843b6e9c8668703e39b0144f6f902a1a7ab6f5afb474fbc6d8032d646db5b8a27d9066359077b7219c55164d0b2d70a4c8278
+  metadata.gz: 8cf10c9d7692cb5554ed704119f68e5f6475e80563c8154cbca6f98d1da1e3053c1db8dae485e9b76119e6efaeede7ecda16a407c0d4cb39e2e75f4aa679ed11
+  data.tar.gz: b44127223ecc9063cb35d05135626cdb20330ebf7da09eee9998919c4555ded9c1b450456f8de467ef90701b57be224ac44a6365a7b9021052432e7def349304

data/README.md

@@ -1,6 +1,6 @@
 # Scaffolding
 
-Generate a rails scaffold from a CSV file and import the data into your rails application.
+Generate a rails scaffold from a CSV, Dat or Txt file and import the data into your rails application.
 
 ## Installation
 
@@ -28,12 +28,12 @@ Or
 
     $ rake scaffolding:folder
 
-then add CSV files to the 'tmp/scaffolding' directory and process them all with:
+then add files to the 'tmp/scaffolding' directory and process them all with:
 
     $ rails g/generate scaffolding
 
 ### Options:
-(You will be asked these at the terminal if not specify)
+(You will be asked these at the terminal if not specified)
 
 Automatically determine the data types of each column:
 

data/Rakefile

@@ -1,5 +1,6 @@
 require "bundler/gem_tasks"
 
-#C:\\Users\\jdejong\\Desktop\\Windows.csv
-#C:\Users\jdejong\Desktop\data.xlsx
+# C:\Users\jdejong\Desktop\Windows.csv
+# C:\Users\jdejong\Desktop\data.xlsx
 # /Users/Jordandejong/Desktop/ht.csv
+# ftp://ftp2.bom.gov.au/anon/gen/fwo/IDY02128.dat

data/lib/generators/scaffolding_generator.rb

@@ -4,19 +4,23 @@ Rails.application.load_tasks
 require 'scaffolding'
 
 class ScaffoldingGenerator < Rails::Generators::Base
-  desc "This generator generates a scaffold based on a CSV file and imports the data"
+  desc "This generator generates a scaffold based on a CSV, Dat or Txt file and imports the data"
   argument :source, :type => :string, :default => ""
   class_option :auto, :type => :boolean, :default => false, :description => "Automatically choose data types"
   class_option :migrate, :type => :boolean, :default => false, :description => "Migrate the database"
   class_option :import, :type => :boolean, :default => false, :description => "Import data from source"
 
   def analyize_sources
+    @auto = options[:auto]
+    @migrate = options[:migrate]
+    @import = options[:import]
+
     if source == "" || source.nil?
       Dir[Rails.root.join('tmp/scaffolding/*')].each do |source|
-        Scaffolding.generate(source, options[:auto], options[:migrate], options[:import])
+        Scaffolding.generate(source, @auto, @migrate, @import)
       end
     else
-      Scaffolding.generate(source, options[:auto], options[:migrate], options[:import])
+      Scaffolding.generate(source, @auto, @migrate, @import)
     end
   end
 

data/lib/scaffolding.rb

@@ -2,17 +2,39 @@ require "scaffolding/version"
 
 module Scaffolding
   require 'scaffolding/railtie'
+  require 'uri'
 
   def self.generate(source, auto, migrate, import)
     @source = source
+    @auto = auto
     @migrate = migrate
     @import = import
-    results = Scaffolding::Parser::Csv.process(@source, auto)
+    @uri = Scaffolding.uri?
+
+    results = Scaffolding.parser
     return if Scaffolding.errors(results)
+
     Rails::Generators::Base.new.generate "scaffold", results
     Scaffolding.import_data if Scaffolding.migrate_database
   end
 
+  def self.uri?
+    @source =~ URI::regexp
+  end
+
+  def self.parser(namespace="")
+    "Scaffolding::Parser#{namespace + "::" + Scaffolding.class_ref}".constantize.process(@source, @auto, @uri)
+  end
+
+  def self.class_ref
+    case File.extname(@source)
+    when ".csv" && @uri == false
+      "Csv"
+    else
+      "Raw"
+    end
+  end
+
   def self.errors(results)
     if results.kind_of?(Array)
       results.each do |error|
@@ -28,6 +50,7 @@ module Scaffolding
       answer = STDIN.gets.chomp.downcase
     end
     if @migrate || answer == "y"
+      Rake::Task["db:migrate"].reenable
       Rake::Task["db:migrate"].invoke
     else
       false
@@ -39,15 +62,12 @@ module Scaffolding
       puts "\n\n\e[32mImport the data from #{@source}?(y/n)\e[0m\n"
       answer = STDIN.gets.chomp.downcase
     end
-    if @import || answer == "y"
-      Scaffolding::Parser::Importer::CsvData.process(@source, false).each do |k,v|
-        puts "#{v} records #{k}"
-      end
-    end
+    Scaffolding.parser("::Importer").each{ |k,v| puts "#{v} records #{k}" } if @import || answer == "y"
   end
 
 end
 
 require 'scaffolding/parser/base'
+require 'scaffolding/parser/raw'
 require 'scaffolding/parser/csv'
-require 'scaffolding/parser/importer/csvdata'
+require 'scaffolding/parser/importer/base'

data/lib/scaffolding/parser/base.rb

@@ -4,30 +4,45 @@ module Scaffolding
       require 'rails'
       require 'csv'
 
-      def initialize(file="", auto)
-        @errors = []
-        @data = valid_data?(file)
-        @file_name = File.basename(file, ".*" ).to_s.split.join.camelize.strip.singularize
-        @scaffold_builder = @file_name
+      def initialize(source="", auto, uri)
+        @source = source
         @auto = auto
+        @uri = uri
+        @errors = []
         @scaffolding = {}
+        @row_number = 0
+        setup
+      end
+
+      def setup
+        return unless @data = valid_data?
+        @col_seperator = col_seperator
+        @source_name = File.basename(@source, ".*" ).to_s.split.join.camelize.strip.singularize
+        @scaffold_builder = @source_name
       end
 
       def errors
         @errors
       end
 
-      def valid_data?(file)
-        unless file == "" || file.nil?
-          # if [".csv", ".xls", ".xlsx"].include? File.extname(file)
-          if File.extname(file) == ".csv"
-            File.read(utf8_encode(file))
-          else
-            @errors << "Unknown file type #{File.extname(file)} for #{file}"
-            false
-          end
+      def valid_data?
+        if @source == "" || @source.nil?
+          @errors << "No source selected"
+          return false
+        end
+        @uri ? web : file
+      end
+
+      def web
+        `curl "#{@source}"`
+      end
+
+      def file
+        ext = File.extname(@source)
+        if [".csv", ".dat", ".txt"].include? ext
+          File.read(utf8_encode(@source))
         else
-          @errors << "No file selected"
+          @errors << "Unknown source type #{ext} for #{@source}"
           false
         end
       end
@@ -54,13 +69,44 @@ module Scaffolding
           data_type = :time if Time.parse(Time.strptime(data, '%H:%M:%S')) rescue data_type
           data_type = :datetime if DateTime.parse(DateTime.strptime(data, '%m/%d/%Y %H:%M:%S')) || DateTime.parse(DateTime.strptime(data, '%d/%m/%Y %H:%M:%S')) rescue data_type
           data_type = :integer if Integer(data) rescue data_type
-          data_type = :decimal if ((data.to_f - data.to_i).abs > 0.0) rescue data_type
+          data_type = :decimal if (data =~ (/[-]?\d*[,]?\d*[.]\d*[%]?$/)) == 0 rescue data_type
           @scaffolding[column.to_sym][data_type] += 1 unless data == ""
         end
       end
 
-      def self.process(file, auto)
-        importer = self.new(file, auto)
+      def scaffold_rank
+        unless @auto
+          puts "\n\e[33mManually choose data types?(y/n) for #{@source_name}\e[0m"
+          manual = STDIN.gets.chomp
+        end
+        @scaffolding.each do |scaffold, data_types|
+          data_type = data_types.max_by{|k,v| v}[0]
+            unless @auto || manual != "y"
+              puts "\n\e[32m#{scaffold}\e[0m is a \e[33m#{data_type}\e[0m? ([Enter]/string/integer/date ect)"
+              answer = STDIN.gets.chomp.downcase
+              data_type = answer if data_types.keys.include?(answer.to_sym)
+            end
+          @scaffolding[scaffold] = data_type
+        end
+      end
+
+      def build_string
+        @scaffolding.each do |k, v|
+          @scaffold_builder << " #{k}:#{v}" unless k.to_s.downcase == "id"
+        end
+        @scaffold_builder
+      end
+
+      def results
+        groom_data
+        process_data
+        return @errors unless @errors.count == 0
+        scaffold_rank
+        build_string
+      end
+
+      def self.process(source, auto, uri)
+        importer = self.new(source, auto, uri)
         return importer.errors unless importer.errors.count == 0
         importer.results
       end

data/lib/scaffolding/parser/csv.rb

@@ -2,20 +2,19 @@ module Scaffolding
   module Parser
     class Csv < Scaffolding::Parser::Base
 
-      def initialize(file = "", auto)
-        super
+      def groom_data
         @headers = true
-        @row_number = 0
+        setup_columns
       end
 
       def setup_columns
-        CSV.parse(@data, headers: @headers, col_sep: col_seperator, skip_blanks: true).first.each do |column|
+        CSV.parse(@data, headers: @headers, col_sep: @col_seperator, skip_blanks: true).first.each do |column|
           @scaffolding[column[0].downcase.to_sym] = data_types
         end
       end
 
       def process_data
-        CSV.parse(@data, headers: @headers, col_sep: col_seperator, skip_blanks: true) do |row|
+        CSV.parse(@data, headers: @headers, col_sep: @col_seperator, skip_blanks: true) do |row|
           @row_number += 1
           begin
             process_row(row.inject({}){|row,(k,v)| row[k.downcase.to_sym] = v; row})
@@ -25,33 +24,6 @@ module Scaffolding
         end
       end
 
-      def scaffold_rank
-        unless @auto
-          puts "\n\e[33mManually choose data types?(y/n)\e[0m"
-          manual = STDIN.gets.chomp
-        end
-        @scaffolding.each do |scaffold, data_types|
-          data_type = data_types.max_by{|k,v| v}[0]
-            unless @auto || manual != "y"
-              puts "\n\e[32m#{scaffold}\e[0m is a \e[33m#{data_type}\e[0m? (y/string/integer/date ect)"
-              answer = STDIN.gets.chomp.downcase
-              data_type = answer unless answer == "y" || answer == ""
-            end
-          @scaffolding[scaffold] = data_type
-        end
-      end
-
-      def results
-        setup_columns
-        process_data
-        return @errors unless @errors.count == 0
-        scaffold_rank
-        @scaffolding.each do |k, v|
-          @scaffold_builder << " #{k}:#{v}" unless k.to_s.downcase == "id"
-        end
-        @scaffold_builder
-      end
-
     end
   end
 end

data/lib/scaffolding/parser/importer/base.rb

@@ -0,0 +1,30 @@
+module Scaffolding
+  module Parser
+    module Importer
+      ["Csv", "Raw"].each do |type|
+        importer_class = Class.new("Scaffolding::Parser::#{type}".constantize) do
+
+          def initialize(file="", auto, uri)
+            super
+            @saved = 0
+            @failed = 0
+          end
+
+          def process_row(row)
+             model = @source_name.classify.constantize.new(row.except!(:id))
+             model.save ? @saved += 1 : @failed += 1
+          end
+
+          def results
+            groom_data
+            process_data
+            return @errors unless @errors.count == 0
+            {saved: @saved, failed: @failed}
+          end
+
+        end
+        Scaffolding::Parser::Importer.const_set(type, importer_class)
+      end
+    end
+  end
+end

data/lib/scaffolding/parser/raw.rb

@@ -0,0 +1,60 @@
+module Scaffolding
+  module Parser
+    class Raw < Scaffolding::Parser::Base
+
+      def groom_data
+        @data = @data.split("\n")
+        find_headers
+        setup_columns
+        hashed_data
+      end
+
+      def find_headers
+        hc = header_count
+        rows = 0
+        @data.map do |row|
+          rows += 1
+          if row.split(@col_seperator).count == hc
+            @headers = row.split(@col_seperator).map{ |header| header.strip.downcase.gsub(/(\W|\d)/, "") }
+            rows.times{ @data.shift }
+            return
+          end
+        end
+      end
+
+      def header_count
+        row_counts = []
+        @data.map{ |row| row_counts << row.split(@col_seperator).count }
+        row_counts.uniq.max_by{ |i| row_counts.count(i) }
+      end
+
+      def setup_columns
+        @headers.each{ |column| @scaffolding[column.to_sym] = data_types }
+      end
+
+      def hashed_data
+        @data = @data.map do |row|
+        fields  = row.split(@col_seperator)
+        hash    = {}
+        @headers.each_with_index do |header, index|
+          hash[header] = fields[index].to_s.strip
+        end
+        hash
+        end
+      end
+
+      def process_data
+        @data.each do |row|
+          @row_number += 1
+          begin
+            process_row(row)
+          rescue => e
+            @errors << "Unable to process row #{@row_number} Error: #{e}\n"
+          end
+        end
+      end
+
+    end
+
+  end
+end

data/lib/scaffolding/railtie.rb

@@ -5,7 +5,7 @@ module Scaffolding
   class Generators < Rails::Railtie
     rake_tasks do
       load "tasks/generators.rake"
-    end
+    end    
     generators do
       require "generators/scaffolding_generator.rb"
     end

data/lib/scaffolding/version.rb

@@ -1,3 +1,3 @@
 module Scaffolding
-  VERSION = "0.3.1"
+  VERSION = "0.3.2"
 end

data/lib/tasks/generators.rake

@@ -5,45 +5,3 @@ namespace :scaffolding do
     Dir.mkdir(File.join(Rails.root, 'tmp/scaffolding'))
   end
 end
-
-# namespace :g do
-#   desc 'generate a scaffold from a csv file'
-#   task :scaffold, [:file] do |t, args|
-#     require 'rails/generators'
-#     require 'scaffolding'
-#
-#     system "clear" or system "cls"
-#     puts "\n***** Scaffolding Gem *****\n\n\n"
-#
-#     if args[:file]
-#       file = args[:file]
-#     else
-#       puts "\e[32mEnter a relative file name to build the scaffold\e[0m\n"
-#       puts "You are currently here: #{Dir.pwd}\n\n"
-#       file = STDIN.gets.chomp
-#     end
-#
-#     results = Scaffolding.generate(file)
-#     if results.kind_of?(Array)
-#       results.each do |error|
-#         puts "\e[31m#{error}\e[0m"
-#       end
-#       puts ""
-#       next
-#     end
-#
-#     Rails::Generators::Base.new.generate "scaffold", results
-#
-#     puts "\n\n\e[32mMigrate the database?(y/n)\e[0m\n"
-#     if STDIN.gets.chomp == "y"
-#       Rake::Task["db:migrate"].invoke
-#
-#       puts "\n\n\e[32mImport the data from #{file}?(y/n)\e[0m\n"
-#       if STDIN.gets.chomp == "y"
-#         Scaffolding.import_data(file).each do |k,v|
-#           puts "#{v} records #{k}"
-#         end
-#       end
-#     end
-#   end
-# end

data/usage.rb

@@ -0,0 +1,24 @@
+require 'net/http'
+require 'json'
+
+gem_name = "scaffolding"
+
+def rubygems_get(gem_name: "", endpoint: "")
+  path = File.join("/api/v1/gems/", gem_name, endpoint).chomp("/") + ".json"
+  JSON.parse(Net::HTTP.get("rubygems.org", path))
+end
+
+results = rubygems_get(gem_name: gem_name, endpoint: "reverse_dependencies")
+
+weighted_results = {}
+results.each do |name|
+  begin
+    weighted_results[name] = rubygems_get(gem_name: name)["downloads"]
+  rescue => e
+    puts "#{name} #{e.message}"
+  end
+end
+
+weighted_results.sort {|(k1, v1), (k2, v2)| v2 <=> v1 }.first(50).each_with_index do |(k, v), i|
+  puts "#{i}) #{k}: #{v}"
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scaffolding
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
 platform: ruby
 authors:
 - Jordan-deJong
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-10-05 00:00:00.000000000 Z
+date: 2015-10-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -57,11 +57,13 @@ files:
 - lib/scaffolding.rb
 - lib/scaffolding/parser/base.rb
 - lib/scaffolding/parser/csv.rb
-- lib/scaffolding/parser/importer/csvdata.rb
+- lib/scaffolding/parser/importer/base.rb
+- lib/scaffolding/parser/raw.rb
 - lib/scaffolding/railtie.rb
 - lib/scaffolding/version.rb
 - lib/tasks/generators.rake
 - scaffolding.gemspec
+- usage.rb
 homepage: https://github.com/Jordan-deJong/scaffolding
 licenses:
 - MIT

data/lib/scaffolding/parser/importer/csvdata.rb

@@ -1,26 +0,0 @@
-module Scaffolding
-  module Parser
-    module Importer
-      class CsvData < Scaffolding::Parser::Csv
-
-        def initialize(file="", auto)
-          super
-          @saved = 0
-          @failed = 0
-        end
-
-        def process_row(row)
-           model = @file_name.classify.constantize.new(row.except!(:id))
-           model.save ? @saved += 1 : @failed += 1
-        end
-
-        def results
-          process_data
-          return @errors unless @errors.count == 0
-          {saved: @saved, failed: @failed}
-        end
-
-      end
-    end
-  end
-end