jsonsql 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 96c7bbaa0d3e9df043848e47c4ba1d77d8ebba76
-  data.tar.gz: 019089ab9c083a07822a0238d5fd196fa6867c9e
+  metadata.gz: 8903ce2ba94d38ef654e8f863739e1b332c4667d
+  data.tar.gz: c460087e79b014e09fe967ba8b77c8d034a2b036
 SHA512:
-  metadata.gz: 5be271218b893fa326ba37948e739dad396e24da1a481b41926de5332de9aaf1ce9195c4ec7e893f9f0fe7ef18d74d6ca5f292f4d5a2b07238741c2a2e97f905
-  data.tar.gz: 784fc08a97341ffdc1ab189613aea32c1e8b69119fa195df4a623e10e42e01c2f60c44dfae64cb465143d43ec7d2763b91566b633e6698855d8de4a5394ca281
+  metadata.gz: 02f61e1cb22c045b35f4690750c148ea64e8319f3be10d011351129e04523c265bff4192c4a489d9c5064be8e89d1c9bf9da2ab650d36693398ff72ac5492b28
+  data.tar.gz: e4ec9071902be1306d62351e5d654b27e7859520e1cd17848e3dd3168bd4a5d8796f9525c4d64c013128abb8137880e3b2a696e8518e4ae78e382f7214ef436b

data/README.md

@@ -79,7 +79,7 @@ If you omit the ``--save-to`` option, the database will be discarded after the c
 
 ## Contributing
 
-1. Fork it ( http://github.com/<my-github-username>/jsonsql/fork )
+1. Fork it ( https://github.com/siuying/jsonsql/fork )
 2. Create your feature branch (`git checkout -b my-new-feature`)
 3. Commit your changes (`git commit -am 'Add some feature'`)
 4. Push to the branch (`git push origin my-new-feature`)

data/lib/jsonsql/command.rb

@@ -23,6 +23,7 @@ module Jsonsql
         @jsons << argument
       end
 
+      @verbose = argv.flag?('verbose', false)
       @console = argv.flag?('console', true)
       @filename = argv.option('save-to') || ':memory:'
       @table_name = argv.option('table-name') || 'table'
@@ -38,7 +39,7 @@ module Jsonsql
 
     def run
       sequel = Sequel.sqlite(@filename)
-      importer = Importer.new(database: sequel, table_name: @table_name)
+      importer = Importer.new(database: sequel, table_name: @table_name, verbose: @verbose)
       importer.import @jsons
 
       if @console

data/lib/jsonsql/importer.rb

@@ -1,5 +1,6 @@
 require 'sequel'
 require 'json'
+require 'logger'
 
 module Jsonsql
   class Importer
@@ -7,11 +8,12 @@ module Jsonsql
 
     attr_reader :database, :table_name, :transformer
 
-    def initialize(database: Sequel.sqlite, table_name: "table", transformer: nil)
+    def initialize(database: Sequel.sqlite, table_name: "table", transformer: nil, verbose: false)
       @table_name = table_name
       @database   = database
       @transformer = transformer
       @table_created = false
+      @database.loggers << Logger.new($stdout) if verbose
     end
 
     def table
@@ -50,20 +52,31 @@ module Jsonsql
       # filter to only supported fields
       row = row.select { |k, v| Jsonsql::Importer.class_supported?(v.class) }
 
-      create_table_if_needed(row)
+      create_or_alter_table_if_needed(row)
       table.insert(row)
     end
 
-    # create a table using row data, if it has not been created
-    def create_table_if_needed(row)
+    def create_or_alter_table_if_needed(row)
       unless @table_created
-        create_columns = columns_with_row(row)
+        columns = columns_with_row(row)
         database.create_table(table_name.to_sym) do
-          create_columns.each do |name, clazz|
+          columns.each do |name, clazz|
             column name.to_sym, clazz
           end
         end
+        @columns = columns
         @table_created = true
+      else
+        updated_columns = columns_with_row(row)
+        added_keys = updated_columns.keys - @columns.keys
+        if added_keys.count > 0
+          database.alter_table(table_name.to_sym) do
+            added_keys.each do |key|
+              add_column key.to_sym, updated_columns[key], :default => nil
+            end
+          end
+          @columns = updated_columns
+        end
       end
     end
 
@@ -74,8 +87,6 @@ module Jsonsql
         clazz = value.class
         if self.class.class_supported?(clazz)
           columns[name] = value.class
-        else
-          $stderr.puts "Class #{clazz} not supported, skipped"
         end
       end
       columns

data/lib/jsonsql/transformer.rb

@@ -8,7 +8,7 @@ module Jsonsql
       self.instance_exec(row, &@transformation)
     end
 
-    def self.transformer_with_block(string)
+    def self.transformer_with_string(string)
       block = eval(string)
       Transformer.new(&block)
     end

data/lib/jsonsql/version.rb

@@ -1,3 +1,3 @@
 module Jsonsql
-  VERSION = "0.2.0"
+  VERSION = "0.2.1"
 end

data/spec/importer_spec.rb

@@ -6,6 +6,27 @@ require 'pry'
 describe Jsonsql::Importer do
   subject { Jsonsql::Importer.new }
 
+  context "using transformer" do
+    it "import JSON with transformer" do
+      transformer = Jsonsql::Transformer.transformer_with_string("Proc.new { |row| row['created_at'] = Time.parse(row['created_at']) if row['created_at']; row } ")
+      expect(transformer).to_not be_nil
+
+      subject = Jsonsql::Importer.new(transformer: transformer)
+      subject.import_jsonfile("samples/mtr/437582112921640960.json")
+
+      expect(subject.table).to_not be_nil
+      expect(subject.table.count).to eq(1)
+      expected_row = {
+        :id => 437582112921640960,
+        :text => "@kaede19940908 圖片係港鐵官方的",
+        :created_at => Time.parse("2014-02-23 21:38:00 +0800"),
+        :lang => "zh",
+        :reply_to => "kaede19940908"
+      }
+      expect(subject.table.where(:id => 437582112921640960).first).to eq(expected_row)
+    end
+  end
+
   context "#import_jsonfile" do
     it "import one json Hash file to db" do
       subject.import_jsonfile("samples/mtr/437582112921640960.json")

data/spec/transformer_spec.rb

@@ -15,9 +15,9 @@ describe Jsonsql::Transformer do
     end
   end
 
-  context "::transformer_with_block" do
+  context "::transformer_with_string" do
     it "create a transformer with block" do
-      transformer = Jsonsql::Transformer.transformer_with_block "
+      transformer = Jsonsql::Transformer.transformer_with_string "
       Proc.new { |row| row[:name] = row[:name].downcase if row[:name]; row } 
       "
       data = transformer.transform({:name => "Peter", :id => 1})

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: jsonsql
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Francis Chong
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-04 00:00:00.000000000 Z
+date: 2014-03-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -169,7 +169,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.2.0
 signing_key: 
 specification_version: 4
 summary: Execute SQL against set of JSON files.