bookshark 1.0.0.beta.3 → 1.0.0.beta.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1a457d8fc6d56572ae211107913ac4ec260745cc
-  data.tar.gz: 310068a2c23c898c732fa6243fe4697339e39dea
+  metadata.gz: a592c6e055f501c19f4a7dea23bad79446b6b28c
+  data.tar.gz: 797820d896b398e6294f9804a4a42f151682a027
 SHA512:
-  metadata.gz: 81074707b067456d71340d23cfde9b6441dc69ffa860291025e5644d44698b10a065872b57d66ca1207275c5507815a95de329983f49d301a37095da0cf88290
-  data.tar.gz: 292cf7099f3f62e264e0bc113747f611bec87d49ce87f8890e49c9f174cb89b3cf5899ff8915d0874d250307185bef1d8ecd71b9c466342f9d53269e8da292a6
+  metadata.gz: 1f8f7b1b0b0f7312549964153afe8198aab00b0acabdb9693c5e9f7c8242e90a19357fb3507397df6cd5b3278cb1ceb22d2b45dfb4646851e698b5cf2bfeb96d
+  data.tar.gz: 73e4f60098595c1e5b20101bf66c5d24d3cc2e2659c5776a7973635a9789149e5a6d2a2a4a69ff7af126314f25f9b2e31824fd131879f0b3b5939eff55462604

data/.gitignore

@@ -8,7 +8,12 @@
 /pkg/
 /spec/reports/
 /tmp/
+/lib/bookshark/storage/html_book_pages/
+/lib/bookshark/storage/html_author_pages/
 /lib/bookshark/storage/html_publisher_pages/
+/lib/bookshark/storage/json_book_pages/
+/lib/bookshark/storage/json_author_pages/
+/lib/bookshark/storage/json_publisher_pages/
 /lib/bookshark/logs/*.log
 *.bundle
 *.so

data/.travis.yml

@@ -1,3 +1,5 @@
+notifications:
+  slack: bibliography:gIEyQx7yEqCDHlXaBxBLsKlo
 language: ruby
 rvm:
   - 2.2.0

data/bookshark.gemspec

@@ -25,7 +25,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "json", "~> 1.8"
   spec.add_dependency "htmlentities", "~> 4.3"
 
-  spec.add_development_dependency "bundler", "~> 1.9"
+  spec.add_development_dependency "bundler", ">= 1.6"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency 'rspec', "~> 3.2"
 end

data/lib/bookshark.rb

@@ -119,6 +119,54 @@ module Bookshark
       return response       
     end
 
+    def books_from_storage
+      extract_from_storage_and_save('book', 'html_book_pages', 'json_book_pages')
+    end
+
+    def authors_from_storage
+      extract_from_storage_and_save('author', 'html_author_pages', 'json_author_pages')
+    end
+
+    def publishers_from_storage
+      extract_from_storage_and_save('publisher', 'html_publisher_pages', 'json_publisher_pages')
+    end
+
+    def categories_from_storage
+      extract_from_storage_and_save('category', 'html_category_pages', 'json_category_pages')
+    end
+
+    def extract_from_storage_and_save(metadata_type, source_dir, target_dir)      
+      list_directories(path: Bookshark.path_to_storage + '/' + source_dir).each do |dir|
+        dir_to_save = dir.gsub(source_dir, target_dir)        
+
+        list_files(path: dir, extension: 'html', all:true).each do |file|
+          puts "Extracting from file: " + file.to_s          
+
+          # Extract publisher metadata form local file.
+          options = {uri: file, format: 'pretty_json', local: true}          
+          
+          case metadata_type
+          when 'author'
+            record = author(options)
+          when 'publisher'
+            record = publisher(options)
+          when 'book'
+            record = book(options)
+          when 'category'
+            record = category(options)       
+          end  
+
+          # Prepare a path to save the new file.
+          filename  = File.basename(file,".*")
+          path_to_save = "#{dir_to_save}#{filename}.json"
+      
+          # Save to file.        
+          save_to("#{path_to_save}", record)
+          
+        end # unless File.directory?(dir_to_save) # if dir.end_with? '/195/'
+      end
+    end
+
     def parse_all_categories(will_save=false)
       # list_directories('raw_ddc_pages').each do |dir|
         # p dir
@@ -166,9 +214,8 @@ module Bookshark
     private
 
     def process_options(options = {}, caller = nil)
-      # puts caller_locations(1,1)[0].label
-      # options[:format] ||= @format
-      puts caller
+      # puts "Called from method: " + caller.to_s
+
       id = options[:id]
 
       if id

data/lib/bookshark/extractors/publisher_extractor.rb

@@ -28,7 +28,7 @@ module Biblionet
                 
         headquarters                    = page.headquarters
         bookstores                      = page.bookstores
-        bookstores['Έδρα']              = headquarters 
+        bookstores['Έδρα']              = headquarters unless headquarters.all? {|k,v| v.nil? or v.empty?}
 
         publisher_hash = {}
         publisher_hash[:name]          = page.name
@@ -97,7 +97,7 @@ module Biblionet
         # Change keys. Use the same as in bookstores.
         mappings                      = {"Διεύθυνση" => :address, "Τηλ" => :telephone, "FAX" => :fax, "E-mail" => :email, "Web site" => :website}
         headquarters_hash             = Hash[headquarters_hash.map {|k, v| [mappings[k], v] }]
-        headquarters_hash[:telephone] = [headquarters_hash[:telephone]] unless headquarters_hash[:telephone].kind_of?(Array)
+        headquarters_hash[:telephone] = [headquarters_hash[:telephone]] unless headquarters_hash[:telephone].kind_of?(Array) or headquarters_hash[:telephone].nil?
         headquarters_hash[:website]   = headquarters_hash[:website].split(',').map(&:strip) if (headquarters_hash[:website] and headquarters_hash[:website].include? ',')
 
         return headquarters_hash                

data/lib/bookshark/version.rb

@@ -1,3 +1,3 @@
 module Bookshark
-  VERSION = "1.0.0.beta.3"
+  VERSION = "1.0.0.beta.5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bookshark
 version: !ruby/object:Gem::Version
-  version: 1.0.0.beta.3
+  version: 1.0.0.beta.5
 platform: ruby
 authors:
 - Dimitris Klisiaris
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-13 00:00:00.000000000 Z
+date: 2015-04-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -76,16 +76,16 @@ dependencies:
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.9'
+        version: '1.6'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.9'
+        version: '1.6'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement