shp2geocouch 0.0.3 → 0.0.4

data/README.textile

@@ -23,10 +23,10 @@ Install the @shp2geocouch@ gem:
 
 h2. Usage
 
-@shp2geocouch <path_to_shapefile> [your-geocouch-root-url (optional, default: http://localhost:5984)]@
+@shp2geocouch <path_to_shapefile> [geocouch-url (optional, default: http://localhost:5984/zip_filename)]@
 
 example: @shp2geocouch /sweet_geo_data/Portland_Oregon_Public_Libraries.shp@
-another possible example: @shp2geocouch /zipped_shapfiles/US_Railroads.zip http://myusername.couchone.com@
+another possible example: @shp2geocouch /zipped_shapfiles/US_Railroads.zip http://myusername.couchone.com/railroads@
 
 Once completed, you can run bounding box queries against your data! Visit the following link to receive a dump of your entire dataset as GeoJSON (a bounding box that covers the entire planet):
 @http://localhost:5984/your_db_name/_design/geojson/_spatial/points?bbox=-180,180,-90,90@
@@ -38,6 +38,7 @@ h3. Arguments:
 * You can feed in either a @.shp@ file or a @.zip@ that contains a @.shp@
 * @-v@ for verbose
 * @--no-cleanup@ if you want @shp2geocouch@ to leave you a copy of unzipped Shapefile and converted JSON.
+* @--chunksize@ to customize the number of docs that it sends in each PUT to CouchDB's @_bulk_docs@ interface (default is 50)
 
 h2. What?
 

data/VERSION

@@ -1 +1 @@
-0.0.3
+0.0.4

data/bin/shp2geocouch

@@ -5,7 +5,7 @@ require 'optparse'
 require 'iconv'
 
 class ShapefileToGeoCouch
-  attr_accessor :path, :extension, :name, :couch_url, :cleanup, :verbose
+  attr_accessor :path, :extension, :name, :couch_url, :cleanup, :verbose, :chunksize
   
   def initialize(options)
     set_accessors(options)
@@ -69,13 +69,29 @@ class ShapefileToGeoCouch
     %x!sed -e '/^\"type\": \"FeatureCollection\",$/d' -e '/^\"features\": \\[$/d' -e '/^{$/d' -e '/^,$/d' -e '/^}$/d' -e '/^]$/d' -e '/^$/d' -e 's/$/,/' #{json} > #{bulk}!
   end
   
-  def upload
+  def post(string)
     ic = Iconv.new('UTF-8//IGNORE', 'UTF-8') # disregard all UTF8 characters
+    valid_string = ic.iconv(string[0..-3] + '  ')[0..-3] # http://po-ru.com/diary/fixing-invalid-utf-8-in-ruby-revisited/
+    HTTParty.post(database_url + '/_bulk_docs', :body => '{"docs": [' + valid_string + "]}", :headers => {'content-type' => "application/json"})
+  end
+  
+  def upload
     puts "Bulk loading data into GeoCouch... view progress at #{@couch_url}/_utils" if @verbose
-    File.open(bulk).each_line do |line|
-      valid_string = ic.iconv(line[0..-3] + '  ')[0..-3] # http://po-ru.com/diary/fixing-invalid-utf-8-in-ruby-revisited/
-      HTTParty.post(database_url + '/_bulk_docs', :body => '{"docs": [' + valid_string + "]}", :headers => {'content-type' => "application/json"})
-     end
+    group = []
+    length = File.open(bulk).lines.count - 1
+    File.open(bulk).each_with_index do |line, index|
+      if (index % @chunksize == 0)
+        post(group.join(''))
+        group = [line]
+        next
+      end
+      if (index == length)
+        group << line
+        post(group.join(''))
+        next
+      end
+      group << line
+    end
   end
   
   def add_ddoc
@@ -97,13 +113,17 @@ class ShapefileToGeoCouch
   end
 end  
 
-defaults = {:cleanup => true, :verbose => false}
+defaults = {:cleanup => true, :verbose => false, :chunksize => 50}
 OptionParser.new do |opts|
   opts.banner = "Usage: #{__FILE__} [path-to-shapefile] [your-geocouch-url (optional, default: http://localhost:5984/zip_filename)]"
   opts.on("--no-cleanup", "Don't remove converted files after upload") do |v|
     defaults[:cleanup] = false
   end
   
+  opts.on("--chunksize [SIZE]", "Couch bulk docs interface lines per chunk PUT (default 50)") do |size|
+    defaults[:chunksize] = size.to_i
+  end
+  
   opts.on('-v') do |v|
     defaults[:verbose] = true
   end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: shp2geocouch
 version: !ruby/object:Gem::Version 
-  hash: 25
+  hash: 23
   prerelease: false
   segments: 
   - 0
   - 0
-  - 3
-  version: 0.0.3
+  - 4
+  version: 0.0.4
 platform: ruby
 authors: 
 - Max Ogden
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-10-23 00:00:00 -07:00
+date: 2010-10-26 00:00:00 -07:00
 default_executable: shp2geocouch
 dependencies: 
 - !ruby/object:Gem::Dependency