citysdk 0.1.2.5 → 1.0

data/lib/citysdk/importer.rb

@@ -1,9 +1,10 @@
+
 module CitySDK
   
   class Importer
     attr_reader :filereader, :api, :params
     
-    def initialize(pars)
+    def initialize(pars, fr = nil)
       @params = pars
       
       raise Exception.new("Missing :host in Importer parameters.") if @params[:host].nil?
@@ -11,22 +12,16 @@ module CitySDK
       raise Exception.new("Missing :file_path in Importer parameters.") if @params[:file_path].nil?
 
       @api = CitySDK::API.new(@params[:host])
-      if @params[:email]
-        raise Exception.new("Missing :passw in Importer parameters.") if @params[:passw].nil?
-        raise Exception.new("Failure to authenticate '#{@params[:email]}' with api.") if not @api.authenticate(@params[:email],@params[:passw])
+      if @params[:name]
+        raise Exception.new("Missing :password in Importer parameters.") if @params[:password].nil?
+        raise Exception.new("Failure to authenticate '#{@params[:name]}' with api.") if not @api.authenticate(@params[:name],@params[:password])
         @api.release
       end
 
       @params[:addresslayer] = 'bag.vbo' if @params[:addressleyer].nil?
       @params[:addressfield] = 'postcode_huisnummer' if @params[:addressfield].nil?
       
-      @params[:create_type] = 'create' if @params[:create_type].nil?
-      # CREATE_TYPE_UPDATE = 'update' 
-      # CREATE_TYPE_ROUTES = 'routes'
-      # CREATE_TYPE_CREATE = 'create'
-
-
-      @filereader = FileReader.new(@params)
+      @filereader = fr || FileReader.new(@params)
     end
     
     
@@ -34,16 +29,6 @@ module CitySDK
       return @filereader.write(path)
     end
     
-    def setLayerStatus(m)
-      begin
-        sign_in
-        @api.set_layer_status(m.gsub("\n","<br/>"))
-        sign_out
-      rescue Exception => e
-        puts "File Importer setLayerStatus Exception #{e.message}"
-      end
-    end
-    
     def setParameter(k,v)
       begin
         @params[(k.to_sym rescue k) || k] = v
@@ -53,28 +38,12 @@ module CitySDK
       nil
     end
 
-    def setMatchParameter(l,f,v)
-      begin
-        @params[:match] = [] if @params[:match].nil?
-        @params[:match] << [l,f,v] 
-        return true
-      rescue
-      end
-      nil
-    end
-
     def sign_in
-      if @params[:email].nil?
-        raise Exception.new("No credentials provided..")
-      end
-        
       begin
         sign_out if @signed_in
         @api.set_host(@params[:host])
         @api.set_layer(@params[:layername])
-        @api.set_matchTemplate(@params[:match_tpl]) if @params[:match_tpl]
-        @api.set_createTemplate(@params[:create_tpl]) if @params[:create_tpl]
-        @api.authenticate(@params[:email],@params[:passw])
+        @api.authenticate(@params[:name],@params[:password])
         @signed_in = true
       rescue => e
         @api.release
@@ -89,35 +58,32 @@ module CitySDK
     end
   
     def filterFields(h)
-      
-      # puts "h: \n#{h}"
-      # puts "@params[:fields]: \n#{@params[:fields]}"
-      
       data = {}
       h.each_key do |k|
         k = (k.to_sym rescue k) || k
-        data[k] = h[k] if @params[:fields].include?(k)
+        j = @params[:alternate_fields][k]
+        data[j] = h[k] if @params[:fields].include?(k)
       end
-      # puts "data: \n#{data}"
       data
     end
     
     
+    def geomToJSON(a)
+      # puts JSON.pretty_generate(a)
+      a
+    end
     
-    
-  
-    def doImport(dryrun=false, &block)
+    def doImport(&block)
       result = {
-        :updated => 0,
         :created => 0,
         :not_added => 0
       }
       
       failed = nil
       
-      if @params[:hasaddress] == 'certain'
-        failed = addToAddress(dryrun, &block)
-      end
+#      if @params[:hasaddress] == 'certain'
+#        failed = addToAddress(&block)
+#      end
       
       # TODO: add possibility to add node to postal code
 
@@ -130,72 +96,30 @@ module CitySDK
         result[:updated] += (@filereader.content.length - failed.length)
       end
       
-      nodes = failed || @filereader.content
-      
-      count = nodes.length
-      
-      @api.set_createTemplate(
-        {
-          :create => {
-            :params => { #TODO other create types!!
-              :create_type => @params[:create_type],
-              :srid => @params[:srid] || 4326
-            }
-          }
-        }
-      )
-      
-      match_tpl = {
-        :match => {
-          :params => {
-            :debug => true,
-            :layers => {}
-          }
-        }
-      }
+      objects = failed || @filereader.content
+      count = objects.length
 
       begin
         sign_in
-        if @params[:unique_id] and @params[:match] and (@params[:hasgeometry] != 'unknown')
-
-          match_tpl[:match][:params][:radius] = @params[:radius] || 200
-          match_tpl[:match][:params][:geometry_type] = @params[:geometry_type] || :point
-          @params[:match].each do |a|
-            match_tpl[:match][:params][:layers][a[0]] = {} if match_tpl[:match][:params][:layers][a[0]].nil?
-            match_tpl[:match][:params][:layers][a[0]][a[1]] = a[2]
-          end
-          @api.set_matchTemplate(match_tpl)
-          nodes.each do |rec|
-            node = {
-              :geom => rec[:geometry],
-              :id   => rec[:id]
-            }
-            node[:name] = rec[:properties][@params[:name]] if @params[:name]
-            node[:data] = filterFields(rec[:properties])
-
-            # puts JSON.pretty_generate(node)
-            
-            yield(node) if block_given?
-            
-            @api.match_create_node(node) if not dryrun
-            count -= 1
-          end
-          @api.match_create_flush
-      
-        elsif @params[:unique_id] and (@params[:hasgeometry] != 'unknown')
-
+        
+        if @params[:hasgeometry]
           begin
-            nodes.each do |rec|
+            objects.each do |record|
+              
               node = {
-                :geom => rec[:geometry],
-                :id   => rec[:id]
+                type: 'Feature',
+                properties: record[:properties],
+                geometry: geomToJSON(record[:geometry]),
               }
-              node[:name] = rec[:properties][@params[:name].to_sym] if @params[:name]
-              node[:data] = filterFields(rec[:properties])
+              
+              node[:properties][:data] = filterFields(record[:properties][:data])
+              node[:crs] = { type: 'EPSG', properties: { code: @params[:srid]  } } if @params[:srid] != 4326
+              
+              node[:properties][:title] = node[:properties][:data][@params[:title]] if @params[:title]
 
-              yield(node) if block_given?
+              yield(node[:properties]) if block_given?
 
-              @api.create_node(node) if not dryrun
+              @api.create_object(node)
               count -= 1
             end
             @api.create_flush
@@ -209,14 +133,13 @@ module CitySDK
         raise Exception.new(e.message)
       ensure
         a = sign_out
-        result[:updated] += a[0]
-        result[:created] += a[1]
+        result[:created] += a[:created]
         result[:not_added] = count
       end
       return result
     end
 
-    def addToAddress(dryrun=false)
+    def addToAddress()
       failed = []
       if @params[:postcode] and @params[:housenumber]
         begin
@@ -240,7 +163,7 @@ module CitySDK
               url = '/' + qres[:results][0][:cdk_id] + '/' + @params[:layername]
               data = filterFields(row)
               yield({addto: qres[:results][0][:cdk_id], data: data}) if block_given?
-              n = @api.put(url,{'data'=>data}) if not dryrun
+              n = @api.put(url,{'data'=>data})
             else
               failed << rec
             end

data/lib/citysdk/util.rb

@@ -1,10 +1,32 @@
 require 'json'
 
-class NilClass
-  def empty?; true; end
+class String
+  def remove_non_ascii
+    self.encode( "UTF-8", "binary", :invalid => :replace, :undef => :replace, :replace => '§')
+  end
+  def starts_with?(aString)
+    index(aString) == 0
+  end
 end
 
+class Object
+  def deep_copy
+    Marshal.load(Marshal.dump(self))
+  end
+  def blank?
+    return false if self.class == Symbol
+    self.nil? or (self.class==String and self.strip == '') or (self.respond_to?(:empty?) ? self.empty? : false)
+  end
+end
+
+
 module CitySDK
+  
+  # for debugging purposes...
+  def jsonlog(o)
+    puts JSON.pretty_generate({ o.class.to_s => o })
+  end
+
 
   class Exception < ::Exception
     def initialize(message,parms=nil,srcfile=nil,srcline=nil)
@@ -21,9 +43,9 @@ module CitySDK
 
   def self.parseJson(jsonstring)
     begin
-      return JSON.parse(jsonstring,{:symbolize_names => true})
+      return jsonstring.blank? ? {} : JSON.parse(jsonstring,symbolize_names: true)
     rescue Exception => e
-      raise CitySDK::Exception.new(e.message)
+      raise CitySDK::Exception.new("#{e.message}; input: #{jsonstring}")
     end
   end
 

data/lib/citysdk.rb

@@ -4,6 +4,6 @@ require 'citysdk/file_reader.rb'
 require 'citysdk/importer.rb'
 
 module CitySDK
-  VERSION = "0.1.2.5"
+  VERSION = "1.0"
 end
 

data/spec/api_spec.rb

@@ -1,62 +1,69 @@
 
 
 module CitySDK
-  TEST_HOST = 'test-api.citysdk.waag.org'
-  TEST_USER = 'test@waag.org'
+  TEST_HOST = 'localhost:9292'
+  # TEST_HOST = 'test-api.citysdk.waag.org'
+  TEST_USER = 'test'
   TEST_PASS = '123Test321'
 
+  # name=test.rspec title='test layer' owner=citysdk description='for testing' rdf_type=csdk\:Test category=security
+
   TEST_LAYER = {
-    :data => {
-      :name => 'test.rspec',
-      :description => 'for testing',
-      :organization => 'waag',
-      :category => 'security.test'
-    }
+    name: "test.rspec",
+    title: "test layer",
+    owner: TEST_USER,
+    description: "for testing",
+    data_sources: [""],
+    rdf_type: "csdk:Test",
+    category: "security",
+    subcategory: "test",
+    licence: "CC0",
+    fields: []
   }
   
-  describe API do
+  $api = API.new(TEST_HOST)
 
-    before(:each) do
-      @api = API.new(TEST_HOST)
-      TEST_USER.should_not == 'citysdk@waag.org'
-    end
+  
+  
+  describe API do
 
-    after(:each) do
-      @api.release
-    end
 
     it "can be connected to" do
-      @api.should_not == nil
+      expect($api.class).to be(CitySDK::API)
     end
 
     it "can be queried" do
-      @api.get('/layers')[:status].should == 'success'
+      expect($api.get('/layers').class).to be(Hash)
     end
 
     it "can be authorized against" do
-      @api.authenticate(TEST_USER,TEST_PASS).should == true
+      expect($api.authenticate(TEST_USER,TEST_PASS)).to be(true)
+      expect($api.release.class).to be(Array)
+    end
+
+    it "can create and delete a test layer" do
+      expect($api.authenticate(TEST_USER,TEST_PASS)).to be(true)
+      expect($api.post('/layers',TEST_LAYER)[:features].class).to be(Array)
+      expect($api.delete('/layers/test.rspec')).to eq({})
+      expect($api.release.class).to be(Array)
     end
-    
-    # it "can create and delete a test layer" do
-    #   @api.authenticate(TEST_USER,TEST_PASS).should == true
-    #   @api.put('/layers',TEST_LAYER)[:status].should == 'success'
-    #   @api.delete('/layer/test.rspec?delete_layer=true')[:status].should == 'success'
-    # end
 
     it "can not create a layer in an unauthorized domain" do
       h = TEST_LAYER.dup
-      h[:data][:name] = 'unauthorized.rspec'
-      @api.authenticate(TEST_USER,TEST_PASS).should == true
-      expect { @api.put('/layers',h) }.to raise_error(HostException,"Not authorized for domain 'unauthorized'.")
-      TEST_LAYER[:data][:name] = 'test.rspec'
+      h[:name] = 'unauthorized.rsped'
+      expect($api.authenticate(TEST_USER,TEST_PASS)).to be(true)
+      expect { $api.post('/layers',h) }.to raise_error(HostException,"Owner has no access to domain 'unauthorized'")
+      h[:name] = "test.rspec"
+      $api.release
     end
 
     it "can not add data to a layer not owned" do
-      res = @api.get('/nodes?per_page=1')
-      res[:status].should == 'success'
-      cdk = res[:results][0][:cdk_id]
-      @api.authenticate(TEST_USER,TEST_PASS).should == true
-      expect { @api.put("/#{cdk}/osm",{:data=>{:plop => 'pipo'}}) }.to raise_error(HostException,"Not authorized for layer 'osm'.")
+      res = $api.get('/objects?per_page=1')
+      expect(res[:type]).to eq('FeatureCollection')
+      cdk = res[:features][0][:properties][:cdk_id]
+      expect($api.authenticate(TEST_USER,TEST_PASS)).to be(true)
+      expect { $api.patch("/objects/#{cdk}/layers/osm",{:plop => 'pipo'}) }.to raise_error(HostException,"Operation requires owners' authorization")
+      $api.release
     end
 
   end

data/spec/import_spec.rb

@@ -3,82 +3,48 @@
 module CitySDK
 
   describe "Importer" do
-    
-    
+
     def newImporter(f)
       @importer = Importer.new({
-        :host=>TEST_HOST, 
-        :layername=>'test.rspec',
-        :file_path => f
+        file_path: f,
+        host: TEST_HOST, 
+        layername: TEST_LAYER[:name]
       })
     end
-    
-    after(:each) do
-      if @layerCreated
-        @importer.api.authenticate(TEST_USER,TEST_PASS) do 
-          @importer.api.delete("/layer/test.rspec?delete_layer=true")
-        end.should_not == nil
-      end
-    end
-    
-    
+
     it "checks parameters" do
       expect { Importer.new({:a=>1}) }.to raise_error(CitySDK::Exception)
     end
 
     it "succesfully creates a FileReader" do
       newImporter('./spec/files/wkb.csv')
-      @importer.should_not == nil 
-      @importer.filereader.params[:unique_id].should == :gid
+      expect(@importer.class).to be(CitySDK::Importer)
+      expect(@importer.filereader.params[:unique_id]).to be(:gid)
     end
 
-    it "can add a (match) parameter" do
-      newImporter('./spec/files/wkb.csv')
-      @importer.setParameter('freut',:nix).should be_true
-      @importer.setMatchParameter('freut','pipo',nil).should be_true
-    end
-    
     it "needs authorization to import" do
       newImporter('./spec/files/hotels.csv')
       expect { @importer.doImport }.to raise_error(CitySDK::Exception)
     end
 
-    it "can make a match and add data to osm obj" do
-      newImporter('./spec/files/rk.csv')
-      
-      @importer.api.authenticate(TEST_USER,TEST_PASS) do 
-        @importer.api.put('/layers',TEST_LAYER)[:status].should == 'success'
-      end.should_not == nil
-      @layerCreated = true
-      @importer.setParameter(:email,TEST_USER).should be_true
-      @importer.setParameter(:passw,TEST_PASS).should be_true
-
-      @importer.setMatchParameter('osm','amenity','fuel').should be_true
-
-      res = @importer.doImport[:not_added].should == 0
-      res = @importer.api.get("/nodes?count&layer=test.rspec")
-      res[:record_count].should == 5
-      # puts JSON.pretty_generate(res)
-    end
+    it "can make a layer and delete" do
 
-
-    it "can make a layer and add data to and address" do
-      
       newImporter('./spec/files/hotels.csv')
       
+      @importer.setParameter(:name,TEST_USER)
+      @importer.setParameter(:password,TEST_PASS)
+      @importer.sign_in
+      res = @importer.api.post('/layers',TEST_LAYER)
+      expect( (!!res[:features] and res[:features].length == 1) ).to be(true)
+      @importer.sign_out
+
+      expect { @importer.doImport }.to raise_error(CitySDK::Exception)
+
+      res = @importer.api.get("/layers/test.rspec/objects")
+      expect( (!!res[:features] and res[:features].length == 0) ).to be(true)
+
       @importer.api.authenticate(TEST_USER,TEST_PASS) do 
-        @importer.api.put('/layers',TEST_LAYER)[:status].should == 'success'
-      end.should_not == nil
-      @layerCreated = true
-
-      @importer.setParameter(:email,TEST_USER).should be_true
-      @importer.setParameter(:passw,TEST_PASS).should be_true
-      if TEST_HOST == 'api.citysdk.waag.org'
-        @importer.doImport[:not_added].should == 0
-        res = @importer.api.get("/nodes?count&layer=test.rspec")
-        res[:record_count].should == 4
-      else
-        @importer.doImport[:not_added].should == 4
+        expect( @importer.api.delete("/layers/#{TEST_LAYER[:name]}") ).to eq({})
       end
 
     end
@@ -89,49 +55,48 @@ module CitySDK
 
     it "can parse json" do
       j = CitySDK::parseJson('{ "arr" : [0,1,1,1], "hash": {"aap": "noot"}, "num": 0 }')
-      j[:arr].length.should == 4
-      j[:num].class.should == Fixnum
-      j[:hash][:aap].should == "noot"
+      expect(j[:arr].length).to be(4)
+      expect(j[:num].class).to be(Fixnum)
+      expect(j[:hash][:aap]).to eq("noot")
     end
 
     it "can read json files" do
       fr = FileReader.new({:file_path => './spec/files/stations.json'})
-      File.basename(fr.file).should == 'stations.json'
-      fr.params[:geomtry_type].should == 'Point'
-      fr.params[:srid].should == 4326
-      fr.params[:unique_id].should == :code
+      expect(File.basename(fr.file)).to eq('stations.json')
+      expect(fr.params[:geomtry_type]).to eq('Point')
+      expect(fr.params[:srid]).to be(4326)
+      expect(fr.params[:unique_id]).to be(:code)
     end
 
     it "can read geojson files" do
       fr = FileReader.new({:file_path => './spec/files/geojsonTest.GeoJSON'})
-      File.basename(fr.file).should == 'geojsonTest.GeoJSON'
-      fr.params[:unique_id].should == nil
+      expect(File.basename(fr.file)).to eq('geojsonTest.GeoJSON')
+      expect(fr.params[:unique_id]).to be(:id)#self generated unique id
     end
 
     it "can read csv files" do
       fr = FileReader.new({:file_path => './spec/files/csvtest.zip'})
-      File.basename(fr.file).should == 'akw.csv'
-      fr.params[:srid].should == 4326
-      fr.params[:colsep].should == ';'
-      fr.params[:unique_id].should == :ID
+      expect(File.basename(fr.file)).to eq('akw.csv')
+      expect(fr.params[:srid]).to be(4326)
+      expect(fr.params[:colsep]).to eq(';')
+      expect(fr.params[:unique_id]).to be(:ID)
     end
 
     it "can read csv files with wkb geometry" do
       fr = FileReader.new({:file_path => './spec/files/wkb.csv'})
-      File.basename(fr.file).should == 'wkb.csv'
-      fr.params[:srid].should == 4326
-      fr.params[:colsep].should == ';'
-      fr.params[:unique_id].should == :gid
+      expect(File.basename(fr.file)).to eq('wkb.csv')
+      expect(fr.params[:srid]).to be(4326)
+      expect(fr.params[:colsep]).to eq(';')
+      expect(fr.params[:unique_id]).to be(:gid)
     end
 
     it "can read zipped shape files" do
       fr = FileReader.new({:file_path => './spec/files/shapeTest.zip'})
-      fr.params[:srid].should == 2100
+      expect(fr.params[:srid]).to be(2100)
     end
 
   end
-  
+
 end
 
 
-# puts JSON.pretty_generate(fr.content[0])