citysdk 0.1.1

data/lib/citysdk/importer.rb

@@ -0,0 +1,230 @@
+module CitySDK
+  
+  class Importer
+    attr_reader :filereader, :api
+    
+    def initialize(pars)
+      @params = pars
+      
+      raise Exception.new("Missing :host in Importer parameters.") if @params[:host].nil?
+      raise Exception.new("Missing :layername in Importer parameters.") if @params[:layername].nil?
+      raise Exception.new("Missing :file_path in Importer parameters.") if @params[:file_path].nil?
+
+      @api = CitySDK::API.new(@params[:host])
+      if @params[:email]
+        raise Exception.new("Missing :passw in Importer parameters.") if @params[:passw].nil?
+        raise Exception.new("Failure to authenticate '#{@params[:email]}' with api.") if not @api.authenticate(@params[:email],@params[:passw])
+        @api.release
+      end
+
+      @params[:addresslayer] = 'bag.vbo' if @params[:addressleyer].nil?
+      @params[:addressfield] = 'postcode_huisnummer' if @params[:addressfield].nil?
+      
+      @params[:create_type] = 'create' if @params[:create_type].nil?
+      # CREATE_TYPE_UPDATE = 'update' 
+      # CREATE_TYPE_ROUTES = 'routes'
+      # CREATE_TYPE_CREATE = 'create'
+      
+
+      @filereader = FileReader.new(@params)
+    end
+    
+    def setLayerStatus(m)
+      begin
+        sign_in
+        @api.set_layer_status(m.gsub("\n","<br/>"))
+        sign_out
+      rescue Exception => e
+        puts "File Importer setLayerStatus Exception #{e.message}"
+      end
+    end
+    
+    def setParameter(k,v)
+      begin
+        @params[(k.to_sym rescue k) || k] = v
+        return true
+      rescue
+      end
+      nil
+    end
+
+    def setMatchParameter(l,f,v)
+      begin
+        @params[:match] = [] if @params[:match].nil?
+        @params[:match] << [l,f,v] 
+        return true
+      rescue
+      end
+      nil
+    end
+
+    def sign_in
+      if @params[:email].nil?
+        raise Exception.new("No credentials provided..")
+      end
+        
+      begin
+        sign_out if @signed_in
+        @api.set_host(@params[:host])
+        @api.set_layer(@params[:layername])
+        @api.set_matchTemplate(@params[:match_tpl]) if @params[:match_tpl]
+        @api.set_createTemplate(@params[:create_tpl]) if @params[:create_tpl]
+        @api.authenticate(@params[:email],@params[:passw])
+        @signed_in = true
+      rescue => e
+        @api.release
+        raise e
+      ensure
+      end
+    end
+
+    def sign_out
+      @signed_in = false
+      return @api.release
+    end
+  
+    def filterFields(h)
+      data = {}
+      h.each_key do |k|
+        data[k] = h[k] if @params[:fields].include?(k)
+      end
+      data
+    end
+  
+    def doImport(dryrun=false)
+      result = {
+        :updated => 0,
+        :created => 0,
+        :not_added => 0
+      }
+      
+      failed = nil
+      
+      if @params[:hasaddress] == 'certain'
+        failed = addToAddress(dryrun)
+      # elsif @params[:hasaddress] == 'maybe'
+      #   failed = addToAddress(dryrun)
+      end
+
+      if failed == []
+        result[:updated] += @filereader.content.length
+        return result
+      end
+
+      if failed 
+        result[:updated] += (@filereader.content.length - failed.length)
+      end
+      
+      nodes = failed || @filereader.content
+      
+      count = nodes.length
+      
+      @api.set_createTemplate(
+        {
+          :create => {
+            :params => { #TODO other create types!!
+              :create_type => @params[:create_type],
+              :srid => @params[:srid] || 4326
+            }
+          }
+        }
+      )
+      
+      match_tpl = {
+        :match => {
+          :params => {
+            :debug => true,
+            :layers => {}
+          }
+        }
+      }
+
+      begin
+        sign_in
+        if @params[:unique_id] and @params[:match] and (@params[:hasgeometry] != 'unknown')
+
+          match_tpl[:match][:params][:radius] = @params[:radius] || 200
+          match_tpl[:match][:params][:geometry_type] = @params[:geometry_type] || :point
+          @params[:match].each do |a|
+            match_tpl[:match][:params][:layers][a[0]] = {} if match_tpl[:match][:params][:layers][a[0]].nil?
+            match_tpl[:match][:params][:layers][a[0]][a[1]] = a[2]
+          end
+          @api.set_matchTemplate(match_tpl)
+          nodes.each do |rec|
+            node = {
+              :geom => rec[:geometry],
+              :id   => rec[:properties][@params[:unique_id]]
+            }
+            node[:name] = rec[:properties][@params[:name]] if @params[:name]
+            node[:data] = rec[:properties]
+            @api.match_create_node(node) if not dryrun
+            count -= 1
+          end
+          @api.match_create_flush
+      
+        elsif @params[:unique_id] and (@params[:hasgeometry] != 'unknown')
+          
+          begin
+            nodes.each do |rec|
+              node = {
+                :geom => rec[:geometry],
+                :id   => rec[:properties][@params[:unique_id]]
+              }
+              node[:name] = rec[:properties][@params[:name]] if @params[:name]
+              node[:data] = rec[:properties]
+              @api.create_node(node) if not dryrun
+              count -= 1
+            end
+            @api.create_flush
+          rescue => e
+            raise e
+          end
+        else
+          raise Exception.new("Cannot import. Geometry or uniuque id is not known for records.") if failed.nil?
+        end
+      ensure
+        a = sign_out
+        result[:updated] += a[0]
+        result[:created] += a[1]
+        result[:not_added] = count
+      end
+      return result
+    end
+
+    def addToAddress(dryrun=false)
+      failed = []
+      if @params[:postcode] and @params[:housenumber]
+        begin
+          sign_in
+          @filereader.content.each do |rec|
+            row = rec[:properties]
+            pc = row[@params[:postcode]].downcase.gsub(/[^a-z0-9]/,'')
+            hn = row[@params[:housenumber]]
+            qres = {}
+            if not (pc.empty? or hn.empty?)
+              hn.scan(/\d+/).reverse.each { |n|
+                qres = @api.get("/nodes?#{@params[:addresslayer]}::#{@params[:addressfield]}=#{pc + n}")
+                break if qres[:status]=='success' and qres[:record_count].to_i >= 1
+              }
+            else
+              qres[:status]='nix'
+            end
+            if qres[:status]=='success' and qres[:results] and qres[:results][0]
+              url = '/' + qres[:results][0][:cdk_id] + '/' + @params[:layername]
+              n = @api.put(url,{'data'=>filterFields(row)}) if not dryrun
+            else
+              failed << rec
+            end
+          end
+        rescue => e
+          raise e
+        ensure
+          sign_out
+        end
+        return failed
+      end
+      raise Exception.new("Addresses not well defined in dataset.")
+    end
+
+  end
+end

data/lib/citysdk/util.rb

@@ -0,0 +1,33 @@
+require 'json'
+
+class NilClass
+  def empty?; true; end
+end
+
+module CitySDK
+
+  class Exception < ::Exception
+    def initialize(message,parms=nil,srcfile=nil,srcline=nil)
+      if parms and srcfile and srcline
+        file = File.basename( parms[:originalfile] ? parms[:originalfile] : ( parms[:filepath] || '-' ) )
+        m = "#{Time.now.strftime("%b %M %Y, %H:%M")}; CitySDK, processing file: #{file}\n Exception in #{File.basename(srcfile)}, #{srcline}\n #{message}"
+      else
+        m = "#{Time.now.strftime("%b %M %Y, %H:%M")}; CitySDK Exception: #{message}"
+      end
+      super(m)
+      $stderr.puts(m) if parms and parms[:verbose]
+    end
+  end
+
+  def self.parseJson(jsonstring)
+    begin
+      return JSON.parse(jsonstring,{:symbolize_names => true})
+    rescue Exception => e
+      raise CitySDK::Exception.new(e.message)
+    end
+  end
+
+end
+
+
+

data/spec/api_spec.rb

@@ -0,0 +1,64 @@
+
+
+module CitySDK
+  TEST_HOST = 'test-api.citysdk.waag.org'
+  TEST_USER = 'test@waag.org'
+  TEST_PASS = '123Test321'
+
+  TEST_LAYER = {
+    :data => {
+      :name => 'test.rspec',
+      :description => 'for testing',
+      :organization => 'waag',
+      :category => 'security.test'
+    }
+  }
+  
+  describe API do
+
+    before(:each) do
+      @api = API.new(TEST_HOST)
+      TEST_USER.should_not == 'citysdk@waag.org'
+    end
+
+    after(:each) do
+      @api.release
+    end
+
+    it "can be connected to" do
+      @api.should_not == nil
+    end
+
+    it "can be queried" do
+      @api.get('/layers')[:status].should == 'success'
+    end
+
+    it "can be authorized against" do
+      @api.authenticate(TEST_USER,TEST_PASS).should == true
+    end
+    
+    # it "can create and delete a test layer" do
+    #   @api.authenticate(TEST_USER,TEST_PASS).should == true
+    #   @api.put('/layers',TEST_LAYER)[:status].should == 'success'
+    #   @api.delete('/layer/test.rspec?delete_layer=true')[:status].should == 'success'
+    # end
+
+    it "can not create a layer in an unauthorized domain" do
+      h = TEST_LAYER.dup
+      h[:data][:name] = 'unauthorized.rspec'
+      @api.authenticate(TEST_USER,TEST_PASS).should == true
+      expect { @api.put('/layers',h) }.to raise_error(HostException,"Not authorized for domain 'unauthorized'.")
+      TEST_LAYER[:data][:name] = 'test.rspec'
+    end
+
+    it "can not add data to a layer not owned" do
+      res = @api.get('/nodes?per_page=1')
+      res[:status].should == 'success'
+      cdk = res[:results][0][:cdk_id]
+      @api.authenticate(TEST_USER,TEST_PASS).should == true
+      expect { @api.put("/#{cdk}/osm",{:data=>{:plop => 'pipo'}}) }.to raise_error(HostException,"Not authorized for layer 'osm'.")
+    end
+
+  end
+  
+end

data/spec/files/geojsonTest.GeoJSON

@@ -0,0 +1,25 @@
+{
+  "type": "FeatureCollection",
+  "features": [
+    { "type": "Feature", 
+      "id": "TrajectSensor_Route059", 
+      "properties": { 
+        "DSSObjectType": "TRAJECT_SENSOR", 
+        "LOCATION": "TrajectSensor_Route059", 
+        "TIMESTAMP": "13-07-2013 15:45:30", 
+        "COLOR": "#00FF00", 
+        "VELOCITY": 33, 
+        "LENGTH": 961, 
+        "TRAVELTIME": 104, 
+        "TRAVELTIME_FF": 961
+      },  
+      "geometry": { 
+        "type": "LineString", 
+        "coordinates": [ [123289, 487663], [123606, 487587], [123691, 487565], [123792, 487540], [123830, 487520], [123866, 487500], [123888, 487488], [123901, 487479], [123978, 487426], [123982, 487422], [124019, 487394], [124024, 487385], [124037, 487373], [124051, 487358], [124059, 487346], [124067, 487336], [124072, 487316]]
+      }
+    },
+    { "type": "Feature", "id": "TrajectSensor_Route058", "properties": { "DSSObjectType": "TRAJECT_SENSOR", "LOCATION": "TrajectSensor_Route058", "TIMESTAMP": "13-07-2013 15:45:27", "COLOR": "#00FF00", "VELOCITY": 45, "LENGTH": 961, "TRAVELTIME": 76, "TRAVELTIME_FF": 69},  "geometry": { "type": "LineString", "coordinates": [ [124153, 487283], [124138, 487340], [124119, 487371], [124110, 487384], [124095, 487401], [124091, 487404], [124053, 487430], [124009, 487466], [123929, 487521], [123914, 487531], [123890, 487544], [123853, 487564], [123809, 487586], [123703, 487613], [123618, 487635], [123349, 487699]]}},
+    { "type": "Feature", "id": "TrajectSensor_Route057", "properties": { "DSSObjectType": "TRAJECT_SENSOR", "LOCATION": "TrajectSensor_Route057", "TIMESTAMP": "13-07-2013 15:44:51", "COLOR": "#FF9900", "VELOCITY": 18, "LENGTH": 3960, "TRAVELTIME": 775, "TRAVELTIME_FF": 285},  "geometry": { "type": "LineString", "coordinates": [ [118925, 483606], [118912, 483647], [118913, 483654], [118918, 483697], [118921, 483731], [118928, 483773], [118933, 483814], [118936, 483834], [118937, 483875], [118937, 483900], [118935, 483921], [118928, 483952], [118921, 483987], [118918, 483997], [118908, 484046], [118889, 484130], [118879, 484176], [118876, 484193], [118874, 484204], [118873, 484206], [118875, 484206], [118873, 484205], [118877, 484207], [118898, 484217], [118916, 484226], [118921, 484228], [118955, 484252], [118958, 484314], [118955, 484460], [118954, 484490], [118953, 484548], [118953, 484647], [118945, 484678], [118931, 484727], [118936, 484719], [118953, 484731], [118965, 484750], [118967, 484762], [118970, 484773], [118964, 484799], [118951, 484817], [118933, 484828], [118910, 484834], [118907, 484824], [118902, 484876], [118891, 484914], [118888, 484928], [118886, 484934], [118879, 484958], [118872, 484986], [118866, 485006], [118861, 485026], [118851, 485072], [118846, 485098], [118843, 485111], [118836, 485145], [118831, 485169], [118819, 485224], [118812, 485254], [118807, 485282], [118803, 485301], [118801, 485318], [118796, 485352], [118793, 485366], [118788, 485390], [118758, 485505], [118753, 485526], [118750, 485535], [118744, 485545], [118750, 485551], [118755, 485575], [118761, 485592], [118776, 485610], [118777, 485612], [118789, 485628], [118800, 485651], [118791, 485659], [118810, 485655], [118909, 485692], [118933, 485702], [119016, 485737], [119056, 485754], [119097, 485771], [119108, 485776], [119141, 485791], [119190, 485809], [119218, 485820], [119319, 485859], [119414, 485897], [119477, 485921], [119588, 485961], [119636, 485980], [119681, 485998], [119738, 486021], [119769, 486034], [119870, 486077], [120009, 486134], [120030, 486142], [120135, 486184], [120143, 486187], [120151, 486191], [120208, 486216], [120330, 486273]]}}
+  ]
+}
+

data/spec/files/hotels.csv

@@ -0,0 +1,5 @@
+nummer;name;stars;rooms;beds;camping;municipality;town;postcode;street;number;buurt2005;wijk;buurt2010;quarter
+1;Eden Amsterdam American Hotel;4;175;354; ;Amsterdam;Amsterdam;1017PN;Leidsekade;97;A07b;A;A07b;A
+2;Amstel Intercontinental Amsterdam;5;79;165; ;Amsterdam;Amsterdam;1018GX;Professor Tulpplein;1;A08b;A;A08b;A
+3;Amsterdam Centre,nh;4;232;424; ;Amsterdam;Amsterdam;1054ES;Stadhouderskade;7;D22a;D;E22a;E
+4;Barbizon Palace,nh;5;274;548; ;Amsterdam;Amsterdam;1012AD;Prins Hendrikkade;59-72;A00a;A;A00a;A

data/spec/files/rk.csv

@@ -0,0 +1,6 @@
+gml_id;bevoegd_gezag;rrgs_id;naam_inrichting;straat;huisnummer;postcode;plaats;gemeente;wm_vergunning;date_changed;longitude;latitude
+ID_23;Gemeente ALBRANDSWAARD;12438;RIJNPOORT BV;Ambachtsstraat;5;X176PR;POORTUGAAL;ALBRANDSWAARD;315599;15-05-2009;4.406100015089653;51.85913344374868
+ID_24;Gemeente HARDENBERG;3870;Autobedrijf Wessels Dedemsvaart BV.;Langewijk;198; ;DEDEMSVAART;HARDENBERG;90740;07-04-2011;6.464028693395486;52.60686467883884
+ID_63;Gemeente PUTTEN;7445;Tankstation Pieper;Broekermolenweg;3;X882MG;PUTTEN;PUTTEN;M07/09;31-01-2011;5.55024558211801;52.2490702463568
+ID_9676;Gemeente BOXTEL;1818;Tankstation Texaco Ooiendonk;Rijksweg ;2; ; ;BOXTEL;481;02-02-2011;5.36197249261972;51.55510273583312
+ID_9533;Gemeente BEEMSTER;9961;Tankstation B.P. Kruisoord;Rijksweg A7;; ;NOORDBEEMSTER;BEEMSTER;2003-6032;16-01-2012;4.977775378849885;52.58074726262354

data/spec/files/stations.json

@@ -0,0 +1,42 @@
+[
+    {
+        "alias": "true", 
+        "code": "GVC", 
+        "country": "NL", 
+        "lat": "52.080276", 
+        "long": "4.325", 
+        "name": "'s-Gravenhage"
+    }, 
+    {
+        "alias": "true", 
+        "code": "ALM", 
+        "country": "NL", 
+        "lat": "52.37503", 
+        "long": "5.21764", 
+        "name": "Almere"
+    }, 
+    {
+        "alias": "true", 
+        "code": "APN", 
+        "country": "NL", 
+        "lat": "52.124443", 
+        "long": "4.657778", 
+        "name": "Alphen aan den Rijn"
+    }, 
+    {
+        "alias": "true", 
+        "code": "ASD", 
+        "country": "NL", 
+        "lat": "52.378887", 
+        "long": "4.9002776", 
+        "name": "Amsterdam"
+    }, 
+    {
+        "alias": "true", 
+        "code": "SHL", 
+        "country": "NL", 
+        "lat": "52.309444", 
+        "long": "4.7619443", 
+        "name": "Amsterdam Airport"
+    }
+]

data/spec/files/wkb.csv

@@ -0,0 +1,3 @@
+"gid";"verkoop_pu";"b_dat_verk";"e_dat_verk";"oms";"domein_cod";"gmrotation";"geom"
+18;19184;"2007-02-12";"";"Hofwijckstraat 42 BORD";363;0.259811880644949;"0104000020E61000000100000001010000001960CC67A2661340142840BDAD304A40"
+19;19191;"2007-02-12";"";"Spaanse Brabanderstraat 31 BORD";363;0.260326709794243;"0104000020E6100000010000000101000000B7EDEED58C65134069AFCF2E90304A40"

data/spec/import_spec.rb

@@ -0,0 +1,137 @@
+
+
+module CitySDK
+
+  describe "Importer" do
+    
+    
+    def newImporter(f)
+      @importer = Importer.new({
+        :host=>TEST_HOST, 
+        :layername=>'test.rspec',
+        :file_path => f
+      })
+    end
+    
+    after(:each) do
+      if @layerCreated
+        @importer.api.authenticate(TEST_USER,TEST_PASS) do 
+          @importer.api.delete("/layer/test.rspec?delete_layer=true")
+        end.should_not == nil
+      end
+    end
+    
+    
+    it "checks parameters" do
+      expect { Importer.new({:a=>1}) }.to raise_error(CitySDK::Exception)
+    end
+
+    it "succesfully creates a FileReader" do
+      newImporter('./spec/files/wkb.csv')
+      @importer.should_not == nil 
+      @importer.filereader.params[:unique_id].should == :gid
+    end
+
+    it "can add a (match) parameter" do
+      newImporter('./spec/files/wkb.csv')
+      @importer.setParameter('freut',:nix).should be_true
+      @importer.setMatchParameter('freut','pipo',nil).should be_true
+    end
+    
+    it "needs authorization to import" do
+      newImporter('./spec/files/hotels.csv')
+      expect { @importer.doImport }.to raise_error(CitySDK::Exception)
+    end
+
+    it "can make a match and add data to osm obj" do
+      newImporter('./spec/files/rk.csv')
+      
+      @importer.api.authenticate(TEST_USER,TEST_PASS) do 
+        @importer.api.put('/layers',TEST_LAYER)[:status].should == 'success'
+      end.should_not == nil
+      @layerCreated = true
+      @importer.setParameter(:email,TEST_USER).should be_true
+      @importer.setParameter(:passw,TEST_PASS).should be_true
+
+      @importer.setMatchParameter('osm','amenity','fuel').should be_true
+
+      res = @importer.doImport[:not_added].should == 0
+      res = @importer.api.get("/nodes?count&layer=test.rspec")
+      res[:record_count].should == 5
+      puts JSON.pretty_generate(res)
+    end
+
+
+    it "can make a layer and add data to and address" do
+      
+      newImporter('./spec/files/hotels.csv')
+      
+      @importer.api.authenticate(TEST_USER,TEST_PASS) do 
+        @importer.api.put('/layers',TEST_LAYER)[:status].should == 'success'
+      end.should_not == nil
+      @layerCreated = true
+
+      @importer.setParameter(:email,TEST_USER).should be_true
+      @importer.setParameter(:passw,TEST_PASS).should be_true
+      if TEST_HOST == 'api.citysdk.waag.org'
+        @importer.doImport[:not_added].should == 0
+        res = @importer.api.get("/nodes?count&layer=test.rspec")
+        res[:record_count].should == 4
+      else
+        @importer.doImport[:not_added].should == 4
+      end
+
+    end
+
+  end
+  
+  describe "FileReader" do
+
+    it "can parse json" do
+      j = CitySDK::parseJson('{ "arr" : [0,1,1,1], "hash": {"aap": "noot"}, "num": 0 }')
+      j[:arr].length.should == 4
+      j[:num].class.should == Fixnum
+      j[:hash][:aap].should == "noot"
+    end
+
+    it "can read json files" do
+      fr = FileReader.new({:file_path => './spec/files/stations.json'})
+      File.basename(fr.file).should == 'stations.json'
+      fr.params[:geomtry_type].should == 'Point'
+      fr.params[:srid].should == 4326
+      fr.params[:unique_id].should == :code
+    end
+
+    it "can read geojson files" do
+      fr = FileReader.new({:file_path => './spec/files/geojsonTest.GeoJSON'})
+      File.basename(fr.file).should == 'geojsonTest.GeoJSON'
+      fr.params[:unique_id].should == nil
+    end
+
+    it "can read csv files" do
+      fr = FileReader.new({:file_path => './spec/files/csvtest.zip'})
+      File.basename(fr.file).should == 'akw.csv'
+      fr.params[:srid].should == 4326
+      fr.params[:colsep].should == ';'
+      fr.params[:unique_id].should == :ID
+    end
+
+    it "can read csv files with wkb geometry" do
+      fr = FileReader.new({:file_path => './spec/files/wkb.csv'})
+      File.basename(fr.file).should == 'wkb.csv'
+      fr.params[:srid].should == 4326
+      fr.params[:colsep].should == ';'
+      fr.params[:unique_id].should == :gid
+    end
+
+    it "can read zipped shape files" do
+      fr = FileReader.new({:file_path => './spec/files/shapeTest.zip'})
+      fr.params[:srid].should == 2100
+    end
+
+  end
+  
+end
+
+
+# puts JSON.pretty_generate(fr.content[0])