inegi-geo 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5809f4c7ab83ec51d1f2f394bf646ad17b1924c3a501668fa790130e08c795db
-  data.tar.gz: a4e4ff7f78fa14e656aef924f8b533eabedb865edbf6e8c402f1b4756ca498b1
+  metadata.gz: 3386d962c2a0810d6899facc342e594fd2c1ff2e62c5dada4ea8f3c9e9eeb4b7
+  data.tar.gz: 437bd3a648cb88623cce3a7b25c7f153773ef504de854f163d40107a6ca2f8a6
 SHA512:
-  metadata.gz: b7ab14fffe1fcf0891037370fe7daa9c3a237ec3d6dd7fec44895bbd0c64f94dfbaa4e0a5fdc682dda887fcaa2be99e70244ab81c432859bd49f5737e12cd90b
-  data.tar.gz: 28a8953792cd930983698611a12f0bd3b1fb458ed69be39a981771c9150049c84525f04a3c132b03fb5e21098c9eeda607437884f7e70d1e05accde5b4a5bdb5
+  metadata.gz: 8f71a53284746a4140a8ad6a58cba9fd57bb85f85bf5a008932af71a82d1c66b23706169159f3d890756f34281e28d4c55113ec229cf6174d9052a952d4de2f9
+  data.tar.gz: 4ddc344749385d437892c7be9aca860bbef026a0832601fca510278d5c73f391618f4fe0b117147565928a8495ee86459dadff0b3198e47fc02d842acdc42a92

data/README.md

@@ -48,10 +48,58 @@ rake inegi:geo:download inegi:geo:transform
 # CONVERTED: states.dbf -> states.csv
 ```
 
+Delete the `.zip` and `.dbf` or put them on your `.gitignore` if you want
+to save them locally.
+
+```bash
+rm -f datasets/*[^.csv]
+```
+
 After editing the CSV to select the columns that you want, you can use
 [activerecord-import][activerecord-import] to insert the data in the CSVs into
 your database.
 
+```bash
+# head -n 5 states.csv
+
+"code","name"
+"01","Aguascalientes"
+"02","Baja California"
+"03","Baja California Sur"
+"04","Campeche"
+```
+
+```bash
+# head -n 5 municipalities.csv
+
+"state_code","code","name"
+"01","001","Aguascalientes"
+"01","002","Asientos"
+"01","003","Calvillo"
+"01","004","Cosío"
+```
+
+```bash
+# head -n 5 localities.csv
+
+"state_code","municipality_code","code","name"
+"01","001","0001","Aguascalientes"
+"01","001","0094","Granja Adelita"
+"01","001","0096","Agua Azul"
+"01","001","0100","Rancho Alegre"
+```
+
+The size of the CSVs:
+
+```txt
+Size Name
+---- ----
+ 10M localities.csv
+ 68k municipalities.csv
+ 612 states.csv
+```
+
+
 _:warning: Review and understand the code before copying and pasting it._
 
 In this example, I chose to store `State`, `Municipality`, and `Locality`.
@@ -77,18 +125,64 @@ rails generate model locality code:string \
 require 'activerecord-import/base'
 require 'activerecord-import/active_record/adapters/postgresql_adapter'
 
-def import_states
-  State.transaction do
-    dataset = Rails.root.join('datasets', 'states.csv')
-    rows = CSV.read(dataset)
+def import_from_csv(dataset)
+  file = Rails.root.join('datasets', "#{dataset}.csv")
+  klass = dataset.classify.constantize
+
+  return if klass.count > 0
+
+  print "\rImporting #{dataset.pluralize}..."
+
+  klass.transaction do
+    rows = CSV.read(file)
     headers = rows.delete_at(0)
-    State.import(headers, rows, validate: false)
-  end unless State.count > 0
+    klass.import(headers, rows, validate: false)
+  end
+
+  puts "\rImported #{dataset.pluralize}: #{klass.count}"
+end
 
-  puts "Imported states: #{State.count}"
+def update_references
+  return unless Municipality.where(state_id: nil).exists? ||
+                Locality.where(municipality_id: nil).exists?
+
+  print "\rUpdating references..."
+
+  states = <<~SQL
+    UPDATE municipalities
+    SET state_id = states.id
+    FROM states
+    WHERE municipalities.state_code = states.code;
+
+    UPDATE localities
+    SET state_id = states.id
+    FROM states
+    WHERE localities.state_code = states.code;
+  SQL
+
+  municipalities = <<~SQL
+    UPDATE localities
+    SET municipality_id = municipalities.id
+    FROM municipalities
+    WHERE localities.state_code = municipalities.state_code
+      AND localities.municipality_code = municipalities.code;
+  SQL
+
+  ActiveRecord::Base.connection.execute(states)
+  ActiveRecord::Base.connection.execute(municipalities)
+
+  puts "\rUpdating references: DONE"
 end
+
+import_from_csv 'states'
+import_from_csv 'municipalities'
+import_from_csv 'localities'
+update_references
 ```
 
+The code above imports the data from the CSV to the models, and update
+the references. The time to run this (in my computer) is below **1 second**.
+
 ## References
 - [guivaloz/INEGI](https://github.com/guivaloz/INEGI)
 - [evilmartians/fias](https://github.com/evilmartians/fias)

data/inegi-geo.gemspec

@@ -2,7 +2,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = 'inegi-geo'
-  s.version     = '0.0.2'
+  s.version     = '0.0.3'
   s.authors     = ['mroutis']
   s.email       = ['outis@civica.digital']
   s.summary     = 'Imports INEGI geospatial database into a readable CSV'

data/lib/inegi/geo/transformer.rb

@@ -53,20 +53,25 @@ class Inegi::Geo::Transformer
 
   def translate_header(header)
     dictionary = {
-      CVE_ENT: 'state_code',
-      CVE_MUN: 'municipality_code',
+      AMBITO: 'area_type',
+      ALTITUD: 'altitude',
+      LATITUD: 'latitude',
+      LONGITUD: 'longitude',
       CVE_CAB: 'head_code',
-      CVE_LOC: 'locality_code',
       CVE_CAP: 'capital_code',
-      NOM_ENT: 'state_name',
+      CVE_CARTA: 'map_code',
+      CVE_ENT: 'state_code',
+      CVE_LOC: 'locality_code',
+      CVE_MUN: 'municipality_code',
       NOM_ABR: 'state_abbreviation',
-      NOM_CAP: 'capital_name',
-      NOM_MUN: 'municipality_name',
       NOM_CAB: 'head_name',
+      NOM_CAP: 'capital_name',
+      NOM_ENT: 'state_name',
       NOM_LOC: 'locality_name',
-      PTOT: 'total_population',
-      PMAS: 'masculine_population',
+      NOM_MUN: 'municipality_name',
       PFEM: 'femenine_population',
+      PMAS: 'masculine_population',
+      PTOT: 'total_population',
       VTOT: 'inhabited_residences',
     }
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: inegi-geo
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - mroutis