fias_parser 1.0.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    MmUxMDRhMjRmMmNhY2Y3ZDM1Y2M1MjM4NjYxYmNlOTBmNjY3NWM3Zg==
+  data.tar.gz: !binary |-
+    NDgyOTVhMTM4MzE4NjIwMjQ4NTRiNDQyNDhjMzAyNDhjYTc0YzEwMQ==
+SHA512:
+  metadata.gz: !binary |-
+    MzgxODRmOTE0MzM5ZTA5NzBjMTkxYjFiNTM1Mzc2NzI3Zjg0OGNjOWM2MzA4
+    YWY2NWI1NTVlNmJhODIyOWJkNDQ4OTE2NmJkYzc5YzJkZWZkZDc4Nzk3NDYz
+    N2U2MzE5YTQxZDAyM2VhMjMwMzQ3ZmU5ZjczNTFhYTk5NWUxMzM=
+  data.tar.gz: !binary |-
+    MDhjNmM1YzdiNDA4MWMzOWI4MjNjOTM4OTkxMTI3ZDVlMTkzNWEwMTIwNjNh
+    MWQyYjIxZGZjMGJkZjMxODhkOGRmMWU2YTVlZTJmZjBmOWZkOGUwNGZkNjJh
+    ZTI3MTUzMjFlZTdjMzViYzgzMGNjNTFlNzQ2ZjRhOWMwNmZkMWU=

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.ruby-gemset

@@ -0,0 +1 @@
+fias_parser

data/.ruby-version

@@ -0,0 +1 @@
+2.0.0-p353

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fias_parser.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Sergey Malykh
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,94 @@
+# FiasParser
+
+This gem grabs FIAS database from official site (http://fias.nalog.ru/Public/DownloadPage.aspx) and parses data to usable format. 
+
+## Installation
+
+1. Install system dependecies:
+
+    $ sudo apt-get install unar wget
+
+2. Add this line to your application's Gemfile:
+
+```ruby
+  gem 'fias_parser'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install fias_parser
+
+## Usage
+
+Example rake-task (Mongoid):
+
+```ruby
+  # app/models/street.rb
+  class Street
+    include Mongoid::Document
+
+    field :name
+    field :prefix
+    field :guid
+    field :okato
+  end
+
+  # lib/tasks/fias.rake
+  namespace :fias do
+    desc "Stores streets to database"
+
+    task install: :environment do
+      parser = FiasParser::Parser.new( base_dir: Rails.root.join( 'tmp' ) )
+
+      parser.get_latest
+
+      parser.process( 'as_addrobj', batch_size: 100 ) do |batch|
+        batch.each do |item|
+          # only active items
+          next if item[:ACTSTATUS].to_i != 1
+          # streets
+          next if item[:AOLEVEL].to_i != 7
+          # just Moscow
+          next unless /^45/ =~ item[:OKATO]
+
+          Street.create!(
+            name: item[:OFFNAME],
+            prefix: item[:SHORTNAME],
+            okato: item[:OKATO],
+            guid: item[:AOGUID]
+          )
+        end
+      end
+    end
+  end
+```
+
+Available xmls:
+  AS_ACTSTAT
+  AS_ADDROBJ
+  AS_CENTERST
+  AS_CURENTST
+  AS_ESTSTAT
+  AS_HOUSE
+  AS_HSTSTAT
+  AS_INTVSTAT
+  AS_LANDMARK
+  AS_NDOCTYPE
+  AS_NORMDOC
+  AS_OPERSTAT
+  AS_SOCRBASE
+  AS_STRSTAT
+
+XSD schemas look on official site (http://fias.nalog.ru/Public/DownloadPage.aspx)
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/fias_parser.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'fias_parser/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "fias_parser"
+  spec.version       = FiasParser::VERSION
+  spec.authors       = ["Sergey Malykh"]
+  spec.email         = ["xronos.i.am@gmail.com"]
+  spec.description   = %q{Parse and install FIAS database}
+  spec.summary       = %q{Parse and install FIAS database http://fias.nalog.ru/Public/DownloadPage.aspx}
+  spec.homepage      = "https://github.com/xronos-i-am/fias_parser"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "mechanize"
+  spec.add_runtime_dependency "ox"
+  spec.add_runtime_dependency "cocaine"
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+end

data/lib/fias_parser/version.rb

@@ -0,0 +1,3 @@
+module FiasParser
+  VERSION = "1.0.0"
+end

data/lib/fias_parser.rb

@@ -0,0 +1,164 @@
+require "fias_parser/version"
+require 'mechanize'
+require 'ox'
+require 'cocaine'
+
+module FiasParser
+  class Document
+    def initialize( options, block )
+      @block = block
+      @batch = []
+      @batch_size = options[:batch_size] || 10
+      
+      @root_name = nil
+      @item_name = nil
+    end
+
+    def start_element( name )
+      @item = nil
+
+      if @root_name.nil?
+        @root_name = name
+        
+        return
+      end
+      
+      @item_name = name
+      @item = {}
+    end
+
+    def attr( name, value )
+      @item[ name ] = value unless @item.nil?
+    end
+
+    def end_element( name )
+      self.yield_batch if name == @root_name && @batch.any?
+
+      return if @item.nil? || name != @item_name
+      
+      @batch << @item
+
+      self.yield_batch if @batch.size >= @batch_size
+    end
+
+    def yield_batch
+      @block.call( @batch )      
+      @batch = []      
+    end
+  end
+
+  class Parser
+    attr_accessor :date
+
+    def initialize( options = {} )
+      @base_dir = options[:base_dir] || Dir.tmpdir
+    end
+
+    def get_latest
+      return unless self.check_executables
+
+      self.download_latest
+      self.unpack
+    end
+
+    def process( term, options = {}, &block ) 
+      file_name = Dir.entries( self.archive_path ).find { |f| f =~ /#{term}/i }
+
+      if file_name.nil?
+        puts "File /#{term}/ not found in '#{self.archive_path}'"
+
+        return
+      end
+
+      File.open( File.join( self.archive_path, file_name ) ) do |file|
+        ::Ox.sax_parse( Document.new( options, block ), file )
+      end
+    end
+
+    def check_executables
+      line = Cocaine::CommandLine.new( "wget", "-h" )
+
+      begin
+        line.run
+      rescue Cocaine::CommandNotFoundError => e
+        puts 'wget is not installed. Run "sudo apt-get install wget"'
+
+        return false
+      end 
+
+      line = Cocaine::CommandLine.new( "unar", "-h" )
+
+      begin
+        line.run
+      rescue Cocaine::CommandNotFoundError => e
+        puts 'unar is not installed. Run "sudo apt-get install unar"'
+
+        return false
+      end 
+
+      true
+    end
+
+    def download_latest
+      agent = Mechanize.new
+      agent.user_agent = 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:10.0.1) Gecko/20100101 Firefox/10.0.1'
+
+      url = 'http://fias.nalog.ru/Public/DownloadPage.aspx'
+
+      doc = agent.get( url ).root
+
+      post_data = {}
+
+      doc.css( 'form input[type="hidden"]' ).each do |input|
+        post_data[ input['name'] ] = input['value']
+      end
+
+      @date = doc.css( '#ctl00_contentPlaceHolder_downloadRadGrid_ctl00__0 tr td:first-child' ).text.gsub( /[^\d]/, '' )
+
+      post_data['__EVENTTARGET'] = 'ctl00$contentPlaceHolder$downloadRadGrid$ctl00$ctl04$fullSZLinkButton'
+
+      if File.exists?( self.archive_file_path )
+        puts "File '#{self.archive_file_path}' already exists."        
+        
+        return
+      end
+
+      line = Cocaine::CommandLine.new( "wget", "--output-document=:out --post-data=:post_data :url" )
+
+      line.run( {
+        out: self.archive_file_path,
+        post_data: Mechanize::Util.build_query_string( post_data ),
+        url: url,
+      } )
+    end
+
+    def unpack
+      dir = File.join( @base_dir )
+
+      FileUtils.mkdir_p ( dir ) unless File.exists?( dir )
+
+      line = Cocaine::CommandLine.new( "unar", "-o :dir :archive" )
+
+      line.run( {
+        dir: dir,
+        archive: self.archive_file_path,
+      } )
+    end
+
+    def archive_path
+      @archive_path ||= get_archive_path
+    end
+
+    def get_archive_path
+      File.join( @base_dir, @date )
+    end
+
+    def archive_file_path
+      @archive_file_path ||= get_archive_file_path
+    end
+
+    def get_archive_file_path
+      File.join( @base_dir,  "#{@date}.rar" )
+    end
+  end
+end

metadata

@@ -0,0 +1,124 @@
+--- !ruby/object:Gem::Specification
+name: fias_parser
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Sergey Malykh
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-11-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: mechanize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ox
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: cocaine
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Parse and install FIAS database
+email:
+- xronos.i.am@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .ruby-gemset
+- .ruby-version
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- fias_parser.gemspec
+- lib/fias_parser.rb
+- lib/fias_parser/version.rb
+homepage: https://github.com/xronos-i-am/fias_parser
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.1.11
+signing_key: 
+specification_version: 4
+summary: Parse and install FIAS database http://fias.nalog.ru/Public/DownloadPage.aspx
+test_files: []