fias 0.0.2 → 1.0.0

data/lib/fias/config.rb

@@ -0,0 +1,74 @@
+module Fias
+  class Config
+    def initialize
+      @index = {}
+      @longs = {}
+      @shorts = {}
+      @aliases = {}
+      @exceptions = {}
+      @proper_names = []
+      @synonyms = []
+      @synonyms_index = {}
+
+      yield(self)
+
+      finalize_index
+    end
+
+    attr_reader :index, :longs, :shorts, :aliases, :exceptions
+    attr_reader :proper_names, :synonyms, :synonyms_index
+
+    def add_name(long, short, aliases = [])
+      @longs[Unicode.downcase(short)] = long
+      @shorts[Unicode.downcase(long)] = short
+      @aliases[Unicode.downcase(long)] = aliases
+
+      populate_index(long, short, aliases)
+    end
+
+    def add_exception(long, short)
+      @exceptions[Unicode.downcase(short)] = [long, short]
+      @exceptions[Unicode.downcase(long)] = [long, short]
+    end
+
+    def add_proper_name(name)
+      @proper_names << name
+    end
+
+    def add_synonym(*names)
+      @synonyms << names
+      populate_synonyms_index(names)
+    end
+
+    private
+
+    def populate_index(long, short, aliases)
+      long_downcase = Unicode.downcase(long)
+      short_downcase = Unicode.downcase(short)
+
+      populate_long_permutations(long)
+
+      if long_downcase != short_downcase
+        @index[short_downcase] = long
+        @index[short_downcase[0..-2]] = long if short_downcase[-1] == '.'
+      end
+
+      aliases.each { |al| @index[Unicode.downcase(al)] = long }
+    end
+
+    def populate_long_permutations(long)
+      Unicode.downcase(long).split(' ').permutation.each do |variant|
+        @index[variant.join(' ')] = long
+      end
+    end
+
+    def finalize_index
+      @index = @index.sort_by { |key, _| key.size }.reverse
+      @index = Hash[*@index.flatten]
+    end
+
+    def populate_synonyms_index(names)
+      names.each { |name| @synonyms_index[name] = names }
+    end
+  end
+end

data/lib/fias/import/copy.rb

@@ -0,0 +1,62 @@
+module Fias
+  module Import
+    class Copy
+      attr_reader :dbf, :table_name
+
+      def initialize(db, table_name, dbf, types = {})
+        @db = db
+        @table_name = table_name.to_sym
+        @dbf = dbf
+        @encoder = PgDataEncoder::EncodeForCopy.new(
+          column_types: map_types(types)
+        )
+      end
+
+      def encode
+        @dbf.each do |record|
+          line = record.to_a.map { |v| v == '' ? nil : v }
+          @encoder.add(line)
+          yield if block_given?
+        end
+      end
+
+      def copy
+        prepare
+        copy_into
+      end
+
+      private
+
+      def map_types(types)
+        types = types.map do |name, type|
+          index = columns.index(name.to_sym)
+          [index, type] if index
+        end
+        Hash[*types.compact.flatten]
+      end
+
+      def columns
+        @columns ||= @dbf.columns.map(&:name).map(&:downcase).map(&:to_sym)
+      end
+
+      def prepare
+        @db[@table_name].truncate
+        @db.run('SET client_min_messages TO warning;')
+      end
+
+      def copy_into
+        io = @encoder.get_io
+
+        @db.copy_into(@table_name.to_sym, columns: columns, format: :binary) do
+          begin
+            io.readpartial(BLOCK_SIZE)
+          rescue EOFError => _e
+            nil
+          end
+        end
+      end
+
+      BLOCK_SIZE = 65_536 # 10_240
+    end
+  end
+end

data/lib/fias/import/dbf.rb

@@ -0,0 +1,81 @@
+module Fias
+  module Import
+    class Dbf
+      def initialize(path, encoding = DEFAULT_ENCODING)
+        @path = path
+        @files = {}
+
+        unless Dir.exist?(@path)
+          fail ArgumentError, "FIAS database path #{@path} does not exists"
+        end
+
+        open_files(encoding)
+      end
+
+      def only(*names)
+        return @files if names.empty?
+
+        names = names.map do |name|
+          name = name.to_sym
+          name == :houses ? HOUSE_TABLES.keys : name
+          name == :nordocs ? NORDOC_TABLES.keys : name
+        end
+
+        names.flatten!
+
+        @files.slice(*names)
+      end
+
+      attr_reader :files
+
+      private
+
+      def open_files(encoding)
+        TABLES.each do |accessor, dbf_filename|
+          filename = File.join(@path, dbf_filename)
+
+          next unless File.exist?(filename)
+
+          dbf = DBF::Table.new(filename, nil, encoding)
+          @files[accessor] = dbf if dbf
+        end
+      end
+
+      def self.n_tables(title)
+        tables = (1..99).map do |n|
+          [
+            format('%s%0.2d', title, n).to_sym,
+            format('%s%0.2d.DBF', title.upcase, n)
+          ]
+        end
+
+        tables.flatten!
+
+        Hash[*tables]
+      end
+
+      HOUSE_TABLES = n_tables('house')
+      NORDOC_TABLES = n_tables('nordoc')
+
+      TABLES = {
+        address_object_types: 'SOCRBASE.DBF',
+        current_statuses: 'CURENTST.DBF',
+        actual_statuses: 'ACTSTAT.DBF',
+        operation_statuses: 'OPERSTAT.DBF',
+        center_statuses: 'CENTERST.DBF',
+        interval_statuses: 'INTVSTAT.DBF',
+        estate_statues: 'ESTSTAT.DBF',
+        structure_statuses: 'STRSTAT.DBF',
+        address_objects: 'ADDROBJ.DBF',
+        house_intervals: 'HOUSEINT.DBF',
+        landmarks: 'LANDMARK.DBF'
+      }.merge(
+        HOUSE_TABLES
+      ).merge(
+        NORDOC_TABLES
+      )
+
+      DEFAULT_ENCODING = Encoding::CP866
+    end
+  end
+end

data/lib/fias/import/download_service.rb

@@ -0,0 +1,37 @@
+module Fias
+  module Import
+    module DownloadService
+      def url
+        response = HTTParty.post(
+          'http://fias.nalog.ru/WebServices/Public/DownloadService.asmx',
+          OPTIONS
+        )
+
+        matches =
+          response.body.match(/<FiasCompleteDbfUrl>(.*)<\/FiasCompleteDbfUrl>/)
+
+        matches[1] if matches
+      end
+
+      OPTIONS = {
+        body: %(<?xml version="1.0" encoding="utf-8"?>
+<soap:Envelope
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xmlns:xsd="http://www.w3.org/2001/XMLSchema"
+  xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/">
+<soap:Body>
+<GetLastDownloadFileInfo
+  xmlns="http://fias.nalog.ru/WebServices/Public/DownloadService.asmx/" />
+</soap:Body>
+</soap:Envelope>
+),
+        headers: {
+          'SOAPAction' => 'http://fias.nalog.ru/WebServices/Public/DownloadService.asmx/GetLastDownloadFileInfo',
+          'Content-Type' => 'text/xml; encoding=utf-8'
+        }
+      }
+
+      module_function :url
+    end
+  end
+end

data/lib/fias/import/restore_parent_id.rb

@@ -0,0 +1,51 @@
+module Fias
+  module Import
+    class RestoreParentId
+      def initialize(scope, options = {})
+        @scope = scope
+        @key = options.fetch(:key, :aoguid)
+        @parent_key = options.fetch(:parent_key, :parentguid)
+        @id = options.fetch(:id, :id)
+        @parent_id = options.fetch(:parent_id, :parent_id)
+      end
+
+      def restore
+        id_grouped_by_parent_id.each do |parent_id, ids|
+          @scope.where(id: ids).update(parent_id: parent_id)
+        end
+      end
+
+      private
+
+      def records
+        @records ||= @scope.select_map([@id, @key, @parent_key])
+      end
+
+      def records_by_key
+        @records_by_key ||= records.index_by { |r| r[1] }
+      end
+
+      def id_parent_id_tuples
+        records.map do |row|
+          id, _, key = row
+
+          if key
+            parent_id = records_by_key[key]
+            parent_id = parent_id[0] if parent_id
+          end
+
+          [id, parent_id]
+        end
+      end
+
+      def id_grouped_by_parent_id
+        {}.tap do |rows|
+          id_parent_id_tuples.each do |(id, parent_id)|
+            rows[parent_id] ||= []
+            rows[parent_id] << id
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fias/import/tables.rb

@@ -0,0 +1,74 @@
+module Fias
+  module Import
+    class Tables
+      def initialize(db, files, prefix = DEFAULT_PREFIX)
+        @db = db
+        @files = files
+        @prefix = prefix
+      end
+
+      attr_reader :files
+
+      def create
+        @files.each do |name, dbf|
+          next if dbf.blank?
+          create_table(name, dbf)
+        end
+      end
+
+      def copy
+        @files.map do |name, dbf|
+          Copy.new(@db, table_name(name), dbf, uuid_column_types(name))
+        end
+      end
+
+      private
+
+      def table_name(name)
+        [@prefix, name].delete_if(&:blank?).join('_').to_sym
+      end
+
+      def create_table(name, dbf)
+        columns = columns_for(name, dbf)
+        @db.create_table(table_name(name)) do
+          primary_key :id
+          columns.each { |args| column(*args) }
+        end
+      end
+
+      def columns_for(name, dbf)
+        dbf.columns.map do |column|
+          column_for(name, column)
+        end
+      end
+
+      def column_for(name, column)
+        alter = UUID[name]
+        column_name = column.name.downcase
+
+        parse_c_def(column.schema_definition).tap do |c_def|
+          c_def[1] = :uuid if alter && alter.include?(column_name)
+          c_def[1] = :text if c_def[1] == :string
+        end
+      end
+
+      def parse_c_def(c_def)
+        c_def = c_def.strip.split(',').map(&:strip)
+        name = c_def[0][1..-2]
+        type = c_def[1][1..-1]
+        [name, type].map(&:to_sym)
+      end
+
+      def uuid_column_types(name)
+        uuid = UUID[name] || []
+        Hash[*uuid.zip([:uuid] * uuid.size).flatten]
+      end
+
+      UUID = {
+        address_objects: %w(aoguid aoid previd nextid parentguid)
+      }
+
+      DEFAULT_PREFIX = 'fias'
+    end
+  end
+end

data/lib/fias/name/append.rb

@@ -0,0 +1,30 @@
+module Fias
+  module Name
+    module Append
+      class << self
+        def append(name, short_name)
+          long, _, short, _ = Canonical.canonical(short_name)
+
+          exception = Fias.config.exceptions[Unicode.downcase(name)]
+          return exception.reverse if exception
+
+          [concat(short, name), concat(long, name)]
+        end
+
+        private
+
+        def concat(status, name)
+          must_append?(name) ? "#{name} #{status}" : "#{status} #{name}"
+        end
+
+        def must_append?(name)
+          ending = name[-2..-1]
+          ENDINGS_TO_APPEND.include?(ending) || name =~ JUST_NUMBER
+        end
+      end
+
+      ENDINGS_TO_APPEND = %w(ая ий ый)
+      JUST_NUMBER = /^\d+([\-А-Яа-яе]{1,3})?$/u
+    end
+  end
+end

data/lib/fias/name/canonical.rb

@@ -0,0 +1,42 @@
+module Fias
+  module Name
+    module Canonical
+      class << self
+        def canonical(name)
+          result = search(name) || search_exception(name)
+          result || fail("Unknown abbrevation: #{name}")
+          fix_republic_case(result)
+        end
+
+        private
+
+        def search(key)
+          long = Fias.config.index[Unicode.downcase(key)]
+          return nil unless long
+          short = short_for(long)
+          short_stripped = short_for(long).gsub(/\.$/, '')
+          [long, short_stripped, short, aliases_for(long)].flatten.compact
+        end
+
+        def short_for(long)
+          Fias.config.shorts[Unicode.downcase(long)]
+        end
+
+        def aliases_for(long)
+          Fias.config.aliases[Unicode.downcase(long)]
+        end
+
+        def search_exception(name)
+          Fias.config.exceptions[Unicode.downcase(name)]
+        end
+
+        def fix_republic_case(canonical)
+          return canonical unless canonical[0] == REPUBLIC
+          canonical.map { |n| Unicode.upcase(n[0]) + n[1..-1] }
+        end
+      end
+
+      REPUBLIC = 'республика'
+    end
+  end
+end