TimezoneParser 0.4.0 → 1.0.0

data/lib/timezone_parser/data.rb

@@ -13,107 +13,5 @@ module TimezoneParser
         DataDir = RootDir + 'data'
         # Path to Vendor directory
         VendorDir = RootDir + 'vendor'
-
-        attr_reader :Offsets
-        attr_reader :Timezones
-        attr_reader :Types
-        attr_reader :Metazones
-        def initialize
-            @Offsets = SortedSet.new
-            @Timezones = SortedSet.new
-            @Types = SortedSet.new
-            @Metazones = SortedSet.new
-        end
-
-        def processEntry(entry, toTime, fromTime, regions = [])
-            @Timezones += entry['Timezones'] if entry['Timezones']
-            @Offsets << entry['Offset'] if entry['Offset']
-            @Types += entry['Types'].map(&:to_sym) if entry['Types']
-            if entry.has_key?('Metazones')
-                entry['Metazones'].each do |zone|
-                    @Metazones << zone
-                    @Timezones += Storage.getTimezones(zone, toTime, fromTime, regions)
-                end
-            end
-            self
-        end
-
-        def findOffsets(toTime, fromTime, regions = nil, types = nil)
-            types = @Types.to_a unless types
-            types = [:daylight, :standard] if types.empty?
-            @Timezones.each do |timezone|
-                if regions and not regions.empty?
-                    timezoneRegions = Data::Storage.TimezoneCountries[timezone]
-                    next if timezoneRegions and (timezoneRegions & regions).empty?
-                end
-                begin
-                    tz = TZInfo::Timezone.get(timezone)
-                rescue TZInfo::InvalidTimezoneIdentifier
-                    tz = nil
-                end
-                next unless tz
-                offsets = []
-                self.class.addOffset(offsets, tz.period_for_utc(fromTime).offset, types)
-                tz.transitions_up_to(toTime, fromTime).each do |transition|
-                    self.class.addOffset(offsets, transition.offset, types)
-                end
-                @Offsets += offsets
-            end
-            @Offsets
-        end
-
-        # Load data entries which match specified time
-        # @param data [Array<Hash>] array of entries
-        # @param toTime [DateTime] entries before this date, exclusive
-        # @param fromTime [DateTime] entries after/at this date, inclusive
-        # @return [Array<Hash>] resulting array containing filtered entries
-        def self.loadEntries(data, toTime, fromTime, offsets = false)
-            result = []
-            data.each do |entry|
-                result << entry if (entry['From'] && entry['To'] and toTime > entry['From'] and fromTime < entry['To']) or
-                (entry['From'] && !entry['To'] and toTime > entry['From']) or
-                (!entry['From'] && entry['To'] and fromTime < entry['To']) or
-                (!entry['From'] && !entry['To'])
-            end
-            result.each do |entry|
-                return result if not offsets or (offsets and entry['Offset'])
-            end
-            data.each_index do |i|
-                entry = data[i]
-                nextentry = offsets ? getNextEntry(data, i) : data[i+1]
-                result << entry if ( entry['From'] && entry['To'] and toTime > entry['From'] and fromTime < entry['To'] ) or
-                ( entry['To'] and ( (nextentry.nil? and fromTime >= entry['To']) or
-                (nextentry and nextentry['From'] and fromTime >= entry['To'] and toTime <= nextentry['From']) or
-                (!entry['From'] and fromTime < entry['To']) ) ) or
-                ( entry['From'] and ( (i.zero? and toTime <= entry['From']) or (!entry['To'] and toTime > entry['From']) ) )
-            end
-            result
-        end
-
-        def self.filterData(data, toTime, fromTime, type, regions)
-            entries = []
-            data.each do |entry|
-                next if type and entry['Types'] and not entry['Types'].include?(type)
-                next if not regions.empty? and entry['Countries'] and (entry['Countries'] & regions).empty?
-                entries << entry
-            end
-            loadEntries(entries, toTime, fromTime, true)
-        end
-
-        protected
-
-        def self.getNextEntry(data, i)
-            j = 1
-            begin
-                entry = data[i+j]
-                j += 1
-            end until entry.nil? or (entry and entry['Offset'])
-            entry
-        end
-
-        def self.addOffset(offsets, offset, types)
-            offsets << offset.utc_total_offset if (offset.dst? and types.include?(:daylight)) or (not offset.dst? and types.include?(:standard))
-        end
-
     end
 end

data/lib/timezone_parser/data/exporter.rb

@@ -0,0 +1,242 @@
+require 'yaml'
+require 'sqlite3'
+require_relative 'tzinfo'
+require_relative '../zone_info'
+
+module TimezoneParser
+    class Data
+        # Timezone data Exporter class
+        class Exporter
+            Database = 'timezones.db'
+
+            public
+
+            def initialize(location)
+                path = location + Database
+                File.delete(path) if File.exist?(path)
+                @Database = SQLite3::Database.new(path.to_s)
+                @DataDir = Data::DataDir
+            end
+
+            def exportDatabase
+                configure
+                loadSchema
+                loadLocales
+                loadTerritories
+                loadTimezones
+                loadTimezoneTerritories
+                loadMetazones
+                loadTimezoneNames
+                loadAbbreviations
+                loadRailsTimezones
+                loadRailsI18N
+                loadWindowsZones
+                loadWindowsZoneTimezones
+                loadWindowsZoneNames
+                finalize
+            end
+
+            private
+
+            @Database = nil
+            @LocaleIds = {}
+            @TerritoryIds = {}
+            @TimezoneIds = {}
+            @MetazoneIds = {}
+            @RailsTimezoneIds = {}
+            @WindowsZoneIds = {}
+
+            def configure
+                @Database.application_id = 'TZPR'.unpack('l>').first.to_s
+                @Database.user_version = TimezoneParser::VERSION.split('.').map(&:to_i).pack('CS>C').unpack('l>').first.to_s
+                @Database.foreign_keys = true
+                @Database.journal_mode = 'off'
+                @Database.temp_store = 'memory'
+                @Database.locking_mode = 'exclusive'
+                @Database.synchronous = 'off'
+            end
+
+            def finalize
+                @Database.execute('ANALYZE')
+                @Database.execute('VACUUM')
+            end
+
+            def normalizeLocale(locale)
+                locale = locale.gsub('_', '-')
+                case locale
+                when 'zh-CN'
+                    locale = 'zh-Hans-CN'
+                when 'zh-TW'
+                    locale = 'zh-Hant-TW'
+                when 'ha-Latn-NG'
+                    locale = 'ha-NG'
+                end
+                locale
+            end
+
+            def getLocaleId(locale)
+                @LocaleIds[normalizeLocale(locale)]
+            end
+
+            def loadSchema
+                schema = File.read(@DataDir + 'schema.sql')
+                @Database.execute_batch(schema)
+            end
+
+            def loadLocales
+                @LocaleIds = {}
+                locales = YAML.load_file(@DataDir + 'locales.yaml')
+                locales.each do |locale|
+                    locale = normalizeLocale(locale)
+                    @Database.execute('INSERT INTO `Locales` (`Name`) VALUES (?)', locale)
+                    @LocaleIds[locale] = @Database.last_insert_row_id
+                end
+                locales.each do |locale|
+                    locale = normalizeLocale(locale)
+                    if locale.include?('-')
+                        parent = locale.split('-')[0..-2].join('-')
+                        @Database.execute('UPDATE `Locales` SET `Parent` = ? WHERE `ID` = ?', getLocaleId(parent), getLocaleId(locale))
+                    end
+                end
+            end
+
+            def loadTerritories
+                @TerritoryIds = {}
+                territories = YAML.load_file(@DataDir + 'territories.yaml')
+                (territories.to_a.flatten + ['ZZ']).sort.uniq.each do |territory|
+                    @Database.execute('INSERT INTO `Territories` (`Territory`) VALUES (?)', territory)
+                    @TerritoryIds[territory] = @Database.last_insert_row_id
+                end
+                territories.each do |parent, entries|
+                    entries.each do |territory|
+                        @Database.execute('INSERT INTO `TerritoryContainment` (`Parent`, `Territory`) VALUES (?, ?)', @TerritoryIds[parent], @TerritoryIds[territory])
+                    end
+                end
+            end
+
+            def loadTimezones
+                @TimezoneIds = {}
+                TimezoneParser::TZInfo.getTimezones.each do |timezone|
+                    @Database.execute('INSERT INTO `Timezones` (`Name`) VALUES (?)', timezone)
+                    @TimezoneIds[timezone] = @Database.last_insert_row_id
+                end
+            end
+
+            def loadTimezoneTerritories
+                YAML.load_file(@DataDir + 'countries.yaml').each do |timezone, countries|
+                    countries.each do |country|
+                        @Database.execute('INSERT INTO TimezoneTerritories (Timezone, Territory) VALUES (?, ?)', [@TimezoneIds[timezone], @TerritoryIds[country]])
+                    end
+                end
+            end
+
+            def loadMetazones
+                @MetazoneIds = {}
+                YAML.load_file(@DataDir + 'metazones.yaml').each do |metazone, entries|
+                    @Database.execute('INSERT INTO `Metazones` (`Name`) VALUES (?)', [metazone])
+                    @MetazoneIds[metazone] = @Database.last_insert_row_id
+                    entries.each do |entry|
+                        @Database.execute('INSERT INTO `MetazonePeriods` (`Metazone`, `From`, `To`) VALUES (?, ?, ?)', [@MetazoneIds[metazone], entry['From'], entry['To']])
+                        period = @Database.last_insert_row_id
+                        entry['Timezones'].each do |timezone|
+                            @Database.execute('INSERT INTO `MetazonePeriod_Timezones` (`MetazonePeriod`, `Timezone`) VALUES (?, ?)', [period, @TimezoneIds[timezone]])
+                        end
+                    end
+                end
+            end
+
+            def getTypes(data)
+                types = nil
+                if data['Types']
+                    types = types.to_i | TimezoneParser::ZoneInfo::TIMEZONE_TYPE_STANDARD if data['Types'].include?('standard')
+                    types = types.to_i | TimezoneParser::ZoneInfo::TIMEZONE_TYPE_DAYLIGHT if data['Types'].include?('daylight')
+                end
+                types
+            end
+
+            def loadTimezoneNames
+                YAML.load_file(@DataDir + 'timezones.yaml').each do |locale, localeData|
+                    localeData.each do |name, data|
+                        @Database.execute('INSERT INTO TimezoneNames (`Locale`, `Name`, `NameLowercase`, Types) VALUES (?, ?, ?, ?)', [getLocaleId(locale), name, name.downcase, getTypes(data)])
+                        nameId = @Database.last_insert_row_id
+                        data['Timezones'].to_a.each do |timezone|
+                            @Database.execute('INSERT INTO TimezoneName_Timezones (`Name`, `Timezone`) VALUES (?, ?)', [nameId, @TimezoneIds[timezone]])
+                        end
+                        data['Metazones'].to_a.each do |metazone|
+                            @Database.execute('INSERT INTO TimezoneName_Metazones (`Name`, `Metazone`) VALUES (?, ?)', [nameId, @MetazoneIds[metazone]])
+                        end
+                    end
+                end
+            end
+
+            def loadAbbreviations
+                YAML.load_file(@DataDir + 'abbreviations.yaml').each do |abbreviation, entries|
+                    @Database.execute('INSERT INTO `Abbreviations` (`Name`, `NameLowercase`) VALUES (?, ?)', [abbreviation, abbreviation.downcase])
+                    abbreviationId = @Database.last_insert_row_id
+                    entries.each do |entry|
+                        @Database.execute('INSERT INTO `AbbreviationOffsets` (`Abbreviation`, `Offset`, `Types`, `From`, `To`) VALUES (?, ?, ?, ?, ?)', [abbreviationId, entry['Offset'], getTypes(entry), entry['From'], entry['To']])
+                        offset = @Database.last_insert_row_id
+                        entry['Timezones'].to_a.each do |timezone|
+                            @Database.execute('INSERT INTO `AbbreviationOffset_Timezones` (`Offset`, `Timezone`) VALUES (?, ?)', [offset, @TimezoneIds[timezone]])
+                        end
+                        entry['Metazones'].to_a.each do |timezone|
+                            @Database.execute('INSERT INTO `AbbreviationOffset_Metazones` (`Offset`, `Metazone`) VALUES (?, ?)', [offset, @MetazoneIds[timezone]])
+                        end
+                    end
+                end
+            end
+
+            def loadRailsTimezones
+                @RailsTimezoneIds = {}
+                YAML.load_file(@DataDir + 'rails.yaml').each do |name, timezone|
+                    @Database.execute('INSERT INTO `RailsTimezones` (`Name`, `Timezone`) VALUES (?, ?)', [name, @TimezoneIds[timezone]])
+                    @RailsTimezoneIds[name] = @Database.last_insert_row_id
+                    @Database.execute('INSERT INTO `RailsI18N` (`Locale`, `Name`, `NameLowercase`, `Zone`) VALUES (?, ?, ?, ?)', [getLocaleId('en'), name, name.downcase, @RailsTimezoneIds[name]])
+                end
+            end
+
+            def loadRailsI18N
+                YAML.load_file(@DataDir + 'rails_i18n.yaml').each do |locale, data|
+                    data.each do |name, zone|
+                        @Database.execute('INSERT INTO `RailsI18N` (`Locale`, `Name`, `NameLowercase`, `Zone`) VALUES (?, ?, ?, ?)', [getLocaleId(locale), name, name.downcase, @RailsTimezoneIds[zone]])
+                    end
+                end
+            end
+
+            def loadWindowsZones
+                @WindowsZoneIds = {}
+                YAML.load_file(@DataDir + 'windows_offsets.yaml').each do |zone, data|
+                    @Database.execute('INSERT INTO `WindowsZones` (`Name`, `Standard`, `Daylight`) VALUES (?, ?, ?)', [zone, data['standard'], data['daylight']])
+                    @WindowsZoneIds[zone] = @Database.last_insert_row_id
+                end
+            end
+
+            def loadWindowsZoneTimezones
+                YAML.load_file(@DataDir + 'windows_timezones.yaml').each do |zone, data|
+                    unless @WindowsZoneIds.has_key?(zone)
+                        puts "Warning! Need to update windows_offsets.yaml! No timezone offset found for '#{zone}'"
+                        next
+                    end
+                    data.each do |territory, entries|
+                        entries.each do |timezone|
+                            @Database.execute('INSERT INTO `WindowsZone_Timezones` (`Zone`, `Territory`, `Timezone`) VALUES (?, ?, ?)', [@WindowsZoneIds[zone], @TerritoryIds[territory], @TimezoneIds[timezone]])
+                        end
+                    end
+                end
+            end
+
+            def loadWindowsZoneNames
+                YAML.load_file(@DataDir + 'windows_zonenames.yaml').each do |locale, localeData|
+                    localeData.each do |name, data|
+                        @Database.execute('INSERT INTO `WindowsZoneNames` (`Locale`, `Name`, `NameLowercase`, `Types`) VALUES (?, ?, ?, ?)', [getLocaleId(locale), name, name.downcase, getTypes(data)])
+                        nameid = @Database.last_insert_row_id
+                        data['Metazones'].to_a.each do |zone|
+                            @Database.execute('INSERT INTO `WindowsZoneName_Zones` (`Name`, `Zone`) VALUES (?, ?)', [nameid, @WindowsZoneIds[zone]])
+                        end
+                    end
+                end
+            end
+
+        end
+    end
+end

data/lib/timezone_parser/data/storage.rb

@@ -1,172 +1,34 @@
 # encoding: utf-8
 require 'date'
 require 'insensitive_hash'
+require 'sqlite3'
 
 module TimezoneParser
     class Data
         # Timezone data Storage class
         class Storage
             protected
-            @@Abbreviations = nil
-            @@Timezones = nil
-            @@TimezoneCountries = nil
-            @@Metazones = nil
-            @@WindowsZones = nil
-            @@WindowsTimezones = nil
-            @@WindowsOffsets = nil
-            @@RailsZones = nil
-            @@RailsTranslated = nil
+            @@Database = nil
+            @@Statements = {}
 
             public
 
-            def self.Abbreviations
-                unless @@Abbreviations
-                    @@Abbreviations = Marshal.load(File.open(Data::DataDir + 'abbreviations.dat'))
-                    @@Abbreviations.each do |abbr, data|
-                        proccessData(data)
-                    end
-                end
-                @@Abbreviations
-            end
-
-            def self.Timezones
-                unless @@Timezones
-                    @@Timezones = Marshal.load(File.open(Data::DataDir + 'timezones.dat')).insensitive
-                end
-                @@Timezones
-            end
-
-            def self.TimezoneCountries
-                unless @@TimezoneCountries
-                    @@TimezoneCountries = Marshal.load(File.open(Data::DataDir + 'countries.dat')).insensitive
-                end
-                @@TimezoneCountries
-            end
-
-            def self.Metazones
-                unless @@Metazones
-                    @@Metazones = Marshal.load(File.open(Data::DataDir + 'metazones.dat'))
-                    @@Metazones.each do |zone, data|
-                        proccessData(data)
-                    end
-                end
-                @@Metazones
-            end
-
-            def self.WindowsZones
-                unless @@WindowsZones
-                    @@WindowsZones = Marshal.load(File.open(Data::DataDir + 'windows_zonenames.dat')).insensitive
-                end
-                @@WindowsZones
-            end
-
-            def self.WindowsTimezones
-                unless @@WindowsTimezones
-                    @@WindowsTimezones = Marshal.load(File.open(Data::DataDir + 'windows_timezones.dat')).insensitive
-                end
-                @@WindowsTimezones
-            end
-
-            def self.WindowsOffsets
-                unless @@WindowsOffsets
-                    @@WindowsOffsets = Marshal.load(File.open(Data::DataDir + 'windows_offsets.dat')).insensitive
-                end
-                @@WindowsOffsets
-            end
-
-            def self.RailsZones
-                unless @@RailsZones
-                    @@RailsZones = Marshal.load(File.open(Data::DataDir + 'rails.dat')).insensitive
-                end
-                @@RailsZones
-            end
-
-            def self.RailsTranslated
-                unless @@RailsTranslated
-                    @@RailsTranslated = Marshal.load(File.open(Data::DataDir + 'rails_i18n.dat')).insensitive
-                end
-                @@RailsTranslated
-            end
-
-            def self.preload(modules)
-                preloaded = false
-                modules.each do |m|
-                    case m
-                    when :Abbreviations
-                        self.Abbreviations
-                        self.Metazones
-                        preloaded = true
-                    when :Timezones
-                        self.Timezones
-                        self.Metazones
-                        preloaded = true
-                    when :WindowsZones
-                        self.WindowsZones
-                        self.WindowsTimezones
-                        self.WindowsOffsets
-                        preloaded = true
-                    when :RailsZones
-                        self.RailsZones
-                        self.RailsTranslated
-                        preloaded = true
-                    end
-                end
-                preloaded
-            end
-
-            def self.getTimezones(metazone, toTime, fromTime, regions = [])
-                timezones = SortedSet.new
-                if self.Metazones.has_key?(metazone)
-                    entries = Data::loadEntries(self.Metazones[metazone], toTime, fromTime)
-                    entries.each do |entry|
-                        add = true
-                        timezones += entry['Timezones'].select do |timezone|
-                            if regions.empty?
-                                true
-                            else
-                                timezoneRegions = self.TimezoneCountries[timezone]
-                                timezoneRegions && !(regions & timezoneRegions).empty?
-                            end
-                        end
-                    end
-                end
-                timezones
-            end
+            DatabaseName = 'timezones.db'
 
-            def self.getTimezones2(zone, regions = [])
-                timezones = SortedSet.new
-                if self.WindowsTimezones.has_key?(zone)
-                    entries = self.WindowsTimezones[zone]
-                    regions = entries.keys if regions.empty?
-                    regions.each do |region|
-                        next unless entries.has_key?(region)
-                        timezones += entries[region]
-                    end
+            def self.Database
+                unless @@Database
+                    @@Database = SQLite3::Database.new((Data::DataDir + DatabaseName).to_s, { readonly: true } )
                 end
-                timezones
+                @@Database
             end
 
-            def self.getOffsets(zone, types = [])
-                offsets = SortedSet.new
-                if self.WindowsOffsets.has_key?(zone)
-                    data = self.WindowsOffsets[zone]
-                    types = [:standard, :daylight] if types.empty?
-                    types.each do |type|
-                        next unless data.has_key?(type)
-                        offsets << data[type]
-                    end
+            def self.getStatement(statement)
+                unless @@Statements.has_key?(statement)
+                    @@Statements[statement] = self.Database.prepare(statement)
                 end
-                offsets
+                @@Statements[statement]
             end
 
-            protected
-
-            def self.proccessData(data)
-                data.each do |entry|
-                    entry['From'] = DateTime.parse(entry['From']) if entry['From']
-                    entry['To'] = DateTime.parse(entry['To']) if entry['To']
-                end
-            end
         end
     end
 end