data_resurrection 0.1.0

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2011 Rodrigo Manhães
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,37 @@
+= data_resurrection
+
+Bring your data, buried in decrepit formats, back to life! Convert data from old formats to modern ones. Currently supports DBF.
+
+== How to use
+
+If you have a PostgreSQL database and a DBF file called 'decrepit.dbf', you must call:
+
+    r = DataResurrection::Resuscitator.new(:dbf, :postgresql => :settings)
+    r.resurrect('decrepit.dbf', :target => 'new_beautiful_table',
+      :from => 'WINDOWS-1252', :to => 'UTF-8')
+
+and you will have a table called 'new_beautiful_table' in your PostgreSQL database.
+
+The hash parameter to constructor should be the your database settings for ActiveRecord (i.e., those definitions within config/database.yml in a Rails application).
+
+The method "resurrect" accepts ":target", ":from" and ":to" as options. Target is the name of the table to be created in the pointed database, and :from and :to (both optional) forces a encoding conversion for all fields. Parameter :from may be a list in case of multiple encodings for the same table (yes, this kind of freaky thing really exists).
+
+In some cases, field types in original tables are not compatible with ones in the target table. By example, in a real case, a DBF table has an integer field containing a 12-digit value, what caused an overflow when trying to copy the value to an integer field on a PostGreSQL table. This can be handled with an additional option to the method "resurrect":
+
+    @data_resurrection.resurrect(@dbf_file_path, :target => 'nationality',
+      :from => ['WINDOWS-1252', 'CP850'], :to => 'UTF-8',
+      :field_types => {:nr => :string })
+
+The option :field_types is a hash in which each key is the field name and the value is the field type in the target table.
+
+If a field name equals to SQL reserved words or core Ruby methods like "class", the field name are appended with an underscore in the new table.
+
+
+== But gem dbf does this work!
+
+Data resurrection heavily uses this amazing gem internally, but currently it doesn't support encoding conversion, field type conversion and does not handle fields whose name equals to SQL reserved words and Ruby methods.
+
+== Risk disclaimer
+
+This project was built for my own use, and it works for me. Remember that database migrations are a very critical issue. Anything you do with this software is at your own risk.
+

data/lib/data_resurrection/adapters.rb

@@ -0,0 +1,113 @@
+# coding: utf-8
+
+require 'dbf'
+require 'iconv'
+
+module DataResurrection
+  module Adapters
+    module DBF
+      def resurrect(origin_table, options)
+        target_table_name, from, to = options[:target], options[:from], options[:to]
+        field_types = options[:field_types]
+        table = ::DBF::Table.new(origin_table)
+        data = get_data(table, {from: from, to: to}, reserved_words)
+        create_table(table, target_table_name, data, field_types)
+        copy_data(target_table_name, data)
+      end
+
+      def get_data(table, encodings=nil, reserved_words=[])
+        result = get_raw_data(table, reserved_words)
+        result = handle_encodings(result, encodings) if encodings
+        result
+      end
+
+      private
+
+      def create_table(table, table_name, data, field_types)
+        schema = mark_name_clashed_fields(table.schema, data)
+        schema = replace_types(schema, field_types) if field_types
+        eval(schema)
+      end
+
+      def copy_data(table_name, data)
+        ARObject.instance_eval do
+          self.table_name = table_name
+          reset_column_information
+        end
+        data.each {|record| ARObject.create! record }
+      end
+
+      def get_raw_data(table, reserved_words)
+        result = table.map {|record|
+          table.columns.map {|c|
+            { generated_field_name(c.name.downcase, reserved_words) => record.attributes[c.name.downcase] } if record
+          }.
+            compact.
+            reduce({}) {|h, e| h.merge! e }
+        }.compact.reject {|v| v.empty? }
+      end
+
+      def handle_encodings(data, encodings)
+        from = encodings[:from].clone || []
+        from = [from] unless from.kind_of? Array
+        to = encodings[:to]
+        data.each do |record|
+          record.each do |k, v|
+            if v.kind_of?(String)
+              value = v
+              from.each do |encoding|
+                begin
+                  ic = Iconv.new(to, encoding)
+                  value = ic.iconv(v) if v.kind_of?(String)
+                  break if all_valid?(value)
+                rescue Iconv::IllegalSequence
+                  raise if encoding == from.last
+                end
+              end
+              record[k] = value
+            end
+          end
+        end
+      end
+
+      def mark_name_clashed_fields(schema, data)
+        data.first.keys.each do |field|
+          if !schema.include?('column "%s"' % field)
+            schema['column "%s"' % field.chop] = 'column "%s"' % field
+          end
+        end
+        schema
+      end
+
+      def replace_types(schema, field_types)
+        field_types.each do |field, new_type|
+          schema =~ /column "#{field}", :(.+)/
+          tail = $1.split(':')
+          old_type = tail.shift
+          tail = tail.empty? ? "" : ":#{tail.join(':')}"
+          schema['column "%s", :%s%s' % [field, old_type, tail]] =
+            'column "%s", :%s%s' % [field, new_type, tail]
+        end
+        schema
+      end
+
+      def all_valid?(string)
+        string.chars.all? {|c| VALID_CHARS.include? c }
+      end
+
+      def generated_field_name(field_name, reserved_words)
+        reserved_words.include?(field_name.upcase) ? "#{field_name}_" : field_name
+      end
+
+      class ARObject < ActiveRecord::Base
+      end
+
+      REGULAR_LETTERS = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
+      DIGITS = '1234567890'
+      ACCENTED_LETTERS = 'ÀàÁÉÍÓÚáéíìóúÂÊÔâêôÃÑÕãñõÖÜöüÇç'
+      SYMBOLS = "\"'!@#$\%&*()-_+=`{}[]^~,<>.:;/?|\\ "
+      VALID_CHARS = [REGULAR_LETTERS, DIGITS, ACCENTED_LETTERS, SYMBOLS].join
+    end
+  end
+end
+

data/lib/data_resurrection/reserved_words

@@ -0,0 +1,338 @@
+ABSOLUTE
+ACTION
+ADD
+AFTER
+ALL
+ALLOCATE
+ALTER
+AND
+ANY
+ARE
+ARRAY
+AS
+ASC
+ASENSITIVE
+ASSERTION
+ASYMMETRIC
+AT
+ATOMIC
+AUTHORIZATION
+AVG
+BEFORE
+BEGIN
+BETWEEN
+BIGINT
+BINARY
+BIT
+BIT_LENGTH
+BLOB
+BOOLEAN
+BOTH
+BREADTH
+BY
+CALL
+CALLED
+CASCADE
+CASCADED
+CASE
+CAST
+CATALOG
+CHAR
+CHAR_LENGTH
+CHARACTER
+CHARACTER_LENGTH
+CHECK
+CLASS
+CLOB
+CLOSE
+COALESCE
+COLLATE
+COLLATION
+COLUMN
+COMMIT
+CONDITION
+CONNECT
+CONNECTION
+CONSTRAINT
+CONSTRAINTS
+CONSTRUCTOR
+CONTAINS
+CONTINUE
+CONVERT
+CORRESPONDING
+COUNT
+CREATE
+CROSS
+CUBE
+CURRENT
+CURRENT_DATE
+CURRENT_DEFAULT_TRANSFORM_GROUP
+CURRENT_PATH
+CURRENT_ROLE
+CURRENT_TIME
+CURRENT_TIMESTAMP
+CURRENT_TRANSFORM_GROUP_FOR_TYPE
+CURRENT_USER
+CURSOR
+CYCLE
+DATA
+DATE
+DAY
+DEALLOCATE
+DEC
+DECIMAL
+DECLARE
+DEFAULT
+DEFERRABLE
+DEFERRED
+DELETE
+DEPTH
+DEREF
+DESC
+DESCRIBE
+DESCRIPTOR
+DETERMINISTIC
+DIAGNOSTICS
+DISCONNECT
+DISTINCT
+DO
+DOMAIN
+DOUBLE
+DROP
+DYNAMIC
+EACH
+ELEMENT
+ELSE
+ELSEIF
+END
+EQUALS
+ESCAPE
+EXCEPT
+EXCEPTION
+EXEC
+EXECUTE
+EXISTS
+EXIT
+EXTERNAL
+EXTRACT
+FALSE
+FETCH
+FILTER
+FIRST
+FLOAT
+FOR
+FOREIGN
+FOUND
+FREE
+FROM
+FULL
+FUNCTION
+GENERAL
+GET
+GLOBAL
+GO
+GOTO
+GRANT
+GROUP
+GROUPING
+HANDLER
+HAVING
+HOLD
+HOUR
+IDENTITY
+IF
+IMMEDIATE
+IN
+INDICATOR
+INITIALLY
+INNER
+INOUT
+INPUT
+INSENSITIVE
+INSERT
+INT
+INTEGER
+INTERSECT
+INTERVAL
+INTO
+IS
+ISOLATION
+ITERATE
+JOIN
+KEY
+LANGUAGE
+LARGE
+LAST
+LATERAL
+LEADING
+LEAVE
+LEFT
+LEVEL
+LIKE
+LOCAL
+LOCALTIME
+LOCALTIMESTAMP
+LOCATOR
+LOOP
+LOWER
+MAP
+MATCH
+MAX
+MEMBER
+MERGE
+METHOD
+MIN
+MINUTE
+MODIFIES
+MODULE
+MONTH
+MULTISET
+NAMES
+NATIONAL
+NATURAL
+NCHAR
+NCLOB
+NEW
+NEXT
+NO
+NONE
+NOT
+NULL
+NULLIF
+NUMERIC
+OBJECT
+OCTET_LENGTH
+OF
+OLD
+ON
+ONLY
+OPEN
+OPTION
+OR
+ORDER
+ORDINALITY
+OUT
+OUTER
+OUTPUT
+OVER
+OVERLAPS
+PAD
+PARAMETER
+PARTIAL
+PARTITION
+PATH
+POSITION
+PRECISION
+PREPARE
+PRESERVE
+PRIMARY
+PRIOR
+PRIVILEGES
+PROCEDURE
+PUBLIC
+RANGE
+READ
+READS
+REAL
+RECURSIVE
+REF
+REFERENCES
+REFERENCING
+RELATIVE
+RELEASE
+REPEAT
+RESIGNAL
+RESTRICT
+RESULT
+RETURN
+RETURNS
+REVOKE
+RIGHT
+ROLE
+ROLLBACK
+ROLLUP
+ROUTINE
+ROW
+ROWS
+SAVEPOINT
+SCHEMA
+SCOPE
+SCROLL
+SEARCH
+SECOND
+SECTION
+SELECT
+SENSITIVE
+SESSION
+SESSION_USER
+SET
+SETS
+SIGNAL
+SIMILAR
+SIZE
+SMALLINT
+SOME
+SPACE
+SPECIFIC
+SPECIFICTYPE
+SQL
+SQLCODE
+SQLERROR
+SQLEXCEPTION
+SQLSTATE
+SQLWARNING
+START
+STATE
+STATIC
+SUBMULTISET
+SUBSTRING
+SUM
+SYMMETRIC
+SYSTEM
+SYSTEM_USER
+TABLE
+TABLESAMPLE
+TEMPORARY
+THEN
+TIME
+TIMESTAMP
+TIMEZONE_HOUR
+TIMEZONE_MINUTE
+TO
+TRAILING
+TRANSACTION
+TRANSLATE
+TRANSLATION
+TREAT
+TRIGGER
+TRIM
+TRUE
+UNDER
+UNDO
+UNION
+UNIQUE
+UNKNOWN
+UNNEST
+UNTIL
+UPDATE
+UPPER
+USAGE
+USER
+USING
+VALUE
+VALUES
+VARCHAR
+VARYING
+VIEW
+WHEN
+WHENEVER
+WHERE
+WHILE
+WINDOW
+WITH
+WITHIN
+WITHOUT
+WORK
+WRITE
+YEAR
+ZONE

data/lib/data_resurrection/resurrector.rb

@@ -0,0 +1,21 @@
+module DataResurrection
+  class Resuscitator
+    def initialize(adapter, active_record_settings)
+      extend adapters[adapter]
+      ActiveRecord::Base.establish_connection(active_record_settings)
+      @connection = ActiveRecord::Base.connection
+    end
+
+    private
+
+    def adapters
+      @adapters ||= {:dbf => DataResurrection::Adapters::DBF }
+    end
+
+    def reserved_words
+      @reserved_words ||= File.read(File.expand_path(File.join(File.dirname(__FILE__), 'reserved_words'))).
+        each_line.map(&:chomp)
+    end
+  end
+end
+

data/lib/data_resurrection.rb

@@ -0,0 +1,3 @@
+data_resurrection_folder = File.join(File.dirname(__FILE__), 'data_resurrection', '*.rb')
+Dir.glob(data_resurrection_folder).each {|f| require f }
+

metadata

@@ -0,0 +1,98 @@
+--- !ruby/object:Gem::Specification
+name: data_resurrection
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Rodrigo Manhães
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-08-27 00:00:00.000000000 -03:00
+default_executable: 
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: &19880520 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *19880520
+- !ruby/object:Gem::Dependency
+  name: dbf
+  requirement: &19880060 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *19880060
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: &19879600 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+  type: :development
+  prerelease: false
+  version_requirements: *19879600
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &19918680 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.6.0
+  type: :development
+  prerelease: false
+  version_requirements: *19918680
+description: Converts DBF to modern formats.
+email: rmanhaes@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/data_resurrection.rb
+- lib/data_resurrection/adapters.rb
+- lib/data_resurrection/resurrector.rb
+- README.rdoc
+- LICENSE.txt
+- lib/data_resurrection/reserved_words
+has_rdoc: true
+homepage: https://github.com/rodrigomanhaes/data_resurrection
+licenses: []
+post_install_message: 
+rdoc_options:
+- --charset=UTF-8
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.6.2
+signing_key: 
+specification_version: 3
+summary: Bring your data, buried in decrepit formats, back to life! Convert data from
+  old formats to modern ones. Currently supports DBF.
+test_files: []