feidee_utils 0.0.4 → 0.0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ef73043dafb067cba94f418475c9b87acfd1b064
-  data.tar.gz: 1740946281e699563aec46dce68bb6c9d97e837c
+  metadata.gz: 293b859c218934656fc1b096076284f44fe7e17b
+  data.tar.gz: bf5e50c65f90d910f75b39bc667f63d9ff61fbf3
 SHA512:
-  metadata.gz: 33c241f66d3457a4879b1ef18226186b122cefac1e64e734e13f087af2ec6ff874e0f3dc8197f1512b51340b568d00c00a44f3e0a2300e86f6577df4ee95652b
-  data.tar.gz: 6805623ecc37f3a3b70e1f81a84c5921c08b3d4c55f6c8b2e28338da0cbeefea050fc32ec21282f5c8e8ad66d9d5b9a4a8766a530fdd01a26b503ec243d5b9c6
+  metadata.gz: 30ee3bc1c7e4b1fa59ec6a2a5e29b5750261f08f2f335e0dbf3fc6e3fd462c5958803df7e07fc04c162750dcee5a8482dd16ce851bc5533a63363fda0613286f
+  data.tar.gz: 101e8d20b5db4048f9f9f994d35fe285f65d0f0b2881a7e236f5fc16e9fbc6fbb7fa2afbc5f58a80643c6794931e05b999624ca0f2e675c07f0df3dd9958787f

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/MIT-LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Liqing Muyi (muyiliqing@gmail.com)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,67 @@
+Feidee Utils
+============
+[![Build Status](https://travis-ci.org/muyiliqing/feidee_utils.svg?branch=master)](https://travis-ci.org/muyiliqing/feidee_utils)
+
+Free users' data from [Feidee](http://www.feidee.com) private backups (.kbf).
+
+Feidee And The KBF Format
+-----------
+[Feidee MyMoney](http://www.feidee.com/money/) is a set of popular book-keeping software in China. It includes Android, iOS, web apps and several desktop software. The Android and iOS apps produces backup in kbf format.
+
+A kbf file is in fact a zip file contains a modified SQLite database and other attachments, such as photos.
+The first 16 bits of the SQLite database is modified so that it could not be read directly by SQLite.
+The database itself is NOT encrypted. Almost all useful information is in the SQLite database.
+
+Install
+---------
+```bash
+gem install feidee_utils
+```
+
+Usage
+----------
+A set of ActiveRecord-like classes are provided to access the information in the backup. See the quick example below.
+
+```ruby
+require 'feidee_utils'
+
+kbf = FeideeUtils::Kbf.open_file(path_to_kbf_file)
+database = kbf.sqlite_db
+all_accounts = database.namespaced::Account.all
+all_transactions = database.namespaced::Transaction.all
+```
+
+For more examples see ```examples/``` (To be added).
+
+Supported Entities
+-----------------
+
+*  Account
+*  Transaction
+*  AccountGroup
+*  Category
+
+Chinese Characters
+-----------------
+
+The database contains many Chinese characters such as builtin category names. Some of the characters are also included in tests. The gem is developed under OSX so presumably the gem should work fine in Unix-like environments with Unicode/UTF8 or whatever the encoding is.
+
+Why not ActiveRecord
+----------------
+Sometimes we have to compare the content of two backups and must open them at the same time.
+Only one database can be opened using ActiveRecord. It is not designed to be used in such a way.
+
+Why Feidee Utils At All
+-----------
+Originally the Feidee Android and iOS app let users export their personal data recorded by the app.
+Since some version last year, the functionality is removed from the app and user data is trapped inside Feidee's private system forever. The uses may pay to get pro version, or upload their data to Feidee's server in order to export.
+
+As a user of Feidee MyMoney, I'm truelly grateful that such great apps are available free of charge. However I also believe that users' data belongs to users and should be controlled by its owner. Thus I decided to build the utils to help Feidee MyMoney users access their own data.
+
+Disclaimer
+---------
+Use at your own risk. Study purpose only. Please do NOT use for any illegal purpose. For details see MIT-LICENSE in the repo.
+
+This software DOES NOT involve any kind of jail break, reverse engineering or crack of any Feidee's software or app.
+
+The trademark Feidee, Feidee MyMoney, kbf file format and database design are intellecture properties of Feidee, or whoever the owners are.

data/Rakefile

@@ -0,0 +1,24 @@
+require 'rake/testtask'
+require 'fileutils'
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+end
+
+namespace :sync do
+  desc "Sync source file with the main project."
+  task :source do
+    puts "copying source code..."
+    FileUtils.cp_r Dir.glob("lib/*"), "/Users/muyiliqing/Git/cloud-ruby-dev/lib/"
+  end
+  task :scallion do
+    puts "copying source code..."
+    FileUtils.cp_r Dir.glob("lib/*"), "/Users/muyiliqing/Git/scallion/lib/"
+  end
+end
+
+desc "Run tests"
+task :default => :test
+task :deploy => :'sync:source'
+task :scallion => :'sync:scallion'

data/lib/feidee_utils/account.rb

@@ -0,0 +1,132 @@
+require 'feidee_utils/record'
+require 'feidee_utils/account_group'
+require 'bigdecimal'
+
+module FeideeUtils
+  class Account < Record
+    def validate_integrity
+      raise "Account type should always be 0, but it's #{field["type"]}.\n" + inspect unless field["type"] == 0
+      raise "Account usedCount should always be 0, but it's #{field["usedCount"]}.\n" + inspect unless field["usedCount"] == 0
+      raise "Account uuid should always be empty, but it's #{field["uuid"]}.\n" + inspect unless field["uuid"].to_s.empty?
+      raise "Account hierachy contains more than 2 levels.\n" + inspect unless flat_parent_hierachy?
+      raise "Account hidden should be either 0 or 1, but it's #{raw_hidden}.\n" + inspect unless (raw_hidden == 1 or raw_hidden == 0)
+    end
+
+    def self.validate_global_integrity
+      if self.find_by_id(-1) != nil
+        raise "-1 is used as the parent POID placeholder of a parent account. " +
+          "Account of POID -1 should not exist."
+      end
+    end
+
+    FieldMappings = {
+      name:                 "name",
+      raw_balance:          "balance",
+      raw_credit:           "amountOfCredit",
+      raw_debit:            "amountOfLiability",
+      currency:             "currencyType",
+      parent_poid:          "parent",
+      memo:                 "memo",
+      ordered:              "ordered",
+      # Examples: saving accounts, credit cards, cash, insurances and so on.
+      account_group_poid:   "accountGroupPOID",
+      raw_hidden:           "hidden",
+    }.freeze
+
+    IgnoredFields = [
+      "tradingEntityPOID",
+      "type",             # Always 0
+      "usedCount",        # Always 0
+      "uuid",             # Always empty.
+      "code",             # WTF
+      "clientID",         # WTF
+    ].freeze
+
+    define_accessors(FieldMappings)
+
+    # NOTE: balance is not set for credit cards etc. Instead
+    # credit/debit are used.
+    # Guess: The special behavior is be controlled by
+    # account_group_poid. Again, the code can work in all cases,
+    # thus no check is done.
+    def balance
+      to_bigdecimal(raw_balance) + credit - debit
+    end
+
+    def credit
+      to_bigdecimal(raw_credit)
+    end
+
+    def debit
+      to_bigdecimal(raw_debit)
+    end
+
+    def hidden?
+      raw_hidden == 1
+    end
+
+    def account_group
+      self.class.environment::AccountGroup.find_by_id(account_group_poid)
+    end
+
+    # Parent related.
+    def parent
+      self.class.find_by_id(parent_poid)
+    end
+
+    def has_parent?
+      parent_poid != 0 && !flagged_as_parent?
+    end
+
+    def flagged_as_parent?
+      # Account with POID -1 doesn't exist. It's just a special
+      # POID used to indicate that this account itself is the parent
+      # of some other accounts.
+      parent_poid == -1
+    end
+
+    def flat_parent_hierachy?
+      !has_parent? or parent.flagged_as_parent?
+    end
+
+    def children
+      arr = []
+      self.class.database.query("SELECT * FROM #{self.class.table_name} WHERE parent = ?", poid) do |result|
+        result.each do |raw_row|
+          arr << self.class.new(result.columns, result.types, raw_row)
+        end
+      end
+      arr
+    end
+
+    class ModifiedAccount < Record::ModifiedRecord
+      define_custom_methods([
+        :balance,
+        :credit,
+        :debit,
+        :parent,
+      ])
+      define_default_methods(FieldMappings)
+    end
+
+    # Schema:
+    # accountPOID LONG NOT NULL,
+    # name varchar(100) NOT NULL,
+    # tradingEntityPOID integer NOT NULL,
+    # lastUpdateTime] LONG,
+    # usedCount integer DEFAULT 0,
+    # accountGroupPOID integer,
+    # balance decimal(12, 2),
+    # currencyType varchar(50) default 'CNY',
+    # memo varchar(200),
+    # type integer DEFAULT 0,
+    # amountOfLiability decimal(12, 2)DEFAULT 0,
+    # amountOfCredit decimal(12, 2)DEFAULT 0,
+    # ordered integer default 0,
+    # code VARCHAR(20),
+    # parent LONG default 0,
+    # hidden integer default 0,
+    # clientID LONG default 0,
+    # uuid varchar(200),
+  end
+end

data/lib/feidee_utils/account_group.rb

@@ -0,0 +1,54 @@
+require 'feidee_utils/record'
+require 'feidee_utils/mixins/parent_and_path'
+require 'feidee_utils/mixins/type'
+
+module FeideeUtils
+  class AccountGroup < Record
+    include FeideeUtils::Mixins::ParentAndPath
+    include FeideeUtils::Mixins::Type
+
+    def validate_integrity
+      validate_depth_integrity
+      validate_one_level_path_integrity
+
+      unless (poid == 1 and raw_type == -1) or (raw_type >= 0 and raw_type <= 2)
+        raise "Unexpected account group type #{raw_type}.\n" + inspect
+      end
+    end
+
+    FieldMappings = {
+      name:                   "name",
+      parent_poid:            "parentAccountGroupPOID",
+      raw_path:               "path",
+      depth:                  "depth",
+      raw_type:               "type",
+      ordered:                "ordered",
+    }.freeze
+
+    IgnoredFields = [
+      "userTradingEntityPOID", # WTF
+      "_tempIconName",         # Icon name in the app
+      "clientID",              # WTF
+    ].freeze
+
+    define_accessors(FieldMappings)
+
+    define_type_enum({
+      0 => :asset,
+      1 => :liability,
+      2 => :claim,
+    })
+
+    # Schema
+    # accountGroupPOID long not null
+    # name varchar(100) not null
+    # parentAccountGroupPOID long not null
+    # path varchar(200)
+    # depth integer
+    # lastUpdateTime long
+    # userTradingEntityPOID long
+    # _tempIconName varchar(100) default defaultAccountGroupIcon
+    # type integer default 1
+    # ordered integer default 0
+  end
+end

data/lib/feidee_utils/category.rb

@@ -0,0 +1,76 @@
+require 'feidee_utils/record'
+require 'feidee_utils/mixins/parent_and_path'
+require 'feidee_utils/mixins/type'
+
+module FeideeUtils
+  class Category < Record
+    include FeideeUtils::Mixins::ParentAndPath
+    include FeideeUtils::Mixins::Type
+
+    def validate_integrity
+      validate_depth_integrity
+      validate_one_level_path_integrity
+      raise "Category usedCount should always be 0, but it's #{field["usedCount"]}.\n" + inspect unless field["usedCount"] == 0
+    end
+
+    def self.validate_global_integrity
+      project_root_code = 2
+      if TypeEnum[project_root_code] != :project_root
+        raise "The type code of project root has been changed, please update the code."
+      end
+
+      rows = self.database.execute <<-SQL
+        SELECT #{id_field_name}, #{FieldMappings[:name]} FROM #{table_name}
+        WHERE #{FieldMappings[:raw_type]}=#{project_root_code};
+      SQL
+
+      if rows.length > 1
+        poids = rows.map do |row| row[0] end
+        raise "More than one category have type project_root. IDs are #{poids.inspect}."
+      elsif rows.length == 1
+        category_name = rows[0][1]
+        if category_name != "projectRoot" and category_name != "root"
+          raise "Category #{category_name} has type project_root. ID: #{rows[0][0]}."
+        end
+      end
+    end
+
+    FieldMappings = {
+      name:                   "name",
+      parent_poid:            "parentCategoryPOID",
+      raw_path:               "path",
+      depth:                  "depth",
+      raw_type:               "type",
+      ordered:                "ordered",
+    }.freeze
+
+    IgnoredFields = [
+      "userTradingEntityPOID", # WTF
+      "_tempIconName",         # Icon name in the app
+      "usedCount",             # Always 0.
+      "clientID",              # WTF
+    ].freeze
+
+    define_accessors(FieldMappings)
+
+    define_type_enum({
+      0 => :expenditure,
+      1 => :income,
+      2 => :project_root, # unkown
+    })
+
+    # Schema
+    # categoryPOID LONG NOT NULL
+    # name varchar(100) NOT NULL
+    # parentCategoryPOID LONG NOT NULL
+    # path VARCHAR(200)
+    # depth INTEGER
+    # lastUpdateTime LONG
+    # userTradingEntityPOID LONG
+    # _tempIconName VARCHAR(100) DEFAULT defaultIcon,
+    # usedCount integer default 0,
+    # type integer default 0,
+    # ordered integer default 0,
+    # clientID LONG default 0,
+  end
+end

data/lib/feidee_utils/database.rb

@@ -0,0 +1,190 @@
+require 'sqlite3'
+require 'feidee_utils/record'
+
+# A thin wrapper around SQLite3
+module FeideeUtils
+  class Database < SQLite3::Database
+    Tables = {
+      accounts: "t_account",
+      account_groups: "t_account_group",
+      categories: "t_category",
+      transactions: "t_transaction",
+
+      metadata: "t_metadata",
+      profile: "t_profile",
+    }.freeze
+
+    PotentialUsefulTables = %w(
+      t_account_book
+      t_account_info
+      t_budget_item
+      t_fund_holding
+      t_fund_trans
+      t_module_stock_holding
+      t_module_stock_tran
+    ).freeze
+
+    attr_reader :sqlite_file
+    attr_reader :platform, :sqlite_name, :sqlite_timestamp
+    attr_reader :missing_tables
+    attr_reader :namespaced
+
+    def initialize(private_sqlite, strip = false)
+      @sqlite_file = Database.feidee_to_sqlite(private_sqlite)
+
+      super(@sqlite_file.path)
+
+      extract_metadata
+      drop_unused_tables if strip
+
+      @namespaced = Record.generate_namespaced_record_classes(self)
+    end
+
+    def sqlite_backup(dest_file_path)
+      self.execute("vacuum;")
+
+      backup_sqlite_db = SQLite3::Database.new(dest_file_path.to_s)
+      backup_obj = SQLite3::Backup.new(backup_sqlite_db, "main", self, "main")
+      backup_obj.step(-1)
+      backup_obj.finish
+      backup_sqlite_db.close
+    end
+
+    def validate_global_integrity
+      @namespaced.constants.each do |const|
+        @namespaced.const_get(const).validate_global_integrity if const != :Database
+      end
+    end
+
+    private
+    def all_tables
+      rows = self.execute <<-SQL
+        SELECT name FROM sqlite_master
+        WHERE type IN ('table','view') AND name NOT LIKE 'sqlite_%'
+        UNION ALL
+        SELECT name FROM sqlite_temp_master
+        WHERE type IN ('table','view')
+        ORDER BY 1
+      SQL
+      rows.map do |row| row[0] end.sort
+    end
+
+    def drop_unused_tables
+      useful_tables = Tables.values + PotentialUsefulTables
+      useful_tables = (useful_tables + useful_tables.map do |x| self.class.trash_table_name(x) end).sort
+
+      # TODO: Record all tables droped.
+      (all_tables - useful_tables).each do |table|
+        self.execute("DROP TABLE IF EXISTS #{table}");
+      end
+
+      @missing_tables = Tables.values - all_tables
+      if !@missing_tables.empty?
+        raise "Missing tables: #{@missing_tables} from kbf backup."
+      end
+    end
+
+    def extract_metadata
+      @platform = self.execute("SELECT platform from #{Tables[:metadata]}")[0][0];
+
+      @sqlite_name = self.get_first_row("SELECT accountBookName FROM #{Tables[:profile]};")[0];
+
+      # This is not recorded in the database, so the lastest lastUpdateTime of all
+      # transactions is chosen.
+      timestamp = self.get_first_row("SELECT max(lastUpdateTime) FROM #{Tables[:transactions]};")[0]
+      @sqlite_timestamp = timestamp == nil ? Time.at(0) : Time.at(timestamp / 1000)
+    end
+
+    class << self
+      def open_file(file_name)
+        Database.new(File.open(file_name))
+      end
+
+      Header = "SQLite format 3\0".force_encoding("binary")
+      FeideeHeader_iOS = "%$^#&!@_@- -!F\xff\0".force_encoding('binary')
+      FeideeHeader_Android = ("\0" * 13 + "F\xff\0").force_encoding("binary")
+
+      def feidee_to_sqlite(private_sqlite, sqlite_file = nil)
+        # Discard the first a few bytes content.
+        private_header = private_sqlite.read(Header.length)
+
+        unless [FeideeHeader_iOS, FeideeHeader_Android, Header].include? private_header
+          raise "Unexpected header #{private_header.inspect} in private sqlite file."
+        end
+
+        # Write the rest to a tempfile.
+        sqlite_file ||= Tempfile.new("kingdee_sqlite", binmode: true)
+        sqlite_file.binmode
+        sqlite_file.write(Header)
+        sqlite_file.write(private_sqlite.read)
+        sqlite_file.fsync
+        sqlite_file
+      end
+    end
+
+    class << self
+      NoDeleteSuffixTables = %w(account category tag tradingEntity transaction transaction_template)
+
+      def trash_table_name name
+        NoDeleteSuffixTables.each do |core_name|
+          if name == "t_" + core_name then
+            return "t_" + "deleted_" + core_name;
+          end
+        end
+
+        name + "_delete"
+      end
+    end
+
+    AllKnownTables = {
+      t_account:                "As named",
+      t_account_book:           "A group of accounts, travel accounts etc.",
+      t_account_extra:          "Extra Feidee account configs, key/value pair.",
+      t_account_group:          "A group of accounts, saving/chekcing etc.",
+      t_account_info:           "Additional info of accounts: banks etc.",
+      t_accountgrant:           "Feidee account VIP related stuff.",
+      t_budget_item:            "Used to create budgets. An extension of category.",
+      t_binding:                "Netbank / credit card / Taobao bindings.",
+      t_category:               "Transaction categories.",
+      t_currency:               "Currency types.",
+      t_exchange:               "Currency exchange rates.",
+      t_fund:                   "List of money manage institute names. Abandoned.",
+      t_fund_holding:           "Fund accounts.",
+      t_fund_trans:             "Fund transactions.",
+      t_fund_price_history:     "Fund price history",
+      t_id_seed:                "ID seeds for all tables.",
+      t_import_history:         "As named.",
+      t_import_source:          "Import data from text messages etc.",
+
+      # JCT stands for Jia Cai Tong, a software for family book keeping.
+      # See http://www.feidee.com/jct/
+      # JCT is quite obsolete. All the related tables can be safely ignored.
+      t_jct_clientdeviceregist: "Client devices.",
+      t_jct_clientdevicestatus: "Client devices status.",
+      t_jct_syncbookfilelist:   "Name of files synced from other devices.",
+      t_jct_usergrant:          "Maybe if the user has purchased any service.",
+      t_jct_userlog:            "As named.",
+
+      t_local_recent:           "Local merchandise used recently",
+      t_message:                "Kingdee ads.",
+      t_metadata:               "Database version, client version etc.",
+      t_module_stock_holding:   "Stock accounts.",
+      t_module_stock_info:      "Stock rates.",
+      t_module_stock_trans:     "Stock transactions.",
+      t_notification:           "If and when a notification has been delivered.",
+      t_profile:                "User profile, default stuff when configuring account books.",
+      t_property:               "Data collected on user settings, key/value pair.",
+      t_syncResource:           "???",
+      t_sync_logs:              "As named.",
+      t_tag:                    "Other support like roles/merchandise.",
+      t_tradingEntity:          "Merchandise. Used together with t_user in Debt Center.",
+      t_trans_debt:             "Transactions in Debt Center.",
+      t_trans_debt_group:       "Transaction groups in Debt Center.",
+      t_transaction:            "As named.",
+      t_transaction_projectcategory_map: "Transaction project.",
+      t_transaction_template:   "As named. UI.",
+      t_user:                   "Multi-user support.",
+      t_usage_count:            "As named. Abandoned."
+    }
+  end
+end

data/lib/feidee_utils/kbf.rb

@@ -0,0 +1,36 @@
+require 'zip'
+require 'sqlite3'
+require 'feidee_utils/database'
+require 'feidee_utils/record'
+require 'feidee_utils/transaction'
+require 'feidee_utils/account'
+
+module FeideeUtils
+  class Kbf
+    DatabaseName = 'mymoney.sqlite'
+
+    attr_reader :zipfile, :sqlite_db
+
+    def initialize(input_stream)
+      @zipfile = Zip::File.open_buffer(input_stream) do |zipfile|
+        zipfile.each do |entry|
+          if entry.name == DatabaseName
+            # Each call to get_input_stream will create a new stream
+            @original_sqlite_db_entry = entry
+            @sqlite_db = FeideeUtils::Database.new(entry.get_input_stream, true)
+          end
+        end
+      end
+    end
+
+    def extract_original_sqlite(dest_file_path = nil)
+      FeideeUtils::Database.feidee_to_sqlite(@original_sqlite_db_entry.get_input_stream, dest_file_path)
+    end
+
+    class << self
+      def open_file(file_name)
+        return Kbf.new(File.open(file_name))
+      end
+    end
+  end
+end

data/lib/feidee_utils/mixins/parent_and_path.rb

@@ -0,0 +1,69 @@
+module FeideeUtils
+  module Mixins
+    # Requires:
+    #   instance methods: poid, parent_poid, raw_path
+    #   class methods: find_by_id
+    module ParentAndPath
+      class InconsistentDepthException < Exception
+      end
+
+      class InconsistentPathException < Exception
+      end
+
+      def validate_depth_integrity
+        path_depth = path.length - 1
+        if path_depth != depth
+          raise InconsistentDepthException,
+            "Path is #{path}, but the given depth is #{depth}.\n" +
+            inspect
+        end
+      end
+
+      def validate_one_level_path_integrity
+        path_array = path.clone
+        last_poid = path_array.pop
+
+        if last_poid != poid
+          raise InconsistentPathException,
+            "The last node in path is #{last_poid}, but the current poid is #{poid}.\n" +
+            inspect
+        end
+
+        if has_parent? and path_array != parent.path
+          raise InconsistentPathException,
+            "Path is #{path}, but path of parent is #{parent.path}.\n" +
+            inspect
+        end
+      end
+
+      def validate_path_integrity_hard
+        cur = self
+        step = 0
+        while cur or step != path.length
+          step += 1
+          poid = path[-step]
+          if !cur or poid == nil or poid != cur.poid
+            raise InconsistentPathException,
+              "Reverse path and trace are different at step #{step}. " +
+              "Path shows #{poid}, but trace shows #{cur and cur.poid}.\n" +
+              inspect
+          end
+
+          cur = cur.has_parent? && cur.parent
+        end
+      end
+
+      def path
+        @path ||= (raw_path.split("/").map do |poid| poid.to_i end)[1..-1]
+      end
+
+      def parent
+        self.class.find_by_id(parent_poid)
+      end
+
+      def has_parent?
+        parent_poid != 0
+      end
+    end
+  end
+end

data/lib/feidee_utils/mixins/type.rb

@@ -0,0 +1,34 @@
+module FeideeUtils
+  module Mixins
+    # Requires:
+    #   instance methods: raw_type
+    module Type
+      module ClassMethods
+        def define_type_enum type_enum, reverse_lookup = true
+          const_set :TypeEnum, type_enum.freeze
+
+          if reverse_lookup
+            enum_values = type_enum.values
+            if enum_values.size != enum_values.uniq.size
+              raise "Duplicate values in enum #{type_enum}."
+            end
+
+            const_set :TypeCode, type_enum.invert.freeze
+            define_singleton_method :type_code do |type_enum_value|
+              self::TypeCode[type_enum_value]
+            end
+          end
+        end
+      end
+
+      def self.included klass
+        klass.extend ClassMethods
+      end
+
+      def type
+       self.class::TypeEnum[raw_type]
+      end
+    end
+  end
+end
+

data/lib/feidee_utils/record/accessors.rb

@@ -0,0 +1,22 @@
+module FeideeUtils
+  class Record
+    module Accessors
+      def poid
+        @field[self.class.id_field_name]
+      end
+
+      def last_update_time
+        timestamp_to_time(@field["lastUpdateTime"])
+      end
+
+      module ClassMethods
+        def define_accessors field_mappings
+          field_mappings.each do |name, key|
+            raise "Accessor #{name} already exists in #{self.name}." if method_defined? name
+            define_method name do field[key] end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/feidee_utils/record/modified_record.rb

@@ -0,0 +1,76 @@
+module FeideeUtils
+  class Record
+    # TODO: Reconsider this class and ship full support to all entities.
+    class ModifiedRecord
+      attr_reader :poid
+      attr_reader :base, :head
+      attr_reader :modified_fields
+
+      def initialize(poid, base, head)
+        raise "Base row doesn't have the given poid." if base.poid != poid
+        raise "Head row doesn't have the given poid." if head.poid != poid
+        @poid = poid
+        @base = base
+        @head = head
+        @modified_fields = self.class.fields_diff(base.field, head.field)
+      end
+
+      class ValuePair
+        attr_reader :old_value, :new_value
+        def initialize(old_value, new_value)
+          @old = old_value
+          @new = new_value
+        end
+      end
+
+      def self.fields_diff base, head
+        (base.keys.sort | head.keys.sort).inject({}) do |hash, key|
+          if base[key] != head[key]
+            hash[key] = ValuePair.new(base[key], head[key])
+          end
+          hash
+        end
+      end
+
+      def touched?
+        !modified_fields.empty?
+      end
+
+      def changed?
+        methods.inject(false) do |acc, name|
+          if name.to_s.end_with? "_changed?"
+            acc ||= send name
+          end
+          acc
+        end
+      end
+
+      protected
+      def self.define_custom_methods fields
+        fields.each do |name|
+          if !respond_to? name
+            define_method name do
+              ValuePair.new((base.send name), (head.send name))
+            end
+            define_method (name.to_s + "_changed?").to_sym do
+              (base.send name) != (head.send name)
+            end
+          end
+        end
+      end
+
+      def self.define_default_methods field_mappings
+        field_mappings.each do |name, key|
+          if !respond_to? name
+            define_method name do
+              modified_fields[key]
+            end
+            define_method (name.to_s + "_changed?").to_sym do
+              modified_fields.has_key? key
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/feidee_utils/record/namespaced.rb

@@ -0,0 +1,39 @@
+require 'set'
+
+module FeideeUtils
+  class Record
+    module Namespaced
+      module ClassMethods
+        attr_reader :child_classes
+
+        # Must be invoked by Record.inherited
+        def collect_subclass(child_class)
+          @child_classes ||= Set.new
+          @child_classes.add(child_class)
+        end
+
+        # To use Record with different databases, generate a set of classes for each db
+        def generate_namespaced_record_classes(db)
+          @child_classes ||= Set.new
+          this = self
+          Module.new do |mod|
+            const_set(:Database, Module.new {
+              define_method("database") { db }
+              define_method("environment") { mod }
+            })
+
+            this.child_classes.each do |child_class|
+              if child_class.name.start_with? FeideeUtils.name
+                class_name = child_class.name.sub(/#{FeideeUtils.name}::/, '')
+                # Generate a const for the child class
+                const_set(class_name, Class.new(child_class) {
+                  extend mod::Database
+                })
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/feidee_utils/record/persistent.rb

@@ -0,0 +1,55 @@
+
+module FeideeUtils
+  class Record
+    module Persistent
+      module ClassMethods
+        # Names
+        # Must be invoked by Record.inherited
+        def genereate_names subclass
+          entity_name =
+            if i = subclass.name.rindex("::")
+              subclass.name[(i+2)..-1]
+            else
+              subclass.name
+            end
+
+          id_field_name = entity_name.sub(/^[A-Z]/) { $&.downcase } + "POID"
+          table_name = "t_" + entity_name.gsub(/([a-z\d])([A-Z\d])/, '\1_\2').downcase
+          subclass.class_exec do
+            define_singleton_method :entity_name do entity_name end
+            define_singleton_method :id_field_name do id_field_name end
+            define_singleton_method :table_name do table_name end
+          end
+        end
+
+        # Persistent
+        def all
+          arr = []
+          database.query("SELECT * FROM #{self.table_name}") do |result|
+            result.each do |raw_row|
+              arr << self.new(result.columns, result.types, raw_row)
+            end
+          end
+          arr
+        end
+
+        def find_by_id(id)
+          raw_result = database.query("SELECT * FROM #{self.table_name} WHERE #{self.id_field_name} = ?", id)
+
+          raw_row = raw_result.next
+          return nil if raw_row == nil
+
+          if raw_result.next != nil
+            raise "Getting more than one result with the same ID #{id} in table #{self.table_name}."
+          end
+
+          self.new(raw_result.columns, raw_result.types, raw_row)
+        end
+
+        def find(id)
+          find_by_id(id) or raise "No #{self.name} of poid #{id} found"
+        end
+      end
+    end
+  end
+end

data/lib/feidee_utils/record/utils.rb

@@ -0,0 +1,15 @@
+module FeideeUtils
+  class Record
+    module Utils
+      protected
+      def timestamp_to_time num
+        Time.at(num / 1000.0, num % 1000)
+      end
+
+      def to_bigdecimal number
+        # Be aware of the precision lost from String -> Float -> BigDecimal.
+        BigDecimal.new(number, 12).round(2)
+      end
+    end
+  end
+end

data/lib/feidee_utils/record.rb

@@ -0,0 +1,66 @@
+require 'feidee_utils/record/accessors'
+require 'feidee_utils/record/namespaced'
+require 'feidee_utils/record/persistent'
+require 'feidee_utils/record/utils'
+require 'feidee_utils/record/modified_record'
+
+module FeideeUtils
+  # The implementation here is wired.
+  # The goal is to create a class hierachy similar to ActiveRecord, where every table is represented by a
+  # subclass of ActiveRecord::Base. Class methods, attribute accessors and almost all other functionalities
+  # are provided by ActiveRecord::Base. For example, Base.all(), Base.find_by_id() are tied to a specific
+  # table in a specific database.
+  # The problem we are solving here is not the same as ActiveRecord. In ActiveRecord, the databases
+  # are static, i.e. they won't be changed at runtime. Meanwhile, in our case, new databases can be created
+  # at runtime, when a new KBF backup file is uploaded. Furthermore, multiple instances of different databases
+  # can co-exist at the same time. To provide the same syntax as ActiveRecord, a standalone "Base" class has
+  # to be created for each database.
+  # In our implementation, when a new database is created, a subclass of Record is created in a new namepsace.
+  # For each subclass of Record, a new subclass is copied to the new namespace, with it's database method
+  # overloaded.
+  class Record
+    attr_reader :field, :field_type
+
+    public
+    def initialize(columns, types, raw_row)
+      @field = Hash[ columns.zip(raw_row) ]
+      @field_type = Hash[ columns.zip(types) ]
+
+      validate_integrity
+    end
+
+    def validate_integrity
+      # Do nothing.
+    end
+
+    def self.validate_global_integrity
+      # Do nothing.
+    end
+
+    class << self
+      protected
+      def database
+        raise NotImplementedError.new("Subclasses must set database")
+      end
+
+      private
+      def inherited subclass
+        if subclass.name != nil and subclass.name.start_with? FeideeUtils.name
+          collect_subclass subclass
+          genereate_names subclass
+        end
+      end
+    end
+
+    # Basic accessors, poid, last update time, etc.
+    include Accessors
+    # Helper methods to define accessors
+    extend Accessors::ClassMethods
+    # Helper methods to define new classes in a given namespace.
+    extend Namespaced::ClassMethods
+    # Helper methods to look up records.
+    extend Persistent::ClassMethods
+    # Helper methods to convert data types.
+    include Utils
+  end
+end

data/lib/feidee_utils/transaction.rb

@@ -0,0 +1,232 @@
+require 'feidee_utils/record'
+require 'feidee_utils/mixins/type'
+
+module FeideeUtils
+  class Transaction < Record
+    include FeideeUtils::Mixins::Type
+
+    class TransferLackBuyerOrSellerException < Exception
+    end
+
+    class TransferWithCategoryException < Exception
+    end
+
+    class InconsistentBuyerAndSellerSetException < Exception
+    end
+
+    class InconsistentCategoryException < Exception
+    end
+
+    class InconsistentAmountException < Exception
+    end
+
+    def validate_integrity
+      if is_transfer?
+        unless buyer_account_poid != 0 and seller_account_poid != 0
+          raise TransferLackBuyerOrSellerException,
+            "Both buyer and seller should be set in a transfer. " +
+            "Buyer account POID: #{buyer_account_poid}. Seller account POID: #{seller_account_poid}.\n" +
+            inspect
+        end
+        unless buyer_category_poid == 0 and seller_category_poid == 0
+          raise TransferWithCategoryException,
+            "Neither buyer or seller category should be set in a transfer. " +
+            "Buyer category POID: #{buyer_category_poid}. Seller category POID: #{seller_category_poid}.\n" +
+            inspect
+        end
+      else
+        unless (buyer_account_poid == 0) ^ (seller_account_poid == 0)
+          raise InconsistentBuyerAndSellerSetException,
+            "Exactly one of buyer and seller should be set in a non-transfer transaction. " +
+            "Buyer account POID: #{buyer_account_poid}. Seller account POID: #{seller_account_poid}.\n" +
+            inspect
+        end
+
+        # We could enforce that category is set to the matching party (buyer or seller) of account.
+        # However the implementation could handle all situations, as long as only one of them is set.
+        # Thus no extra check is done here.
+        unless buyer_category_poid == 0 or seller_category_poid == 0
+          raise InconsistentCategoryException,
+            "Only one of buyer and seller category should be set in a non-transfer transaction. "
+            "Buyer category POID: #{buyer_category_poid}. Seller category POID: #{seller_category_poid}.\n" +
+            inspect
+        end
+      end
+
+      unless buyer_deduction == seller_addition
+        raise InconsistentAmountException,
+          "Buyer and seller should have the same amount set. " +
+          "Buyer deduction: #{buyer_deduction}, seller_addition: #{seller_addition}.\n" +
+        inspect
+      end
+    end
+
+    class TransfersNotPaired < Exception
+    end
+
+    def self.validate_global_integrity
+      uuids_map = all.inject({}) do |uuids, transaction|
+        if transaction.is_transfer?
+          uuid = transaction.uuid
+          uuids[uuid] ||= [nil, nil]
+          uuids[uuid][transaction.raw_type - 2] = transaction
+        end
+        uuids
+      end
+
+      uuids_map.each do |uuid, transfers|
+        valid = true
+        valid &&= transfers[0] != nil
+        valid &&= transfers[1] != nil
+        valid &&= transfers[0].buyer_account_poid == transfers[1].buyer_account_poid
+        valid &&= transfers[0].seller_account_poid == transfers[1].seller_account_poid
+        raise TransfersNotPaired.new([uuid] + transfers) unless valid
+      end
+    end
+
+    FieldMappings = {
+      raw_created_at:         "createdTime",
+      raw_modified_at:        "modifiedTime",
+      raw_trade_at:           "tradeTime",
+      raw_type:               "type",
+      memo:                   "memo",
+      buyer_account_poid:     "buyerAccountPOID",
+      buyer_category_poid:    "buyerCategoryPOID",
+      seller_account_poid:    "sellerAccountPOID",
+      seller_category_poid:   "sellerCategoryPOID",
+      raw_buyer_deduction:    "buyerMoney",
+      raw_seller_addition:    "sellerMoney",
+      uuid:                   "relation",
+    }.freeze
+
+    IgnoredFields = [
+      "creatorTradingEntityPOID",
+      "modifierTradingEntityPOID",
+      "ffrom",                # The signature of the App writting this transaction.
+      "photoName",            # To be added
+      "photoNeedUpload",      # To be added
+      "relationUnitPOID",     # WTF
+      "clientID",             # WTF
+      "FSourceKey",           # WTF
+    ].freeze
+
+    define_accessors(FieldMappings)
+
+    define_type_enum({
+      0 => :expenditure,
+      1 => :income,
+      2 => :transfer_buyer,
+      3 => :transfer_seller,
+      8 => :positive_initial_balance,
+      9 => :negative_initial_balance,
+    })
+
+    def created_at
+      timestamp_to_time(raw_created_at)
+    end
+
+    def modified_at
+      timestamp_to_time(raw_modified_at)
+    end
+
+    def trade_at
+      timestamp_to_time(raw_trade_at)
+    end
+
+    def has_category?
+      category_poid != 0
+    end
+
+    def category_poid
+      # At least one of those two must be 0.
+      buyer_category_poid + seller_category_poid
+    end
+
+    # Amount accessors
+
+    def buyer_deduction
+      sign_by_type(raw_buyer_deduction)
+    end
+
+    def seller_addition
+      sign_by_type(raw_seller_addition)
+    end
+
+    def amount
+      # Buyer deduction is always equal to seller addition.
+      (buyer_deduction + seller_addition) / 2
+    end
+
+    def is_transfer?
+      type == :transfer_buyer or type == :transfer_seller
+    end
+
+    def is_initial_balance?
+      type == :positive_initial_balance or type == :negative_initial_balance
+    end
+
+    def revised_account_poid
+      if type == :transfer_buyer
+        buyer_account_poid
+      elsif type == :transfer_seller
+        seller_account_poid
+      else
+        buyer_account_poid + seller_account_poid
+      end
+    end
+
+    def revised_amount
+      account_poid = revised_account_poid
+      if account_poid == buyer_account_poid
+        -buyer_deduction
+      elsif account_poid == seller_account_poid
+        seller_addition
+      else
+        raise "Unexpected revised account poid #{account_poid}."
+      end
+    end
+
+    class ModifiedTransaction < ModifiedRecord
+      define_custom_methods([
+        :created_at,
+        :modified_at,
+        :trade_at,
+        :type,
+        :category_poid,
+        :buyer_deduction,
+        :seller_addition,
+        :amount,
+      ])
+      define_default_methods(FieldMappings)
+    end
+
+    private
+    def sign_by_type num
+      raw_type == 9 ? -num : num
+    end
+
+    # Schema:
+    # transactionPOID LONG NOT NULL,
+    # createdTime LONG NOT NULL,
+    # modifiedTime LONG NOT NULL,
+    # tradeTime LONG NOT NULL,
+    # memo varchar(100),
+    # type integer NOT NULL,
+    # creatorTradingEntityPOID LONG,
+    # modifierTradingEntityPOID LONG,
+    # buyerAccountPOID LONG,
+    # buyerCategoryPOID LONG default 0,
+    # buyerMoney decimal(12, 2),
+    # sellerAccountPOID LONG,
+    # sellerCategoryPOID LONG default 0,
+    # sellerMoney decimal(12, 2),
+    # lastUpdateTime LONG,
+    # photoName VARCHAR(100),
+    # photoNeedUpload integer default 0,
+    # relation varchar(200) default '',
+    # relationUnitPOID LONG,
+    # ffrom varchar(250) default '',
+    # clientID LONG default 0,
+    # FSourceKey varchar(100) DEFAULT NULL,
+  end
+end

data/lib/feidee_utils/version.rb

@@ -0,0 +1,3 @@
+module FeideeUtils
+    VERSION = '0.0.4.1'
+end

data/lib/feidee_utils.rb

@@ -0,0 +1,9 @@
+require 'feidee_utils/kbf'
+require 'feidee_utils/database'
+require 'feidee_utils/record'
+require 'feidee_utils/account'
+require 'feidee_utils/transaction'
+require 'feidee_utils/category'
+
+module FeideeUtils
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: feidee_utils
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.4.1
 platform: ruby
 authors:
 - Liqing Muyi
@@ -97,7 +97,27 @@ email: muyiliqing@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
-files: []
+files:
+- Gemfile
+- MIT-LICENSE
+- README.md
+- Rakefile
+- lib/feidee_utils.rb
+- lib/feidee_utils/account.rb
+- lib/feidee_utils/account_group.rb
+- lib/feidee_utils/category.rb
+- lib/feidee_utils/database.rb
+- lib/feidee_utils/kbf.rb
+- lib/feidee_utils/mixins/parent_and_path.rb
+- lib/feidee_utils/mixins/type.rb
+- lib/feidee_utils/record.rb
+- lib/feidee_utils/record/accessors.rb
+- lib/feidee_utils/record/modified_record.rb
+- lib/feidee_utils/record/namespaced.rb
+- lib/feidee_utils/record/persistent.rb
+- lib/feidee_utils/record/utils.rb
+- lib/feidee_utils/transaction.rb
+- lib/feidee_utils/version.rb
 homepage: http://github.com/muyiliqing/feidee_utils
 licenses:
 - MIT