cns 0.1.0 → 0.1.5

data/lib/cns/bigquery1.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+require('google/cloud/bigquery')
+require('bigdecimal/util')
+
+# @author Hernani Rodrigues Vaz
+module Cns
+  BD = 'hernanirvaz.coins'
+
+  # (see Bigquery)
+  class Cns::Bigquery
+    # @return [Google::Cloud::Bigquery] API bigquery
+    attr_reader :api
+    # @return [Google::Cloud::Bigquery::QueryJob] job bigquery
+    attr_reader :job
+    # @return [Thor::CoreExt::HashWithIndifferentAccess] opcoes trabalho
+    attr_reader :ops
+    # @return (see sql)
+    attr_reader :sqr
+
+    # @param [Thor::CoreExt::HashWithIndifferentAccess] pop opcoes trabalho
+    # @option pop [Hash] :h ({}) configuracao ajuste reposicionamento temporal
+    # @option pop [Boolean] :v (false) mostra transacoes trades & ledger?
+    # @option pop [Boolean] :t (false) mostra transacoes todas ou somente novas?
+    # @return [Bigquery] API bigquery
+    def initialize(pop)
+      # usa env GOOGLE_APPLICATION_CREDENTIALS para obter credentials
+      # @see https://cloud.google.com/bigquery/docs/authentication/getting-started
+      @api = Google::Cloud::Bigquery.new
+      @ops = pop
+    end
+
+    # mostra situacao completa entre kraken/bitcoinde/paymium/therock/etherscan/greymass & bigquery
+    def mostra_tudo
+      apius.mostra_resumo
+      apide.mostra_resumo
+      apifr.mostra_resumo
+      apimt.mostra_resumo
+      apies.mostra_resumo
+      apigm.mostra_resumo
+    end
+
+    # insere (caso existam) transacoes novas kraken/bitcoinde/paymium/therock/etherscan/greymass no bigquery
+    def processa_tudo
+      processa_us
+      processa_de
+      processa_fr
+      processa_mt
+      processa_eth
+      processa_eos
+    end
+
+    private
+
+    # insere transacoes exchange kraken novas nas tabelas ust (trades), usl (ledger)
+    def processa_us
+      puts(format("%<n>2i TRADES KRAKEN INSERIDAS #{BD}.ust", n: apius.trades.empty? ? 0 : dml(ust_ins)))
+      puts(format("%<n>2i LEDGER KRAKEN INSERIDAS #{BD}.usl", n: apius.ledger.empty? ? 0 : dml(usl_ins)))
+    end
+
+    # insere transacoes exchange bitcoinde novas nas tabelas det (trades), del (ledger)
+    def processa_de
+      puts(format("%<n>2i TRADES BITCOINDE INSERIDAS #{BD}.det", n: apide.trades.empty? ? 0 : dml(det_ins)))
+      puts(format("%<n>2i LEDGER BITCOINDE INSERIDAS #{BD}.del", n: apide.ledger.empty? ? 0 : dml(del_ins)))
+    end
+
+    # insere transacoes exchange paymium novas na tabela fr (ledger)
+    def processa_fr
+      puts(format("%<n>2i LEDGER PAYMIUM INSERIDAS #{BD}.fr", n: apifr.ledger.empty? ? 0 : dml(frl_ins)))
+    end
+
+    # insere transacoes exchange therock novas na tabela mt (ledger)
+    def processa_mt
+      puts(format("%<n>2i LEDGER THEROCK INSERIDAS #{BD}.mt", n: apimt.ledger.empty? ? 0 : dml(mtl_ins)))
+    end
+
+    # insere transacoes blockchain novas nas tabelas etht (norml), ethk (token)
+    def processa_eth
+      puts(format("%<n>2i TRANSACOES ETH INSERIDAS #{BD}.etht", n: apies.novtx.empty? ? 0 : dml(etht_ins)))
+      puts(format("%<n>2i TOKEN EVENTS ETH INSERIDAS #{BD}.ethk", n: apies.novkx.empty? ? 0 : dml(ethk_ins)))
+    end
+
+    # insere transacoes blockchain novas na tabela eos
+    def processa_eos
+      puts(format("%<n>2i TRANSACOES EOS INSERIDAS #{BD}.eos ", n: apigm.novax.empty? ? 0 : dml(eost_ins)))
+    end
+
+    # cria job bigquery & verifica execucao
+    #
+    # @param cmd (see sql)
+    # @return [Boolean] job ok?
+    def job?(cmd)
+      @job = api.query_job(cmd)
+      @job.wait_until_done!
+      puts(@job.error['message']) if @job.failed?
+      @job.failed?
+    end
+
+    # cria Structured Query Language (SQL) job bigquery
+    #
+    # @param [String] cmd comando SQL a executar
+    # @param [String] res resultado quando SQL tem erro
+    # @return [Google::Cloud::Bigquery::Data] resultado do SQL
+    def sql(cmd, res = [])
+      @sqr = job?(cmd) ? res : job.data
+    end
+
+    # cria Data Manipulation Language (DML) job bigquery
+    #
+    # @param cmd (see sql)
+    # @return [Integer] numero linhas afetadas
+    def dml(cmd)
+      job?(cmd) ? 0 : job.num_dml_affected_rows
+    end
+  end
+end

data/lib/cns/bigquery2.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+# @author Hernani Rodrigues Vaz
+module Cns
+  # (see Bigquery)
+  class Bigquery
+    private
+
+    # @return [Etherscan] API blockchain ETH
+    def apies
+      @apies ||= Etherscan.new(
+        {
+          wb: sql("select * from #{BD}.walletEth order by 2"),
+          nt: sql("select itx,iax from #{BD}.ethtx"),
+          nk: sql("select itx,iax from #{BD}.ethkx")
+        },
+        ops
+      )
+    end
+
+    # @return [Greymass] API blockchain EOS
+    def apigm
+      @apigm ||= Greymass.new(
+        {
+          wb: sql("select * from #{BD}.walletEos order by 2"),
+          nt: sql("select itx,iax from #{BD}.eostx")
+        },
+        ops
+      )
+    end
+
+    # @return [Kraken] API exchange kraken
+    def apius
+      @apius ||= Kraken.new(
+        {
+          sl: sql("select sum(btc) xxbt,sum(eth) xeth,sum(eos) eos,sum(eur) zeur from #{BD}.ussl")[0],
+          nt: sql("select * from #{BD}.ustx order by time,txid"),
+          nl: sql("select * from #{BD}.uslx order by time,txid")
+        },
+        ops
+      )
+    end
+
+    # @return [Bitcoinde] API exchange bitcoinde
+    def apide
+      @apide ||= Bitcoinde.new(
+        {
+          sl: sql("select sum(btc) btc from #{BD}.desl")[0],
+          nt: sql("select * from #{BD}.detx order by time,txid"),
+          nl: sql("select * from #{BD}.delx order by time,txid")
+        },
+        ops
+      )
+    end
+
+    # @return [Paymium] API exchange paymium
+    def apifr
+      @apifr ||= Paymium.new(
+        {
+          sl: sql("select sum(btc) btc,sum(eur) eur from #{BD}.frsl")[0],
+          nl: sql("select * from #{BD}.frlx order by time,txid")
+        },
+        ops
+      )
+    end
+
+    # @return [TheRock] API exchange therock
+    def apimt
+      @apimt ||= TheRock.new(
+        {
+          sl: sql("select sum(btc) btc,sum(eur) eur from #{BD}.mtsl")[0],
+          nl: sql("select * from #{BD}.mtlx order by time,txid")
+        },
+        ops
+      )
+    end
+  end
+end

data/lib/cns/bigquery3.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+require('google/cloud/bigquery')
+require('bigdecimal/util')
+
+# @author Hernani Rodrigues Vaz
+module Cns
+  # (see Bigquery)
+  class Bigquery
+    private
+
+    # @return [String] comando insert SQL formatado etht (norml)
+    def etht_ins
+      "insert #{BD}.etht(blocknumber,timestamp,txhash,nonce,blockhash,transactionindex,axfrom,axto,iax," \
+      'value,gas,gasprice,gasused,iserror,txreceipt_status,input,contractaddress,dias' \
+      ") VALUES#{apies.novtx.map { |e| etht_val1(e) }.join(',')}"
+    end
+
+    # @example (see Apibc#norml_es)
+    # @param [Hash] htx transacao norml etherscan
+    # @return [String] valores formatados etht (norml parte1)
+    def etht_val1(htx)
+      "(#{Integer(htx[:blockNumber])}," \
+      "#{Integer(htx[:timeStamp])}," \
+      "'#{htx[:hash]}'," \
+      "#{Integer(htx[:nonce])}," \
+      "'#{htx[:blockHash]}'," \
+      "#{Integer(htx[:transactionIndex])}," \
+      "'#{htx[:from]}'," \
+      "'#{htx[:to]}'," \
+      "'#{htx[:iax]}'," \
+      "#{etht_val2(htx)}"
+    end
+
+    # @param (see etht_val1)
+    # @return [String] valores formatados etht (norml parte2)
+    def etht_val2(htx)
+      "cast('#{htx[:value]}' as numeric)," \
+      "cast('#{htx[:gas]}' as numeric)," \
+      "cast('#{htx[:gasPrice]}' as numeric)," \
+      "cast('#{htx[:gasUsed]}' as numeric)," \
+      "#{Integer(htx[:isError])}," \
+      "#{htx[:txreceipt_status].length.zero? ? 'null' : htx[:txreceipt_status]}," \
+      "#{etht_val3(htx)}"
+    end
+
+    # @param (see etht_val1)
+    # @return [String] valores formatados etht (norml parte3)
+    def etht_val3(htx)
+      "#{htx[:input].length.zero? ? 'null' : "'#{htx[:input]}'"}," \
+      "#{htx[:contractAddress].length.zero? ? 'null' : "'#{htx[:contractAddress]}'"}," \
+      "#{Integer(ops[:h][htx[:blockNumber]] || 0)})"
+    end
+
+    # @return [String] comando insert SQL formatado ethk (token)
+    def ethk_ins
+      "insert #{BD}.ethk(blocknumber,timestamp,txhash,nonce,blockhash,transactionindex,axfrom,axto,iax," \
+      'value,tokenname,tokensymbol,tokendecimal,gas,gasprice,gasused,input,contractaddress,dias' \
+      ") VALUES#{apies.novkx.map { |e| ethk_val1(e) }.join(',')}"
+    end
+
+    # @example (see Apibc#token_es)
+    # @param [Hash] hkx token event etherscan
+    # @return [String] valores formatados ethk (token parte1)
+    def ethk_val1(hkx)
+      "(#{Integer(hkx[:blockNumber])}," \
+      "#{Integer(hkx[:timeStamp])}," \
+      "'#{hkx[:hash]}'," \
+      "#{Integer(hkx[:nonce])}," \
+      "'#{hkx[:blockHash]}'," \
+      "#{Integer(hkx[:transactionIndex])}," \
+      "'#{hkx[:from]}'," \
+      "'#{hkx[:to]}'," \
+      "'#{hkx[:iax]}'," \
+      "#{ethk_val2(hkx)}"
+    end
+
+    # @param (see ethk_val1)
+    # @return [String] valores formatados ethk (token parte2)
+    def ethk_val2(hkx)
+      "cast('#{hkx[:value]}' as numeric)," \
+      "'#{hkx[:tokenName]}'," \
+      "'#{hkx[:tokenSymbol]}'," \
+      "#{Integer(hkx[:tokenDecimal])}," \
+      "cast('#{hkx[:gas]}' as numeric)," \
+      "cast('#{hkx[:gasPrice]}' as numeric)," \
+      "cast('#{hkx[:gasUsed]}' as numeric)," \
+      "#{ethk_val3(hkx)}"
+    end
+
+    # @param (see ethk_val1)
+    # @return [String] valores formatados ethk (token parte3)
+    def ethk_val3(hkx)
+      "#{hkx[:input].length.zero? ? 'null' : "'#{hkx[:input]}'"}," \
+      "#{hkx[:contractAddress].length.zero? ? 'null' : "'#{hkx[:contractAddress]}'"}," \
+      "#{Integer(ops[:h][hkx[:blockNumber]] || 0)})"
+    end
+
+    # @return [String] comando insert SQL formatado eos
+    def eost_ins
+      "insert #{BD}.eos(gseq,aseq,bnum,time,contract,action,acfrom,acto,iax,amount,moeda,memo,dias" \
+      ") VALUES#{apigm.novax.map { |e| eost_val1(e) }.join(',')}"
+    end
+
+    # @example (see Apibc#ledger_gm)
+    # @param [Hash] hlx ledger greymass
+    # @return [String] valores formatados para insert eos (parte1)
+    def eost_val1(hlx)
+      a = hlx[:action_trace][:act]
+      "(#{hlx[:global_action_seq]}," \
+      "#{hlx[:account_action_seq]}," \
+      "#{hlx[:block_num]}," \
+      "DATETIME(TIMESTAMP('#{hlx[:block_time]}'))," \
+      "'#{a[:account]}'," \
+      "'#{a[:name]}'," \
+      "#{eost_val2(hlx, a)}"
+    end
+
+    # @param (see eost_val1)
+    # @param [Hash] act dados da acao
+    # @return [String] valores formatados para insert eos (parte2)
+    def eost_val2(hlx, act)
+      d = act[:data]
+      q = d[:quantity].to_s
+      s = d[:memo].inspect
+      "'#{d[:from]}'," \
+      "'#{d[:to]}'," \
+      "'#{hlx[:iax]}'," \
+      "#{q.to_d},'#{q[/[[:upper:]]+/]}'," \
+      "nullif('#{s.gsub(/['"]/, '')}','nil')," \
+      "#{ops[:h][String(hlx[:itx])] || 0})"
+    end
+  end
+end

data/lib/cns/bigquery4.rb

@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+
+# @author Hernani Rodrigues Vaz
+module Cns
+  # classe para processar bigquery
+  class Bigquery
+    private
+
+    # @return [String] comando insert SQL formatado det (trades)
+    def det_ins
+      "insert #{BD}.det(txid,time,tp,user,btc,eur,dtc,dias) VALUES#{apide.trades.map { |h| det_val1(h) }.join(',')}"
+    end
+
+    # @example (see Apice#trades_de)
+    # @param [Hash] htx trade bitcoinde
+    # @return [String] valores formatados det (trades parte1)
+    def det_val1(htx)
+      "('#{htx[:trade_id]}'," \
+      "DATETIME(TIMESTAMP('#{htx[:successfully_finished_at]}'))," \
+      "'#{htx[:type]}'," \
+      "'#{htx[:trading_partner_information][:username]}'," \
+      "#{det_val2(htx)}"
+    end
+
+    # @param (see det_val1)
+    # @return [String] valores formatados det (trades parte2)
+    def det_val2(htx)
+      'cast(' \
+      "#{htx[:type] == 'buy' ? htx[:amount_currency_to_trade_after_fee] : "-#{htx[:amount_currency_to_trade]}"}" \
+      ' as numeric),' \
+      "cast(#{htx[:volume_currency_to_pay_after_fee]} as numeric)," \
+      "DATETIME(TIMESTAMP('#{htx[:trade_marked_as_paid_at]}'))," \
+      "#{Integer(ops[:h][htx[:trade_id]] || 0)})"
+    end
+
+    # @return [String] comando insert SQL formatado del (ledger)
+    def del_ins
+      "insert #{BD}.del(txid,time,tp,add,moe,qt,fee) VALUES#{apide.ledger.map { |h| del_val(h) }.join(',')}"
+    end
+
+    # @example (see Apice#deposits_de)
+    # @example (see Apice#withdrawals_de)
+    # @param [Hash] hlx ledger (deposits + withdrawals) bitcoinde
+    # @return [String] valores formatados del (ledger)
+    def del_val(hlx)
+      "(#{hlx[:txid]}," \
+      "DATETIME(TIMESTAMP('#{hlx[:time].iso8601}'))," \
+      "'#{hlx[:tp]}'," \
+      "'#{hlx[:add]}'," \
+      "'#{hlx[:moe]}'," \
+      "cast(#{hlx[:tp] =='withdrawal' ? '-' : ''}#{hlx[:qt]} as numeric)," \
+      "cast(#{hlx[:fee]} as numeric))"
+    end
+
+    # @return [String] comando insert SQL formatado ust (trades)
+    def ust_ins
+      "insert #{BD}.ust(txid,ordertxid,pair,time,type,ordertype,price,cost,fee,vol,margin,misc,ledgers,dias) " \
+      "VALUES#{apius.trades.map { |k, v| ust_val1(k, v) }.join(',')}"
+    end
+
+    # @example (see Apice#trades_us)
+    # @param [String] idx identificador transacao
+    # @param [Hash] htx trade kraken
+    # @return [String] valores formatados ust (trades parte1)
+    def ust_val1(idx, htx)
+      "('#{idx}'," \
+      "'#{htx[:ordertxid]}'," \
+      "'#{htx[:pair]}'," \
+      "PARSE_DATETIME('%s', '#{String(htx[:time].round)}')," \
+      "'#{htx[:type]}'," \
+      "'#{htx[:ordertype]}'," \
+      "cast(#{htx[:price]} as numeric)," \
+      "cast(#{htx[:cost]} as numeric)," \
+      "cast(#{htx[:fee]} as numeric)," \
+      "#{ust_val2(idx, htx)}"
+    end
+
+    # @param (see ust_val1)
+    # @return [String] valores formatados ust (trades parte2)
+    def ust_val2(idx, htx)
+      "cast(#{htx[:vol]} as numeric)," \
+      "cast(#{htx[:margin]} as numeric)," \
+      "#{htx[:misc].to_s.empty? ? 'null' : "'#{htx[:misc]}'"}," \
+      "'#{apius.ledger.select { |_, v| v[:refid] == idx }.keys.join(',') || ''}'," \
+      "#{Integer(ops[:h][idx] || 0)})"
+    end
+
+    # @return [String] comando insert SQL formatado usl (ledger)
+    def usl_ins
+      "insert #{BD}.usl(txid,refid,time,type,aclass,asset,amount,fee) " \
+      "VALUES#{apius.ledger.map { |k, v| usl_val(k, v) }.join(',')}"
+    end
+
+    # @example (see Apice#ledger_us)
+    # @param idx (see ust_val1)
+    # @param [Hash] hlx ledger kraken
+    # @return [String] valores formatados usl (ledger)
+    def usl_val(idx, hlx)
+      "('#{idx}'," \
+      "'#{hlx[:refid]}'," \
+      "PARSE_DATETIME('%s', '#{String(hlx[:time].round)}')," \
+      "'#{hlx[:type]}'," \
+      "#{hlx[:aclass].to_s.empty? ? 'null' : "'#{hlx[:aclass]}'"}," \
+      "'#{hlx[:asset]}'," \
+      "cast(#{hlx[:amount]} as numeric)," \
+      "cast(#{hlx[:fee]} as numeric))"
+    end
+
+    # @return [String] comando insert SQL formatado fr (ledger)
+    def frl_ins
+      "insert #{BD}.fr(uuid,tipo,valor,moe,time,dias) VALUES#{apifr.ledger.map { |h| frl_val(h) }.join(',')}"
+    end
+
+    # @example (see Apice#ledger_fr)
+    # @param [Hash] hlx ledger paymium
+    # @return [String] valores formatados frl (ledger)
+    def frl_val(hlx)
+      "('#{hlx[:uuid]}'," \
+      "'#{hlx[:name]}'," \
+      "cast(#{hlx[:amount]} as numeric)," \
+      "'#{hlx[:currency]}'," \
+      "PARSE_DATETIME('%s', '#{hlx[:created_at_int]}')," \
+      "#{Integer(ops[:h][hlx[:uuid]] || 0)})"
+    end
+
+    # @return [String] comando insert SQL formatado fr (ledger)
+    def mtl_ins
+      "insert #{BD}.mt(id,time,type,valor,moe,pair,note,trade_id,dias) " \
+      "VALUES#{apimt.ledger.map { |h| mtl_val1(h) }.join(',')}"
+    end
+
+    # @example (see Apice#ledger_mt)
+    # @param [Hash] hlx ledger therock
+    # @return [String] valores formatados mtl (ledger parte1)
+    def mtl_val1(hlx)
+      "(#{hlx[:id]}," \
+      "DATETIME(TIMESTAMP('#{hlx[:date]}'))," \
+      "'#{hlx[:type]}'," \
+      "cast(#{hlx[:price]} as numeric)," \
+      "'#{hlx[:currency]}'," \
+      "#{hlx[:fund_id].to_s.empty? ? 'null' : "'#{hlx[:fund_id]}'"}," \
+      "#{mtl_val2(hlx)}"
+    end
+
+    # @param (see mtl_val1)
+    # @return [String] valores formatados mtl (ledger parte2)
+    def mtl_val2(hlx)
+      "#{hlx[:note].to_s.empty? ? 'null' : "'#{hlx[:note]}'"}," \
+      "#{hlx[:trade_id].to_s.empty? ? 'null' : (hlx[:trade_id]).to_s}," \
+      "#{Integer(ops[:h][String(hlx[:id])] || 0)})"
+    end
+  end
+end