cns 0.1.0 → 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,116 @@
1
+ # frozen_string_literal: true
2
+
3
+ require('google/cloud/bigquery')
4
+ require('bigdecimal/util')
5
+
6
+ # @author Hernani Rodrigues Vaz
7
+ module Cns
8
+ BD = 'hernanirvaz.coins'
9
+
10
+ # (see Bigquery)
11
+ class Cns::Bigquery
12
+ # @return [Google::Cloud::Bigquery] API bigquery
13
+ attr_reader :api
14
+ # @return [Google::Cloud::Bigquery::QueryJob] job bigquery
15
+ attr_reader :job
16
+ # @return [Thor::CoreExt::HashWithIndifferentAccess] opcoes trabalho
17
+ attr_reader :ops
18
+ # @return (see sql)
19
+ attr_reader :sqr
20
+
21
+ # @param [Thor::CoreExt::HashWithIndifferentAccess] pop opcoes trabalho
22
+ # @option pop [Hash] :h ({}) configuracao ajuste reposicionamento temporal
23
+ # @option pop [Boolean] :v (false) mostra transacoes trades & ledger?
24
+ # @option pop [Boolean] :t (false) mostra transacoes todas ou somente novas?
25
+ # @return [Bigquery] API bigquery
26
+ def initialize(pop)
27
+ # usa env GOOGLE_APPLICATION_CREDENTIALS para obter credentials
28
+ # @see https://cloud.google.com/bigquery/docs/authentication/getting-started
29
+ @api = Google::Cloud::Bigquery.new
30
+ @ops = pop
31
+ end
32
+
33
+ # mostra situacao completa entre kraken/bitcoinde/paymium/therock/etherscan/greymass & bigquery
34
+ def mostra_tudo
35
+ apius.mostra_resumo
36
+ apide.mostra_resumo
37
+ apifr.mostra_resumo
38
+ apimt.mostra_resumo
39
+ apies.mostra_resumo
40
+ apigm.mostra_resumo
41
+ end
42
+
43
+ # insere (caso existam) transacoes novas kraken/bitcoinde/paymium/therock/etherscan/greymass no bigquery
44
+ def processa_tudo
45
+ processa_us
46
+ processa_de
47
+ processa_fr
48
+ processa_mt
49
+ processa_eth
50
+ processa_eos
51
+ end
52
+
53
+ private
54
+
55
+ # insere transacoes exchange kraken novas nas tabelas ust (trades), usl (ledger)
56
+ def processa_us
57
+ puts(format("%<n>2i TRADES KRAKEN INSERIDAS #{BD}.ust", n: apius.trades.empty? ? 0 : dml(ust_ins)))
58
+ puts(format("%<n>2i LEDGER KRAKEN INSERIDAS #{BD}.usl", n: apius.ledger.empty? ? 0 : dml(usl_ins)))
59
+ end
60
+
61
+ # insere transacoes exchange bitcoinde novas nas tabelas det (trades), del (ledger)
62
+ def processa_de
63
+ puts(format("%<n>2i TRADES BITCOINDE INSERIDAS #{BD}.det", n: apide.trades.empty? ? 0 : dml(det_ins)))
64
+ puts(format("%<n>2i LEDGER BITCOINDE INSERIDAS #{BD}.del", n: apide.ledger.empty? ? 0 : dml(del_ins)))
65
+ end
66
+
67
+ # insere transacoes exchange paymium novas na tabela fr (ledger)
68
+ def processa_fr
69
+ puts(format("%<n>2i LEDGER PAYMIUM INSERIDAS #{BD}.fr", n: apifr.ledger.empty? ? 0 : dml(frl_ins)))
70
+ end
71
+
72
+ # insere transacoes exchange therock novas na tabela mt (ledger)
73
+ def processa_mt
74
+ puts(format("%<n>2i LEDGER THEROCK INSERIDAS #{BD}.mt", n: apimt.ledger.empty? ? 0 : dml(mtl_ins)))
75
+ end
76
+
77
+ # insere transacoes blockchain novas nas tabelas etht (norml), ethk (token)
78
+ def processa_eth
79
+ puts(format("%<n>2i TRANSACOES ETH INSERIDAS #{BD}.etht", n: apies.novtx.empty? ? 0 : dml(etht_ins)))
80
+ puts(format("%<n>2i TOKEN EVENTS ETH INSERIDAS #{BD}.ethk", n: apies.novkx.empty? ? 0 : dml(ethk_ins)))
81
+ end
82
+
83
+ # insere transacoes blockchain novas na tabela eos
84
+ def processa_eos
85
+ puts(format("%<n>2i TRANSACOES EOS INSERIDAS #{BD}.eos ", n: apigm.novax.empty? ? 0 : dml(eost_ins)))
86
+ end
87
+
88
+ # cria job bigquery & verifica execucao
89
+ #
90
+ # @param cmd (see sql)
91
+ # @return [Boolean] job ok?
92
+ def job?(cmd)
93
+ @job = api.query_job(cmd)
94
+ @job.wait_until_done!
95
+ puts(@job.error['message']) if @job.failed?
96
+ @job.failed?
97
+ end
98
+
99
+ # cria Structured Query Language (SQL) job bigquery
100
+ #
101
+ # @param [String] cmd comando SQL a executar
102
+ # @param [String] res resultado quando SQL tem erro
103
+ # @return [Google::Cloud::Bigquery::Data] resultado do SQL
104
+ def sql(cmd, res = [])
105
+ @sqr = job?(cmd) ? res : job.data
106
+ end
107
+
108
+ # cria Data Manipulation Language (DML) job bigquery
109
+ #
110
+ # @param cmd (see sql)
111
+ # @return [Integer] numero linhas afetadas
112
+ def dml(cmd)
113
+ job?(cmd) ? 0 : job.num_dml_affected_rows
114
+ end
115
+ end
116
+ end
@@ -0,0 +1,78 @@
1
+ # frozen_string_literal: true
2
+
3
+ # @author Hernani Rodrigues Vaz
4
+ module Cns
5
+ # (see Bigquery)
6
+ class Bigquery
7
+ private
8
+
9
+ # @return [Etherscan] API blockchain ETH
10
+ def apies
11
+ @apies ||= Etherscan.new(
12
+ {
13
+ wb: sql("select * from #{BD}.walletEth order by 2"),
14
+ nt: sql("select itx,iax from #{BD}.ethtx"),
15
+ nk: sql("select itx,iax from #{BD}.ethkx")
16
+ },
17
+ ops
18
+ )
19
+ end
20
+
21
+ # @return [Greymass] API blockchain EOS
22
+ def apigm
23
+ @apigm ||= Greymass.new(
24
+ {
25
+ wb: sql("select * from #{BD}.walletEos order by 2"),
26
+ nt: sql("select itx,iax from #{BD}.eostx")
27
+ },
28
+ ops
29
+ )
30
+ end
31
+
32
+ # @return [Kraken] API exchange kraken
33
+ def apius
34
+ @apius ||= Kraken.new(
35
+ {
36
+ sl: sql("select sum(btc) xxbt,sum(eth) xeth,sum(eos) eos,sum(eur) zeur from #{BD}.ussl")[0],
37
+ nt: sql("select * from #{BD}.ustx order by time,txid"),
38
+ nl: sql("select * from #{BD}.uslx order by time,txid")
39
+ },
40
+ ops
41
+ )
42
+ end
43
+
44
+ # @return [Bitcoinde] API exchange bitcoinde
45
+ def apide
46
+ @apide ||= Bitcoinde.new(
47
+ {
48
+ sl: sql("select sum(btc) btc from #{BD}.desl")[0],
49
+ nt: sql("select * from #{BD}.detx order by time,txid"),
50
+ nl: sql("select * from #{BD}.delx order by time,txid")
51
+ },
52
+ ops
53
+ )
54
+ end
55
+
56
+ # @return [Paymium] API exchange paymium
57
+ def apifr
58
+ @apifr ||= Paymium.new(
59
+ {
60
+ sl: sql("select sum(btc) btc,sum(eur) eur from #{BD}.frsl")[0],
61
+ nl: sql("select * from #{BD}.frlx order by time,txid")
62
+ },
63
+ ops
64
+ )
65
+ end
66
+
67
+ # @return [TheRock] API exchange therock
68
+ def apimt
69
+ @apimt ||= TheRock.new(
70
+ {
71
+ sl: sql("select sum(btc) btc,sum(eur) eur from #{BD}.mtsl")[0],
72
+ nl: sql("select * from #{BD}.mtlx order by time,txid")
73
+ },
74
+ ops
75
+ )
76
+ end
77
+ end
78
+ end
@@ -0,0 +1,134 @@
1
+ # frozen_string_literal: true
2
+
3
+ require('google/cloud/bigquery')
4
+ require('bigdecimal/util')
5
+
6
+ # @author Hernani Rodrigues Vaz
7
+ module Cns
8
+ # (see Bigquery)
9
+ class Bigquery
10
+ private
11
+
12
+ # @return [String] comando insert SQL formatado etht (norml)
13
+ def etht_ins
14
+ "insert #{BD}.etht(blocknumber,timestamp,txhash,nonce,blockhash,transactionindex,axfrom,axto,iax," \
15
+ 'value,gas,gasprice,gasused,iserror,txreceipt_status,input,contractaddress,dias' \
16
+ ") VALUES#{apies.novtx.map { |e| etht_val1(e) }.join(',')}"
17
+ end
18
+
19
+ # @example (see Apibc#norml_es)
20
+ # @param [Hash] htx transacao norml etherscan
21
+ # @return [String] valores formatados etht (norml parte1)
22
+ def etht_val1(htx)
23
+ "(#{Integer(htx[:blockNumber])}," \
24
+ "#{Integer(htx[:timeStamp])}," \
25
+ "'#{htx[:hash]}'," \
26
+ "#{Integer(htx[:nonce])}," \
27
+ "'#{htx[:blockHash]}'," \
28
+ "#{Integer(htx[:transactionIndex])}," \
29
+ "'#{htx[:from]}'," \
30
+ "'#{htx[:to]}'," \
31
+ "'#{htx[:iax]}'," \
32
+ "#{etht_val2(htx)}"
33
+ end
34
+
35
+ # @param (see etht_val1)
36
+ # @return [String] valores formatados etht (norml parte2)
37
+ def etht_val2(htx)
38
+ "cast('#{htx[:value]}' as numeric)," \
39
+ "cast('#{htx[:gas]}' as numeric)," \
40
+ "cast('#{htx[:gasPrice]}' as numeric)," \
41
+ "cast('#{htx[:gasUsed]}' as numeric)," \
42
+ "#{Integer(htx[:isError])}," \
43
+ "#{htx[:txreceipt_status].length.zero? ? 'null' : htx[:txreceipt_status]}," \
44
+ "#{etht_val3(htx)}"
45
+ end
46
+
47
+ # @param (see etht_val1)
48
+ # @return [String] valores formatados etht (norml parte3)
49
+ def etht_val3(htx)
50
+ "#{htx[:input].length.zero? ? 'null' : "'#{htx[:input]}'"}," \
51
+ "#{htx[:contractAddress].length.zero? ? 'null' : "'#{htx[:contractAddress]}'"}," \
52
+ "#{Integer(ops[:h][htx[:blockNumber]] || 0)})"
53
+ end
54
+
55
+ # @return [String] comando insert SQL formatado ethk (token)
56
+ def ethk_ins
57
+ "insert #{BD}.ethk(blocknumber,timestamp,txhash,nonce,blockhash,transactionindex,axfrom,axto,iax," \
58
+ 'value,tokenname,tokensymbol,tokendecimal,gas,gasprice,gasused,input,contractaddress,dias' \
59
+ ") VALUES#{apies.novkx.map { |e| ethk_val1(e) }.join(',')}"
60
+ end
61
+
62
+ # @example (see Apibc#token_es)
63
+ # @param [Hash] hkx token event etherscan
64
+ # @return [String] valores formatados ethk (token parte1)
65
+ def ethk_val1(hkx)
66
+ "(#{Integer(hkx[:blockNumber])}," \
67
+ "#{Integer(hkx[:timeStamp])}," \
68
+ "'#{hkx[:hash]}'," \
69
+ "#{Integer(hkx[:nonce])}," \
70
+ "'#{hkx[:blockHash]}'," \
71
+ "#{Integer(hkx[:transactionIndex])}," \
72
+ "'#{hkx[:from]}'," \
73
+ "'#{hkx[:to]}'," \
74
+ "'#{hkx[:iax]}'," \
75
+ "#{ethk_val2(hkx)}"
76
+ end
77
+
78
+ # @param (see ethk_val1)
79
+ # @return [String] valores formatados ethk (token parte2)
80
+ def ethk_val2(hkx)
81
+ "cast('#{hkx[:value]}' as numeric)," \
82
+ "'#{hkx[:tokenName]}'," \
83
+ "'#{hkx[:tokenSymbol]}'," \
84
+ "#{Integer(hkx[:tokenDecimal])}," \
85
+ "cast('#{hkx[:gas]}' as numeric)," \
86
+ "cast('#{hkx[:gasPrice]}' as numeric)," \
87
+ "cast('#{hkx[:gasUsed]}' as numeric)," \
88
+ "#{ethk_val3(hkx)}"
89
+ end
90
+
91
+ # @param (see ethk_val1)
92
+ # @return [String] valores formatados ethk (token parte3)
93
+ def ethk_val3(hkx)
94
+ "#{hkx[:input].length.zero? ? 'null' : "'#{hkx[:input]}'"}," \
95
+ "#{hkx[:contractAddress].length.zero? ? 'null' : "'#{hkx[:contractAddress]}'"}," \
96
+ "#{Integer(ops[:h][hkx[:blockNumber]] || 0)})"
97
+ end
98
+
99
+ # @return [String] comando insert SQL formatado eos
100
+ def eost_ins
101
+ "insert #{BD}.eos(gseq,aseq,bnum,time,contract,action,acfrom,acto,iax,amount,moeda,memo,dias" \
102
+ ") VALUES#{apigm.novax.map { |e| eost_val1(e) }.join(',')}"
103
+ end
104
+
105
+ # @example (see Apibc#ledger_gm)
106
+ # @param [Hash] hlx ledger greymass
107
+ # @return [String] valores formatados para insert eos (parte1)
108
+ def eost_val1(hlx)
109
+ a = hlx[:action_trace][:act]
110
+ "(#{hlx[:global_action_seq]}," \
111
+ "#{hlx[:account_action_seq]}," \
112
+ "#{hlx[:block_num]}," \
113
+ "DATETIME(TIMESTAMP('#{hlx[:block_time]}'))," \
114
+ "'#{a[:account]}'," \
115
+ "'#{a[:name]}'," \
116
+ "#{eost_val2(hlx, a)}"
117
+ end
118
+
119
+ # @param (see eost_val1)
120
+ # @param [Hash] act dados da acao
121
+ # @return [String] valores formatados para insert eos (parte2)
122
+ def eost_val2(hlx, act)
123
+ d = act[:data]
124
+ q = d[:quantity].to_s
125
+ s = d[:memo].inspect
126
+ "'#{d[:from]}'," \
127
+ "'#{d[:to]}'," \
128
+ "'#{hlx[:iax]}'," \
129
+ "#{q.to_d},'#{q[/[[:upper:]]+/]}'," \
130
+ "nullif('#{s.gsub(/['"]/, '')}','nil')," \
131
+ "#{ops[:h][String(hlx[:itx])] || 0})"
132
+ end
133
+ end
134
+ end
@@ -0,0 +1,153 @@
1
+ # frozen_string_literal: true
2
+
3
+ # @author Hernani Rodrigues Vaz
4
+ module Cns
5
+ # classe para processar bigquery
6
+ class Bigquery
7
+ private
8
+
9
+ # @return [String] comando insert SQL formatado det (trades)
10
+ def det_ins
11
+ "insert #{BD}.det(txid,time,tp,user,btc,eur,dtc,dias) VALUES#{apide.trades.map { |h| det_val1(h) }.join(',')}"
12
+ end
13
+
14
+ # @example (see Apice#trades_de)
15
+ # @param [Hash] htx trade bitcoinde
16
+ # @return [String] valores formatados det (trades parte1)
17
+ def det_val1(htx)
18
+ "('#{htx[:trade_id]}'," \
19
+ "DATETIME(TIMESTAMP('#{htx[:successfully_finished_at]}'))," \
20
+ "'#{htx[:type]}'," \
21
+ "'#{htx[:trading_partner_information][:username]}'," \
22
+ "#{det_val2(htx)}"
23
+ end
24
+
25
+ # @param (see det_val1)
26
+ # @return [String] valores formatados det (trades parte2)
27
+ def det_val2(htx)
28
+ 'cast(' \
29
+ "#{htx[:type] == 'buy' ? htx[:amount_currency_to_trade_after_fee] : "-#{htx[:amount_currency_to_trade]}"}" \
30
+ ' as numeric),' \
31
+ "cast(#{htx[:volume_currency_to_pay_after_fee]} as numeric)," \
32
+ "DATETIME(TIMESTAMP('#{htx[:trade_marked_as_paid_at]}'))," \
33
+ "#{Integer(ops[:h][htx[:trade_id]] || 0)})"
34
+ end
35
+
36
+ # @return [String] comando insert SQL formatado del (ledger)
37
+ def del_ins
38
+ "insert #{BD}.del(txid,time,tp,add,moe,qt,fee) VALUES#{apide.ledger.map { |h| del_val(h) }.join(',')}"
39
+ end
40
+
41
+ # @example (see Apice#deposits_de)
42
+ # @example (see Apice#withdrawals_de)
43
+ # @param [Hash] hlx ledger (deposits + withdrawals) bitcoinde
44
+ # @return [String] valores formatados del (ledger)
45
+ def del_val(hlx)
46
+ "(#{hlx[:txid]}," \
47
+ "DATETIME(TIMESTAMP('#{hlx[:time].iso8601}'))," \
48
+ "'#{hlx[:tp]}'," \
49
+ "'#{hlx[:add]}'," \
50
+ "'#{hlx[:moe]}'," \
51
+ "cast(#{hlx[:tp] =='withdrawal' ? '-' : ''}#{hlx[:qt]} as numeric)," \
52
+ "cast(#{hlx[:fee]} as numeric))"
53
+ end
54
+
55
+ # @return [String] comando insert SQL formatado ust (trades)
56
+ def ust_ins
57
+ "insert #{BD}.ust(txid,ordertxid,pair,time,type,ordertype,price,cost,fee,vol,margin,misc,ledgers,dias) " \
58
+ "VALUES#{apius.trades.map { |k, v| ust_val1(k, v) }.join(',')}"
59
+ end
60
+
61
+ # @example (see Apice#trades_us)
62
+ # @param [String] idx identificador transacao
63
+ # @param [Hash] htx trade kraken
64
+ # @return [String] valores formatados ust (trades parte1)
65
+ def ust_val1(idx, htx)
66
+ "('#{idx}'," \
67
+ "'#{htx[:ordertxid]}'," \
68
+ "'#{htx[:pair]}'," \
69
+ "PARSE_DATETIME('%s', '#{String(htx[:time].round)}')," \
70
+ "'#{htx[:type]}'," \
71
+ "'#{htx[:ordertype]}'," \
72
+ "cast(#{htx[:price]} as numeric)," \
73
+ "cast(#{htx[:cost]} as numeric)," \
74
+ "cast(#{htx[:fee]} as numeric)," \
75
+ "#{ust_val2(idx, htx)}"
76
+ end
77
+
78
+ # @param (see ust_val1)
79
+ # @return [String] valores formatados ust (trades parte2)
80
+ def ust_val2(idx, htx)
81
+ "cast(#{htx[:vol]} as numeric)," \
82
+ "cast(#{htx[:margin]} as numeric)," \
83
+ "#{htx[:misc].to_s.empty? ? 'null' : "'#{htx[:misc]}'"}," \
84
+ "'#{apius.ledger.select { |_, v| v[:refid] == idx }.keys.join(',') || ''}'," \
85
+ "#{Integer(ops[:h][idx] || 0)})"
86
+ end
87
+
88
+ # @return [String] comando insert SQL formatado usl (ledger)
89
+ def usl_ins
90
+ "insert #{BD}.usl(txid,refid,time,type,aclass,asset,amount,fee) " \
91
+ "VALUES#{apius.ledger.map { |k, v| usl_val(k, v) }.join(',')}"
92
+ end
93
+
94
+ # @example (see Apice#ledger_us)
95
+ # @param idx (see ust_val1)
96
+ # @param [Hash] hlx ledger kraken
97
+ # @return [String] valores formatados usl (ledger)
98
+ def usl_val(idx, hlx)
99
+ "('#{idx}'," \
100
+ "'#{hlx[:refid]}'," \
101
+ "PARSE_DATETIME('%s', '#{String(hlx[:time].round)}')," \
102
+ "'#{hlx[:type]}'," \
103
+ "#{hlx[:aclass].to_s.empty? ? 'null' : "'#{hlx[:aclass]}'"}," \
104
+ "'#{hlx[:asset]}'," \
105
+ "cast(#{hlx[:amount]} as numeric)," \
106
+ "cast(#{hlx[:fee]} as numeric))"
107
+ end
108
+
109
+ # @return [String] comando insert SQL formatado fr (ledger)
110
+ def frl_ins
111
+ "insert #{BD}.fr(uuid,tipo,valor,moe,time,dias) VALUES#{apifr.ledger.map { |h| frl_val(h) }.join(',')}"
112
+ end
113
+
114
+ # @example (see Apice#ledger_fr)
115
+ # @param [Hash] hlx ledger paymium
116
+ # @return [String] valores formatados frl (ledger)
117
+ def frl_val(hlx)
118
+ "('#{hlx[:uuid]}'," \
119
+ "'#{hlx[:name]}'," \
120
+ "cast(#{hlx[:amount]} as numeric)," \
121
+ "'#{hlx[:currency]}'," \
122
+ "PARSE_DATETIME('%s', '#{hlx[:created_at_int]}')," \
123
+ "#{Integer(ops[:h][hlx[:uuid]] || 0)})"
124
+ end
125
+
126
+ # @return [String] comando insert SQL formatado fr (ledger)
127
+ def mtl_ins
128
+ "insert #{BD}.mt(id,time,type,valor,moe,pair,note,trade_id,dias) " \
129
+ "VALUES#{apimt.ledger.map { |h| mtl_val1(h) }.join(',')}"
130
+ end
131
+
132
+ # @example (see Apice#ledger_mt)
133
+ # @param [Hash] hlx ledger therock
134
+ # @return [String] valores formatados mtl (ledger parte1)
135
+ def mtl_val1(hlx)
136
+ "(#{hlx[:id]}," \
137
+ "DATETIME(TIMESTAMP('#{hlx[:date]}'))," \
138
+ "'#{hlx[:type]}'," \
139
+ "cast(#{hlx[:price]} as numeric)," \
140
+ "'#{hlx[:currency]}'," \
141
+ "#{hlx[:fund_id].to_s.empty? ? 'null' : "'#{hlx[:fund_id]}'"}," \
142
+ "#{mtl_val2(hlx)}"
143
+ end
144
+
145
+ # @param (see mtl_val1)
146
+ # @return [String] valores formatados mtl (ledger parte2)
147
+ def mtl_val2(hlx)
148
+ "#{hlx[:note].to_s.empty? ? 'null' : "'#{hlx[:note]}'"}," \
149
+ "#{hlx[:trade_id].to_s.empty? ? 'null' : (hlx[:trade_id]).to_s}," \
150
+ "#{Integer(ops[:h][String(hlx[:id])] || 0)})"
151
+ end
152
+ end
153
+ end