arquivo 0.2.4 → 0.2.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 37306fc6850e0daf5c5ad74f3c65dff487323afe2060ebfd9a0e4d042755af1e
-  data.tar.gz: 76d29adf388c92e326cf49244da298d9627b79b0b1514024b0f68e3478359c4a
+  metadata.gz: 6d030623f8fd4780d6207d8ad206a06f9b7eb35e1cf2e484d10697bfe5ec1aa4
+  data.tar.gz: 5b3c49c43d6b607290db7e0582e0b84a7a87b1d5c6bece2cf2b697bab60e9ce9
 SHA512:
-  metadata.gz: 10c0761fa284205166169b931ebd5bff2ab412740b25934b348a15f026251220de6fad7de10e803c6107a8fcc5cba0cba40e93a968cda0a77b5f4c1b41ea7cac
-  data.tar.gz: 2d25a6cba97f28a7289bfca7610011c42f29700a871ecb6ebe15fc45fc4d9de0c8717ca2310be91f295ef7f74bd37596609121c955c8fb8348348d9ac607cf14
+  metadata.gz: 5c6046db2f720435c3548a9df7cf96d2846d572c3fd5ca259cd3dcbc233d99e7c97414594390d211426b60eea268e3d20d09acb1aa69359c4be7f4990387b26a
+  data.tar.gz: 89c01519dcedbfa9cda6b4fe05887156ab93a31358e15eccde40796531655f4ecb6d23fd7b6127d7203bf52620e25c7a96631d6c50415d3def7ad0522804b1a1

data/Gemfile.lock

@@ -1,9 +1,11 @@
 PATH
   remote: .
   specs:
-    arquivo (0.2.4)
+    arquivo (0.2.9)
       fastimage (~> 2.1)
       google-api-client (~> 0.34)
+      google-cloud-bigquery
+      i18n
       pdf-reader (~> 2.3)
       thor (~> 0.1)
       yard (~> 0.9)
@@ -15,12 +17,13 @@ GEM
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
     afm (0.2.2)
+    concurrent-ruby (1.1.6)
     declarative (0.0.10)
     declarative-option (0.1.0)
-    faraday (0.17.1)
+    faraday (1.0.1)
       multipart-post (>= 1.2, < 3)
     fastimage (2.1.7)
-    google-api-client (0.36.3)
+    google-api-client (0.40.0)
       addressable (~> 2.5, >= 2.5.1)
       googleauth (~> 0.9)
       httpclient (>= 2.8.1, < 3.0)
@@ -28,44 +31,58 @@ GEM
       representable (~> 3.0)
       retriable (>= 2.0, < 4.0)
       signet (~> 0.12)
-    googleauth (0.10.0)
-      faraday (~> 0.12)
+    google-cloud-bigquery (1.21.1)
+      concurrent-ruby (~> 1.0)
+      google-api-client (~> 0.33)
+      google-cloud-core (~> 1.2)
+      googleauth (~> 0.9)
+      mini_mime (~> 1.0)
+    google-cloud-core (1.5.0)
+      google-cloud-env (~> 1.0)
+      google-cloud-errors (~> 1.0)
+    google-cloud-env (1.3.2)
+      faraday (>= 0.17.3, < 2.0)
+    google-cloud-errors (1.0.1)
+    googleauth (0.12.0)
+      faraday (>= 0.17.3, < 2.0)
       jwt (>= 1.4, < 3.0)
       memoist (~> 0.16)
       multi_json (~> 1.11)
       os (>= 0.9, < 2.0)
-      signet (~> 0.12)
+      signet (~> 0.14)
     hashery (2.1.2)
     httpclient (2.8.3)
+    i18n (1.8.2)
+      concurrent-ruby (~> 1.0)
     jwt (2.2.1)
     memoist (0.16.2)
     mini_mime (1.0.2)
     multi_json (1.14.1)
     multipart-post (2.1.1)
-    os (1.0.1)
+    os (1.1.0)
     pdf-reader (2.4.0)
       Ascii85 (~> 1.0.0)
       afm (~> 0.2.1)
       hashery (~> 2.0)
       ruby-rc4
       ttfunk
-    public_suffix (4.0.2)
-    rake (10.5.0)
+    public_suffix (4.0.5)
+    rake (13.0.1)
     representable (3.0.4)
       declarative (< 0.1.0)
       declarative-option (< 0.2.0)
       uber (< 0.2.0)
     retriable (3.1.2)
     ruby-rc4 (0.1.5)
-    signet (0.12.0)
+    signet (0.14.0)
       addressable (~> 2.3)
-      faraday (~> 0.9)
+      faraday (>= 0.17.3, < 2.0)
       jwt (>= 1.5, < 3.0)
       multi_json (~> 1.10)
     thor (0.20.3)
-    ttfunk (1.5.1)
+    ttfunk (1.6.2.1)
     uber (0.1.0)
-    yard (0.9.20)
+    yard (0.9.25)
 
 PLATFORMS
   ruby
@@ -73,7 +90,7 @@ PLATFORMS
 DEPENDENCIES
   arquivo!
   bundler (~> 1.17)
-  rake (~> 10.0)
+  rake
 
 BUNDLED WITH
-   1.17.2
+   1.17.3

data/README.md

@@ -1,6 +1,6 @@
 # Arquivo
 
-Processa documentos do condominio ph1341c118 para arquivo. Pode tambem segmentar PDFs e MINUTAS. Tendo os documentos em pastas separadas, pode ainda criar arquivos apropriados.
+Processa documentos do condominio ph1341c118 para arquivo. Pode tambem segmentar PDFs e MINUTAS. Tendo os documentos em pastas separadas, pode ainda criar arquivos apropriados. Parmite ainda processar dados do condominio ph1341c118 no bigquery para arquivo historico.
 
 ## Installation
 

data/arquivo.gemspec

@@ -34,10 +34,12 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_development_dependency 'bundler', '~> 1.17'
-  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rake'
 
   spec.add_dependency 'fastimage', '~> 2.1'
   spec.add_dependency 'google-api-client', '~> 0.34'
+  spec.add_dependency 'google-cloud-bigquery'
+  spec.add_dependency 'i18n'
   spec.add_dependency 'pdf-reader', '~> 2.3'
   spec.add_dependency 'thor', '~> 0.1'
   spec.add_dependency 'yard', '~> 0.9'

data/lib/arquivo.rb

@@ -2,6 +2,8 @@
 
 require 'thor'
 require 'arquivo/version'
+require 'arquivo/sheets'
+require 'arquivo/bigquery'
 require 'arquivo/dir'
 require 'arquivo/pdf'
 require 'arquivo/jpg'
@@ -64,6 +66,12 @@ module Arquivo
       C118pdf.new(dashboard, options).faz_dashboard
     end
 
+    desc 'big', 'processa dados bigquery c118'
+    # processa bigquery c118
+    def big
+      C118bigquery.new.processa_big
+    end
+
     desc 'dir PASTA', 'processa faturas/recibos/extratos/minutas ' \
                       'e cria arquivos c118'
     option :fuzz, type: :numeric, default: 29,

data/lib/arquivo/bigquery.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require 'google/cloud/bigquery'
+
+module Arquivo
+  # permite arquivar dados c118 no bigquery
+  class C118bigquery < C118sheets
+    # @return [Google::Cloud::Bigquery] API bigquery c118
+    attr_reader :big
+
+    # @return [C118bigquery] acesso bigquery c118
+    def initialize
+      # inicializar API sheets com ID cliente & credenciais
+      sheets_credentials
+
+      # This uses Application Default Credentials to authenticate.
+      # @see https://cloud.google.com/bigquery/docs/authentication/getting-started
+      @big = Google::Cloud::Bigquery.new
+    end
+
+    # obtem dados da folha c118 & processa no bigquery
+    def processa_big
+      # folha c118-contas
+      s = '1PbiMrtTtqGztZMhe3AiJbDS6NQE9o3hXebnQEFdt954'
+      a = folhas.get_spreadsheet_values(s, 'cbd!AJ2')
+                .values.flatten[0]
+      i = folhas.get_spreadsheet_values(s, 'bal!R2:R')
+                .values.flatten.join(',')
+      puts 'processamento bigquery feito para ano ' + a +
+           ": [del_bal,del_hise,ins_bal,ins_hise] #{sql_big(a, i)}"
+    end
+
+    # executa comandos DML (Data Manipulation Language) no bigquery
+    #
+    # @return [Array<Integer>] numero linhas afetadas pelos DMLs
+    def sql_big(ano, lst)
+      [dml('delete FROM arquivo.bal  WHERE ano=' + ano),
+       dml('delete FROM arquivo.hise WHERE ano=' + ano),
+       dml('INSERT arquivo.bal (data,entidade,documento,descricao,valor,tag,' \
+           'dr,banco,conta,ano,id4,dref,daa,paga,desb) VALUES' + lst),
+       dml('INSERT arquivo.hise(ano,dr,tag,descricao,valor) ' \
+           'select * from arquivo.vhe where ano=' + ano)]
+    end
+
+    # @return [Integer] numero linhas afetadas pelo DML
+    def dml(sql)
+      job = big.query_job(sql)
+      job.wait_until_done!
+      puts job.error if job.failed?
+      job.num_dml_affected_rows
+    end
+  end
+end

data/lib/arquivo/dir.rb

@@ -1,9 +1,5 @@
 # frozen_string_literal: true
 
-require 'google/apis/sheets_v4'
-require 'googleauth'
-require 'googleauth/stores/file_token_store'
-
 require 'arquivo/noise'
 
 module Arquivo
@@ -73,49 +69,20 @@ module Arquivo
       @item = nil
     end
 
-    # @!group dados online
-    # @return [Hash] dados oficiais para reclassificacao de faturas e recibos
+    # @!group dados folhas-calculo c118
+    # @return [Hash] dados oficiais para classificacao de faturas e recibos
     def obtem_dados
       @dados = {}
       # somente faturas e recibos necessitam reclassificacao
       return unless %i[fft frc].include?(contem)
 
-      # sheet c118-contas
-      dg = '1PbiMrtTtqGztZMhe3AiJbDS6NQE9o3hXebnQEFdt954'
-      @dados = c118_sheets.get_spreadsheet_values(dg, contem.to_s + '!A2:E')
-                          .values.group_by { |k| k[0][/\w+/] }
+      # folha c118-contas
+      s = '1PbiMrtTtqGztZMhe3AiJbDS6NQE9o3hXebnQEFdt954'
+      @dados = C118sheets.new.folhas
+                         .get_spreadsheet_values(s, contem.to_s + '!A2:E')
+                         .values.group_by { |k| k[0][/\w+/] }
     rescue StandardError
       @dados = {}
     end
-
-    # assegura credenciais validas, obtidas dum ficheiro de credencias
-    #
-    # @return [Google::Apis::SheetsV4::SheetsService] c118 sheets_v4
-    def c118_sheets
-      p = '/home/c118/c118-'
-      # file obtido console.cloud.google.com/apis OAuth 2.0 client IDs
-      i = Google::Auth::ClientId.from_file("#{p}credentials.json")
-      s = Google::Apis::SheetsV4::AUTH_SPREADSHEETS_READONLY
-      # file criado aquando new_credentials is executed
-      f = Google::Auth::Stores::FileTokenStore.new(file: "#{p}token.yaml")
-      z = Google::Auth::UserAuthorizer.new(i, s, f)
-
-      sheets = Google::Apis::SheetsV4::SheetsService.new
-      sheets.client_options.application_name = 'c118-arquivo'
-      sheets.authorization = z.get_credentials('default') ||
-                             new_credentials(z, 'urn:ietf:wg:oauth:2.0:oob')
-      sheets
-    end
-
-    # inicializar OAuth2 authorization abrindo URL e copiando novo codigo
-    #
-    # @return [Google::Auth::UserAuthorizer] OAuth2 credentials
-    def new_credentials(aut, oob)
-      puts 'Open URL and copy code after authorization, in <codigo-aqui>',
-           aut.get_authorization_url(base_url: oob)
-      aut.get_and_store_credentials_from_code(user_id: 'default',
-                                              code: '<codigo-aqui>',
-                                              base_url: oob)
-    end
   end
 end

data/lib/arquivo/sheets.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require 'google/apis/sheets_v4'
+require 'googleauth'
+require 'googleauth/stores/file_token_store'
+
+module Arquivo
+  # acede a folhas-calculo c118
+  class C118sheets
+    # @return (see #sheets_credentials)
+    attr_reader :folhas
+
+    # @return [C118sheets] acesso folhas-calculo c118
+    def initialize
+      sheets_credentials
+    end
+
+    # inicializar API sheets com ID cliente & credenciais
+    #
+    # @return [Google::Apis::SheetsV4::SheetsService] API folhas-calculo c118
+    def sheets_credentials
+      l = '/home/c118/.sheets.'
+      # file obtido console.cloud.google.com/apis OAuth 2.0 client IDs
+      i = Google::Auth::ClientId.from_file(l + 'json')
+      s = Google::Apis::SheetsV4::AUTH_SPREADSHEETS_READONLY
+      # file criado aquando new_credentials is executed
+      f = Google::Auth::Stores::FileTokenStore.new(file: l + 'yaml')
+      z = Google::Auth::UserAuthorizer.new(i, s, f)
+      @folhas = Google::Apis::SheetsV4::SheetsService.new
+      @folhas.client_options.application_name = 'c118-arquivo'
+      @folhas.authorization = z.get_credentials('default') ||
+                              new_credentials(z, 'urn:ietf:wg:oauth:2.0:oob')
+    end
+
+    # inicializar OAuth2 authorization abrindo URL e copiando novo codigo
+    #
+    # @return [Google::Auth::UserAuthorizer] OAuth2 credentials
+    def new_credentials(aut, oob)
+      puts 'Open URL and copy code after authorization, in <codigo-aqui>',
+           aut.get_authorization_url(base_url: oob)
+      aut.get_and_store_credentials_from_code(user_id: 'default',
+                                              code: '<codigo-aqui>',
+                                              base_url: oob)
+    end
+  end
+end

data/lib/arquivo/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Arquivo
-  VERSION = '0.2.4'
+  VERSION = '0.2.9'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: arquivo
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.2.9
 platform: ruby
 authors:
 - Hernâni Rodrigues Vaz
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-01-02 00:00:00.000000000 Z
+date: 2020-06-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -28,16 +28,16 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: fastimage
   requirement: !ruby/object:Gem::Requirement
@@ -66,6 +66,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.34'
+- !ruby/object:Gem::Dependency
+  name: google-cloud-bigquery
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: i18n
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: pdf-reader
   requirement: !ruby/object:Gem::Requirement
@@ -129,12 +157,14 @@ files:
 - bin/setup
 - exe/arquivo
 - lib/arquivo.rb
+- lib/arquivo/bigquery.rb
 - lib/arquivo/dir.rb
 - lib/arquivo/extrato.rb
 - lib/arquivo/jpg.rb
 - lib/arquivo/mp3.rb
 - lib/arquivo/noise.rb
 - lib/arquivo/pdf.rb
+- lib/arquivo/sheets.rb
 - lib/arquivo/version.rb
 homepage: https://github.com/ph1341c118/arquivo
 licenses:
@@ -157,7 +187,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.0.8
 signing_key: 
 specification_version: 4
 summary: Processa documentos do condominio ph1341c118 para arquivo.