ba_upload 0.3.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 15804867b016b8c972bbb18385a2d1caec4a9729776ea2d1877dcddaeb73b15d
-  data.tar.gz: 930ef0bf4b3817c43616873144a43e46e482d5baa1efc5f5db63038dd44fd6e0
+  metadata.gz: a97b2695bf77d8a8693710f458561034c58ae2a43e737863f283e65301e8aa65
+  data.tar.gz: 8e3da29f1068f76f6f5320110ab68a99a0d3240e1b8d4ea589bb82771b02a676
 SHA512:
-  metadata.gz: 4b906aa0b7c5860142c9c2d01f1f6955b8ccf8217448fd9164e293081f697ff9d43de2411989ff266a4729eb79a087a73fd56124e79159ec146fb29bd72fe6a2
-  data.tar.gz: fa9aaa0351fdf5758cb2a770e2b0105c6f599bae516f416ffa1ef25de58727f20b0180574086079474050042b253789dded4d58ce78e0edb55f81c146107e2cf
+  metadata.gz: 53b3dcee1f488d9b4758a03d9263bc6787a7b9b74680ef999d1ba1c01eb9625c3d2ea9210a2681a3804dd8727b43c9f745374e0f94567694a4470d9214c3d88a
+  data.tar.gz: 262b6794309cf833bc0d2a6301c0a757d036a3ac917a54c2eb9cea1301c02456359fa4a886e33b028ca8a264ac64dfeaadee8e3b9c67a2f30b057645e2b1c587

data/README.md

@@ -33,7 +33,8 @@ require 'ba_upload'
 connection = BaUpload.open_connection(file_path: 'config/Zertifikat-1XXXX.p12', passphrase: 'YOURPASSPHRASE')
 
 # Upload a xml-file
-connection.upload(file: File.open('/opt/vam-transfer/data/DSP000132700_2016-08-08_05-00-09.xml'))
+file_path = "/opt/vam-transfer/data/DSP000132700_2016-08-08_05-00-09.xml"
+connection.upload(file: file_path))
 
 # later cronjob to download all error files
 
@@ -50,11 +51,23 @@ end
 ```ruby
 #!/usr/bin/env ruby
 require 'ba_upload'
-connection = BaUpload.open_connection(file_path: 'config/Zertifikat-1XXXX.p12', passphrase: 'YOURPASSPHRASE')
+BaUpload.open_connection(file_path: 'config/Zertifikat-1XXXX.p12', passphrase: 'YOURPASSPHRASE')
 connection.upload(file: File.open(ARGV[0]))
 ```
 
-Save to a file and just run it with the xml file as argument
+Save to a file and just run it with the xml file as argument. It's important, because the file upload of BA validates the original file name.
+
+
+### Memory safety
+
+because we are using Mechanize under the hood, it's a good idea to always close sockets, (with connection.shutdown), or use the open helper:
+
+```ruby
+BaUpload.open(file_path: 'config/Zertifikat-1XXXX.p12', passphrase:) do |c|
+    c.upload(file: file_path)
+    c.error_files....
+end
+```
 
 
 ### Downloading "misc" files
@@ -70,6 +83,19 @@ connection.misc.each do |link|
 end
 ```
 
+### Downloading 'Statistiken' xlsx reports
+
+Download XLSX reports (validation errors and "Stellenübersicht") from BA; You might use another Gem, like roo, axlsx etc. to parse those files if necessary.
+
+```ruby
+connection.statistics.each do |link|
+  link.tempfile
+
+  # or:
+  link.read
+end
+```
+
 ### Usage with multiple client certificats
 
 Since September 2022, users with multiple client certificats issued to the same email address need to provide their respective partner ID when using the API.
@@ -87,6 +113,190 @@ connection.misc(partner_id: 'P000XXXXXX')
 
 ```
 
+## Appendix: Berufe
+
+Sooner or later, you have to provide a TitleCode = Vocation "Beruf" for each job. To fetch and process the Berufe, we create a ActiveRecord Model in our database:
+
+Here an example of a implementation at Empfehlungsbund. You can also use our [search mask](https://login.empfehlungsbund.de/arbeitsagentur) to search for occupations.
+
+We put the "help" / "validation" messages, that we found in the appropriate scopes, too, as "Ausbildungen" and "Duale Studiengänge" need different types of professions.
+
+<details>
+<summary>ActiveRecord Model for Ba::Profession</summary>
+
+```ruby 
+# migration:
+create_table :ba_professions do |t|
+ t.string "bkz"
+ t.string "typ"
+ t.string "lbkgruppe"
+ t.string "hochschulberuf"
+ t.string "kuenstler"
+ t.string "bezeichnung_nl"
+ t.string "bezeichnung_nk"
+ t.string "suchname_nl"
+ t.datetime "created_at"
+ t.datetime "updated_at"
+ t.integer "ebene"
+ t.integer "qualifikationsniveau"
+ t.datetime "deleted_on"
+end
+
+class Ba::Profession < ApplicationRecord
+  has_many :jobs
+
+  scope :undeleted, -> { where 'deleted_on is null' }
+  scope :berufe, -> { where typ: 'B' }
+  scope :ausbildungen, -> { where typ: 'A' }
+  scope :sorted, -> { order(Arel.sql('deleted_on is not null, bezeichnung_nl')) }
+  # Bei Auswahl von „Ausbildung“ (EducationType=0) sind die Berufe mit dem
+  # Qualifikationsniveau 2 zulässig. Zusätzlich sind hier alle Berufe folgender
+  # berufskundlicher Gruppen erlaubt: [...]
+  scope :reine_ausbildungen, -> {
+    where(qualifikationsniveau: 2).or(
+      where(lbkgruppe: [1150, 3110, 5130])
+    ).ausbildungen
+  }
+  # Wird ein Stellenangebot vom Typ „Duales Studium“ (EducationType=1) übermittelt, sind der
+  # Studiengang und der ggf. vorhandene Ausbildungsberuf getrennt anzugeben. Als
+  # Studiengang (Course) sind Berufe mit ausschließlich dem Qualifikationsniveau 4 zulässig.
+  # Diese Berufe entstammen alle der berufskundlichen Gruppe 3120 („A Grundständige
+  # Studienfächer/-gänge“). Der als Ausbildung (TitleCode) angegebene Beruf darf
+  # dementsprechend nicht ausschließlich das Qualifikationsniveau 4 haben.
+  scope :duale_studiengaenge, -> { ausbildungen.where ebene: 3, qualifikationsniveau: 4 }
+
+  def duales_studium?
+    ebene == 3 && qualifikationsniveau == 4 && typ == 'A'
+  end
+
+  def self.download_from_ba
+    require 'tty/prompt'
+    prompt = TTY::Prompt.new
+    link = Ba::Distributor.ba_connection.misc.last do |link|
+    link.click
+    target = "public/ba/#{link.href}"
+    response = link.click
+    File.open(target, "wb+") { |f| f.write(response.body) }
+
+    puts "Unzipping vam_beruf_kurz.xml..."
+    `unzip -o -d public/ba/ #{target} vam_beruf_kurz.xml`
+  end
+
+  def self.import(path: 'public/ba/vam_beruf_kurz.xml')
+    doc = Nokogiri::XML.parse(File.open(path))
+    berufe_vorher = Ba::Beruf.undeleted.pluck(:id)
+    doc.search('beruf').each do |beruf_doc|
+      beruf = where(id: beruf_doc['id']).first_or_initialize
+
+      beruf.bkz = beruf_doc['bkz']
+
+      beruf.typ = beruf_doc.at('typ').text == 't' ? 'B' : 'A'
+      beruf.qualifikationsniveau = beruf_doc.at('qualifikationsNiveau[niveau]')['niveau']
+      beruf_doc.search(*%w[lbkgruppe hochschulberuf ebene kuenstler bezeichnung_nl bezeichnung_nk suchname_nl]).each do |i|
+        beruf.send("#{i.name}=", i.text)
+      end
+      beruf.save
+      berufe_vorher.delete(beruf.id)
+    end
+    Ba::Beruf.where(id: berufe_vorher).update_all deleted_on: Time.zone.now if berufe_vorher.any?
+  end
+  scope :duale_studiengaenge, -> { where ebene: 3, qualifikationsniveau: 4 }
+
+  def display_name
+    prefix = if deleted_on?
+               "[!VERALTET!] "
+             end
+    if typ == 'A'
+      if ebene == 3 && qualifikationsniveau == 4
+        "#{prefix}#{bezeichnung_nk} (DUALES STUDIUM/praxisorientiert)"
+      else
+        "#{prefix}#{bezeichnung_nk} (AUSBILDUNG)"
+      end
+    else
+      "#{prefix}#{bezeichnung_nk}"
+    end
+  end
+
+  def as_json(opts = {})
+    {
+      id: id,
+      display_name: display_name
+    }
+  end
+```
+
+</details>
+
+## Appendix: How to construct a Job-Posting XML file to upload
+
+- Download the most recent JobPosting xsd from https://baxml.arbeitsagentur.de/geschuetzt/download/
+- You can visualize the xsd here: http://www.xml-tools.net/schemaviewer.html
+- Now, you can construct the file with xml-builder:
+
+<details>
+<summary>Example for constructing a feed using XmlBuilder</summary>
+
+```ruby
+    xml = Builder::XmlMarkup.new(indent: 1)
+      xml.instruct!
+      xml.tag!("HRBAXMLJobPositionPosting") do
+        xml.tag!("Header") do
+          xml.tag!("SupplierId", SUPPLIER_ID)
+          xml.tag!("Timestamp", Time.zone.now.to_s(:db).tr(" ", "T"))
+          xml.tag!("Amount", obs.count)
+          # F: Full
+          # D: Diff
+          if @only_jobs
+            xml.tag!("TypeOfLoad", "D")
+          else
+            xml.tag!("TypeOfLoad", "F")
+          end
+        end
+        xml.tag!("Data") do
+          jobs.each do |job|
+            generate_xml_for_job(xml, job)
+          end
+          
+          jobs_to_delete.each do |job|
+            xml.tag! "DeleteEntry" do
+              xml.tag! "EntryId", id
+            end
+          end
+        end
+      end
+      xml
+```
+</details>
+
+- Then, you should validate your feed:
+
+```ruby
+xsd = Nokogiri::XML::Schema(File.open("vendor/ba/HRBAXML_JobPosition_Current.xsd"))
+doc = Nokogiri::XML(xml.to_s)
+xsd.validate(doc)
+```
+
+- Then, you can put that into a file - so you will need to generate a filename **according to the spec**:
+
+<details>
+<summary>Generate a filename</summary>
+
+```ruby
+# for historic reasons, you could transmit a bunch of files with the same timestamp using an index/offset, but usually, just putting 0 here should be enought
+index = 0
+number_of_feeds_to_push_now = 1
+ended = index == (number_of_feeds_to_push_now - 1)
+flag = ended ? "E" : "C"
+date = Time.zone.now.strftime "%Y-%m-%d_%H-%M-%S_F#{'%03d' % (index + 1)}#{flag}"
+"DS#{SUPPLIER_ID}_#{date}.xml"
+```
+
+</details>
+
+- Upload the file using this Gem. You should wait a "couple of minutes" (tip: enqueue a activeJob for 10 minutes later), to fetch the resulting **error file**, and analyse that.
+
+
+
 ## License
 
 The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/ba_upload.gemspec

@@ -20,6 +20,6 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "mechanize"
-  spec.add_development_dependency "bundler", "~> 1.11"
+  spec.add_development_dependency "bundler", "~> 2.0"
   spec.add_development_dependency "rake", "~> 10.0"
 end

data/lib/ba_upload/connection.rb

@@ -1,4 +1,5 @@
 require 'ba_upload/error_file'
+require 'ba_upload/statistic_file'
 module BaUpload
   class Connection
     attr_reader :m
@@ -16,7 +17,7 @@ module BaUpload
 
     def upload(file: nil, partner_id: nil)
       url = base_url(partner_id) + "in/"
-      m.get url
+      m.get(url)
       form = m.page.forms.first
       form.file_uploads.first.file_name = file
       form.submit
@@ -24,13 +25,21 @@ module BaUpload
 
     def error_files(partner_id: nil)
       url = base_url(partner_id)
-      m.get url
+      m.get(url)
       links = m.page.links_with(text: /ESP|ESV/)
       links.map do |link|
         ErrorFile.new(link)
       end
     end
 
+    def statistics(partner_id: nil)
+      url = base_url(partner_id) + "Statistiken"
+      m.get(url)
+      m.page.links_with(text: /xlsx/).map do |link|
+        StatisticFile.new(link)
+      end
+    end
+
     def misc(partner_id: nil)
       url = base_url(partner_id)
       m.get url

data/lib/ba_upload/statistic_file.rb

@@ -0,0 +1,18 @@
+require 'ba_upload/error_file'
+module BaUpload
+  class StatisticFile < ErrorFile
+    def tempfile
+      tf = Tempfile.new(['statistic_file', '.xlsx'])
+      tf.binmode
+      tf.write(read)
+      tf.flush
+      tf.rewind
+      tf
+    end
+
+    def read
+      @mechanize_link.click.body
+    end
+  end
+end
+

data/lib/ba_upload/version.rb

@@ -1,3 +1,3 @@
 module BaUpload
-  VERSION = "0.3.0"
+  VERSION = "0.5.0"
 end

data/lib/ba_upload.rb

@@ -1,6 +1,11 @@
 require "ba_upload/version"
 require "openssl"
 
+if OpenSSL::VERSION >= '3.0.0' && defined?(OpenSSL::Provider)
+  # import legacy
+  OpenSSL::Provider.load("legacy")
+end
+
 module BaUpload
   def self.export_certificate(file_path:, passphrase:)
     cert = OpenSSL::PKCS12.new(File.read(file_path), passphrase)
@@ -15,6 +20,13 @@ module BaUpload
     cert = BaUpload.export_certificate(file_path: file_path, passphrase: passphrase)
     BaUpload::Connection.new(cert[:key], cert[:cert], cert[:ca_cert])
   end
+
+  def self.open(file_path:, passphrase:, &block)
+    conn = BaUpload.open_connection(file_path: file_path, passphrase: passphrase)
+    block.call(conn)
+  ensure
+    conn.shutdown
+  end
 end
 
 require 'ba_upload/connection'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ba_upload
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Stefan Wienert
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-09-08 00:00:00.000000000 Z
+date: 2024-10-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.11'
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.11'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -70,6 +70,7 @@ files:
 - lib/ba_upload.rb
 - lib/ba_upload/connection.rb
 - lib/ba_upload/error_file.rb
+- lib/ba_upload/statistic_file.rb
 - lib/ba_upload/version.rb
 homepage: https://github.com/pludoni/ba_upload
 licenses:
@@ -90,7 +91,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.5.21
 signing_key: 
 specification_version: 4
 summary: Upload API for Bundesagentur fuer Arbeit (hrbaxml.arbeitsagentur)