oneacct-export 0.4.6 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fbb13a1a7e4fb9551033c6f76366219b375a9a8f
-  data.tar.gz: c1dbe229079dd112924d6baadeb3634a8a82657f
+  metadata.gz: ad91a0345fee62abbe9f35d2d80c7825b9086fd8
+  data.tar.gz: 593dfc69ba583587902d759ba689618db9230ad7
 SHA512:
-  metadata.gz: 399a1af0c6d7d6bc63344948f2213e062a4031c34fdd9e25f639b4d4583b839273104f6f8d0b5bb0da433a4eee21b3f3a886cd55fbe67c7173423aedde91949d
-  data.tar.gz: 3ae5121fa9963562628e16b445fea8780cad47b45a9787c91f1d7e356c4db4e3624967a0a52776698d0b0ff4e1f705c834ec3c038ef7e37524b075e1a08be051
+  metadata.gz: 444f35564d46a870f321da7153966a0357a9ab1fa6dd95c1f813e280fb086884b1fd1f77ad172d910537bcc86317cb1f6d269b4fcb1ef5a95fda072954696065
+  data.tar.gz: 1aa96aa47c7affbbb5293451a1e30e523a0e7746fa89e44adaa3723e7b2cdf09e6d8d34f3d1a773706783ebf78bb6c685f892fb013c13b8e217ae8671ead8a2f

data/.circleci/config.yml

@@ -0,0 +1,40 @@
+version: 2
+jobs:
+  build:
+    branches:
+      ignore:
+        - /.*/
+    docker:
+      - image: docker:stable
+    working_directory: /root/oneacct-export
+    steps:
+      - run: apk add --no-cache git openssh
+      - checkout
+      - setup_remote_docker
+      - run: |
+          git config --global --replace-all versionsort.prereleasesuffix ".alpha"
+          git config --global --add versionsort.prereleasesuffix ".beta"
+      - run: |
+          TAG=${CIRCLE_TAG#v}
+          BRANCH=${TAG/%.*/.x}
+          VERSION=${TAG}
+          LATEST=$(git tag --sort=-version:refname | head -n 1)
+
+          docker login -u $DOCKER_USER -p $DOCKER_PASS
+
+          docker build --build-arg branch=$BRANCH --build-arg version="$VERSION" -t oneacctexport/oneacct-export:$TAG -f docker/Dockerfile_oneacct-export .
+          docker push oneacctexport/oneacct-export:$TAG
+          docker build --build-arg branch=$BRANCH --build-arg version="$VERSION" -t oneacctexport/sidekiq:$TAG -f docker/Dockerfile_sidekiq .
+          docker push oneacctexport/sidekiq:$TAG
+
+          if [ "$LATEST" == "$CIRCLE_TAG" ]; then
+            docker tag oneacctexport/oneacct-export:$TAG oneacctexport/oneacct-export:latest
+            docker push oneacctexport/oneacct-export:latest
+            docker tag oneacctexport/sidekiq:$TAG oneacctexport/sidekiq:latest
+            docker push oneacctexport/sidekiq:latest
+          fi
+deployment:
+  fake_deploy_for_cci2:
+    tag: /v.*/
+    commands:
+      - echo "make tags run in 2.0"

data/README.md

@@ -1,22 +1,23 @@
 # OneacctExport
 Exporting OpenNebula accounting data.
 
-[![Build Status](https://secure.travis-ci.org/EGI-FCTF/oneacct_export.png)](http://travis-ci.org/EGI-FCTF/oneacct_export)
-[![Dependency Status](https://gemnasium.com/EGI-FCTF/oneacct_export.png)](https://gemnasium.com/EGI-FCTF/oneacct_export)
-[![Gem Version](https://fury-badge.herokuapp.com/rb/oneacct-export.png)](https://badge.fury.io/rb/oneacct-export)
-[![Code Climate](https://codeclimate.com/github/EGI-FCTF/oneacct_export.png)](https://codeclimate.com/github/EGI-FCTF/oneacct_export)
+[![Travis](https://img.shields.io/travis/the-oneacct-export-project/oneacct-export.svg?style=flat-square)](http://travis-ci.org/the-oneacct-export-project/oneacct-export)
+[![Gemnasium](https://img.shields.io/gemnasium/the-oneacct-export-project/oneacct-export.svg?style=flat-square)](https://gemnasium.com/the-oneacct-export-project/oneacct-export)
+[![Gem](https://img.shields.io/gem/v/oneacct-export.svg?style=flat-square)](https://rubygems.org/gems/oneacct-export)
+[![Code Climate](https://img.shields.io/codeclimate/github/the-oneacct-export-project/oneacct-export.svg?style=flat-square)](https://codeclimate.com/github/the-oneacct-export-project/oneacct-export)
+[![DockerHub](https://img.shields.io/badge/docker-ready-blue.svg?style=flat-square)](https://hub.docker.com/r/oneacctexport/oneacct-export/)
 
 
-##Requirements
+## Requirements
 * Ruby >= 2.0
 * Rubygems
 * Redis server (doesn't have to be present on the same machine)
 * OpenNebula >= 4.4 (doesn't have to be present on the same machine)
 
-##Installation
-###From distribution specific packages
+## Installation
+### From distribution specific packages
 Distribution specific packages can be created with
-[omnibus packaging for OneacctExport](https://github.com/EGI-FCTF/omnibus-oneacct-export).
+[omnibus packaging for OneacctExport](https://github.com/the-oneacct-export-project/omnibus-oneacct-export).
 When installing via packages you don't have to install neither ruby
 nor rubygems. Packages contain embedded ruby and all the necessary gems
 and libraries witch will not effect your system ruby, gems and libraries.
@@ -31,13 +32,13 @@ Currently supported distributions:
 * CentOS 5.10
 * CentOS 6.5
 
-###From RubyGems.org
+### From RubyGems.org
 To install the most recent stable version
 ```bash
 gem install oneacct-export
 ```
 
-###From source (dev)
+### From source (dev)
 **Installation from source should never be your first choice! Especially, if you are not
 familiar with RVM, Bundler, Rake and other dev tools for Ruby!**
 
@@ -46,20 +47,20 @@ familiar with RVM, Bundler, Rake and other dev tools for Ruby!**
 To build and install the bleeding edge version from master
 
 ```bash
-git clone git://github.com/EGI-FCTF/oneacct_export.git
+git clone git://github.com/the-oneacct-export-project/oneacct_export.git
 cd oneacct_export
 gem install bundler
 bundle install
 bundle exec rake spec
 rake install
 ```
-##Configuration
-###Create a new user account
+## Configuration
+### Create a new user account
 Create or use an existing `apel` user account which will be used to run
 the export process. This account must be the same as the user account
 used by the APEL SSM client.
 
-###Create a configuration file for OneacctExport
+### Create a configuration file for OneacctExport
 Configuration file can be read by OneacctExport from these
 three locations:
 
@@ -73,7 +74,7 @@ file you have to follow the division into three environments: `production`,
 `development` and `test`. All the configuration options are described
 in the example configuration file.
 
-###Create a configuration file for Sidekiq
+### Create a configuration file for Sidekiq
 Sidekiq configuration file can be placed anywhere you want since you will provide
 path to the configuration later during the Sidekiq start. How the Sidekiq
 configuration should look like and what options you can use
@@ -84,7 +85,7 @@ The important thing is to set the same queue name in both
 OneacctExport and Sidekiq configuration files. OneacctExport
 is currently supporting adding jobs to only one queue.
 
-###Create required directories
+### Create required directories
 ```bash
 mkdir -p /var/run/oneacct-export
 chown apel:apel /var/run/oneacct-export
@@ -93,7 +94,7 @@ mkdir -p /var/log/oneacct-export
 chown apel:apel /var/log/oneacct-export
 ```
 
-###Configure RPC connection
+### Configure RPC connection
 RPC connection for OpenNebula can be configured in two ways:
 
 * Via OneacctExport configuration file, option xml_rpc and its suboptions
@@ -106,7 +107,7 @@ RPC connection for OpenNebula can be configured in two ways:
  System environment variable `ONE_XMLRPC` contains URL of OpenNebula RPC
  gate. If empty, the same information can be stored in `~/.one/one_endpoint`.
 
-###Configure vmcatcher IMAGE attribute inheritance in OpenNebula
+### Configure vmcatcher IMAGE attribute inheritance in OpenNebula
 In `oned.conf`:
 ~~~
 INHERIT_IMAGE_ATTR = "VMCATCHER_EVENT_AD_MPURI"
@@ -116,7 +117,7 @@ INHERIT_IMAGE_ATTR = "VMCATCHER_EVENT_SL_CHECKSUM_SHA512"
 INHERIT_IMAGE_ATTR = "VMCATCHER_EVENT_HV_VERSION"
 ~~~
 
-###Configure benchmark host attributes in OpenNebula
+### Configure benchmark host attributes in OpenNebula
 In order to recognize and fill `BenchmarkType` and `Benchmark` APEL v0.4 fields,
 two attributes have to be set for every host in OpenNebula:
 * `BENCHMARK_TYPE` - represents benchmark's type. For example: `HEP-SPEC06`
@@ -127,17 +128,17 @@ Both attributes can be set both for clusters and hosts in OpenNebula with hosts'
 attributes taking precedence. If attributes are set only for cluster, all hosts
 within the cluster will be assigned these values.
 
-###Set Rails environment variable according to your environment
+### Set Rails environment variable according to your environment
 You have to set system environment variable `RAILS_ENV` to one of the
 values production, development or test. OneacctExport is not a Rails
 application but we chose the Rails variable for easier possible integration in
 the future.
 
-##Usage
+## Usage
 
 **Both OpenNebula and Redis server must be running prior the next steps.**
 
-###Start sidekiq
+### Start sidekiq
 First you have to start Sidekiq so it can run the jobs from the queue. Since
 OneacctExport is not a Rails application Sidekiq has to be started with
 OneacctExport's worker class as an argument. For example:
@@ -146,7 +147,7 @@ OneacctExport's worker class as an argument. For example:
 sidekiq -r $PATH_TO_GEM_DIR/lib/one_worker.rb -C $PATH_TO_GEM_DIR/config/sidekiq.yml
 ```
 
-###Start OneacctExport
+### Start OneacctExport
 
 OneacctExport is run with executable `oneacct-export`. For a list of all
 available options run `oneacct-export -h`:
@@ -171,8 +172,8 @@ Usage oneacct-export [options]
     -v, --version                    Shows version
 ```
 
-###Package specific scripts
-When installed from packages build via [omnibus packaging for OneacctExport](https://github.com/EGI-FCTF/omnibus-oneacct-export),
+### Package specific scripts
+When installed from packages build via [omnibus packaging for OneacctExport](https://github.com/the-oneacct-export-project/omnibus-oneacct-export),
 both Sidekiq and OneacctExport are automatically registered as cron jobs to run
 periodically. Cron job managing OneacctExport uses a bash script which is
 simplifying OneacctExport interface for most common use cases. After the installation,
@@ -184,21 +185,21 @@ which when present in `/opt/oneacct-export/` directory serves as a configuration
 * `groups.include` - contains list of groups to include (same as combination of OneacctExport options `--include-groups` and `--group-file`)
 * `groups.exclude` - contains list of groups to exclude (same as combination of OneacctExport options `--exclude-groups` and `--group-file`)
 
-##Code Documentation
-[Code Documentation for OneacctExport by YARD](http://rubydoc.info/github/EGI-FCTF/oneacct_export/)
+## Code Documentation
+[Code Documentation for OneacctExport by YARD](http://rubydoc.info/github/the-oneacct-export-project/oneacct-export/)
 
-##Continuous integration
-[Continuous integration for OneacctExport by Travis-CI](http://travis-ci.org/EGI-FCTF/oneacct_export/)
+## Continuous integration
+[Continuous integration for OneacctExport by Travis-CI](http://travis-ci.org/the-oneacct-export-project/oneacct-export/)
 
-##Development
-###Contributing
-1. Fork it ( https://github.com/EGI-FCTF/oneacct_export/fork )
+## Development
+### Contributing
+1. Fork it ( https://github.com/the-oneacct-export-project/oneacct-export/fork )
 2. Create your feature branch (`git checkout -b my-new-feature`)
 3. Commit your changes (`git commit -am 'Add some feature'`)
 4. Push to the branch (`git push origin my-new-feature`)
 5. Create a new Pull Request
 
-###Debugging
+### Debugging
 To change the log level of `oneacct-export` and `sidekiq` you have to set the environment variable **ONEACCT_EXPORT_LOG_LEVEL** to log level you need. Supported log levels are `DEBUG`, `INFO`, `WARN` and `ERROR`.
 ```bash
 export ONEACCT_EXPORT_LOG_LEVEL=DEBUG

data/Rakefile

@@ -1,18 +1,6 @@
+require 'rspec/core/rake_task'
 require 'bundler/gem_tasks'
-require 'rubygems/tasks'
 
-task :default => 'test'
+RSpec::Core::RakeTask.new(:spec)
 
-desc "Run all tests; includes rspec and coverage reports"
-task :test => 'rcov:rspec'
-
-desc "Run all tests; includes rspec and coverage reports"
-task :spec => 'test'
-
-Gem::Tasks.new(:build => {:tar => true, :zip => true}, :sign => {:checksum => true, :pgp => false})
-
-namespace :rcov do
-  require 'rspec/core/rake_task'
-
-  RSpec::Core::RakeTask.new(:rspec)
-end
+task default: :spec

data/docker/Dockerfile_oneacct-export

@@ -0,0 +1,49 @@
+FROM ruby:2.2-jessie
+
+ARG branch=master
+ARG version
+
+ENV name="oneacct-export" \
+    user="apel"
+ENV runDir="/var/run/${name}" \
+    logDir="/var/log/${name}" \
+    homeDir="/var/lib/${user}" \
+    spoolDir="/var/spool/${user}/outgoing/00000000"
+ENV confDir="${homeDir}/.${name}"
+
+LABEL application=${name} \
+      description="Exporting OpenNebula accounting data" \
+      maintainer="kimle@cesnet.cz" \
+      version=${version} \
+      branch=${branch}
+
+SHELL ["/bin/bash", "-c"]
+
+# oneacct-export
+RUN gem install ${name} -v "${version}" --no-document
+
+# EGI trust anchors
+RUN set -o pipefail && \
+    apt-get update && \
+    apt-get --assume-yes install curl && \
+    curl -s https://dist.eugridpma.info/distribution/igtf/current/GPG-KEY-EUGridPMA-RPM-3 | apt-key add - && \
+    echo $'#### EGI Trust Anchor Distribution ####\n\
+deb http://repository.egi.eu/sw/production/cas/1/current egi-igtf core' > /etc/apt/sources.list.d/egi.list && \
+    apt-get update && \
+    apt-get --assume-yes install ca-policy-egi-core gettext-base
+
+# env
+RUN useradd --system --shell /bin/false --home ${homeDir} --create-home --uid 1000 ${user} && \
+    usermod -L ${user} && \
+    mkdir -p ${runDir} ${logDir} ${spoolDir} ${confDir} && \
+    chown -R ${user}:${user} ${runDir} ${logDir} ${spoolDir} ${confDir}
+
+COPY config/* /oneacct-export/config/
+COPY bin/* /oneacct-export/bin/
+
+VOLUME ${logDir}
+VOLUME ${spoolDir}
+
+USER ${user}
+
+ENTRYPOINT ["/oneacct-export/bin/oneacct-wrapper.sh"]

data/docker/Dockerfile_sidekiq

@@ -0,0 +1,14 @@
+ARG branch=master
+ARG version
+
+FROM misenko/oneacct-export:${version}
+
+ENV name="sidekiq"
+
+LABEL application=${name} \
+      description="Sidekiq" \
+      maintainer="kimle@cesnet.cz" \
+      version=${version} \
+      branch=${branch}
+
+ENTRYPOINT ["/oneacct-export/bin/sidekiq-wrapper.sh"]

data/docker/bin/configuration.sh

@@ -0,0 +1,11 @@
+function configure_oneacct-export {
+  export ONEACCT_NUM_OF_VMS_PER_FILE="${ONEACCT_NUM_OF_VMS_PER_FILE:-500}"
+  export ONEACCT_SITE_NAME="${ONEACCT_SITE_NAME:-Undefined}"
+  export ONEACCT_CLOUD_TYPE="${ONEACCT_CLOUD_TYPE:-OpenNebula}"
+  export ONEACCT_ENDPOINT="${ONEACCT_ENDPOINT:-https://occi.localhost.com:11443/}"
+  export ONEACCT_REDIS_NAMESPACE="${ONEACCT_REDIS_NAMESPACE:-oneacct_export}"
+  export ONEACCT_REDIS_URL="${ONEACCT_REDIS_URL:-redis://localhost:6379}"
+  export ONEACCT_SIDEKIQ_QUEUE="${ONEACCT_SIDEKIQ_QUEUE:-oneacct_export}"
+
+  envsubst < /oneacct-export/config/conf.yml > /var/lib/apel/.oneacct-export/conf.yml
+}

data/docker/bin/oneacct-wrapper.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+source $(dirname $0)/configuration.sh
+
+configure_oneacct-export
+oneacct-export $@

data/docker/bin/sidekiq-wrapper.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+source $(dirname $0)/configuration.sh
+
+configure_oneacct-export
+sidekiq $@

data/docker/config/conf.yml

@@ -0,0 +1,73 @@
+---
+defaults: &defaults
+  output:
+    output_dir: /var/spool/apel/outgoing/00000000 # Directory for outgoing messages
+    output_type: apel-0.4 # Format of outgoing messages. Choices are: apel-0.2, apel-0.4, pbs-0.1, logstash-0.1
+    num_of_vms_per_file: ${ONEACCT_NUM_OF_VMS_PER_FILE} # Maximum number of virtual machine records per one output file
+    apel: # Options for apel output format
+      site_name: ${ONEACCT_SITE_NAME} # Usually a short provider name, e.g. CESNET
+      cloud_type: ${ONEACCT_CLOUD_TYPE} # CMF type, only OpenNebula is supported
+      endpoint: ${ONEACCT_ENDPOINT} # URL of your OCCI endpoint, e.g. https://fqdn.example.com:11443/
+      # cloud_compute_service: # Name identifying cloud resource within the site. Allows multiple cloud resources within a site. i.e. a level of granularity.
+    pbs: # Options for pbs output format
+      realm: REALM # Owner's realm, e.g. META
+      queue: cloud # Queue name
+      scratch_type: local # Data store type
+      host_identifier: on_localhost # Identifier for host OpenNebula is running on
+    logstash: # Options for logstash output format
+      host: localhost # Host OpenNebula is running on
+      port: 11443 # Port OpenNebula's RPC is listening on
+  logging:
+    log_type: file # Two options: file, syslog. Defaults to stdout
+    log_file: /var/log/oneacct-export/oneacct-export.log # Used when type file selected
+  xml_rpc:
+    secret: ${ONEACCT_XML_RPC_SECRET} # If not specified looking for secret in ONE_AUTH and ~/.one/one_auth
+    endpoint: ${ONEACCT_XML_RPC_ENDPOINT} # Defaults to content of ONE_XMLRPC or content of ~/.one/one_endpoint or http://localhost:2633/RPC2
+  redis:
+    namespace: ${ONEACCT_REDIS_NAMESPACE} # Namespace used by redis for sidekiq jobs
+    url: ${ONEACCT_REDIS_URL} # URL of redis server, defaults to redis://localhost:6379
+    password: ${ONEACCT_REDIS_PASSWORD} # Password to access redis server if needed
+  sidekiq:
+    queue: ${ONEACCT_SIDEKIQ_QUEUE} # Name of the redis queue used by sidekiq, must match sidekiq.yml configuration
+
+###############################################
+#######  DO NOT EDIT AFTER THIS POINT  ########
+###############################################
+
+production:
+  <<: *defaults
+
+
+development:
+  <<: *defaults
+
+
+test:
+  output:
+    output_dir: <placeholder>
+    output_type: <placeholder>
+    num_of_vms_per_file: <placeholder>
+    apel:
+      site_name: <placeholder>
+      cloud_type: <placeholder>
+      endpoint: <placeholder>
+    pbs:
+      realm: <placeholder>
+      queue: <placeholder>
+      scratch_type: <placeholder>
+      host_identifier: <placeholder>
+    logstash:
+      host: <placeholder>
+      port: <placeholder>
+  logging:
+    log_type: <placeholder>
+    log_file: <placeholder>
+  xml_rpc:
+    secret: <placeholder>
+    endpoint: <placeholder>
+  redis:
+    namespace: <placeholder>
+    url: http://some.random.url.com
+    password: password
+  sidekiq:
+    queue: oneacct_export

data/lib/oneacct_exporter/version.rb

@@ -1,3 +1,3 @@
 class OneacctExporter
-  VERSION = '0.4.6'
+  VERSION = '0.5.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: oneacct-export
 version: !ruby/object:Gem::Version
-  version: 0.4.6
+  version: 0.5.0
 platform: ruby
 authors:
 - Michal Kimle
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-04-26 00:00:00.000000000 Z
+date: 2017-10-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -172,6 +172,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".circleci/config.yml"
 - ".gitignore"
 - ".rspec"
 - ".travis.yml"
@@ -183,6 +184,12 @@ files:
 - bin/oneacct-export
 - config/conf.yml
 - config/sidekiq.yml
+- docker/Dockerfile_oneacct-export
+- docker/Dockerfile_sidekiq
+- docker/bin/configuration.sh
+- docker/bin/oneacct-wrapper.sh
+- docker/bin/sidekiq-wrapper.sh
+- docker/config/conf.yml
 - examples/etc/cron.d/oneacct-export
 - examples/etc/init.d/oneacct-export-cron
 - examples/etc/init.d/oneacct-export-sidekiq
@@ -273,7 +280,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.10
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: Exporting OpenNebula accounting data.