soracom_summary 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e0fdd3af68a2563ceb97e1276db7a114c1d51c93
+  data.tar.gz: 1b70ced95d1f6adeee8dc4975af46f3f514d5fc2
+SHA512:
+  metadata.gz: d3fe57db7e72cb1eb2bed0f1e4c324ef44534a35f762819492d6ed14537f4fe8a4598c0bdb71bb5814ed12ccd968f674c4944752d8b40ce69880b18ca58c8a7a
+  data.tar.gz: f8ec6b884d6305a7228544d67005b7fc17f4c854972f7f4e0f4b09ad5b26e7a9591d4f5b37bb6d27a624c9709c27d7407ed7aa2fe25b8ab89d69e62da0fdcc95

data/.gitignore

@@ -0,0 +1,12 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
+.vscode/

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,7 @@
+---
+sudo: false
+language: ruby
+cache: bundler
+rvm:
+  - 2.3.3
+before_install: gem install bundler -v 2.0.1

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at funahara@hioki.co.jp. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

data/Gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in soracom_summary.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,42 @@
+PATH
+  remote: .
+  specs:
+    soracom_summary (0.1.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    debase (0.2.4.1)
+      debase-ruby_core_source (>= 0.10.2)
+    debase-ruby_core_source (0.10.7)
+    diff-lcs (1.3)
+    rake (10.5.0)
+    rspec (3.5.0)
+      rspec-core (~> 3.5.0)
+      rspec-expectations (~> 3.5.0)
+      rspec-mocks (~> 3.5.0)
+    rspec-core (3.5.4)
+      rspec-support (~> 3.5.0)
+    rspec-expectations (3.5.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.5.0)
+    rspec-mocks (3.5.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.5.0)
+    rspec-support (3.5.0)
+    ruby-debug-ide (0.7.0)
+      rake (>= 0.8.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 2.0)
+  debase
+  rake (~> 10.0)
+  rspec (~> 3.0)
+  ruby-debug-ide
+  soracom_summary!
+
+BUNDLED WITH
+   2.0.1

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2019 Ippei Funahara
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,86 @@
+# SORACOM Summary
+SORACOM Summary は、ソラコム株式会社の提供する回線サービスSORACOMの使用状況をWeb APIを用いてスクレイピングし、
+取得した使用状況をSORACOM Harvestにアップロードすることにより可視化するツールです。
+
+## インストール
+
+```
+$ gem install soracom_summary
+```
+
+ライブラリ及びsoracom_summaryコマンドがインストールされます。
+
+## 使用方法
+
+### ユーザー認証情報の設定
+環境変数にSAMユーザーの認証キーIDと認証キーシークレットを設定します。
+SAMユーザーの説明はこちら
+https://dev.soracom.io/jp/start/sam/
+
+```
+$ export SORACOM_AUTH_KEY_ID='keyId-XXXX'
+$ export SORACOM_AUTH_KEY='secret-XXXX'
+```
+
+SAMユーザーには、以下の権限が必要です。上記ドキュメントを参照して、権限を設定します。
+- Subscriber:listSubscribers
+- Subscriber:listSessionEvents
+- Log:getLogs
+- Billing:exportBilling
+- Stats:exportAirStats
+
+### デバイス認証情報の設定
+
+環境変数にHarvestにアップロードするためのSORACOM InventoryのデバイスIDとデバイスシークレットを設定します。
+デバイスIDとデバイスシークレットについての説明はこちら
+https://dev.soracom.io/jp/start/inventory_harvest_with_keys/
+
+集計情報用（必須）
+```
+$ export SORACOM_SUMMARY_DEVICE_ID='d-XXXX'
+$ export SORACOM_SUMMARY_DEVICE_SECRET='XXXX'
+```
+
+セッション情報の分布用（オプション）
+```
+$ export SORACOM_SESSION_DEVICE_ID='d-XXXX'
+$ export SORACOM_SESSION_DEVICE_SECRET='XXXX'
+```
+
+請求情報の分布用（オプション）
+```
+$ export SORACOM_BILLING_DEVICE_ID='d-XXXX'
+$ export SORACOM_BILLING_DEVICE_SECRET='XXXX'
+```
+
+通信量の分布用（オプション）
+```
+$ export SORACOM_TRAFFIC_DEVICE_ID='d-XXXX'
+$ export SORACOM_TRAFFIC_DEVICE_SECRET='XXXX'
+```
+
+### 使用方法
+以下のコマンドを実行すると１日前の集計をします
+```
+$ soracom_summary
+```
+
+セッション情報を取得するには時間がかかるため、--sessionオプションで明示的に指定する必要があります
+```
+$ soracom_summary --session
+```
+
+指定したタグごとにSIMの枚数を集計する場合、--tagオプションを使用します
+```
+$ soracom_summary --tag environment
+```
+
+過去にさかのぼって集計する場合、--from、--toオプションを使用します
+```
+$ soracom_summary --from 2019-11-22 --to 2019-11-24
+```
+
+実行後、データはSORACOM Harvestにアップロードされ、SORACOM Lagoonにて可視化することが出来ます。
+
+Lagoonでの可視化は以下を参照して設定します。
+https://dev.soracom.io/jp/start/lagoon-panel/

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "soracom_summary"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/soracom_summary

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+# coding: utf-8
+require 'soracom_summary'
+
+SoracomSummary::CLI.run(ARGV)

data/lib/soracom_summary.rb

@@ -0,0 +1,8 @@
+require "soracom_summary/version"
+require "soracom_summary/const"
+require "soracom_summary/cli"
+require 'soracom_summary/api_client'
+require 'soracom_summary/http_client'
+require 'soracom_summary/subscriber'
+require 'soracom_summary/billing'
+require 'soracom_summary/traffic'

data/lib/soracom_summary/api_client.rb

@@ -0,0 +1,218 @@
+require 'time'
+require 'csv'
+require 'uri'
+require 'logger'
+
+module SoracomSummary
+  class ApiClient
+    BASE_URL = 'https://api.soracom.io/v1/'
+    def initialize(
+      auth_key_id: ENV['SORACOM_AUTH_KEY_ID'],
+      auth_key: ENV['SORACOM_AUTH_KEY']
+      )
+      @logger = Logger.new(STDERR)
+      @logger.level = ENV['SORACOM_SUMMARY_DEBUG'] ? Logger::DEBUG : Logger::WARN
+      @http_client = HttpClient.new(base_url: BASE_URL)
+      @token = authenticate(auth_key_id, auth_key)
+    end
+
+    def get_subscribers
+      @logger.debug('start get subscribers')
+      subscribers = []
+      path = 'subscribers'
+      loop do
+        response = http_access_with_token('GET', path)
+        response_subscribers = response['body'].map do |subscriber|
+          status = subscriber['status']
+          if status == 'active'
+            status = subscriber['sessionStatus']['online'] ? 'online' : 'offline'
+          end
+          Subscriber.new(
+            imsi: subscriber['imsi'],
+            created_at: subscriber['createdAt'],
+            tags: subscriber['tags'],
+            status: status
+          )
+        end
+        subscribers.concat(response_subscribers)
+        next_link = get_next_link(response['headers'])
+        break if next_link.nil?
+        path = next_link
+      end
+
+      @logger.debug('finish get subscribers')
+      subscribers
+    end
+
+    def get_sessions(imsi, from, to)
+      @logger.debug("start get sessions #{imsi}")
+      from_timestamp_ms = from.to_i * 1000
+      to_timestamp_ms = (to.to_i + Const::ONE_DAY_SEC) * 1000
+      sessions = []
+      path = "subscribers/#{imsi}/events/sessions"
+      loop do
+        response = http_access_with_token('GET', path)
+        break if response['body'].empty?
+
+        # Modifiedなど接続/切断以外のイベントは排除
+        response_sessions = response['body'].select { |session| session['event'] == 'Created' || session['event'] == 'Deleted' }
+        sessions.concat(response_sessions)
+
+        # fromの時間に入る直前までのセッションが取得できていたら終了する
+        break if sessions.length > 0 && sessions.last['time'] < from_timestamp_ms
+
+        next_link = get_next_link(response['headers'])
+        break if next_link.nil?
+        path = next_link
+      end
+
+      # to以降のセッションは除外
+      sessions = sessions.reject { |session| session['time'] >= to_timestamp_ms }
+
+      # from以前のセッションは最後のイベントをfromの開始時のイベントとして残し、他は除外する
+      before_from_sessions = sessions.select { |session| session['time'] < from_timestamp_ms }
+      sessions = sessions.reject { |session| session['time'] < from_timestamp_ms }
+      if before_from_sessions.length > 0
+        sessions.push(before_from_sessions.first.merge({'time' => from_timestamp_ms }))
+      else
+        sessions.push({'event' => 'Deleted', 'time' => from_timestamp_ms} )
+      end
+
+      @logger.debug("finish get sessions #{imsi}")
+      sessions
+    end
+
+    # エラーログを取得する
+    # from、toの指定が効いていない？
+    def get_logs(from, to)
+      @logger.debug("start get logs")
+
+      from_timestamp = from.to_i
+      to_timestamp = to.to_i + Const::ONE_DAY_SEC
+      logs = []
+      path = "logs?from=#{from_timestamp}&to=#{to_timestamp}"
+
+      loop do
+        response = http_access_with_token('GET', path)
+        break if response['body'].empty?
+        logs.concat(response['body'])
+        next_link = get_next_link(response['headers'])
+        break if next_link.nil?
+        path = next_link
+      end
+
+      @logger.debug("finish get logs")
+      logs = logs.select { |log| log['time'] >= from_timestamp * 1000 && log['time'] < to_timestamp * 1000 }
+      logs
+    end
+
+    def get_billing(target_months)
+      @logger.debug("start get billings")
+      billings = []
+
+      target_months.each do |month|
+        export_response = http_access_with_token('POST', "bills/#{month}/export?export_mode=sync")
+        export_url = export_response['body']['url']
+
+        # 1文字目はBOMのため除外する
+        csv_text = HttpClient.download(export_url)
+        csv_text.slice!(0)
+        CSV.parse(csv_text, headers: true ) do |ln|
+          billing = Billing.new(
+            imsi: ln['imsi'],
+            device_id: ln['deviceId'],
+            date: ln['date'],
+            bill_item_name: ln['billItemName'],
+            amount: ln['amount']
+          )
+          billings.push(billing)
+        end
+      end
+
+      @logger.debug("finish get billings")
+      billings
+    end
+
+    def get_traffic(target_months)
+      @logger.debug("start get traffics")
+
+      traffics = []
+
+      target_months.each do |month|
+        from = Time.parse(month + '01')
+        # 翌月の1日 - 1
+        to = from + (32 - (from + 31 * Const::ONE_DAY_SEC).mday) * Const::ONE_DAY_SEC - 1
+        body = { 'from' => from.to_i, 'to' => to.to_i, 'period' => 'day'}
+        export_response = http_access_with_token('POST', "stats/air/operators/#{@token['operatorId']}/export?export_mode=sync", nil, body)
+        export_url = export_response['body']['url']
+
+        # 1文字目はBOMのため除外する
+        csv_text = HttpClient.download(export_url)
+        csv_text.slice!(0)
+        CSV.parse(csv_text, headers: true ) do |ln|
+          traffic = Traffic.new(
+            imsi: ln['imsi'],
+            date: ln['date'],
+            type: ln['type'],
+            upload_byte_size_total: ln['uploadByteSizeTotal'],
+            download_byte_size_total: ln['downloadByteSizeTotal'],
+          )
+          traffics.push(traffic)
+        end
+      end
+
+      @logger.debug("finish get traffics")
+      traffics
+    end
+
+    def upload_harvest(device_id, device_secret, time, data)
+      @logger.debug("start upload harvest #{device_id}")
+
+      # device_secretはURLセーフは文字列ではないためエンコードが必要
+      path = "devices/#{device_id}/publish?device_secret=#{URI.encode_www_form_component(device_secret)}"
+      headers = nil
+      unless time.nil?
+        headers = { 'X-SORACOM-TIMESTAMP' => (time.to_i * 1000).to_s }
+      end
+      result = @http_client.post(path, headers, data)
+
+      @logger.debug("finish upload harvest #{device_id}")
+      result
+    end
+
+    private 
+
+    # 認証IDと認証キーで認証する
+    def authenticate(auth_key_id, auth_key)
+      @logger.debug("start authenticate")
+
+      request_body = { authKeyId: auth_key_id, authKey: auth_key }
+      token_response = @http_client.post("auth", {}, request_body)
+
+      @logger.debug("finish authenticate")
+      token_response['body']
+    end
+
+    # トークン情報付でアクセスする
+    def http_access_with_token(method, path, headers = nil, body = nil)
+      headers = {} if headers.nil?
+      headers.merge!({
+          'X-SORACOM-API-KEY' => @token['apiKey'],
+          'X-SORACOM-TOKEN' => @token['token']
+        })
+      case method
+      when 'GET' then @http_client.get(path, headers)
+      when 'POST' then @http_client.post(path, headers, body)
+      else nil
+      end
+    end
+
+    def get_next_link(headers)
+      return nil unless headers.key?('link')
+      link_array = headers['link'].split(/\s*,\s*/)
+      next_link_text = link_array.find { |link| link.include?('rel=next') }
+      return nil if next_link_text.nil?
+      next_link_text[(next_link_text.index('<') + 1)...(next_link_text.index('>'))]
+    end
+  end
+end

data/lib/soracom_summary/billing.rb

@@ -0,0 +1,62 @@
+module SoracomSummary
+  class Billing
+    attr_accessor :imsi, :device_id, :date, :bill_item_name, :amount
+    class << self
+      def summary(billings, time)
+        month_text = time.strftime('%Y%m')
+        day = time.strftime('%Y%m%d').to_i
+
+        # 対象期間の請求をフィルタする
+        month_billings = billings.select { |billing| billing.date[0, 6] == month_text && billing.date.to_i <= day }
+        day_billings = billings.select { |billing| billing.date.to_i == day }
+        billing_summary = { 'billings-month-total' => sum(month_billings), 'billings-day-total' => sum(day_billings) }
+
+        # 請求を項目ごとにグループ化する
+        month_billings_by_item = month_billings
+          .group_by { |billing| billing.bill_item_name }
+          .map { |item, group| ["billings-month-item-#{item}", sum(group) ]}
+          .to_h
+        billing_summary.merge!(month_billings_by_item)
+
+        day_billings_by_item = day_billings
+          .group_by { |billing| billing.bill_item_name }
+          .map { |item, group| ["billings-day-item-#{item}", sum(group) ]}
+          .to_h
+        billing_summary.merge!(day_billings_by_item)
+
+        billing_summary
+      end
+
+      def group_by_origin(billings, time)
+        day = time.strftime('%Y%m%d')
+
+        # 対象期間の請求をフィルタする
+        day_billings = billings.select { |billing| billing.date == day }
+        
+        day_billings_by_origin = day_billings
+        .group_by { |billing| billing.imsi || billing.device_id || billing.bill_item_name }
+        .map { |origin, group| [origin, sum(group)] }
+        .to_h
+
+        day_billings_by_origin
+      end
+
+      def sum(billings)
+        billings.inject(0.0) { |sum, billing| sum + billing.amount.to_f }
+      end
+    end
+
+    def initialize(
+      imsi:,
+      device_id:,
+      date:,
+      bill_item_name:,
+      amount:)
+      @imsi = imsi
+      @device_id = device_id
+      @date = date
+      @bill_item_name = bill_item_name
+      @amount = amount
+    end
+  end
+end

data/lib/soracom_summary/cli.rb

@@ -0,0 +1,145 @@
+require 'optparse'
+require 'date'
+
+module SoracomSummary
+  class CLI
+    class << self
+      def run(argv)
+        # ソラコム内では時間はUTCで表現されるため、タイムゾーンをUTCにしておく
+        ENV['TZ'] = 'UTC'
+        parse!(argv)
+      end
+
+      private
+
+      def parse!(argv)
+        options = {}
+        parser = create_parser(options)
+        parser.order!(argv)
+
+        if options.key?('from')
+          @from = Time.parse(options['from'])
+        else
+          @from = Date.today.to_time - Const::ONE_DAY_SEC
+        end
+
+        if options.key?('to')
+          @to = Time.parse(options['to']) + Const::ONE_DAY_SEC
+        else
+          @to = Date.today.to_time
+        end
+
+        if @from >= @to
+          raise ArgumentError, 'from must be less or equal to'
+        end
+        
+        # 分類に使用するタグの設定
+        if options.key?('category_tag')
+          @category_tag = options['category_tag']
+        end
+
+        # セッション分析を実施するか
+        # 取得に時間を要するためオプションとする
+        if options.key?('session')
+          @session_analyze_enable = options['session']
+        end
+
+        scrape
+      end 
+
+      def create_parser(options)
+        OptionParser.new do |opt|
+          opt.on_head('-v', '--version', 'display version') do
+            puts "soracom_summary #{VERSION}"
+            exit
+          end 
+          opt.on('--from from_date', 'set from_date to scrape(default: today)') { |v| options['from'] = v }
+          opt.on('--to to_date', 'set to_date to scrape(default: today)') { |v| options['to'] = v }
+          opt.on('--tag category_tag', 'select tag to categorize(default: nil)') { |v| options['category_tag'] = v }
+          opt.on('--session', 'enable session analyze(default: false)') { |v| options['session'] = v }
+        end 
+      end
+
+      def scrape
+        raise ArgumentError, 'need SORACOM_AUTH_KEY_ID environment' unless ENV.key?('SORACOM_AUTH_KEY_ID')
+        raise ArgumentError, 'need SORACOM_AUTH_KEY environment' unless ENV.key?('SORACOM_AUTH_KEY')
+        raise ArgumentError, 'need SORACOM_SUMMARY_DEVICE_ID environment' unless ENV.key?('SORACOM_SUMMARY_DEVICE_ID')
+        raise ArgumentError, 'need SORACOM_SUMMARY_DEVICE_SECRET environment' unless ENV.key?('SORACOM_SUMMARY_DEVICE_SECRET')
+
+        client = SoracomSummary::ApiClient.new(
+          auth_key_id: ENV['SORACOM_AUTH_KEY_ID'],
+          auth_key: ENV['SORACOM_AUTH_KEY'])
+
+        subscribers = client.get_subscribers
+
+        # セッション分析を実行する場合
+        if @session_analyze_enable == true
+          subscribers.each do |subscriber|
+            sessions = client.get_sessions(subscriber.imsi, @from, @to )
+            subscriber.sessions = sessions
+          end
+        end
+
+        target_months = get_target_months(@from, @to)
+        billings = client.get_billing(target_months)
+        traffics = client.get_traffic(target_months)
+        logs = client.get_logs(@from, @to)
+        
+        time = @from
+        loop do
+          summary = get_summary(subscribers, billings, traffics, logs, time)
+          client.upload_harvest(ENV['SORACOM_SUMMARY_DEVICE_ID'], ENV['SORACOM_SUMMARY_DEVICE_SECRET'], time, summary)
+
+          if @session_analyze_enable == true && ENV.key?('SORACOM_SESSION_DEVICE_ID') && ENV.key?('SORACOM_SESSION_DEVICE_SECRET')
+            sessions_count_by_imsi = Subscriber.sessions_count_by_imsi(subscribers, time)
+            client.upload_harvest(ENV['SORACOM_SESSION_DEVICE_ID'], ENV['SORACOM_SESSION_DEVICE_SECRET'], time, sessions_count_by_imsi)
+          end
+
+          if ENV.key?('SORACOM_BILLING_DEVICE_ID') && ENV.key?('SORACOM_BILLING_DEVICE_SECRET')
+            billing_by_origin = Billing.group_by_origin(billings, time)
+            client.upload_harvest(ENV['SORACOM_BILLING_DEVICE_ID'], ENV['SORACOM_BILLING_DEVICE_SECRET'], time, billing_by_origin)
+          end
+
+          if ENV.key?('SORACOM_TRAFFIC_DEVICE_ID') && ENV.key?('SORACOM_TRAFFIC_DEVICE_SECRET')
+            traffic_by_imsi = Traffic.group_by_imsi(traffics, time)
+            client.upload_harvest(ENV['SORACOM_TRAFFIC_DEVICE_ID'], ENV['SORACOM_TRAFFIC_DEVICE_SECRET'], time, traffic_by_imsi)
+          end
+
+          time += Const::ONE_DAY_SEC
+          break if time >= @to
+        end
+      end
+
+      def get_summary(subscribers, billings, traffics, logs, time)
+        subscribers_summary = Subscriber.summary(subscribers, time, @category_tag, @session_analyze_enable)
+        billings_summary = Billing.summary(billings, time)
+        traffics_summary = Traffic.summary(traffics, time)
+        logs_summary = summary_log(logs, time)
+        
+        result = subscribers_summary
+          .merge(billings_summary)
+          .merge(traffics_summary)
+          .merge(logs_summary)
+        result
+      end
+
+      def get_target_months(from, to)
+        target_months = []
+        time = from
+        loop do
+          target_months.push(time.strftime('%Y%m'))
+          time += Const::ONE_DAY_SEC
+          break if time >= to
+        end
+        target_months.uniq
+      end
+
+      def summary_log(logs, time)
+        target_logs = logs.select do |log|
+          log['time'] >= time.to_i * 1000 && log['time'] < time.to_i * 1000 + Const::ONE_DAY_MSEC
+        end
+        { 'error-log-count' => target_logs.length }
+      end
+    end
+  end
+end

data/lib/soracom_summary/const.rb

@@ -0,0 +1,6 @@
+module SoracomSummary
+  class Const
+    ONE_DAY_SEC = 24 * 60 * 60
+    ONE_DAY_MSEC = ONE_DAY_SEC * 1000
+  end
+end

data/lib/soracom_summary/http_client.rb

@@ -0,0 +1,100 @@
+require 'net/http'
+require 'uri'
+require 'json'
+require 'logger'
+
+module SoracomSummary
+  class HttpClient
+    class << self
+      def download(url)
+        uri = URI.parse(url)
+        request = Net::HTTP::Get.new(uri)
+        response = Net::HTTP.start(uri.hostname, uri.port, { use_ssl: true }) do |http|
+          http.request(request)
+        end
+        if (response.code.start_with?('2'))
+          return response.body.force_encoding('UTF-8')
+        else
+          Logger.new(STDERR).warn("Cannot download csv")
+          raise
+        end
+      end
+    end
+
+    def initialize(base_url: , interval_second: 1, retry_limit: 5)
+      @logger = Logger.new(STDERR)
+      @logger.level = ENV['SORACOM_SUMMARY_DEBUG'] ? Logger::DEBUG : Logger::WARN
+      @base_url = base_url
+      @interval_second = interval_second
+      @retry_limit = retry_limit
+    end
+
+    def get(path, headers)
+      uri = URI.join(@base_url, path)
+      request = Net::HTTP::Get.new(uri)
+
+      if headers.class == Hash
+        headers.each do |key, value|
+          request[key] = value
+        end
+      end
+
+      access(uri, request)
+    end
+
+    def post(path, headers, request_body)
+      uri = URI.join(@base_url, path)
+      request = Net::HTTP::Post.new(uri)
+      request.content_type = 'application/json'
+
+      if headers.class == Hash
+        headers.each do |key, value|
+          request[key] = value
+        end
+      end
+
+      if !request_body.nil?
+        request.body = JSON.generate(request_body)
+      end
+
+      access(uri, request)
+    end
+
+    private
+
+    def access(uri, request, retry_count = 0)
+      response = Net::HTTP.start(uri.hostname, uri.port, { use_ssl: true }) do |http|
+        http.request(request)
+      end
+
+      if (response.code.start_with?('2'))
+        # 成功した場合はヘッダーとボディをそれぞれ返す
+        # ヘッダーを返すのはlinkヘッダーの取得が必要となるため
+        # ボディは基本的にはJSONでパースできるが、空の場合は出来ないため空文字列で返す
+        # 連続してアクセスしすぎないよう規定時間スリープしてから応答する
+        sleep @interval_second
+        return { 'headers' => response.header, 'body' => response.body.empty? ? '' : JSON.parse(response.body)}
+      elsif response.code == '429'
+        # 429はAPIレート制限のため、１分待ってからエラーを発生させる
+        @logger.debug("#{response.code}: #{response.body}")
+        sleep 60
+        raise
+      else
+        # その他不明なコードが返った場合はエラーを発生させる
+        @logger.warn("#{response.code}: #{response.body}")
+        raise
+      end
+    rescue => e
+      @logger.warn e
+      # エラーが発生した場合はリトライ回数が規定値未満であればリトライする
+      retry_count += 1
+      if retry_count < @retry_limit
+        sleep (2 ** retry_count)
+        retry
+      end
+
+      # リトライ回数が規定値を超えるとエラーを上位に報告する
+      raise
+    end
+  end
+end

data/lib/soracom_summary/subscriber.rb

@@ -0,0 +1,88 @@
+module SoracomSummary
+  class Subscriber
+    attr_accessor :imsi, :created_at, :tags, :status, :sessions
+    class << self
+      def summary(subscribers, time, category_tag = nil, session_analyze_enable = false)
+        # 指定期間内に登録されていたSIMのみ対象とする
+        target_subscribers = subscribers.select do |subscriber|
+          subscriber.created_at < time.to_i * 1000 + Const::ONE_DAY_MSEC
+        end
+
+        subscribers_summary = {
+          'subscribers-total' => target_subscribers.length
+        }
+
+        # ステータスでの分類
+        subscribers_count_by_status = target_subscribers
+          .group_by { |subscriber| subscriber.status }
+          .map { |status, group| ["subscribers-status-#{status}", group.length ]}
+          .to_h
+        subscribers_summary.merge!(subscribers_count_by_status)
+        
+        # 分類対象のタグがセットされていたらタグで分類する
+        if !category_tag.nil?
+          subscribers_count_by_tag = target_subscribers
+            .group_by { |subscriber| subscriber.tags[category_tag] }
+            .map { |tag, group| ["subscribers-#{category_tag}-#{(tag.nil? ? "none" : tag)}", group.length ]}
+            .to_h
+          subscribers_summary.merge!(subscribers_count_by_tag)
+        end
+
+        # セッション状態での分類
+        if session_analyze_enable
+          subscribers_count_by_active = target_subscribers
+            .group_by { |subscriber| subscriber.active?(time) }
+            .map { |active, group| ["subscribers-#{active ? 'active' : 'inactive'}", group.length ]}
+            .to_h
+          subscribers_summary.merge!(subscribers_count_by_active)
+        end
+        subscribers_summary
+      end
+
+      # imsiごとのセッション数を取得する
+      def sessions_count_by_imsi(subscribers, time)
+        subscribers
+          .map { |subscriber| [ subscriber.imsi, subscriber.sessions_count(time) ]}
+          .to_h
+      end
+    end
+
+    def initialize(
+        imsi:,
+        created_at:,
+        tags:,
+        status:)
+      @imsi = imsi
+      @created_at = created_at
+      @tags = tags
+      @status = status
+      @sessions = []
+    end
+
+    def active?(time)
+      from_timestamp_ms = time.to_i * 1000
+      to_timestamp_ms = time.to_i * 1000 + Const::ONE_DAY_MSEC
+
+      # 対象期間内にCreatedのイベントがあればアクティブ
+      judge = @sessions.any? do |session|
+        session['time'] >= from_timestamp_ms && session['time'] < to_timestamp_ms && session['event'] == 'Created'
+      end
+      return true if judge == true
+      
+      # 対象期間内の直前のイベントがCreatedであればアクティブ
+      sessions_before_from = @sessions.select { |session| session['time'] < from_timestamp_ms }
+      return false if sessions_before_from.length == 0
+      sessions_before_from.first['event'] == 'Created'
+    end
+
+    def sessions_count(time)
+      from_timestamp_ms = time.to_i * 1000
+      to_timestamp_ms = time.to_i * 1000 + Const::ONE_DAY_MSEC
+
+      created_sessions = @sessions.select do |session|
+        session['time'] >= from_timestamp_ms && session['time'] < to_timestamp_ms && session['event'] == 'Created'
+      end
+      created_sessions.length
+    end
+  end
+end

data/lib/soracom_summary/traffic.rb

@@ -0,0 +1,90 @@
+module SoracomSummary
+  class Traffic
+    attr_accessor :imsi, :date, :type, :upload_byte_size_total, :download_byte_size_total
+    class << self
+      def summary(traffics, time)
+        month_text = time.strftime('%Y%m')
+        day = time.strftime('%Y%m%d').to_i
+
+        # 対象期間の請求をフィルタする
+        month_traffics = traffics.select { |traffic| traffic.date[0, 6] == month_text && traffic.date.to_i <= day }
+        day_traffics = traffics.select { |traffic| traffic.date.to_i == day }
+
+        traffic_summary = {
+          'traffics-month-total' => sum(month_traffics),
+          'traffics-month-upload' => upload_sum(month_traffics),
+          'traffics-month-download' => download_sum(month_traffics),
+          'traffics-day-total' => sum(day_traffics),
+          'traffics-day-upload' => upload_sum(day_traffics),
+          'traffics-day-download' => download_sum(day_traffics)
+        }
+
+        # 通信量を種類ごとにグループ化する
+        month_traffics_by_type = month_traffics
+          .group_by { |traffic| traffic.type }
+          .map do |type, group|
+            [
+              "traffics-month-total-#{type}", sum(group),
+              "traffics-month-upload-#{type}", upload_sum(group),
+              "traffics-month-download-#{type}", download_sum(group)
+            ]
+          end.flatten
+        traffic_summary.merge!(Hash[*month_traffics_by_type])
+
+        day_traffics_by_type = day_traffics
+          .group_by { |traffic| traffic.type }
+          .map do |type, group|
+            [
+              "traffics-day-total-#{type}", sum(group),
+              "traffics-day-upload-#{type}", upload_sum(group),
+              "traffics-day-download-#{type}", download_sum(group)
+            ]
+          end.flatten
+        traffic_summary.merge!(Hash[*day_traffics_by_type])
+
+        traffic_summary
+      end
+
+      def group_by_imsi(traffics, time)
+        day = time.strftime('%Y%m%d')
+
+        # 対象期間の請求をフィルタする
+        day_traffics = traffics.select { |traffic| traffic.date == day }
+        
+        day_traffics_by_imsi = day_traffics
+        .group_by { |traffic| traffic.imsi }
+        .map { |imsi, group| [imsi, sum(group)] }
+        .to_h
+
+        day_traffics_by_imsi
+      end
+
+      def sum(traffics)
+        traffics.inject(0) do |sum, traffic|
+          sum + traffic.upload_byte_size_total.to_i + traffic.download_byte_size_total.to_i
+        end
+      end
+
+      def upload_sum(traffics)
+        traffics.inject(0) { |sum, traffic| sum + traffic.upload_byte_size_total.to_i }
+      end
+
+      def download_sum(traffics)
+        traffics.inject(0) { |sum, traffic| sum + traffic.download_byte_size_total.to_i }
+      end
+    end
+
+    def initialize(
+      imsi:,
+      date:,
+      type:,
+      upload_byte_size_total:,
+      download_byte_size_total:)
+      @imsi = imsi
+      @date = date
+      @type = type
+      @upload_byte_size_total = upload_byte_size_total
+      @download_byte_size_total = download_byte_size_total
+    end
+  end
+end

data/lib/soracom_summary/version.rb

@@ -0,0 +1,3 @@
+module SoracomSummary
+  VERSION = "0.1.0"
+end

data/soracom_summary.gemspec

@@ -0,0 +1,28 @@
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "soracom_summary/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "soracom_summary"
+  spec.version       = SoracomSummary::VERSION
+  spec.authors       = ["1stship"]
+  spec.email         = ["1peifunyaq@gmail.com"]
+
+  spec.summary       = 'tool to create SORACOM usage summary'
+  spec.description   = 'Scraping SORACOM usage via SORACOM API and uploading these data to SORACOM Harvest'
+  spec.homepage      = 'https://github.com/1stship/soracom-summary/'
+  spec.license       = "MIT"
+
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 2.0"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency 'ruby-debug-ide'
+  spec.add_development_dependency 'debase'
+end

metadata

@@ -0,0 +1,138 @@
+--- !ruby/object:Gem::Specification
+name: soracom_summary
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- 1stship
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2019-11-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: ruby-debug-ide
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: debase
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Scraping SORACOM usage via SORACOM API and uploading these data to SORACOM
+  Harvest
+email:
+- 1peifunyaq@gmail.com
+executables:
+- soracom_summary
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- exe/soracom_summary
+- lib/soracom_summary.rb
+- lib/soracom_summary/api_client.rb
+- lib/soracom_summary/billing.rb
+- lib/soracom_summary/cli.rb
+- lib/soracom_summary/const.rb
+- lib/soracom_summary/http_client.rb
+- lib/soracom_summary/subscriber.rb
+- lib/soracom_summary/traffic.rb
+- lib/soracom_summary/version.rb
+- soracom_summary.gemspec
+homepage: https://github.com/1stship/soracom-summary/
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.12
+signing_key: 
+specification_version: 4
+summary: tool to create SORACOM usage summary
+test_files: []