toggl_cache 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0d9815ad14776766ab2dbcbf4ee4dc059625d071
-  data.tar.gz: 03c21e7d27ee0fc6835074749a6c07d28b9aaf35
+  metadata.gz: e5c97e1c35e17dd0e4e74f0194eefb1b9d100199
+  data.tar.gz: c4b6bfe066da7c539256d39dc98c00539795e7ec
 SHA512:
-  metadata.gz: '0668183f52a17e6928f592e7cd78cc5088b89c6286f927edbf1a44268820a90fd21feb04698ced8f2caa005b858245b57a09113ea97db9c45ab856038fc51c1d'
-  data.tar.gz: 9e59068942a8e45c1811ce82d3c1dc025b0d2dbb9b647b180c52291a89a5d5d6f1563d12f77a7b4c4961d5b8eb2664f1c789ca29de110cd5c2973ba88457d819
+  metadata.gz: d4f4ce467a89a23278e731de36b6a2e0df90d6164eb742a87e496f9c42667c232d7764df84b0a0c2b90a0ad0780d640fd65a7715c098ca70f29a04864fa34f41
+  data.tar.gz: 167afe2ed80546b63b0007e69dab6bd1d5ce2fd7a49deff03cf975a4578bbe16194c0cfcaf9e1d5b88812cc1ad9714230f5368a9b7ad6930c01fdb387c1f02f1

data/Gemfile

@@ -23,4 +23,5 @@ group :test do
   gem 'coveralls', require: false
   gem 'simplecov', require: false
   gem 'simplecov-json', require: false
+  gem "codeclimate-test-reporter", "~> 1.0.0"
 end

data/HISTORY.md

@@ -1,5 +1,32 @@
 # History
 
+## 2017-03-25 - 0.2.0
+
+Synchronizing deleted records and old changes.
+
+Approach #1: calculate aggregates over periods and perform a full sync over the period if there is a diff.
+
+- Pros: more efficient, smarter, also provides a synchronization correctness control.
+- Cons: harder to implement.
+
+Approach #2: brute-force, i.e. simply clear the content for a given period and perform a full sync.
+
+- Pros: simpler to implement.
+- Cons: synchronization is currently to long.
+
+Trying approach #2. Optimizing synchronization:
+
+- [X] processing fetched records once a page has been fetched, not at the end.
+- [ ] perform requests in parallel.
+
+The approach #2 is not applicable. Reviewing Toggl API's documentation, the rate limit is about 1 request per second, so parallelizing does not seem to be an option.
+
+NB: performing a full synchronization leading to about 40K reports takes about 0.65 hour.
+
+The approach #1 has been implemented. The algorithm first checks each full year. If a difference is detected, it checks each month of the year. For each month with a difference, a sync is done by clearing the cache for the month and fetching the reports.
+
+The implementation has been done in `TogglCache.sync_check_and_fix`.
+
 ## 2017-03-18 - 0.1.1
 
 - Changed default period of import to one week from now, instead of one day.

data/README.md

@@ -1,12 +1,12 @@
 # TogglCache
 
-Fetches reports data from Toggl API and caches them in a MongoDB store.
+Fetches reports data from Toggl API and caches them in a PostgreSQL database.
 
 This allows you to build applications performing complex operations on a large number of Toggl content without the Toggl API latency. You may also use it to backup your precious data!
 
 [![Build Status](https://travis-ci.org/rchampourlier/toggl_cache.svg)](https://travis-ci.org/rchampourlier/toggl_cache)
-[![Code Climate](https://codeclimate.com/github/rchampourlier/toggl_cache/badges/gpa.svg)](https://codeclimate.com/github/rchampourlier/toggl_cache)
-[![Coverage Status](https://coveralls.io/repos/github/rchampourlier/toggl_cache/badge.svg?branch=master)](https://coveralls.io/github/rchampourlier/toggl_cache?branch=master)
+[![Code Climate](https://codeclimate.com/repos/58d7ff3b88ccb7027b000baa/badges/182b308109bf20bd9dbf/gpa.svg)](https://codeclimate.com/repos/58d7ff3b88ccb7027b000baa/feed)
+[![Test Coverage](https://codeclimate.com/repos/58d7ff3b88ccb7027b000baa/badges/182b308109bf20bd9dbf/coverage.svg)](https://codeclimate.com/repos/58d7ff3b88ccb7027b000baa/coverage)
 
 ## Installation
 
@@ -39,6 +39,12 @@ client = TogglCache::Client.new(
 TogglCache.sync_reports(client, 'TOGGL-WORKSPACE-ID')
 ```
 
+## The CLI
+
+```
+ruby lib/toggl_cli.rb [batch|help]
+```
+
 ## Contributing
 
 1. Fork it ( https://github.com/rchampourlier/toggl_cache/fork )

data/VERSION

@@ -1 +1 @@
-0.1.1
+0.2.0

data/lib/toggl_api/reports_client.rb

@@ -10,28 +10,26 @@ module TogglAPI
     API_URL = "https://toggl.com/reports/api/v2"
 
     # @param params [Hash]: Toggl API params
-    #   - date_since
-    #   - date_until
+    #   - since
+    #   - until
     #   - workspace_id
     #   - ... more params available, see Toggl API documentation for details
     def fetch_reports(params)
+      raise "Must give a block" unless block_given?
       page = 1
-      all_results = []
+      current_total = 0
       loop do
         results_raw = fetch_reports_details_raw(
           params.merge(page: page)
         )
-        results = results_raw["data"]
+        yield(results_raw["data"])
 
-        all_results += results
-        break if all_results.count == results_raw["total_count"]
+        current_total += results_raw["data"].count
+        break if current_total == results_raw["total_count"]
         page += 1
       end
-      all_results
     end
 
-    private
-
     def fetch_reports_details_raw(params)
       fetch_reports_raw(api_url(:details), params)
     end
@@ -40,6 +38,8 @@ module TogglAPI
       fetch_reports_raw(api_url(:summary), params)
     end
 
+    private
+
     # @param url [String]
     # @param params [Hash]: Toggl API params
     def fetch_reports_raw(url, params)

data/lib/toggl_cache/data/report_repository.rb

@@ -5,8 +5,10 @@ require "active_support/inflector"
 module TogglCache
   module Data
 
-    # Superclass for repositories. Simply provide some shared
-    # methods.
+    # Repository for Toggl reports.
+    #
+    # TODO: should be used through instances
+    # TODO: #table should be private
     class ReportRepository
 
       MAPPED_REPORT_ATTRIBUTES = %w(
@@ -22,7 +24,7 @@ module TogglCache
       # It inserts a new issue row with the specified data.
       # If the issue already exists (unicity key is `id`)
       # the row is updated instead.
-      def self.create_or_update(report)
+      def create_or_update(report)
         id = report["id"].to_s
         if exist_with_id?(id)
           update_where({ id: id }, row(report: report))
@@ -31,39 +33,65 @@ module TogglCache
         end
       end
 
-      def self.find_by_id(id)
+      def find_by_id(id)
         table.where(id: id).first
       end
 
-      def self.exist_with_id?(id)
+      def exist_with_id?(id)
         table.where(id: id).count != 0
       end
 
-      def self.delete_where(where_data)
+      def delete_where(where_data)
         table.where(where_data).delete
       end
 
-      def self.update_where(where_data, values)
+      def update_where(where_data, values)
         table.where(where_data).update(values)
       end
 
-      def self.first_where(where_data)
+      def first(by: :start)
+        table.order(by).first
+      end
+
+      def first_where(where_data)
         table.where(where_data).first
       end
 
-      def self.index
+      def index
         table.entries
       end
 
-      def self.count
+      def count
         table.count
       end
 
-      def self.table
+      # Returns reports whose `start` time is within the specified range.
+      #
+      # @param since: [Time]
+      # @param until: [Time]
+      def starting(time_since:, time_until:)
+        table.where("start >= ? AND start <= ?", time_since, time_until).entries
+      end
+
+      def delete_starting(time_since:, time_until:)
+        table.where("start >= ? AND start <= ?", time_since, time_until).delete
+      end
+
+      # @param pid [Integer]
+      # @param tid [Integer] optional
+      def where(project_id:, task_id: nil)
+        where_criteria = { pid: project_id }
+        where_criteria[:tid] = tid if task_id
+        table.where(where_criteria).entries
+      end
+
+      private
+
+      def table
         DB[:toggl_cache_reports]
       end
 
-      def self.row(report:, insert_created_at: false, insert_updated_at: true)
+      def row(report:, insert_created_at: false, insert_updated_at: true)
         new_report = map_report_attributes(report: report)
         new_report = add_timestamps(
           report: new_report,
@@ -73,7 +101,7 @@ module TogglCache
         new_report
       end
 
-      def self.map_report_attributes(report:)
+      def map_report_attributes(report:)
         new_report = report.select { |k, _| MAPPED_REPORT_ATTRIBUTES.include?(k) }
         new_report = new_report.merge(
           duration: report["dur"] / 1_000,
@@ -85,7 +113,7 @@ module TogglCache
         new_report
       end
 
-      def self.add_timestamps(report:, insert_created_at:, insert_updated_at:)
+      def add_timestamps(report:, insert_created_at:, insert_updated_at:)
         new_report = {}.merge(report)
         new_report["created_at"] = Time.now if insert_created_at
         new_report["updated_at"] = Time.now if insert_updated_at

data/lib/toggl_cache.rb

@@ -20,17 +20,130 @@ module TogglCache
   # The fetched reports either update the already
   # existing ones, or create new ones.
   #
-  # @param client [TogglAPI::Client] a configured client
-  # @param workspace_id [String] Toggl workspace ID (mandatory)
   # @param date_since [Date] Date since when to fetch
   #   the reports.
-  def self.sync_reports(client: default_client,
-                        date_since: default_date_since)
-    reports = fetch_reports(
+  # @param date_until [Date] Date until when to fetch. Defaults to `Time.now`.
+  # @param client [TogglAPI::Client] a configured client
+  def self.sync_reports(
+    date_since: default_date_since,
+    date_until: Time.now,
+    logger: default_logger,
+    client: default_client
+  )
+    logger.info "Syncing reports from #{date_since} to #{date_until}."
+    clear_cache(
+      time_since: Time.parse("#{date_since} 00:00:00Z"),
+      time_until: Time.parse("#{date_until} 23:59:59Z")
+    )
+    fetch_reports(
       client: client,
-      date_since: date_since
+      date_since: date_since,
+      date_until: date_until
+    ) do |reports|
+      process_reports(reports)
+    end
+  end
+
+  # Performs a full synchronization check, from the time of the first
+  # report in the cache to now. Proceeds by comparing reports total
+  # duration from Toggl (using the Reports API) and the total contained
+  # in the cache. If a difference is detected, proceeds monthly and
+  # clear and reconstructs the cache for the invovled month.
+  #
+  # TODO: enable detecting a change in project/task level aggregates.
+  def self.sync_check_and_fix(logger: default_logger)
+    reports = TogglCache::Data::ReportRepository.new
+    first_report = reports.first
+
+    year_start = first_report[:start].year
+    year_end = Time.now.year
+    month_start = first_report[:start].month
+    month_end = Time.now.month
+
+    (year_start..year_end).each do |year|
+      year_toggl = TogglCache.toggl_total(year: year)
+      year_cache = TogglCache.cache_total(year: year)
+      if year_toggl == year_cache
+        logger.info "Checked total for #{year}: ✅ (#{year_toggl})"
+        next
+      end
+      logger.info "Checked total for #{year}: ❌ (Toggl: #{year_toggl}, cache: #{year_cache})"
+      (1..12).each do |month|
+        next if year == year_start && month < month_start
+        next if year == year_end && month > month_end
+        month_toggl = TogglCache.toggl_total(year: year, month: month)
+        month_cache = TogglCache.cache_total(year: year, month: month)
+        if month_toggl == month_cache
+          logger.info "Checked total for #{year}/#{month}: ✅ (#{month_toggl})"
+        else
+          logger.info "Checked total for #{year}/#{month}: ❌ (Toggl: #{month_toggl}, cache: #{month_cache})"
+          TogglCache.clear_cache(year: year, month: month)
+          TogglCache.sync_reports_for_month(year: year, month: month)
+        end
+      end
+    end
+  end
+
+  # An easy-to-use method to sync reports for a given month. Simply
+  # performs a call to `sync_reports`.
+  #
+  # @param year [Integer]
+  # @param month [Integer
+  def self.sync_reports_for_month(year:, month:)
+    date_since = Date.civil(year, month, 1)
+    date_until = Date.civil(year, month, -1)
+    sync_reports(
+      date_since: date_since,
+      date_until: date_until
+    )
+  end
+
+  # Remove TogglCache's reports between the specified dates.
+  def self.clear_cache(time_since:, time_until:)
+    logger.info "Clearing cache from #{date_since} to #{date_until}."
+    reports = Data::ReportRepository.new
+    reports.delete_starting(
+      time_since: time_since,
+      time_until: time_until
+    )
+  end
+
+  # Remove TogglCache's reports for the specified month.
+  def self.clear_cache_for_month(year:, month:, logger: default_logger)
+    date_since = Date.civil(year, month, 1)
+    date_until = Date.civil(year, month, -1)
+    clear_cache(
+      time_since: Time.parse("#{date_since} 00:00:00Z"),
+      time_until: Time.parse("#{date_until} 23:59:59Z")
     )
-    process_reports(reports)
+  end
+
+  # Returns the total duration from Toggl (using Reports API)
+  # for the specified year or month.
+  def self.toggl_total(year:, month: nil)
+    reports_client = TogglAPI::ReportsClient.new
+    date_since = month ? Date.civil(year, month, 1) : Date.civil(year, 1, 1)
+    date_until = month ? Date.civil(year, month, -1) : Date.civil(year, 12, -1)
+    total_grand = reports_client.fetch_reports_summary_raw(
+      since: date_since.to_s,
+      until: date_until.to_s,
+      workspace_id: ENV["TOGGL_WORKSPACE_ID"]
+    )["total_grand"]
+    total_grand ? total_grand / 3600 / 1000 : 0
+  end
+
+  # Returns the total duration in TogglCache reports
+  # for the specified year or month.
+  def self.cache_total(year:, month: nil)
+    reports = TogglCache::Data::ReportRepository.new
+    date_since = month ? Date.civil(year, month, 1).to_s : Date.civil(year, 1, 1)
+    date_until = month ? Date.civil(year, month, -1).to_s : Date.civil(year, 12, -1)
+    time_since = Time.parse("#{date_since} 00:00:00Z")
+    time_until = Time.parse("#{date_until} 23:59:59Z")
+    reports.starting(
+      time_since: time_since,
+      time_until: time_until
+    ).inject(0) { |sum, r| sum + r[:duration] } / 3600
   end
 
   # Fetch from Toggl
@@ -47,32 +160,37 @@ module TogglCache
     client: default_client,
     workspace_id: default_workspace_id,
     date_since:,
-    date_until: Time.now
+    date_until: Time.now,
+    &block
   )
+    raise "You must give a block to process fetched records" unless block_given?
     if date_since && date_until.year > date_since.year
-      fetch_reports(
-        client: client,
-        workspace_id: workspace_id,
-        date_since: date_since,
-        date_until: Date.new(date_since.year, 12, 31)
-      ) + fetch_reports(
-        client: client,
-        workspace_id: workspace_id,
-        date_since: Date.new(date_since.year + 1, 1, 1),
-        date_until: date_until
-      )
+      [
+        [date_since, Date.new(date_since.year, 12, 31)],
+        [Date.new(date_since.year + 1, 1, 1), date_until]
+      ].each do |dates|
+        fetch_reports(
+          client: client,
+          workspace_id: workspace_id,
+          date_since: dates.first,
+          date_until: dates.last,
+          &block
+        )
+      end
     else
       options = {
         workspace_id: workspace_id, until: date_until.strftime("%Y-%m-%d")
       }
       options[:since] = date_since.strftime("%Y-%m-%d") unless date_since.nil?
-      client.fetch_reports(options)
+      client.fetch_reports(options, &block)
     end
   end
 
-  def self.process_reports(reports)
+  def self.process_reports(reports, logger: default_logger)
+    logger.debug "Processing #{reports.count} Toggl reports"
+    repository = Data::ReportRepository.new
     reports.each do |report|
-      Data::ReportRepository.create_or_update(report)
+      repository.create_or_update(report)
     end
   end
 

data/lib/toggl_cli.rb

@@ -3,7 +3,6 @@ require "thor"
 require "tty-pager"
 require "tty-progressbar"
 require "tty-spinner"
-# require "tty-table"
 require File.expand_path("../../config/boot", __FILE__)
 require "toggl_api/client"
 require "toggl_cache/data/report_repository"
@@ -48,7 +47,7 @@ class TogglCLI < Thor
         say("MOVING!!!")
       end
     end
-    # TogglCache::Data::ReportRepository.create_or_update(report)
+    TogglCache::Data::ReportRepository.new.create_or_update(report)
   end
 
   private
@@ -112,10 +111,9 @@ class TogglCLI < Thor
   end
 
   def find_reports(project_id, task_id)
+    reports = TogglCache::Data::ReportRepository.new
     with_spinner("Fetching matching reports...") do
-      where_criteria = { pid: project_id.to_i }
-      where_criteria[:tid] = task_id if task_id
-      TogglCache::Data::ReportRepository.table.where(where_criteria).entries
+      reports.where(project_id: project_id&.to_i, task_id: task_id&.to_i)
     end
   end
 

data/scripts/sync.rb

@@ -2,7 +2,7 @@
 
 # Usage:
 #
-#   - Syncs reports since 1 month:
+#   - Syncs reports since 31 days::
 #       ruby scripts/sync.rb
 #
 #   - Sync reports since the specified date (use Ruby-parseable
@@ -16,5 +16,11 @@ require 'bundler/setup'
 $LOAD_PATH.unshift File.expand_path('../..', __FILE__)
 require 'config/boot'
 
-date_since = Time.parse(ARGV[0]) unless ARGV[0].nil? || ARGV[0].empty?
+date_since = (
+  if ARGV[0].nil? || ARGV[0].empty?
+    Time.now - 31 * 24 * 3600
+  else
+    Time.parse(ARGV[0])
+  end
+)
 TogglCache.sync_reports(date_since: date_since)

data/scripts/sync_check_and_fix.rb

@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Check that reports stored in TogglCache match report summaries
+# provided by toggl Reports API.
+#
+# Usage:
+#
+#     ruby scripts/sync_check_and_fix.rb
+
+# Load dependencies
+require "rubygems"
+require "bundler/setup"
+require "logger"
+
+$LOAD_PATH.unshift File.expand_path("../..", __FILE__)
+require "config/boot"
+
+TogglCache.sync_check_and_fix

data/spec/spec_helper.rb

@@ -35,7 +35,7 @@ RSpec.configure do |config|
   end
 
   config.after(:each) do
-    TogglCache::Data::ReportRepository.delete_where("TRUE")
+    TogglCache::Data::ReportRepository.new.delete_where("TRUE")
   end
 
   config.after(:all) do

data/spec/unit/toggl_api/reports_client_spec.rb

@@ -4,8 +4,13 @@ require "toggl_api/reports_client"
 
 describe TogglAPI::ReportsClient do
   let(:options) { {} }
-  let(:expected_api_call_url) { "https://toggl_api_token:api_token@toggl.com/reports/api/v2/details?page=1&user_agent=TogglAPI" }
-
+  let(:expected_api_call_url) { "https://toggl.com/reports/api/v2/details?page=1&user_agent=TogglAPI" }
+  let(:expected_headers) do
+    {
+      "Authorization" => "Basic #{["toggl_api_token:api_token"].pack('m').delete("\r\n")}",
+      "Content-Type" => "application/json"
+    }
+  end
   let(:api_response_status) { 200 }
   let(:api_response_body) do
     {
@@ -20,24 +25,25 @@ describe TogglAPI::ReportsClient do
 
   before do
     stub_request(:get, expected_api_call_url)
-      .with(headers: { "Content-Type" => "application/json" })
+      .with(headers: expected_headers)
       .to_return(status: api_response_status, body: api_response_body, headers: {})
   end
 
-  describe "#fetch_reports(params)" do
+  describe "#fetch_reports(params, &block)" do
 
     context "only 1 page" do
       let(:total_count) { 50 }
 
       it "fetches the data only once" do
-        described_class.new.fetch_reports(options)
+        described_class.new.fetch_reports(options) {}
         # It will fail if it fetches several times because
         # the second request, with page=2, is not mocked.
       end
 
-      it "returns the received data" do
-        results = described_class.new.fetch_reports(options)
-        expect(results.count).to eq(50)
+      it "passes the received data to the specified block" do
+        results = described_class.new.fetch_reports(options) do |reports|
+          expect(reports.count).to eq(50)
+        end
       end
     end
 
@@ -53,18 +59,16 @@ describe TogglAPI::ReportsClient do
           data: data_page2
         }.to_json
         stub_request(:get, expected_api_call_url.gsub('page=1', 'page=2'))
-          .with(headers: { "Content-Type" => "application/json" })
+          .with(headers: expected_headers)
           .to_return(status: api_response_status, body: api_response_body_page2, headers: {})
       end
 
       it "fetches the additional pages" do
-        results = described_class.new.fetch_reports(options)
-        expect(results.count).to eq(60)
-      end
-
-      it "returns the merged data" do
-        results = described_class.new.fetch_reports(options)
-        expect(results.inject(0) { |s, i| s + i["id"] }).to eq(50 * 1 + 10 * 2)
+        total_count = 0
+        results = described_class.new.fetch_reports(options) do |reports|
+          total_count += reports.count
+        end
+        expect(total_count).to eq(60)
       end
     end
 
@@ -72,7 +76,7 @@ describe TogglAPI::ReportsClient do
       let(:api_response_status) { 500 }
 
       it "raises a TogglAPI::Error" do
-        expect { described_class.new.fetch_reports(options) }
+        expect { described_class.new.fetch_reports(options) {} }
           .to raise_error(TogglAPI::Error)
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: toggl_cache
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Romain Champourlier
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-03-18 00:00:00.000000000 Z
+date: 2017-04-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: i18n
@@ -204,6 +204,7 @@ files:
 - lib/toggl_cache/version.rb
 - lib/toggl_cli.rb
 - scripts/sync.rb
+- scripts/sync_check_and_fix.rb
 - spec/spec_helper.rb
 - spec/unit/toggl_api/reports_client_spec.rb
 - toggl_cache.gemspec