clockker 0.1.4 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a5917fe8052498f65072fe075093b1af5d725ea0
-  data.tar.gz: 0c09c0cbb7b630660b8f0326ba799a5c08fea671
+  metadata.gz: c7404b361d1f21b145d39ff872ed9901c19f7944
+  data.tar.gz: 7af9d8d18fdbfaf1779250f579a24fa8ef7e57c7
 SHA512:
-  metadata.gz: 4c674892ffb7178e58fa8430582014ae8d456e92dce3c303aa344f0e8c53cab6897298638b5b0c58a243beb6f4d8aa821ea0a58d639f3a22b9610d77d7574f2d
-  data.tar.gz: efe5566b8355c97fc08f2015047aae7450231c39b8001e8658c582488a587afe388bfe75bcc33171b4392dac4936f519be0492c950d97a3e12c36924eff155c4
+  metadata.gz: 66432b991bcbd96026f1761bd87211afad2b626e47df12b9627367cb767574ea2eb34f255b58fca4e17c369970e4bcbd7b1327d95f8b0eb981283317d68c7bf8
+  data.tar.gz: 69bd52efc272a2fa17ed05893e234b81a533d148548c5a1da5bb1afbd215d84b76fac85fb7adda1775cae0eefe5096ff38de45e084bff90cc106ea06b1e5ece6

data/README.md

@@ -8,6 +8,18 @@ Install it with:
 
     $ gem install clockker
 
+To access your Safari history when using macOS Mojave (10.14) and higher, you must give the Terminal app "Full Disk Access".
+
+1. Settings
+2. Security & Privacy
+3. Privacy
+4. Full Disk Access
+5. <unlock>
+6. + to add
+7. Browse to find /Applications/Utilities/Terminal
+8. Quit and restart Terminal
+
+
 ## Usage
 
 Run it with:
@@ -68,6 +80,8 @@ After checking out the repo, run `bin/setup` to install dependencies. You can al
 
 To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 
+To run a development version run `bundle exec exe/clockker <params>`.
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/liquidmedia/clockker. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.

data/clockker.gemspec

@@ -32,6 +32,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'rb-fsevent', '>= 0.10.2'
   spec.add_dependency 'thor'
+  spec.add_dependency 'sqlite3'
 
   spec.add_development_dependency "bundler", "~> 1.15"
   spec.add_development_dependency "rake", "~> 10.0"

data/lib/clockker/version.rb

@@ -1,5 +1,5 @@
 module Clockker
-  VERSION = "0.1.4"
+  VERSION = "0.1.5"
 
   def self.macos_version
     @mov ||= `sw_vers -productVersion`

data/lib/clockker/watcher.rb

@@ -1,6 +1,7 @@
 require 'rb-fsevent'
 require 'logger'
 require 'net/http'
+require 'sqlite3'
 
 module Clockker
   class Watcher
@@ -9,21 +10,33 @@ module Clockker
 
       trap "INT" do
         @fsevent.stop
-        # @watcher.kill  # this also works, but the former is a bit more elegant
-        @queue << :exit  # this shuts down the submitter, first letting it submit its changeset
+        @timer.kill  # the timer can just go away; it doesn't need a graceful shutdown.
+        @safari_watcher_queue << :exit
+        @submitter_queue << :exit  # this shuts down the submitter, first letting it submit its changeset
       end
 
       white_black_list = WhiteBlackList.new(config)
 
-      @queue = Queue.new
-      @watcher = Thread.new { create_watcher('/', white_black_list, logger) }
+      @submitter_queue = Queue.new
+      @safari_watcher_queue = Queue.new
+      @timer = Thread.new { create_timer(@safari_watcher_queue) }
+      @file_watcher = Thread.new { create_file_watcher('/', white_black_list, logger) }
+      @safari_watcher = Thread.new { create_safari_watcher(logger) }
       @submitter = Thread.new { create_submitter(config.submit_frequency, config.region, config.token, config.identifier, logger, config.development) }
       
-      @watcher.join
+      @file_watcher.join
+      @safari_watcher.join
       @submitter.join
     end
 
-    def create_watcher(dir, white_black_list, logger)
+    def create_timer(*queues)
+      loop do
+        sleep 5
+        queues.each {|q| q << :run}
+      end
+    end
+
+    def create_file_watcher(dir, white_black_list, logger)
       @fsevent = FSEvent.new
       @fsevent.watch dir, {file_events: true} do |directories, event_meta|
         t = Time.now.utc
@@ -31,9 +44,9 @@ module Clockker
         event_meta['events'].each do |event|
           next if white_black_list.ignore?(event['path'])
           count += 1
-          @queue << {created_at: t, event_id: event['id'], filepath: event['path'], flags: event['flags']}
+          @submitter_queue << {touched_at: t, contents: event['path'], type: "file", metadata: {flags: event['flags'], id: event['id']}}
         end
-        logger.debug "#{Time.now} checked #{event_meta['numEvents']} events and added #{count} to the queue"
+        logger.debug "#{Time.now} checked #{event_meta['numEvents']} events and added #{count} to the submitter_queue"
       end
       logger.info "#{Time.now} now watching /"
       logger.info "#{Time.now} whitelist paths: #{white_black_list.whitelist}"
@@ -42,8 +55,50 @@ module Clockker
       logger.info "#{Time.now} fswatcher done"
     end
 
-    def create_submitter(submit_frequency, region, token, identifier, logger, development)
-      logger.info "#{Time.now} submitting to clockk.com #{"and localhost:4000 " if development}every #{submit_frequency} seconds with token #{token} from identifier #{identifier} with version #{Clockker.version}"
+    def create_safari_watcher(logger)
+      safari_db = SQLite3::Database.new(File.expand_path("~/Library/Safari/History.db"))
+      logger.info "#{Time.now} now watching Safari"
+      last_time = Time.now.to_i - 978307200   # convert to macOS Core Data time
+      loop do
+        submit_now = false
+        exit_now = false
+
+        value = @safari_watcher_queue.pop
+        case value
+        when :exit
+          submit_now = true
+          exit_now = true
+        when :run
+          # we just move forward
+        else
+          # On the off chance the message isn't :run
+        end
+
+        safari_db.execute("
+          SELECT hv.id, datetime(hv.visit_time+978307200, \"unixepoch\") as visited_at, hv.visit_time, hv.title, hi.url
+          FROM history_visits hv, history_items hi
+          WHERE hv.history_item=hi.id
+          AND hv.visit_time > #{last_time}
+          ORDER BY hv.visit_time ASC
+        ") do |row|
+          id, visited_at, visit_time, title, url = row
+          visited_at = Time.parse(visited_at+"Z") # the time is already UTC; let's make doubly sure!
+          # TODO check the whitelisted domain regexes to see if we can submit this one.
+          logger.info " - #{visited_at} - #{url} - #{title}"
+          @submitter_queue << {touched_at: visited_at, contents: url, type: "url", metadata: {title: title}}
+          last_time = visit_time
+        end
+
+        if exit_now
+          logger.info "#{Time.now} Safari watcher preparing to exit"
+          break
+        end
+      end
+      logger.info "#{Time.now} Safari watcher done"
+    end
+
+    def create_submitter(submit_frequency, region, token, identifier, logger, dev_mode)
+      logger.info "#{Time.now} submitting to clockk.com #{"and localhost:4000 " if dev_mode}every #{submit_frequency} seconds with token #{token} from identifier #{identifier} with version #{Clockker.version}"
       t_start = Time.now
       clockk_agent_uri = URI("https://#{region}.clockk.io/api/v1/agent/artifacts")
       clockk_local_uri = URI("http://localhost:4000/api/v1/agent/artifacts")
@@ -52,7 +107,7 @@ module Clockker
         submit_now = false
         exit_now = false
 
-        value = @queue.pop
+        value = @submitter_queue.pop
         case value
         when :submit
           submit_now = true
@@ -68,29 +123,38 @@ module Clockker
         end
 
         if exit_now
-          logger.info "#{Time.now} preparing to exit"
+          logger.info "#{Time.now} Submitter preparing to exit"
         end
 
-        if submit_now
+        if submit_now && @changeset.length > 0
           logger.info "#{Time.now} submitting #{@changeset.length} items"
           @changeset.each_with_index do |cs, idx|
-            logger.info " - #{idx} - #{cs[:created_at].strftime("%H:%M:%S")} #{cs[:filepath]}"
+            logger.info " - #{idx} - #{cs[:touched_at].strftime("%H:%M:%S")} #{cs[:contents]}"
           end
 
           begin
-            data = {artifacts: @changeset.map{|c| {file_path: c[:filepath], touched_at: c[:created_at].strftime("%Y-%m-%d %H:%M:%S"), identifier: identifier, agent: Clockker.version}}}.to_json
+            dev_data = {artifacts: @changeset.map{|c| {contents: c[:contents], touched_at: c[:touched_at].strftime("%Y-%m-%d %H:%M:%S"), type: c[:type], metadata: c[:metadata], identifier: identifier, agent: Clockker.version}}}.to_json
+
+            # temporarily we're keeping the old data structure, until all of our APIs are up to date.
+            prod_data = @changeset.find_all{|c| c[:type] == "file"}
+            prod_data = {artifacts: prod_data.map{|c| {file_path: c[:contents], touched_at: c[:touched_at].strftime("%Y-%m-%d %H:%M:%S"), identifier: identifier, agent: Clockker.version}}}.to_json
+
             header = {"Authorization" => "Bearer #{token}", "Content-Type" => "application/json"}
 
+            # logger.info "PROD submission: #{prod_data.inspect}"
             Net::HTTP.start(clockk_agent_uri.hostname, clockk_agent_uri.port, use_ssl: clockk_agent_uri.scheme == 'https') do |http|
-              http.post(clockk_agent_uri.path, data, header)
+              http.post(clockk_agent_uri.path, prod_data, header)
             end
 
-            begin
-              Net::HTTP.start(clockk_local_uri.hostname, clockk_local_uri.port, use_ssl: clockk_local_uri.scheme == 'https') do |http|
-                http.post(clockk_local_uri.path, data, header)
+            if dev_mode
+              begin
+                # logger.info "DEV submission: #{dev_data.inspect}"
+                Net::HTTP.start(clockk_local_uri.hostname, clockk_local_uri.port, use_ssl: clockk_local_uri.scheme == 'https') do |http|
+                  http.post(clockk_local_uri.path, dev_data, header)
+                end
+              rescue Exception => q
+                logger.info "#{Time.now} Failed to reach localhost:4000; ignoring"
               end
-            rescue Exception => q
-              logger.info "#{Time.now} Failed to reach localhost:4000; ignoring"
             end
 
             @changeset = []
@@ -102,7 +166,7 @@ module Clockker
         end
 
         if exit_now
-          logger.info "#{Time.now} exiting"
+          logger.info "#{Time.now} Submitter done"
           break
         end
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: clockker
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.5
 platform: ruby
 authors:
 - Paul Doerwald
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-10-02 00:00:00.000000000 Z
+date: 2018-10-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb-fsevent
@@ -38,6 +38,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement