clockker 0.1.6 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b3089d570b9f0d1ad6335a44c1a4674c99b94e40
-  data.tar.gz: 82133e63ff5da97a15bcdee6bfcdf76503a62c9c
+  metadata.gz: 2bf862a8d56ae8baa7ee601e5be41275e6ba67c8
+  data.tar.gz: 578fc40ad2a4402cc4e42909f7092fc44eb9dfbd
 SHA512:
-  metadata.gz: 2b794a8121b1b8d2044337d2bf4c44831e7420dca280c19a790d5d849b818fc12266dcf4dce0c81cd4d45e3ef2a77de98ba9ddb58d778525752ddaca7446853d
-  data.tar.gz: 4a4aff4b8b7e5dbe55b3eaeb776bc367c62ab593d8ee0447593ba1df338de5e2515092e6e61dbaab3c83e9f3b753ff385484cb32a8104d2c823b549f6b864b04
+  metadata.gz: c820784d7aabacb0b51def2e4d2f1c6f97af2787c3b3a0709b925b309ccd2fe4c69737baf3a2a31b1b7a5b5ebf4b0f1f9571c5b7f810a10793bd621fea875232
+  data.tar.gz: 7ffc56c83c595f8b6fe9a9aa779f86004cba57328276f72da860df4aeff8c6cb07f2678e2f72fc373d50cea8848da6fb1504f55946ea93b28715a5e285dfc81f

data/README.md

@@ -66,12 +66,26 @@ Instead of using the command-line options, you can specify a default configurati
     "/Users/paul/Library",
     "/Users/paul/Pictures",
     "/Users/paul/.dropbox"
+  ],
+  "url_whitelist": [
+    "facebook.com",
+    "linkedin.com",
+    "trello.com",
+    "other.com"   # or simply "", which matches everything
   ]
 }
 ```
 
 Note: Version 0.1.0 is the Clockker configuration file version.
 
+## A note on privacy
+
+Clockker does its very best to respect what you consider private. By default, nothing is watched - no files, no URLs. You add things to watch by adding them to the whitelist. If you want to hide something that's inside the whitelist, (e.g. /Users/joe/Photos which is inside the tracked folder /Users/joe/), add that path to the blacklist.
+
+URLs work the same way — the Clockker will only report them if they are in the url_whitelist. That can be a bit tedious, so if you do want Clockk to track everything, simply add "" to the whitelist.
+
+Safari Private Browsing and Google Incognito windows are NEVER tracked.
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/lib/clockker/config.rb

@@ -2,7 +2,7 @@ require 'JSON'
 
 module Clockker
   class Config
-    attr_accessor :whitelist, :blacklist, :config_version, :token, :region, :identifier, :submit_frequency, :log_level, :development
+    attr_accessor :whitelist, :blacklist, :url_whitelist, :config_version, :token, :region, :identifier, :submit_frequency, :log_level, :development
     def initialize(overrides = {})
       if File.exists?(File.join(Dir.home, '.clockker'))
         clockker_config = JSON.parse(File.read(File.join(Dir.home, '.clockker')))
@@ -13,6 +13,7 @@ module Clockker
       # Set values from clockker_config
       @whitelist = clockker_config["whitelist"]
       @blacklist = clockker_config["blacklist"]
+      @url_whitelist = clockker_config["url_whitelist"]
       @config_version = clockker_config["version"]
       @token = clockker_config["token"]
       @region = clockker_config["region"]

data/lib/clockker/version.rb

@@ -1,5 +1,5 @@
 module Clockker
-  VERSION = "0.1.6"
+  VERSION = "0.2.0"
 
   def self.macos_version
     @mov ||= `sw_vers -productVersion`

data/lib/clockker/watcher.rb

@@ -21,7 +21,7 @@ module Clockker
       @safari_watcher_queue = Queue.new
       @timer = Thread.new { create_timer(@safari_watcher_queue) }
       @file_watcher = Thread.new { create_file_watcher('/', white_black_list, logger) }
-      @safari_watcher = Thread.new { create_safari_watcher(logger) }
+      @safari_watcher = Thread.new { create_safari_watcher(white_black_list, logger) }
       @submitter = Thread.new { create_submitter(config.submit_frequency, config.region, config.token, config.identifier, logger, config.development) }
       
       @file_watcher.join
@@ -44,7 +44,7 @@ module Clockker
         event_meta['events'].each do |event|
           next if white_black_list.ignore?(event['path'])
           count += 1
-          file_path = Pathname.new(event['path'])
+          file_path = ::Pathname.new(event['path'])
           dirs, name = file_path.split
           last_dir = dirs.split[-1]
           title = (last_dir + name).to_s
@@ -55,11 +55,12 @@ module Clockker
       logger.info "#{Time.now} now watching /"
       logger.info "#{Time.now} whitelist paths: #{white_black_list.whitelist}"
       logger.info "#{Time.now} blacklist paths: #{white_black_list.blacklist}"
+      logger.info "#{Time.now} url whitelist:   #{white_black_list.url_whitelist}"
       @fsevent.run
       logger.info "#{Time.now} fswatcher done"
     end
 
-    def create_safari_watcher(logger)
+    def create_safari_watcher(white_black_list, logger)
       safari_db = SQLite3::Database.new(File.expand_path("~/Library/Safari/History.db"))
       logger.info "#{Time.now} now watching Safari"
       last_time = Time.now.to_i - 978307200   # convert to macOS Core Data time
@@ -79,15 +80,21 @@ module Clockker
         end
 
         safari_db.execute("
-          SELECT hv.id, datetime(hv.visit_time+978307200, \"unixepoch\") as visited_at, hv.visit_time, hv.title, hi.url
+          SELECT hv.id, datetime(hv.visit_time+978307200, \"unixepoch\") as visited_at, hv.visit_time, hv.title, hi.url,
+                hv.load_successful, hv.http_non_get, hv.redirect_source, hv.redirect_destination, hv.origin, hv.generation, hv.attributes, hv.score
           FROM history_visits hv, history_items hi
           WHERE hv.history_item=hi.id
           AND hv.visit_time > #{last_time}
           ORDER BY hv.visit_time ASC
         ") do |row|
-          id, visited_at, visit_time, title, url = row
+          id, visited_at, visit_time, title, url, load_successful, http_non_get, redirect_source, redirect_destination, origin, generation, attributes, score = row
+
+          next if load_successful == 0
+          next if redirect_destination
+          next if white_black_list.ignore_url?(url)
+
           visited_at = Time.parse(visited_at+"Z") # the time is already UTC; let's make doubly sure!
-          # TODO check the whitelisted domain regexes to see if we can submit this one.
+          logger.debug "- #{url} http_non_get: #{http_non_get} origin: #{origin} generation: #{generation} attributes: #{attributes} score: #{score}"
           @submitter_queue << {touched_at: visited_at, contents: url, meta_type: "url", metadata: {url: url, title: title}}
           last_time = visit_time
         end
@@ -137,21 +144,18 @@ module Clockker
 
           begin
             dev_data = {artifacts: @changeset.map{|c| {contents: c[:contents], touched_at: c[:touched_at].strftime("%Y-%m-%d %H:%M:%S"), meta_type: c[:meta_type], metadata: c[:metadata], identifier: identifier, agent: Clockker.version}}}.to_json
-
-            # temporarily we're keeping the old data structure, until all of our APIs are up to date.
-            prod_data = @changeset.find_all{|c| c[:type] == "file"}
-            prod_data = {artifacts: prod_data.map{|c| {file_path: c[:contents], touched_at: c[:touched_at].strftime("%Y-%m-%d %H:%M:%S"), identifier: identifier, agent: Clockker.version}}}.to_json
-
+            prod_data = dev_data   # at this point the same data goes to prod as to dev
+            
             header = {"Authorization" => "Bearer #{token}", "Content-Type" => "application/json"}
 
-            # logger.info "PROD submission: #{prod_data.inspect}"
+            logger.debug "PROD submission: #{prod_data.inspect}"
             Net::HTTP.start(clockk_agent_uri.hostname, clockk_agent_uri.port, use_ssl: clockk_agent_uri.scheme == 'https') do |http|
               http.post(clockk_agent_uri.path, prod_data, header)
             end
 
             if dev_mode
               begin
-                # logger.info "DEV submission: #{dev_data.inspect}"
+                logger.debug "DEV submission: #{dev_data.inspect}"
                 Net::HTTP.start(clockk_local_uri.hostname, clockk_local_uri.port, use_ssl: clockk_local_uri.scheme == 'https') do |http|
                   http.post(clockk_local_uri.path, dev_data, header)
                 end

data/lib/clockker/white_black_list.rb

@@ -5,7 +5,7 @@ module Clockker
     # whitelist_path should return an array of absolute paths that are to be watched and reported to the clockk server.
 
     # blacklist_path should return an array of absolute paths that are to be ignored. Implicitly any path that isn't whitelisted is blacklisted; this method is only useful if it returns subpaths of whitelisted paths. i.e. /Users/paul/Documents is whitelisted, but /Users/paul/Documents/customers/Bell is blacklisted.
-    attr_accessor :whitelist, :blacklist
+    attr_accessor :whitelist, :blacklist, :url_whitelist
 
     DEFAULT_BLACKLIST = [
       "/tmp/",
@@ -19,13 +19,18 @@ module Clockker
     ]
 
     def initialize(clockker_config)
-      @whitelist = clockker_config.whitelist.map{|wl| Regexp.new(wl)}
-      @blacklist = clockker_config.blacklist.map{|bl| Regexp.new(bl)}
-      @blacklist += DEFAULT_BLACKLIST.map{|bl| Regexp.new(bl)}
+      @whitelist = []
+      @blacklist = []
+      @url_whitelist = []
+
+      @whitelist = clockker_config.whitelist.map{|wl| Regexp.new(wl)} if clockker_config.whitelist
+      @blacklist = clockker_config.blacklist.map{|bl| Regexp.new(bl)} if clockker_config.blacklist
+      @url_whitelist = clockker_config.url_whitelist.map{|uwl| Regexp.new("#{uwl}\/")} if clockker_config.url_whitelist
 
       # Now set defaults in the absence of a config file
       @whitelist ||= [ Regexp.new(Pathname.new(Dir.home).to_s) ]
       @blacklist ||= [ Regexp.new(Pathname.new(File.join(Dir.home, 'Library')).to_s) ]
+      @blacklist += DEFAULT_BLACKLIST.map{|bl| Regexp.new(bl)}
     end
 
     def ignore?(filepath)
@@ -35,5 +40,9 @@ module Clockker
       return false if file_matches_whitelist
       return true
     end
+
+    def ignore_url?(url)
+      !@url_whitelist.any? {|uwl| url =~ Regexp.new(uwl)}
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: clockker
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.2.0
 platform: ruby
 authors:
 - Paul Doerwald
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-10-12 00:00:00.000000000 Z
+date: 2018-10-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb-fsevent