clockker 0.1.6 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +14 -0
- data/lib/clockker/config.rb +2 -1
- data/lib/clockker/version.rb +1 -1
- data/lib/clockker/watcher.rb +17 -13
- data/lib/clockker/white_black_list.rb +13 -4
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2bf862a8d56ae8baa7ee601e5be41275e6ba67c8
|
4
|
+
data.tar.gz: 578fc40ad2a4402cc4e42909f7092fc44eb9dfbd
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c820784d7aabacb0b51def2e4d2f1c6f97af2787c3b3a0709b925b309ccd2fe4c69737baf3a2a31b1b7a5b5ebf4b0f1f9571c5b7f810a10793bd621fea875232
|
7
|
+
data.tar.gz: 7ffc56c83c595f8b6fe9a9aa779f86004cba57328276f72da860df4aeff8c6cb07f2678e2f72fc373d50cea8848da6fb1504f55946ea93b28715a5e285dfc81f
|
data/README.md
CHANGED
@@ -66,12 +66,26 @@ Instead of using the command-line options, you can specify a default configurati
|
|
66
66
|
"/Users/paul/Library",
|
67
67
|
"/Users/paul/Pictures",
|
68
68
|
"/Users/paul/.dropbox"
|
69
|
+
],
|
70
|
+
"url_whitelist": [
|
71
|
+
"facebook.com",
|
72
|
+
"linkedin.com",
|
73
|
+
"trello.com",
|
74
|
+
"other.com" # or simply "", which matches everything
|
69
75
|
]
|
70
76
|
}
|
71
77
|
```
|
72
78
|
|
73
79
|
Note: Version 0.1.0 is the Clockker configuration file version.
|
74
80
|
|
81
|
+
## A note on privacy
|
82
|
+
|
83
|
+
Clockker does its very best to respect what you consider private. By default, nothing is watched - no files, no URLs. You add things to watch by adding them to the whitelist. If you want to hide something that's inside the whitelist, (e.g. /Users/joe/Photos which is inside the tracked folder /Users/joe/), add that path to the blacklist.
|
84
|
+
|
85
|
+
URLs work the same way — the Clockker will only report them if they are in the url_whitelist. That can be a bit tedious, so if you do want Clockk to track everything, simply add "" to the whitelist.
|
86
|
+
|
87
|
+
Safari Private Browsing and Google Incognito windows are NEVER tracked.
|
88
|
+
|
75
89
|
## Development
|
76
90
|
|
77
91
|
After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
data/lib/clockker/config.rb
CHANGED
@@ -2,7 +2,7 @@ require 'JSON'
|
|
2
2
|
|
3
3
|
module Clockker
|
4
4
|
class Config
|
5
|
-
attr_accessor :whitelist, :blacklist, :config_version, :token, :region, :identifier, :submit_frequency, :log_level, :development
|
5
|
+
attr_accessor :whitelist, :blacklist, :url_whitelist, :config_version, :token, :region, :identifier, :submit_frequency, :log_level, :development
|
6
6
|
def initialize(overrides = {})
|
7
7
|
if File.exists?(File.join(Dir.home, '.clockker'))
|
8
8
|
clockker_config = JSON.parse(File.read(File.join(Dir.home, '.clockker')))
|
@@ -13,6 +13,7 @@ module Clockker
|
|
13
13
|
# Set values from clockker_config
|
14
14
|
@whitelist = clockker_config["whitelist"]
|
15
15
|
@blacklist = clockker_config["blacklist"]
|
16
|
+
@url_whitelist = clockker_config["url_whitelist"]
|
16
17
|
@config_version = clockker_config["version"]
|
17
18
|
@token = clockker_config["token"]
|
18
19
|
@region = clockker_config["region"]
|
data/lib/clockker/version.rb
CHANGED
data/lib/clockker/watcher.rb
CHANGED
@@ -21,7 +21,7 @@ module Clockker
|
|
21
21
|
@safari_watcher_queue = Queue.new
|
22
22
|
@timer = Thread.new { create_timer(@safari_watcher_queue) }
|
23
23
|
@file_watcher = Thread.new { create_file_watcher('/', white_black_list, logger) }
|
24
|
-
@safari_watcher = Thread.new { create_safari_watcher(logger) }
|
24
|
+
@safari_watcher = Thread.new { create_safari_watcher(white_black_list, logger) }
|
25
25
|
@submitter = Thread.new { create_submitter(config.submit_frequency, config.region, config.token, config.identifier, logger, config.development) }
|
26
26
|
|
27
27
|
@file_watcher.join
|
@@ -44,7 +44,7 @@ module Clockker
|
|
44
44
|
event_meta['events'].each do |event|
|
45
45
|
next if white_black_list.ignore?(event['path'])
|
46
46
|
count += 1
|
47
|
-
file_path = Pathname.new(event['path'])
|
47
|
+
file_path = ::Pathname.new(event['path'])
|
48
48
|
dirs, name = file_path.split
|
49
49
|
last_dir = dirs.split[-1]
|
50
50
|
title = (last_dir + name).to_s
|
@@ -55,11 +55,12 @@ module Clockker
|
|
55
55
|
logger.info "#{Time.now} now watching /"
|
56
56
|
logger.info "#{Time.now} whitelist paths: #{white_black_list.whitelist}"
|
57
57
|
logger.info "#{Time.now} blacklist paths: #{white_black_list.blacklist}"
|
58
|
+
logger.info "#{Time.now} url whitelist: #{white_black_list.url_whitelist}"
|
58
59
|
@fsevent.run
|
59
60
|
logger.info "#{Time.now} fswatcher done"
|
60
61
|
end
|
61
62
|
|
62
|
-
def create_safari_watcher(logger)
|
63
|
+
def create_safari_watcher(white_black_list, logger)
|
63
64
|
safari_db = SQLite3::Database.new(File.expand_path("~/Library/Safari/History.db"))
|
64
65
|
logger.info "#{Time.now} now watching Safari"
|
65
66
|
last_time = Time.now.to_i - 978307200 # convert to macOS Core Data time
|
@@ -79,15 +80,21 @@ module Clockker
|
|
79
80
|
end
|
80
81
|
|
81
82
|
safari_db.execute("
|
82
|
-
SELECT hv.id, datetime(hv.visit_time+978307200, \"unixepoch\") as visited_at, hv.visit_time, hv.title, hi.url
|
83
|
+
SELECT hv.id, datetime(hv.visit_time+978307200, \"unixepoch\") as visited_at, hv.visit_time, hv.title, hi.url,
|
84
|
+
hv.load_successful, hv.http_non_get, hv.redirect_source, hv.redirect_destination, hv.origin, hv.generation, hv.attributes, hv.score
|
83
85
|
FROM history_visits hv, history_items hi
|
84
86
|
WHERE hv.history_item=hi.id
|
85
87
|
AND hv.visit_time > #{last_time}
|
86
88
|
ORDER BY hv.visit_time ASC
|
87
89
|
") do |row|
|
88
|
-
id, visited_at, visit_time, title, url = row
|
90
|
+
id, visited_at, visit_time, title, url, load_successful, http_non_get, redirect_source, redirect_destination, origin, generation, attributes, score = row
|
91
|
+
|
92
|
+
next if load_successful == 0
|
93
|
+
next if redirect_destination
|
94
|
+
next if white_black_list.ignore_url?(url)
|
95
|
+
|
89
96
|
visited_at = Time.parse(visited_at+"Z") # the time is already UTC; let's make doubly sure!
|
90
|
-
|
97
|
+
logger.debug "- #{url} http_non_get: #{http_non_get} origin: #{origin} generation: #{generation} attributes: #{attributes} score: #{score}"
|
91
98
|
@submitter_queue << {touched_at: visited_at, contents: url, meta_type: "url", metadata: {url: url, title: title}}
|
92
99
|
last_time = visit_time
|
93
100
|
end
|
@@ -137,21 +144,18 @@ module Clockker
|
|
137
144
|
|
138
145
|
begin
|
139
146
|
dev_data = {artifacts: @changeset.map{|c| {contents: c[:contents], touched_at: c[:touched_at].strftime("%Y-%m-%d %H:%M:%S"), meta_type: c[:meta_type], metadata: c[:metadata], identifier: identifier, agent: Clockker.version}}}.to_json
|
140
|
-
|
141
|
-
|
142
|
-
prod_data = @changeset.find_all{|c| c[:type] == "file"}
|
143
|
-
prod_data = {artifacts: prod_data.map{|c| {file_path: c[:contents], touched_at: c[:touched_at].strftime("%Y-%m-%d %H:%M:%S"), identifier: identifier, agent: Clockker.version}}}.to_json
|
144
|
-
|
147
|
+
prod_data = dev_data # at this point the same data goes to prod as to dev
|
148
|
+
|
145
149
|
header = {"Authorization" => "Bearer #{token}", "Content-Type" => "application/json"}
|
146
150
|
|
147
|
-
|
151
|
+
logger.debug "PROD submission: #{prod_data.inspect}"
|
148
152
|
Net::HTTP.start(clockk_agent_uri.hostname, clockk_agent_uri.port, use_ssl: clockk_agent_uri.scheme == 'https') do |http|
|
149
153
|
http.post(clockk_agent_uri.path, prod_data, header)
|
150
154
|
end
|
151
155
|
|
152
156
|
if dev_mode
|
153
157
|
begin
|
154
|
-
|
158
|
+
logger.debug "DEV submission: #{dev_data.inspect}"
|
155
159
|
Net::HTTP.start(clockk_local_uri.hostname, clockk_local_uri.port, use_ssl: clockk_local_uri.scheme == 'https') do |http|
|
156
160
|
http.post(clockk_local_uri.path, dev_data, header)
|
157
161
|
end
|
@@ -5,7 +5,7 @@ module Clockker
|
|
5
5
|
# whitelist_path should return an array of absolute paths that are to be watched and reported to the clockk server.
|
6
6
|
|
7
7
|
# blacklist_path should return an array of absolute paths that are to be ignored. Implicitly any path that isn't whitelisted is blacklisted; this method is only useful if it returns subpaths of whitelisted paths. i.e. /Users/paul/Documents is whitelisted, but /Users/paul/Documents/customers/Bell is blacklisted.
|
8
|
-
attr_accessor :whitelist, :blacklist
|
8
|
+
attr_accessor :whitelist, :blacklist, :url_whitelist
|
9
9
|
|
10
10
|
DEFAULT_BLACKLIST = [
|
11
11
|
"/tmp/",
|
@@ -19,13 +19,18 @@ module Clockker
|
|
19
19
|
]
|
20
20
|
|
21
21
|
def initialize(clockker_config)
|
22
|
-
@whitelist =
|
23
|
-
@blacklist =
|
24
|
-
@
|
22
|
+
@whitelist = []
|
23
|
+
@blacklist = []
|
24
|
+
@url_whitelist = []
|
25
|
+
|
26
|
+
@whitelist = clockker_config.whitelist.map{|wl| Regexp.new(wl)} if clockker_config.whitelist
|
27
|
+
@blacklist = clockker_config.blacklist.map{|bl| Regexp.new(bl)} if clockker_config.blacklist
|
28
|
+
@url_whitelist = clockker_config.url_whitelist.map{|uwl| Regexp.new("#{uwl}\/")} if clockker_config.url_whitelist
|
25
29
|
|
26
30
|
# Now set defaults in the absence of a config file
|
27
31
|
@whitelist ||= [ Regexp.new(Pathname.new(Dir.home).to_s) ]
|
28
32
|
@blacklist ||= [ Regexp.new(Pathname.new(File.join(Dir.home, 'Library')).to_s) ]
|
33
|
+
@blacklist += DEFAULT_BLACKLIST.map{|bl| Regexp.new(bl)}
|
29
34
|
end
|
30
35
|
|
31
36
|
def ignore?(filepath)
|
@@ -35,5 +40,9 @@ module Clockker
|
|
35
40
|
return false if file_matches_whitelist
|
36
41
|
return true
|
37
42
|
end
|
43
|
+
|
44
|
+
def ignore_url?(url)
|
45
|
+
!@url_whitelist.any? {|uwl| url =~ Regexp.new(uwl)}
|
46
|
+
end
|
38
47
|
end
|
39
48
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: clockker
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Paul Doerwald
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-10-
|
11
|
+
date: 2018-10-15 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rb-fsevent
|