chronicle-imessage 0.1.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +2 -0
- data/README.md +2 -3
- data/chronicle-imessage.gemspec +3 -3
- data/lib/chronicle/imessage/imessage_extractor.rb +53 -23
- data/lib/chronicle/imessage/imessage_transformer.rb +32 -14
- data/lib/chronicle/imessage/local_contacts.rb +40 -0
- data/lib/chronicle/imessage/version.rb +1 -1
- data/lib/chronicle/imessage.rb +0 -2
- metadata +9 -10
- data/Gemfile.lock +0 -88
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 76cac6e564a07e8e411e2378c43b15b812c4b4631dedcdee3863eedb0e3932e6
|
4
|
+
data.tar.gz: 7f7551136fae37aa832a98f9bd5ba386fceb7d4d1c0e3396eee4d64c0cbc03d0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: fec05237d35c28b1ff9be65c5a60a622abdabec4c3726d7a91583c579891903e5ecd0860422c88bd35ef5b63d9f1c1fea807233dc6997ae6cd8ee3a0557dc4de
|
7
|
+
data.tar.gz: f1d909c69c41258d43d0175ad707dd1069f540527abc122460a587815ff0617fb7b452e2c53ac93de5bc366a6f0fae6ff89905d11edc7eca9b008096ab2f6c80
|
data/.gitignore
CHANGED
data/README.md
CHANGED
@@ -13,7 +13,6 @@ IMessage importer for [chronicle-etl](https://github.com/chronicle-app/chronicle
|
|
13
13
|
|
14
14
|
```bash
|
15
15
|
gem install chronicle-etl
|
16
|
-
chronicle-etl
|
17
|
-
|
18
|
-
chronicle-etl --extractor imessage --extractor-opts load_since:"2022-02-07" --transformer imessage --loader table
|
16
|
+
chronicle-etl plugins:install imessage
|
17
|
+
chronicle-etl --extractor imessage --since "2022-02-07" --transformer imessage
|
19
18
|
```
|
data/chronicle-imessage.gemspec
CHANGED
@@ -35,9 +35,9 @@ Gem::Specification.new do |spec|
|
|
35
35
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
36
36
|
spec.require_paths = ["lib"]
|
37
37
|
|
38
|
-
spec.add_dependency "chronicle-etl", "~> 0.
|
39
|
-
spec.add_dependency "sqlite3", "~> 1.4
|
40
|
-
spec.add_dependency "phonelib", "~> 0.6
|
38
|
+
spec.add_dependency "chronicle-etl", "~> 0.4.2"
|
39
|
+
spec.add_dependency "sqlite3", "~> 1.4"
|
40
|
+
spec.add_dependency "phonelib", "~> 0.6"
|
41
41
|
|
42
42
|
spec.add_development_dependency "bundler", "~> 2.3"
|
43
43
|
spec.add_development_dependency "rake", "~> 13.0.6"
|
@@ -10,14 +10,16 @@ module Chronicle
|
|
10
10
|
r.description = 'a local imessage database'
|
11
11
|
end
|
12
12
|
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
13
|
+
setting :db, default: File.join(Dir.home, 'Library', 'Messages', 'chat.db'), required: true
|
14
|
+
setting :load_attachments, default: false
|
15
|
+
setting :only_attachments, default: false
|
16
|
+
setting :my_phone_number
|
17
|
+
setting :my_name
|
18
|
+
setting :icloud_account_id
|
19
|
+
setting :icloud_account_dsid
|
20
|
+
setting :icloud_account_display_name
|
21
|
+
|
22
|
+
def prepare
|
21
23
|
prepare_data
|
22
24
|
end
|
23
25
|
|
@@ -26,6 +28,8 @@ module Chronicle
|
|
26
28
|
meta = {}
|
27
29
|
meta[:participants] = @chats[message['chat_id']]
|
28
30
|
meta[:attachments] = @attachments[message['message_id']] if @attachments
|
31
|
+
meta[:my_phone_contact] = @my_phone_contact
|
32
|
+
meta[:my_icloud_account] = @my_icloud_account
|
29
33
|
|
30
34
|
yield Chronicle::ETL::Extraction.new(data: message, meta: meta)
|
31
35
|
end
|
@@ -38,34 +42,60 @@ module Chronicle
|
|
38
42
|
private
|
39
43
|
|
40
44
|
def prepare_data
|
41
|
-
@db = SQLite3::Database.new(@
|
42
|
-
@
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
)
|
47
|
-
@contacts = LocalContacts.new.contacts
|
45
|
+
@db = SQLite3::Database.new(@config.db, results_as_hash: true)
|
46
|
+
@local_contacts = LocalContacts.new
|
47
|
+
@contacts = @local_contacts.contacts
|
48
|
+
|
49
|
+
@messages = load_messages
|
48
50
|
@chats = load_chats
|
49
51
|
|
50
|
-
|
52
|
+
@my_phone_contact = load_my_phone_contact(@local_contacts)
|
53
|
+
@my_icloud_account = load_my_icloud_account(@local_contacts)
|
54
|
+
|
55
|
+
if @config.load_attachments
|
51
56
|
@attachments = load_attachments(@messages.map{|m| m['message_id']})
|
52
57
|
end
|
53
58
|
end
|
54
59
|
|
55
|
-
def
|
56
|
-
|
57
|
-
|
60
|
+
def load_my_phone_contact(local_contacts)
|
61
|
+
{
|
62
|
+
phone_number: @config.my_phone_number || local_contacts.my_phone_contact.fetch(:phone_number),
|
63
|
+
name: @config.my_name || local_contacts.my_phone_contact.fetch(:full_name)
|
64
|
+
}
|
65
|
+
end
|
66
|
+
|
67
|
+
def load_my_icloud_account(local_contacts)
|
68
|
+
{
|
69
|
+
id: @config.icloud_account_id || local_contacts.my_icloud_account.fetch(:AccountID),
|
70
|
+
dsid: @config.icloud_account_dsid || local_contacts.my_icloud_account.fetch(:AccountDSID),
|
71
|
+
display_name: @config.icloud_account_display_name || @config.my_name || local_contacts.my_icloud_account.fetch(:DisplayName)
|
72
|
+
}
|
73
|
+
end
|
74
|
+
|
75
|
+
def load_messages
|
76
|
+
conditions = []
|
77
|
+
|
78
|
+
if @config.until
|
79
|
+
load_until_ios = unix_to_ios_timestamp(@config.until.to_i) * 1000000000
|
80
|
+
conditions << "date < #{load_until_ios}"
|
81
|
+
end
|
82
|
+
|
83
|
+
if @config.since
|
84
|
+
load_since_ios = unix_to_ios_timestamp(@config.since.to_i) * 1000000000
|
85
|
+
conditions << "date > #{load_since_ios}"
|
86
|
+
end
|
87
|
+
|
88
|
+
if @config.only_attachments
|
89
|
+
conditions << "cache_has_attachments = true"
|
90
|
+
end
|
58
91
|
|
59
92
|
sql = "SELECT * from message as m
|
60
93
|
LEFT OUTER JOIN handle as h ON m.handle_id=h.ROWID
|
61
94
|
INNER JOIN chat_message_join as cm ON m.ROWID = cm.message_id"
|
62
95
|
|
63
|
-
conditions = []
|
64
|
-
conditions << "date < #{load_until_ios}" if load_until
|
65
|
-
conditions << "date > #{load_since_ios}" if load_since
|
66
96
|
sql += " WHERE #{conditions.join(" AND ")}" if conditions.any?
|
67
97
|
sql += " ORDER BY date DESC"
|
68
|
-
sql += " LIMIT #{limit}" if limit
|
98
|
+
sql += " LIMIT #{@config.limit}" if @config.limit
|
69
99
|
|
70
100
|
messages = @db.execute(sql)
|
71
101
|
end
|
@@ -8,14 +8,6 @@ module Chronicle
|
|
8
8
|
r.description = 'a row from a local imessage database'
|
9
9
|
end
|
10
10
|
|
11
|
-
DEFAULT_OPTIONS = {
|
12
|
-
}.freeze
|
13
|
-
|
14
|
-
def initialize(*args)
|
15
|
-
super(*args)
|
16
|
-
@options = @options.reverse_merge(DEFAULT_OPTIONS)
|
17
|
-
end
|
18
|
-
|
19
11
|
def transform
|
20
12
|
@message = @extraction.data
|
21
13
|
@participants = @extraction.meta[:participants]
|
@@ -123,14 +115,40 @@ module Chronicle
|
|
123
115
|
end
|
124
116
|
|
125
117
|
def build_identity_mine
|
118
|
+
case identity_provider(@message['service'])
|
119
|
+
when 'icloud'
|
120
|
+
build_identity_mine_icloud
|
121
|
+
when 'phone'
|
122
|
+
build_identity_mine_phone
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
def build_identity_mine_icloud
|
127
|
+
icloud_account = @extraction.meta[:my_icloud_account]
|
128
|
+
raise(UntransformableRecordError, "Missing iCloud account information") unless icloud_account
|
129
|
+
|
126
130
|
record = ::Chronicle::ETL::Models::Entity.new({
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
131
|
+
represent: 'identity',
|
132
|
+
provider: 'icloud',
|
133
|
+
provider_id: icloud_account[:dsid],
|
134
|
+
title: icloud_account[:display_name],
|
135
|
+
slug: icloud_account[:id]
|
136
|
+
})
|
137
|
+
record.dedupe_on << [:provider, :represents, :slug]
|
138
|
+
record
|
139
|
+
end
|
140
|
+
|
141
|
+
def build_identity_mine_phone
|
142
|
+
phone_account = @extraction.meta[:my_phone_contact]
|
143
|
+
raise(UntransformableRecordError, "Missing own phone contact information") unless phone_account
|
144
|
+
|
145
|
+
record = ::Chronicle::ETL::Models::Entity.new({
|
146
|
+
represent: 'identity',
|
147
|
+
provider: 'phone',
|
148
|
+
title: phone_account[:name],
|
149
|
+
slug: phone_account[:phone_number]
|
132
150
|
})
|
133
|
-
record.dedupe_on
|
151
|
+
record.dedupe_on << [:provider, :represents, :slug]
|
134
152
|
record
|
135
153
|
end
|
136
154
|
|
@@ -23,6 +23,14 @@ module Chronicle
|
|
23
23
|
end
|
24
24
|
end
|
25
25
|
|
26
|
+
def my_phone_contact
|
27
|
+
@my_phone_contact ||= load_my_phone_contact
|
28
|
+
end
|
29
|
+
|
30
|
+
def my_icloud_account
|
31
|
+
@my_icloud_account ||= load_my_icloud_account
|
32
|
+
end
|
33
|
+
|
26
34
|
# The synced address book doesn't have a stable folder location so we
|
27
35
|
# have to search for it
|
28
36
|
def find_local_icloud_address_book
|
@@ -30,6 +38,38 @@ module Chronicle
|
|
30
38
|
Dir.glob(pattern).first
|
31
39
|
end
|
32
40
|
|
41
|
+
def load_my_icloud_account
|
42
|
+
@my_icloud_account || begin
|
43
|
+
output = `defaults read MobileMeAccounts Accounts | plutil -convert json -r -o - -- -`
|
44
|
+
JSON.parse(output, symbolize_names: true).first
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def load_my_phone_contact
|
49
|
+
sql = <<-SQL
|
50
|
+
SELECT
|
51
|
+
ZABCDPHONENUMBER.ZFULLNUMBER AS identifier,
|
52
|
+
ZABCDRECORD.ZFIRSTNAME as first_name,
|
53
|
+
ZABCDRECORD.ZLASTNAME as last_name
|
54
|
+
FROM
|
55
|
+
ZABCDRECORD,
|
56
|
+
ZABCDPHONENUMBER
|
57
|
+
WHERE
|
58
|
+
ZABCDRECORD.Z_PK = ZABCDPHONENUMBER.ZOWNER
|
59
|
+
AND ZABCDRECORD.zcontainerwherecontactisme IS NOT NULL
|
60
|
+
SQL
|
61
|
+
|
62
|
+
results = @db.execute(sql)
|
63
|
+
|
64
|
+
guessed_number = results.first
|
65
|
+
return unless guessed_number
|
66
|
+
|
67
|
+
{
|
68
|
+
phone_number: Phonelib.parse(guessed_number['identifier'], "US").e164,
|
69
|
+
full_name: "#{guessed_number['first_name']} #{guessed_number['last_name']}"
|
70
|
+
}
|
71
|
+
end
|
72
|
+
|
33
73
|
def load_phone_numbers
|
34
74
|
sql = <<-SQL
|
35
75
|
SELECT
|
data/lib/chronicle/imessage.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: chronicle-imessage
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Andrew Louis
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-
|
11
|
+
date: 2022-03-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: chronicle-etl
|
@@ -16,42 +16,42 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version:
|
19
|
+
version: 0.4.2
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version:
|
26
|
+
version: 0.4.2
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: sqlite3
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
31
|
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: 1.4
|
33
|
+
version: '1.4'
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: 1.4
|
40
|
+
version: '1.4'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: phonelib
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
45
|
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: 0.6
|
47
|
+
version: '0.6'
|
48
48
|
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: 0.6
|
54
|
+
version: '0.6'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: bundler
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
@@ -90,7 +90,6 @@ files:
|
|
90
90
|
- ".gitignore"
|
91
91
|
- CODE_OF_CONDUCT.md
|
92
92
|
- Gemfile
|
93
|
-
- Gemfile.lock
|
94
93
|
- README.md
|
95
94
|
- Rakefile
|
96
95
|
- bin/console
|
@@ -123,7 +122,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
123
122
|
- !ruby/object:Gem::Version
|
124
123
|
version: '0'
|
125
124
|
requirements: []
|
126
|
-
rubygems_version: 3.
|
125
|
+
rubygems_version: 3.3.9
|
127
126
|
signing_key:
|
128
127
|
specification_version: 4
|
129
128
|
summary: iMessage importer for Chronicle
|
data/Gemfile.lock
DELETED
@@ -1,88 +0,0 @@
|
|
1
|
-
PATH
|
2
|
-
remote: .
|
3
|
-
specs:
|
4
|
-
chronicle-imessage (0.1.0)
|
5
|
-
chronicle-etl (~> 0.3)
|
6
|
-
phonelib (~> 0.6.54)
|
7
|
-
sqlite3 (~> 1.4.2)
|
8
|
-
|
9
|
-
GEM
|
10
|
-
remote: https://rubygems.org/
|
11
|
-
specs:
|
12
|
-
activesupport (7.0.1)
|
13
|
-
concurrent-ruby (~> 1.0, >= 1.0.2)
|
14
|
-
i18n (>= 1.6, < 2)
|
15
|
-
minitest (>= 5.1)
|
16
|
-
tzinfo (~> 2.0)
|
17
|
-
chronic_duration (0.10.6)
|
18
|
-
numerizer (~> 0.1.1)
|
19
|
-
chronicle-etl (0.3.0)
|
20
|
-
activesupport
|
21
|
-
chronic_duration (~> 0.10.6)
|
22
|
-
colorize (~> 0.8.1)
|
23
|
-
marcel (~> 1.0.2)
|
24
|
-
mini_exiftool (~> 2.10)
|
25
|
-
nokogiri (~> 1.13)
|
26
|
-
runcom (~> 6.2)
|
27
|
-
sequel (~> 5.35)
|
28
|
-
sqlite3 (~> 1.4)
|
29
|
-
thor (~> 0.20)
|
30
|
-
tty-progressbar (~> 0.17)
|
31
|
-
tty-table (~> 0.11)
|
32
|
-
colorize (0.8.1)
|
33
|
-
concurrent-ruby (1.1.9)
|
34
|
-
i18n (1.9.1)
|
35
|
-
concurrent-ruby (~> 1.0)
|
36
|
-
marcel (1.0.2)
|
37
|
-
mini_exiftool (2.10.2)
|
38
|
-
mini_portile2 (2.7.1)
|
39
|
-
minitest (5.15.0)
|
40
|
-
nokogiri (1.13.1)
|
41
|
-
mini_portile2 (~> 2.7.0)
|
42
|
-
racc (~> 1.4)
|
43
|
-
numerizer (0.1.1)
|
44
|
-
pastel (0.8.0)
|
45
|
-
tty-color (~> 0.5)
|
46
|
-
phonelib (0.6.55)
|
47
|
-
racc (1.6.0)
|
48
|
-
rake (13.0.6)
|
49
|
-
refinements (7.18.0)
|
50
|
-
runcom (6.6.0)
|
51
|
-
refinements (~> 7.16)
|
52
|
-
xdg (~> 4.4)
|
53
|
-
sequel (5.53.0)
|
54
|
-
sqlite3 (1.4.2)
|
55
|
-
strings (0.2.1)
|
56
|
-
strings-ansi (~> 0.2)
|
57
|
-
unicode-display_width (>= 1.5, < 3.0)
|
58
|
-
unicode_utils (~> 1.4)
|
59
|
-
strings-ansi (0.2.0)
|
60
|
-
thor (0.20.3)
|
61
|
-
tty-color (0.6.0)
|
62
|
-
tty-cursor (0.7.1)
|
63
|
-
tty-progressbar (0.18.2)
|
64
|
-
strings-ansi (~> 0.2)
|
65
|
-
tty-cursor (~> 0.7)
|
66
|
-
tty-screen (~> 0.8)
|
67
|
-
unicode-display_width (>= 1.6, < 3.0)
|
68
|
-
tty-screen (0.8.1)
|
69
|
-
tty-table (0.12.0)
|
70
|
-
pastel (~> 0.8)
|
71
|
-
strings (~> 0.2.0)
|
72
|
-
tty-screen (~> 0.8)
|
73
|
-
tzinfo (2.0.4)
|
74
|
-
concurrent-ruby (~> 1.0)
|
75
|
-
unicode-display_width (2.1.0)
|
76
|
-
unicode_utils (1.4.0)
|
77
|
-
xdg (4.5.0)
|
78
|
-
|
79
|
-
PLATFORMS
|
80
|
-
ruby
|
81
|
-
|
82
|
-
DEPENDENCIES
|
83
|
-
bundler (~> 2.3)
|
84
|
-
chronicle-imessage!
|
85
|
-
rake (~> 13.0.6)
|
86
|
-
|
87
|
-
BUNDLED WITH
|
88
|
-
2.3.6
|