chronicle-imessage 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +8 -0
- data/CODE_OF_CONDUCT.md +74 -0
- data/Gemfile +6 -0
- data/Gemfile.lock +88 -0
- data/README.md +19 -0
- data/Rakefile +2 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/chronicle-imessage.gemspec +44 -0
- data/lib/chronicle/imessage/imessage_extractor.rb +125 -0
- data/lib/chronicle/imessage/imessage_transformer.rb +162 -0
- data/lib/chronicle/imessage/local_contacts.rb +75 -0
- data/lib/chronicle/imessage/version.rb +5 -0
- data/lib/chronicle/imessage.rb +11 -0
- metadata +130 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: bcaf59959a65537025a3ec616fc7edfed08aebcd6a9aca59569e8102553ada41
|
|
4
|
+
data.tar.gz: c78cc1e83b6fef4d41b710944484c935409e7833a30c5269792d59872579f4f4
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: a71a3c50cad2861c304b687c291fdcab1ef418c20a384f20c0a71ea2563a75ffec6937630288723d91620c5148224cf13ff143250b07e44bbd62dd893358421f
|
|
7
|
+
data.tar.gz: 6b4948f26b2e9d4aced7326c9f6f7bb5e031b6447c1885be8b85912209fc440a03047af01833e536fd8f31c9cf9205885fa0ea3edd5f69cf1da2f930065b2f0d
|
data/.gitignore
ADDED
data/CODE_OF_CONDUCT.md
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
# Contributor Covenant Code of Conduct
|
|
2
|
+
|
|
3
|
+
## Our Pledge
|
|
4
|
+
|
|
5
|
+
In the interest of fostering an open and welcoming environment, we as
|
|
6
|
+
contributors and maintainers pledge to making participation in our project and
|
|
7
|
+
our community a harassment-free experience for everyone, regardless of age, body
|
|
8
|
+
size, disability, ethnicity, gender identity and expression, level of experience,
|
|
9
|
+
nationality, personal appearance, race, religion, or sexual identity and
|
|
10
|
+
orientation.
|
|
11
|
+
|
|
12
|
+
## Our Standards
|
|
13
|
+
|
|
14
|
+
Examples of behavior that contributes to creating a positive environment
|
|
15
|
+
include:
|
|
16
|
+
|
|
17
|
+
* Using welcoming and inclusive language
|
|
18
|
+
* Being respectful of differing viewpoints and experiences
|
|
19
|
+
* Gracefully accepting constructive criticism
|
|
20
|
+
* Focusing on what is best for the community
|
|
21
|
+
* Showing empathy towards other community members
|
|
22
|
+
|
|
23
|
+
Examples of unacceptable behavior by participants include:
|
|
24
|
+
|
|
25
|
+
* The use of sexualized language or imagery and unwelcome sexual attention or
|
|
26
|
+
advances
|
|
27
|
+
* Trolling, insulting/derogatory comments, and personal or political attacks
|
|
28
|
+
* Public or private harassment
|
|
29
|
+
* Publishing others' private information, such as a physical or electronic
|
|
30
|
+
address, without explicit permission
|
|
31
|
+
* Other conduct which could reasonably be considered inappropriate in a
|
|
32
|
+
professional setting
|
|
33
|
+
|
|
34
|
+
## Our Responsibilities
|
|
35
|
+
|
|
36
|
+
Project maintainers are responsible for clarifying the standards of acceptable
|
|
37
|
+
behavior and are expected to take appropriate and fair corrective action in
|
|
38
|
+
response to any instances of unacceptable behavior.
|
|
39
|
+
|
|
40
|
+
Project maintainers have the right and responsibility to remove, edit, or
|
|
41
|
+
reject comments, commits, code, wiki edits, issues, and other contributions
|
|
42
|
+
that are not aligned to this Code of Conduct, or to ban temporarily or
|
|
43
|
+
permanently any contributor for other behaviors that they deem inappropriate,
|
|
44
|
+
threatening, offensive, or harmful.
|
|
45
|
+
|
|
46
|
+
## Scope
|
|
47
|
+
|
|
48
|
+
This Code of Conduct applies both within project spaces and in public spaces
|
|
49
|
+
when an individual is representing the project or its community. Examples of
|
|
50
|
+
representing a project or community include using an official project e-mail
|
|
51
|
+
address, posting via an official social media account, or acting as an appointed
|
|
52
|
+
representative at an online or offline event. Representation of a project may be
|
|
53
|
+
further defined and clarified by project maintainers.
|
|
54
|
+
|
|
55
|
+
## Enforcement
|
|
56
|
+
|
|
57
|
+
Instances of abusive, harassing, or otherwise unacceptable behavior may be
|
|
58
|
+
reported by contacting the project team at andrew@hyfen.net. All
|
|
59
|
+
complaints will be reviewed and investigated and will result in a response that
|
|
60
|
+
is deemed necessary and appropriate to the circumstances. The project team is
|
|
61
|
+
obligated to maintain confidentiality with regard to the reporter of an incident.
|
|
62
|
+
Further details of specific enforcement policies may be posted separately.
|
|
63
|
+
|
|
64
|
+
Project maintainers who do not follow or enforce the Code of Conduct in good
|
|
65
|
+
faith may face temporary or permanent repercussions as determined by other
|
|
66
|
+
members of the project's leadership.
|
|
67
|
+
|
|
68
|
+
## Attribution
|
|
69
|
+
|
|
70
|
+
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
|
|
71
|
+
available at [http://contributor-covenant.org/version/1/4][version]
|
|
72
|
+
|
|
73
|
+
[homepage]: http://contributor-covenant.org
|
|
74
|
+
[version]: http://contributor-covenant.org/version/1/4/
|
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
PATH
|
|
2
|
+
remote: .
|
|
3
|
+
specs:
|
|
4
|
+
chronicle-imessage (0.1.0)
|
|
5
|
+
chronicle-etl (~> 0.3)
|
|
6
|
+
phonelib (~> 0.6.54)
|
|
7
|
+
sqlite3 (~> 1.4.2)
|
|
8
|
+
|
|
9
|
+
GEM
|
|
10
|
+
remote: https://rubygems.org/
|
|
11
|
+
specs:
|
|
12
|
+
activesupport (7.0.1)
|
|
13
|
+
concurrent-ruby (~> 1.0, >= 1.0.2)
|
|
14
|
+
i18n (>= 1.6, < 2)
|
|
15
|
+
minitest (>= 5.1)
|
|
16
|
+
tzinfo (~> 2.0)
|
|
17
|
+
chronic_duration (0.10.6)
|
|
18
|
+
numerizer (~> 0.1.1)
|
|
19
|
+
chronicle-etl (0.3.0)
|
|
20
|
+
activesupport
|
|
21
|
+
chronic_duration (~> 0.10.6)
|
|
22
|
+
colorize (~> 0.8.1)
|
|
23
|
+
marcel (~> 1.0.2)
|
|
24
|
+
mini_exiftool (~> 2.10)
|
|
25
|
+
nokogiri (~> 1.13)
|
|
26
|
+
runcom (~> 6.2)
|
|
27
|
+
sequel (~> 5.35)
|
|
28
|
+
sqlite3 (~> 1.4)
|
|
29
|
+
thor (~> 0.20)
|
|
30
|
+
tty-progressbar (~> 0.17)
|
|
31
|
+
tty-table (~> 0.11)
|
|
32
|
+
colorize (0.8.1)
|
|
33
|
+
concurrent-ruby (1.1.9)
|
|
34
|
+
i18n (1.9.1)
|
|
35
|
+
concurrent-ruby (~> 1.0)
|
|
36
|
+
marcel (1.0.2)
|
|
37
|
+
mini_exiftool (2.10.2)
|
|
38
|
+
mini_portile2 (2.7.1)
|
|
39
|
+
minitest (5.15.0)
|
|
40
|
+
nokogiri (1.13.1)
|
|
41
|
+
mini_portile2 (~> 2.7.0)
|
|
42
|
+
racc (~> 1.4)
|
|
43
|
+
numerizer (0.1.1)
|
|
44
|
+
pastel (0.8.0)
|
|
45
|
+
tty-color (~> 0.5)
|
|
46
|
+
phonelib (0.6.55)
|
|
47
|
+
racc (1.6.0)
|
|
48
|
+
rake (13.0.6)
|
|
49
|
+
refinements (7.18.0)
|
|
50
|
+
runcom (6.6.0)
|
|
51
|
+
refinements (~> 7.16)
|
|
52
|
+
xdg (~> 4.4)
|
|
53
|
+
sequel (5.53.0)
|
|
54
|
+
sqlite3 (1.4.2)
|
|
55
|
+
strings (0.2.1)
|
|
56
|
+
strings-ansi (~> 0.2)
|
|
57
|
+
unicode-display_width (>= 1.5, < 3.0)
|
|
58
|
+
unicode_utils (~> 1.4)
|
|
59
|
+
strings-ansi (0.2.0)
|
|
60
|
+
thor (0.20.3)
|
|
61
|
+
tty-color (0.6.0)
|
|
62
|
+
tty-cursor (0.7.1)
|
|
63
|
+
tty-progressbar (0.18.2)
|
|
64
|
+
strings-ansi (~> 0.2)
|
|
65
|
+
tty-cursor (~> 0.7)
|
|
66
|
+
tty-screen (~> 0.8)
|
|
67
|
+
unicode-display_width (>= 1.6, < 3.0)
|
|
68
|
+
tty-screen (0.8.1)
|
|
69
|
+
tty-table (0.12.0)
|
|
70
|
+
pastel (~> 0.8)
|
|
71
|
+
strings (~> 0.2.0)
|
|
72
|
+
tty-screen (~> 0.8)
|
|
73
|
+
tzinfo (2.0.4)
|
|
74
|
+
concurrent-ruby (~> 1.0)
|
|
75
|
+
unicode-display_width (2.1.0)
|
|
76
|
+
unicode_utils (1.4.0)
|
|
77
|
+
xdg (4.5.0)
|
|
78
|
+
|
|
79
|
+
PLATFORMS
|
|
80
|
+
ruby
|
|
81
|
+
|
|
82
|
+
DEPENDENCIES
|
|
83
|
+
bundler (~> 2.3)
|
|
84
|
+
chronicle-imessage!
|
|
85
|
+
rake (~> 13.0.6)
|
|
86
|
+
|
|
87
|
+
BUNDLED WITH
|
|
88
|
+
2.3.6
|
data/README.md
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# Chronicle::Imessage
|
|
2
|
+
|
|
3
|
+
IMessage importer for [chronicle-etl](https://github.com/chronicle-app/chronicle-etl)
|
|
4
|
+
|
|
5
|
+
## Available Connectors
|
|
6
|
+
### Extractors
|
|
7
|
+
- `imessage` - Extractor for importing messages and attachments from local macOS iMessage install (`~/Library/Messages/chat.db`)
|
|
8
|
+
|
|
9
|
+
### Transformers
|
|
10
|
+
- `imessage` - Transformer for processing messages into Chronicle Schema
|
|
11
|
+
|
|
12
|
+
## Usage
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
gem install chronicle-etl
|
|
16
|
+
chronicle-etl connectors:install imessage
|
|
17
|
+
|
|
18
|
+
chronicle-etl --extractor imessage --extractor-opts load_since:"2022-02-07" --transformer imessage --loader table
|
|
19
|
+
```
|
data/Rakefile
ADDED
data/bin/console
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
|
|
3
|
+
require "bundler/setup"
|
|
4
|
+
require "chronicle/imessage"
|
|
5
|
+
|
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
|
8
|
+
|
|
9
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
|
10
|
+
# require "pry"
|
|
11
|
+
# Pry.start
|
|
12
|
+
|
|
13
|
+
require "irb"
|
|
14
|
+
IRB.start(__FILE__)
|
data/bin/setup
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
|
|
2
|
+
lib = File.expand_path("../lib", __FILE__)
|
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
|
+
require "chronicle/imessage/version"
|
|
5
|
+
|
|
6
|
+
Gem::Specification.new do |spec|
|
|
7
|
+
spec.name = "chronicle-imessage"
|
|
8
|
+
spec.version = Chronicle::Imessage::VERSION
|
|
9
|
+
spec.authors = ["Andrew Louis"]
|
|
10
|
+
spec.email = ["andrew@hyfen.net"]
|
|
11
|
+
|
|
12
|
+
spec.summary = "iMessage importer for Chronicle"
|
|
13
|
+
spec.description = "Connectors for iMessage"
|
|
14
|
+
spec.homepage = "https://github.com/chronicle-app/chronicle-imessage"
|
|
15
|
+
|
|
16
|
+
# Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
|
|
17
|
+
# to allow pushing to a single host or delete this section to allow pushing to any host.
|
|
18
|
+
if spec.respond_to?(:metadata)
|
|
19
|
+
spec.metadata['allowed_push_host'] = "https://rubygems.org"
|
|
20
|
+
|
|
21
|
+
spec.metadata["homepage_uri"] = spec.homepage
|
|
22
|
+
spec.metadata["source_code_uri"] = "https://github.com/chronicle-app/chronicle-imessage"
|
|
23
|
+
spec.metadata["changelog_uri"] = "https://github.com/chronicle-app/chronicle-imessage"
|
|
24
|
+
else
|
|
25
|
+
raise "RubyGems 2.0 or newer is required to protect against " \
|
|
26
|
+
"public gem pushes."
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# Specify which files should be added to the gem when it is released.
|
|
30
|
+
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
|
31
|
+
spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do
|
|
32
|
+
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
|
33
|
+
end
|
|
34
|
+
spec.bindir = "exe"
|
|
35
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
|
36
|
+
spec.require_paths = ["lib"]
|
|
37
|
+
|
|
38
|
+
spec.add_dependency "chronicle-etl", "~> 0.3"
|
|
39
|
+
spec.add_dependency "sqlite3", "~> 1.4.2"
|
|
40
|
+
spec.add_dependency "phonelib", "~> 0.6.54"
|
|
41
|
+
|
|
42
|
+
spec.add_development_dependency "bundler", "~> 2.3"
|
|
43
|
+
spec.add_development_dependency "rake", "~> 13.0.6"
|
|
44
|
+
end
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
require 'chronicle/etl'
|
|
2
|
+
require 'sqlite3'
|
|
3
|
+
require 'pry'
|
|
4
|
+
|
|
5
|
+
module Chronicle
|
|
6
|
+
module Imessage
|
|
7
|
+
class ImessageExtractor < Chronicle::ETL::Extractor
|
|
8
|
+
register_connector do |r|
|
|
9
|
+
r.provider = 'imessage'
|
|
10
|
+
r.description = 'a local imessage database'
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
DEFAULT_OPTIONS = {
|
|
14
|
+
db: File.join(Dir.home, 'Library', 'Messages', 'chat.db'),
|
|
15
|
+
load_attachments: false,
|
|
16
|
+
load_since: Time.now - 5000000
|
|
17
|
+
}.freeze
|
|
18
|
+
|
|
19
|
+
def initialize(options = {})
|
|
20
|
+
super(DEFAULT_OPTIONS.merge(options))
|
|
21
|
+
prepare_data
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def extract
|
|
25
|
+
@messages.each do |message|
|
|
26
|
+
meta = {}
|
|
27
|
+
meta[:participants] = @chats[message['chat_id']]
|
|
28
|
+
meta[:attachments] = @attachments[message['message_id']] if @attachments
|
|
29
|
+
|
|
30
|
+
yield Chronicle::ETL::Extraction.new(data: message, meta: meta)
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def results_count
|
|
35
|
+
@messages.count
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
private
|
|
39
|
+
|
|
40
|
+
def prepare_data
|
|
41
|
+
@db = SQLite3::Database.new(@options[:db], results_as_hash: true)
|
|
42
|
+
@messages = load_messages(
|
|
43
|
+
load_since: @options[:load_since],
|
|
44
|
+
load_until: @options[:load_until],
|
|
45
|
+
limit: @options[:limit]
|
|
46
|
+
)
|
|
47
|
+
@contacts = LocalContacts.new.contacts
|
|
48
|
+
@chats = load_chats
|
|
49
|
+
|
|
50
|
+
if @options[:load_attachments]
|
|
51
|
+
@attachments = load_attachments(@messages.map{|m| m['message_id']})
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def load_messages(load_since: nil, load_until: nil, limit: nil)
|
|
56
|
+
load_since_ios = unix_to_ios_timestamp(load_since.to_i) * 1000000000 if load_since
|
|
57
|
+
load_until_ios = unix_to_ios_timestamp(load_until.to_i) * 1000000000 if load_until
|
|
58
|
+
|
|
59
|
+
sql = "SELECT * from message as m
|
|
60
|
+
LEFT OUTER JOIN handle as h ON m.handle_id=h.ROWID
|
|
61
|
+
INNER JOIN chat_message_join as cm ON m.ROWID = cm.message_id"
|
|
62
|
+
|
|
63
|
+
conditions = []
|
|
64
|
+
conditions << "date < #{load_until_ios}" if load_until
|
|
65
|
+
conditions << "date > #{load_since_ios}" if load_since
|
|
66
|
+
sql += " WHERE #{conditions.join(" AND ")}" if conditions.any?
|
|
67
|
+
sql += " ORDER BY date DESC"
|
|
68
|
+
sql += " LIMIT #{limit}" if limit
|
|
69
|
+
|
|
70
|
+
messages = @db.execute(sql)
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# In ios message schema, a message belongs to a chat (basically, a thread).
|
|
74
|
+
# A chat has_many handles which represents members of the thread
|
|
75
|
+
# We load the whole list of chats/handles so we can pass along the participants
|
|
76
|
+
# of a message to the Transformer
|
|
77
|
+
def load_chats
|
|
78
|
+
sql = "SELECT * from chat_handle_join as ch
|
|
79
|
+
INNER JOIN handle as h ON ch.handle_id = h.ROWID
|
|
80
|
+
INNER JOIN chat as c ON ch.chat_id = c.ROWID"
|
|
81
|
+
results = @db.execute(sql)
|
|
82
|
+
|
|
83
|
+
# collate in contact name if available
|
|
84
|
+
results = match_contacts(results) if @contacts
|
|
85
|
+
|
|
86
|
+
# group handles by id so we can pick right one easily
|
|
87
|
+
chats = results.group_by{|x| x['chat_id']}
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def load_attachments(message_ids)
|
|
91
|
+
sql = <<-SQL
|
|
92
|
+
SELECT
|
|
93
|
+
*
|
|
94
|
+
FROM
|
|
95
|
+
message_attachment_join
|
|
96
|
+
LEFT JOIN attachment ON attachment_id = attachment.rowid
|
|
97
|
+
WHERE
|
|
98
|
+
message_id IN(#{message_ids.join(",")})
|
|
99
|
+
SQL
|
|
100
|
+
|
|
101
|
+
results = @db.execute(sql)
|
|
102
|
+
results.group_by { |r| r['message_id'] }
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def match_contacts results
|
|
106
|
+
results.map do |chat|
|
|
107
|
+
contact = @contacts[chat['id']]
|
|
108
|
+
if contact
|
|
109
|
+
full_name = "#{contact['first_name']} #{contact['last_name']}".strip
|
|
110
|
+
chat['full_name'] = full_name if full_name
|
|
111
|
+
end
|
|
112
|
+
chat
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def ios_timestamp_to_unix ts
|
|
117
|
+
ts + 978307200
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
def unix_to_ios_timestamp ts
|
|
121
|
+
ts - 978307200
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
end
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
require 'chronicle/etl'
|
|
2
|
+
|
|
3
|
+
module Chronicle
|
|
4
|
+
module Imessage
|
|
5
|
+
class ImessageTransformer < Chronicle::ETL::Transformer
|
|
6
|
+
register_connector do |r|
|
|
7
|
+
r.provider = 'imessage'
|
|
8
|
+
r.description = 'a row from a local imessage database'
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
DEFAULT_OPTIONS = {
|
|
12
|
+
}.freeze
|
|
13
|
+
|
|
14
|
+
def initialize(*args)
|
|
15
|
+
super(*args)
|
|
16
|
+
@options = @options.reverse_merge(DEFAULT_OPTIONS)
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def transform
|
|
20
|
+
@message = @extraction.data
|
|
21
|
+
@participants = @extraction.meta[:participants]
|
|
22
|
+
@attachments = @extraction.meta[:attachments] || []
|
|
23
|
+
|
|
24
|
+
set_actors
|
|
25
|
+
record = build_messaged
|
|
26
|
+
record
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def timestamp
|
|
30
|
+
Time.at(ios_timestamp_to_unix(@message['date'].to_i / 1000000000))
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def id
|
|
34
|
+
@message['guid']
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
private
|
|
38
|
+
|
|
39
|
+
def set_actors
|
|
40
|
+
me = build_identity_mine
|
|
41
|
+
# Figure out the sender / receiver(s) of a message
|
|
42
|
+
case @message['is_from_me']
|
|
43
|
+
when 1
|
|
44
|
+
@actor = me
|
|
45
|
+
@consumers = @participants.collect{|p| build_identity(p)}
|
|
46
|
+
else
|
|
47
|
+
sender = @participants.select{|p| p['id'] == @message['id']}.first
|
|
48
|
+
receivers = @participants - [sender]
|
|
49
|
+
|
|
50
|
+
@consumers = receivers.collect{|p| build_identity(p)}
|
|
51
|
+
@consumers << me
|
|
52
|
+
@actor = build_identity(sender)
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def build_messaged
|
|
57
|
+
record = ::Chronicle::ETL::Models::Activity.new
|
|
58
|
+
record.end_at = timestamp
|
|
59
|
+
record.verb = 'messaged'
|
|
60
|
+
record.provider_id = id
|
|
61
|
+
record.provider = build_provider(@message['service'])
|
|
62
|
+
record.dedupe_on = [[:provider, :verb, :provider_id]]
|
|
63
|
+
|
|
64
|
+
record.involved = build_message
|
|
65
|
+
record.actor = @actor
|
|
66
|
+
|
|
67
|
+
record
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def build_message
|
|
71
|
+
record = ::Chronicle::ETL::Models::Entity.new
|
|
72
|
+
record.body = @message['text']
|
|
73
|
+
record.provider_id = id
|
|
74
|
+
record.represents = 'message'
|
|
75
|
+
record.provider = build_provider(@message['service'])
|
|
76
|
+
record.dedupe_on = [[:represents, :provider, :provider_id]]
|
|
77
|
+
|
|
78
|
+
record.consumers = @consumers
|
|
79
|
+
record.contains = @attachments.map{ |a| build_attachment(a)}.compact
|
|
80
|
+
|
|
81
|
+
record
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def build_attachment(attachment)
|
|
85
|
+
return unless attachment['mime_type']
|
|
86
|
+
|
|
87
|
+
type, subtype = attachment['mime_type'].split("/")
|
|
88
|
+
return unless ['image', 'audio', 'video'].include?(type)
|
|
89
|
+
return unless attachment['filename']
|
|
90
|
+
|
|
91
|
+
attachment_filename = attachment['filename'].gsub("~", Dir.home)
|
|
92
|
+
return unless File.exist?(attachment_filename)
|
|
93
|
+
|
|
94
|
+
attachment_data = ::Chronicle::ETL::Utils::BinaryAttachments.filename_to_base64(filename: attachment_filename, mimetype: attachment['mime_type'])
|
|
95
|
+
recognized_text = ::Chronicle::ETL::Utils::TextRecognition.recognize_in_image(filename: attachment_filename) if type == 'image'
|
|
96
|
+
|
|
97
|
+
record = ::Chronicle::ETL::Models::Entity.new
|
|
98
|
+
record.provider = 'imessage'
|
|
99
|
+
record.provider_id = attachment['guid']
|
|
100
|
+
record.represents = type
|
|
101
|
+
record.title = File.basename(attachment['filename'])
|
|
102
|
+
record.metadata[:ocr_text] = recognized_text if recognized_text
|
|
103
|
+
record.dedupe_on = [[:provider, :provider_id, :represents]]
|
|
104
|
+
|
|
105
|
+
attachment = ::Chronicle::ETL::Models::Attachment.new
|
|
106
|
+
attachment.data = attachment_data
|
|
107
|
+
record.attachments = [attachment]
|
|
108
|
+
|
|
109
|
+
record
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
def build_identity identity
|
|
113
|
+
raise ::Chronicle::ETL::UntransformableRecordError.new("Could not build identity", transformation: self) unless identity
|
|
114
|
+
|
|
115
|
+
record = ::Chronicle::ETL::Models::Entity.new({
|
|
116
|
+
represents: 'identity',
|
|
117
|
+
slug: identity['id'],
|
|
118
|
+
title: identity['full_name'],
|
|
119
|
+
provider: identity_provider(@message['service']),
|
|
120
|
+
})
|
|
121
|
+
record.dedupe_on = [[:represents, :slug, :provider]]
|
|
122
|
+
record
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def build_identity_mine
|
|
126
|
+
record = ::Chronicle::ETL::Models::Entity.new({
|
|
127
|
+
represents: 'identity',
|
|
128
|
+
slug: @options[:my_phone_slug],
|
|
129
|
+
title: @options[:my_name],
|
|
130
|
+
provider: identity_provider(@message['service']),
|
|
131
|
+
provider_id: @message['account_guid']
|
|
132
|
+
})
|
|
133
|
+
record.dedupe_on = [[:represents, :slug, :provider], [:represents, :provider, :provider_id]]
|
|
134
|
+
record
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# in the wild, this is either null or sms
|
|
138
|
+
def build_provider service
|
|
139
|
+
service ? service.downcase : 'imessage'
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
# FIXME: should probably try to preserve imessage ids instead of imessage
|
|
143
|
+
def identity_provider service
|
|
144
|
+
case service
|
|
145
|
+
# an SMS message is on the 'sms' provider but the provider of the identity used to send it is 'phone'
|
|
146
|
+
when 'SMS'then 'phone'
|
|
147
|
+
# similarly, 'imessage' provider for messages, 'icloud' provider for identity of sender
|
|
148
|
+
when 'iMessage' then 'icloud'
|
|
149
|
+
else 'icloud'
|
|
150
|
+
end
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
# FIXME: refactor to shared
|
|
154
|
+
def ios_timestamp_to_unix ts
|
|
155
|
+
ts + 978307200
|
|
156
|
+
end
|
|
157
|
+
def unix_to_ios_timestamp ts
|
|
158
|
+
ts - 978307200
|
|
159
|
+
end
|
|
160
|
+
end
|
|
161
|
+
end
|
|
162
|
+
end
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
require 'find'
|
|
2
|
+
require 'phonelib'
|
|
3
|
+
|
|
4
|
+
module Chronicle
|
|
5
|
+
module Imessage
|
|
6
|
+
# Load contacts from local macOS address book
|
|
7
|
+
class LocalContacts
|
|
8
|
+
def initialize
|
|
9
|
+
filename = find_local_icloud_address_book
|
|
10
|
+
@db = SQLite3::Database.new(filename, results_as_hash: true)
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def contact_identifier_to_details(identifier)
|
|
14
|
+
contacts[identifier]
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def contacts
|
|
18
|
+
@contacts ||= begin
|
|
19
|
+
c = {}
|
|
20
|
+
c.merge!(load_phone_numbers)
|
|
21
|
+
c.merge!(load_email_addresses)
|
|
22
|
+
c
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# The synced address book doesn't have a stable folder location so we
|
|
27
|
+
# have to search for it
|
|
28
|
+
def find_local_icloud_address_book
|
|
29
|
+
pattern = File.join(Dir.home, '/Library/Application Support/AddressBook/Sources', '**/*.abcddb')
|
|
30
|
+
Dir.glob(pattern).first
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def load_phone_numbers
|
|
34
|
+
sql = <<-SQL
|
|
35
|
+
SELECT
|
|
36
|
+
ZABCDPHONENUMBER.ZFULLNUMBER AS identifier,
|
|
37
|
+
ZABCDRECORD.ZFIRSTNAME as first_name,
|
|
38
|
+
ZABCDRECORD.ZLASTNAME as last_name
|
|
39
|
+
FROM
|
|
40
|
+
ZABCDRECORD,
|
|
41
|
+
ZABCDPHONENUMBER
|
|
42
|
+
WHERE
|
|
43
|
+
ZABCDRECORD.Z_PK = ZABCDPHONENUMBER.ZOWNER
|
|
44
|
+
SQL
|
|
45
|
+
|
|
46
|
+
results = @db.execute(sql)
|
|
47
|
+
results.map do |r|
|
|
48
|
+
# We normalize phone numbers (and assume US/Canada country code)
|
|
49
|
+
# so that we can match identifiers from chat.db
|
|
50
|
+
normalized = Phonelib.parse(r['identifier'], "US").e164
|
|
51
|
+
[normalized, r]
|
|
52
|
+
end.to_h
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def load_email_addresses
|
|
56
|
+
sql = <<-SQL
|
|
57
|
+
SELECT
|
|
58
|
+
ZABCDEMAILADDRESS.ZADDRESSNORMALIZED AS identifier,
|
|
59
|
+
ZABCDRECORD.ZFIRSTNAME as first_name,
|
|
60
|
+
ZABCDRECORD.ZLASTNAME as last_name
|
|
61
|
+
FROM
|
|
62
|
+
ZABCDRECORD,
|
|
63
|
+
ZABCDEMAILADDRESS
|
|
64
|
+
WHERE
|
|
65
|
+
ZABCDRECORD.Z_PK = ZABCDEMAILADDRESS.ZOWNER
|
|
66
|
+
SQL
|
|
67
|
+
|
|
68
|
+
results = @db.execute(sql)
|
|
69
|
+
results.map do |r|
|
|
70
|
+
[r['identifier'], r]
|
|
71
|
+
end.to_h
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
require "chronicle/imessage/version"
|
|
2
|
+
require "chronicle/imessage/imessage_extractor"
|
|
3
|
+
require "chronicle/imessage/imessage_transformer"
|
|
4
|
+
require "chronicle/imessage/local_contacts"
|
|
5
|
+
|
|
6
|
+
module Chronicle
|
|
7
|
+
module Imessage
|
|
8
|
+
class Error < StandardError; end
|
|
9
|
+
# Your code goes here...
|
|
10
|
+
end
|
|
11
|
+
end
|
metadata
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: chronicle-imessage
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.1.0
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- Andrew Louis
|
|
8
|
+
autorequire:
|
|
9
|
+
bindir: exe
|
|
10
|
+
cert_chain: []
|
|
11
|
+
date: 2022-02-07 00:00:00.000000000 Z
|
|
12
|
+
dependencies:
|
|
13
|
+
- !ruby/object:Gem::Dependency
|
|
14
|
+
name: chronicle-etl
|
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
|
16
|
+
requirements:
|
|
17
|
+
- - "~>"
|
|
18
|
+
- !ruby/object:Gem::Version
|
|
19
|
+
version: '0.3'
|
|
20
|
+
type: :runtime
|
|
21
|
+
prerelease: false
|
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
23
|
+
requirements:
|
|
24
|
+
- - "~>"
|
|
25
|
+
- !ruby/object:Gem::Version
|
|
26
|
+
version: '0.3'
|
|
27
|
+
- !ruby/object:Gem::Dependency
|
|
28
|
+
name: sqlite3
|
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
|
30
|
+
requirements:
|
|
31
|
+
- - "~>"
|
|
32
|
+
- !ruby/object:Gem::Version
|
|
33
|
+
version: 1.4.2
|
|
34
|
+
type: :runtime
|
|
35
|
+
prerelease: false
|
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
37
|
+
requirements:
|
|
38
|
+
- - "~>"
|
|
39
|
+
- !ruby/object:Gem::Version
|
|
40
|
+
version: 1.4.2
|
|
41
|
+
- !ruby/object:Gem::Dependency
|
|
42
|
+
name: phonelib
|
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
|
44
|
+
requirements:
|
|
45
|
+
- - "~>"
|
|
46
|
+
- !ruby/object:Gem::Version
|
|
47
|
+
version: 0.6.54
|
|
48
|
+
type: :runtime
|
|
49
|
+
prerelease: false
|
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
51
|
+
requirements:
|
|
52
|
+
- - "~>"
|
|
53
|
+
- !ruby/object:Gem::Version
|
|
54
|
+
version: 0.6.54
|
|
55
|
+
- !ruby/object:Gem::Dependency
|
|
56
|
+
name: bundler
|
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
|
58
|
+
requirements:
|
|
59
|
+
- - "~>"
|
|
60
|
+
- !ruby/object:Gem::Version
|
|
61
|
+
version: '2.3'
|
|
62
|
+
type: :development
|
|
63
|
+
prerelease: false
|
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
65
|
+
requirements:
|
|
66
|
+
- - "~>"
|
|
67
|
+
- !ruby/object:Gem::Version
|
|
68
|
+
version: '2.3'
|
|
69
|
+
- !ruby/object:Gem::Dependency
|
|
70
|
+
name: rake
|
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
|
72
|
+
requirements:
|
|
73
|
+
- - "~>"
|
|
74
|
+
- !ruby/object:Gem::Version
|
|
75
|
+
version: 13.0.6
|
|
76
|
+
type: :development
|
|
77
|
+
prerelease: false
|
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
79
|
+
requirements:
|
|
80
|
+
- - "~>"
|
|
81
|
+
- !ruby/object:Gem::Version
|
|
82
|
+
version: 13.0.6
|
|
83
|
+
description: Connectors for iMessage
|
|
84
|
+
email:
|
|
85
|
+
- andrew@hyfen.net
|
|
86
|
+
executables: []
|
|
87
|
+
extensions: []
|
|
88
|
+
extra_rdoc_files: []
|
|
89
|
+
files:
|
|
90
|
+
- ".gitignore"
|
|
91
|
+
- CODE_OF_CONDUCT.md
|
|
92
|
+
- Gemfile
|
|
93
|
+
- Gemfile.lock
|
|
94
|
+
- README.md
|
|
95
|
+
- Rakefile
|
|
96
|
+
- bin/console
|
|
97
|
+
- bin/setup
|
|
98
|
+
- chronicle-imessage.gemspec
|
|
99
|
+
- lib/chronicle/imessage.rb
|
|
100
|
+
- lib/chronicle/imessage/imessage_extractor.rb
|
|
101
|
+
- lib/chronicle/imessage/imessage_transformer.rb
|
|
102
|
+
- lib/chronicle/imessage/local_contacts.rb
|
|
103
|
+
- lib/chronicle/imessage/version.rb
|
|
104
|
+
homepage: https://github.com/chronicle-app/chronicle-imessage
|
|
105
|
+
licenses: []
|
|
106
|
+
metadata:
|
|
107
|
+
allowed_push_host: https://rubygems.org
|
|
108
|
+
homepage_uri: https://github.com/chronicle-app/chronicle-imessage
|
|
109
|
+
source_code_uri: https://github.com/chronicle-app/chronicle-imessage
|
|
110
|
+
changelog_uri: https://github.com/chronicle-app/chronicle-imessage
|
|
111
|
+
post_install_message:
|
|
112
|
+
rdoc_options: []
|
|
113
|
+
require_paths:
|
|
114
|
+
- lib
|
|
115
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
116
|
+
requirements:
|
|
117
|
+
- - ">="
|
|
118
|
+
- !ruby/object:Gem::Version
|
|
119
|
+
version: '0'
|
|
120
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
121
|
+
requirements:
|
|
122
|
+
- - ">="
|
|
123
|
+
- !ruby/object:Gem::Version
|
|
124
|
+
version: '0'
|
|
125
|
+
requirements: []
|
|
126
|
+
rubygems_version: 3.1.2
|
|
127
|
+
signing_key:
|
|
128
|
+
specification_version: 4
|
|
129
|
+
summary: iMessage importer for Chronicle
|
|
130
|
+
test_files: []
|