abbu 0.8.1 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +129 -2
- data/bin/abbu +158 -6
- data/bin/abbu-mcp +36 -0
- data/docs/ABBU.md +28 -0
- data/docs/ALTERNATE_CALENDAR_EVIDENCE.md +82 -0
- data/docs/CHANGELOG.md +98 -0
- data/docs/CONTRIBUTING.md +3 -0
- data/docs/FUZZY_MATCHING.md +72 -0
- data/docs/HISTORY_EVIDENCE.md +98 -0
- data/docs/ICALENDAR.md +101 -0
- data/docs/MACHINE_JSON.md +128 -0
- data/docs/MCP.md +148 -0
- data/docs/MERGE_PLANS.md +78 -0
- data/docs/MY_CARD_EVIDENCE.md +92 -0
- data/docs/PORTABLE_SQLITE.md +142 -0
- data/docs/RELEASING.md +76 -68
- data/docs/SNAPSHOT_DIFF.md +74 -0
- data/docs/SNAPSHOT_HISTORY.md +93 -0
- data/docs/STREAMING.md +73 -0
- data/docs/TODO.md +4 -4
- data/docs/WRITER_DECISION.md +90 -0
- data/lib/abbu/archive.rb +14 -0
- data/lib/abbu/exporters/csv_exporter.rb +7 -0
- data/lib/abbu/exporters/icalendar_encoding.rb +35 -0
- data/lib/abbu/exporters/icalendar_event.rb +61 -0
- data/lib/abbu/exporters/icalendar_exporter.rb +86 -0
- data/lib/abbu/exporters/json_exporter.rb +2 -2
- data/lib/abbu/exporters/jsonl_exporter.rb +23 -0
- data/lib/abbu/exporters/sqlite_exporter.rb +85 -0
- data/lib/abbu/exporters/sqlite_rows.rb +113 -0
- data/lib/abbu/exporters/sqlite_schema.rb +76 -0
- data/lib/abbu/exporters/vcard_exporter.rb +10 -11
- data/lib/abbu/exporters/vcard_photo.rb +50 -0
- data/lib/abbu/history_tracker.rb +96 -0
- data/lib/abbu/live_store.rb +11 -0
- data/lib/abbu/machine_command.rb +159 -0
- data/lib/abbu/machine_output.rb +52 -0
- data/lib/abbu/mcp/read_session.rb +57 -0
- data/lib/abbu/mcp/server.rb +100 -0
- data/lib/abbu/mcp.rb +11 -0
- data/lib/abbu/merge_plan.rb +132 -0
- data/lib/abbu/merge_preview.rb +35 -0
- data/lib/abbu/merge_values.rb +26 -0
- data/lib/abbu/parsers/plist_parser.rb +9 -0
- data/lib/abbu/parsers/sqlite_parser.rb +25 -0
- data/lib/abbu/snapshot_diff.rb +125 -0
- data/lib/abbu/snapshot_history.rb +66 -0
- data/lib/abbu/utils/fuzzy_matcher.rb +116 -0
- data/lib/abbu/version.rb +1 -1
- data/lib/abbu.rb +9 -0
- data/sig/fuzzy_matcher.rbs +22 -0
- data/sig/icalendar_exporter.rbs +13 -0
- data/sig/machine_output.rbs +23 -0
- data/sig/mcp.rbs +19 -0
- data/sig/merge_plan.rbs +16 -0
- data/sig/snapshot_diff.rbs +8 -0
- data/sig/snapshot_history.rbs +10 -0
- data/sig/sqlite_exporter.rbs +10 -0
- data/sig/streaming.rbs +27 -0
- data/sig/vcard_exporter.rbs +1 -1
- metadata +46 -2
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
# lib/abbu/merge_plan.rb
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require_relative 'contact'
|
|
5
|
+
require_relative 'merge_values'
|
|
6
|
+
|
|
7
|
+
module Abbu
|
|
8
|
+
# A detached review artifact, not an assertion that two records are one person.
|
|
9
|
+
# Materialization is opt-in and never writes an archive or live store.
|
|
10
|
+
class MergePlan
|
|
11
|
+
MULTIVALUES = %i[emails phones addresses groups urls notes related_names social_profiles
|
|
12
|
+
dates instant_messages].freeze
|
|
13
|
+
SCALARS = %i[first_name middle_name last_name company nickname prefix suffix job_title department maiden_name
|
|
14
|
+
phonetic_first_name phonetic_middle_name phonetic_last_name phonetic_company pronouns ringtone
|
|
15
|
+
texttone birthday anniversary verification_code lunar_birthday created_at modified_at].freeze
|
|
16
|
+
FIELDS = (MULTIVALUES + SCALARS + %i[image_uri image_path group_memberships source]).freeze
|
|
17
|
+
POLICIES = %i[union_multivalues prefer_newer prefer_source prefer_more_complete].freeze
|
|
18
|
+
|
|
19
|
+
attr_reader :policy, :fields, :sources, :conflicts
|
|
20
|
+
|
|
21
|
+
def initialize(left, right, policy: nil, source: nil)
|
|
22
|
+
validate_policy(policy, source)
|
|
23
|
+
@policy = policy
|
|
24
|
+
@records = [left, right].map { |contact| snapshot(contact) }.freeze
|
|
25
|
+
@sources = @records.map { |record| record[:source] }.freeze
|
|
26
|
+
preferred = preferred_index(source)
|
|
27
|
+
@fields = build_fields(preferred).freeze
|
|
28
|
+
@conflicts = @fields.select { |_, field| field[:conflict] }.freeze
|
|
29
|
+
freeze
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def to_h
|
|
33
|
+
{ policy: policy, sources: sources, inputs: @records, fields: fields, conflicts: conflicts.keys.freeze,
|
|
34
|
+
materializable: !policy.nil? && conflicts.values.none? { |field| field[:unresolved] } }.freeze
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def materialize
|
|
38
|
+
validate_materialization
|
|
39
|
+
Contact.new.tap do |contact|
|
|
40
|
+
fields.each { |name, field| assign_field(contact, name, field[:selected]) }
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
private
|
|
45
|
+
|
|
46
|
+
def validate_policy(policy, source)
|
|
47
|
+
raise ArgumentError, 'unknown merge policy' if policy && !POLICIES.include?(policy)
|
|
48
|
+
raise ArgumentError, 'source requires prefer_source' if source && policy != :prefer_source
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def validate_materialization
|
|
52
|
+
raise ArgumentError, 'an explicit merge policy is required' unless policy
|
|
53
|
+
raise ArgumentError, 'unresolved scalar conflicts require a deciding policy' if
|
|
54
|
+
conflicts.values.any? { |field| field[:unresolved] }
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def assign_field(contact, name, selected)
|
|
58
|
+
values = name == :image ? selected || {} : { name => selected }
|
|
59
|
+
values.each { |key, value| contact.public_send(:"#{key}=", MergeValues.copy(value)) }
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def snapshot(contact)
|
|
63
|
+
FIELDS.to_h { |name| [name, MergeValues.copy(contact.public_send(name), freeze: true)] }.freeze
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def preferred_index(source)
|
|
67
|
+
case policy
|
|
68
|
+
when :prefer_source then source_index(source)
|
|
69
|
+
when :prefer_newer then comparison_index(@records.map { |record| record[:modified_at] }, Time)
|
|
70
|
+
when :prefer_more_complete then comparison_index(@records.map { |record| completeness(record) }, Integer)
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def source_index(source)
|
|
75
|
+
raise ArgumentError, 'prefer_source requires an exact source relative_path' unless source.is_a?(String)
|
|
76
|
+
|
|
77
|
+
indices = sources.each_index.select { |index| sources[index]&.fetch(:relative_path, nil) == source }
|
|
78
|
+
raise ArgumentError, 'preferred source must identify exactly one input' unless indices.one?
|
|
79
|
+
|
|
80
|
+
indices.first
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def comparison_index(values, type)
|
|
84
|
+
return unless values.all?(type)
|
|
85
|
+
return if values[0] == values[1]
|
|
86
|
+
|
|
87
|
+
values[0] > values[1] ? 0 : 1
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def completeness(record)
|
|
91
|
+
(SCALARS - %i[created_at modified_at]).count { |key| present?(record[key]) } +
|
|
92
|
+
MULTIVALUES.count { |key| present?(record[key]) }
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def present?(value)
|
|
96
|
+
!value.nil? && !(value.respond_to?(:empty?) && value.empty?)
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def build_fields(preferred)
|
|
100
|
+
result = union_fields
|
|
101
|
+
SCALARS.each { |name| result[name] = scalar_field(@records.map { |record| record[name] }, preferred) }
|
|
102
|
+
result[:image] = scalar_field(images, preferred)
|
|
103
|
+
result
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def union_fields
|
|
107
|
+
MULTIVALUES.to_h do |name|
|
|
108
|
+
values = @records.map { |record| record.fetch(name) }
|
|
109
|
+
[name, { selected: values.flatten(1).uniq.freeze, reason: :lossless_union, conflict: false }.freeze]
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def images
|
|
114
|
+
@records.map do |record|
|
|
115
|
+
image = record.slice(:image_uri, :image_path)
|
|
116
|
+
image.freeze if image.values.any? { |value| present?(value) }
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
def scalar_field(values, preferred)
|
|
121
|
+
conflict = values[0] != values[1] && values.all? { |value| present?(value) }
|
|
122
|
+
index = conflict ? preferred : values.index { |value| present?(value) }
|
|
123
|
+
selected = index.nil? ? nil : values[index]
|
|
124
|
+
{ selected: selected, alternatives: values.freeze, conflict: conflict,
|
|
125
|
+
unresolved: conflict && preferred.nil?, reason: reason(conflict) }.freeze
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
def reason(conflict)
|
|
129
|
+
conflict ? policy || :review_required : :uncontested
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
end
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# lib/abbu/merge_preview.rb
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require_relative 'merge_plan'
|
|
5
|
+
require_relative 'utils/deduplicator'
|
|
6
|
+
|
|
7
|
+
module Abbu
|
|
8
|
+
class MergePreview
|
|
9
|
+
def self.validate!(options)
|
|
10
|
+
return unless options.values_at(:merge_preview, :merge_policy, :prefer_source).any?
|
|
11
|
+
|
|
12
|
+
raise ArgumentError, '--merge-policy/--prefer-source require --merge-preview' unless options[:merge_preview]
|
|
13
|
+
|
|
14
|
+
allowed = %i[merge_preview merge_policy prefer_source strict live live_path]
|
|
15
|
+
raise ArgumentError, '--merge-preview cannot be combined with other operations' if
|
|
16
|
+
options.any? { |key, value| value && !allowed.include?(key) }
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def self.plans(contacts, policy: nil, source: nil)
|
|
20
|
+
validate_policy!(policy, source)
|
|
21
|
+
Utils::Deduplicator.new(contacts).matches.map do |match|
|
|
22
|
+
{ status: match.status, score: match.score, evidence: match.evidence,
|
|
23
|
+
plan: MergePlan.new(match.left, match.right, policy: policy, source: source).to_h }
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def self.validate_policy!(policy, source)
|
|
28
|
+
# Validate policies even for empty inputs, before enumerating candidate pairs.
|
|
29
|
+
MergePlan.new(Contact.new, Contact.new, policy: policy) unless policy == :prefer_source
|
|
30
|
+
raise ArgumentError, 'source requires prefer_source' if source && policy != :prefer_source
|
|
31
|
+
raise ArgumentError, 'prefer_source requires a source path' if policy == :prefer_source && !source.is_a?(String)
|
|
32
|
+
end
|
|
33
|
+
private_class_method :validate_policy!
|
|
34
|
+
end
|
|
35
|
+
end
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# lib/abbu/merge_values.rb
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require 'date'
|
|
5
|
+
|
|
6
|
+
module Abbu
|
|
7
|
+
module MergeValues
|
|
8
|
+
module_function
|
|
9
|
+
|
|
10
|
+
def copy(value, freeze: false)
|
|
11
|
+
result = duplicate(value, freeze: freeze)
|
|
12
|
+
freeze ? result.freeze : result
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def duplicate(value, freeze:)
|
|
16
|
+
case value
|
|
17
|
+
when Hash then value.to_h { |key, item| [copy(key, freeze: freeze), copy(item, freeze: freeze)] }
|
|
18
|
+
when Array then value.map { |item| copy(item, freeze: freeze) }
|
|
19
|
+
when String, Time, Date then value.dup
|
|
20
|
+
when NilClass, TrueClass, FalseClass, Symbol, Numeric then value
|
|
21
|
+
else raise ArgumentError, 'unsupported contact value'
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
private_constant :MergeValues
|
|
26
|
+
end
|
|
@@ -38,6 +38,15 @@ module Abbu
|
|
|
38
38
|
@paths.filter_map { |file| parse_file(file) }
|
|
39
39
|
end
|
|
40
40
|
|
|
41
|
+
def each_contact
|
|
42
|
+
return enum_for(:each_contact) unless block_given?
|
|
43
|
+
|
|
44
|
+
@paths.each do |file|
|
|
45
|
+
contact = parse_file(file)
|
|
46
|
+
yield contact if contact
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
|
|
41
50
|
private
|
|
42
51
|
|
|
43
52
|
def resolve_paths(paths)
|
|
@@ -49,8 +49,33 @@ module Abbu
|
|
|
49
49
|
end
|
|
50
50
|
end
|
|
51
51
|
|
|
52
|
+
def each_contact(&block)
|
|
53
|
+
return enum_for(:each_contact) unless block
|
|
54
|
+
|
|
55
|
+
@db_paths.each do |path|
|
|
56
|
+
stream_db(path, &block)
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
|
|
52
60
|
private
|
|
53
61
|
|
|
62
|
+
def stream_db(path)
|
|
63
|
+
@active_db_path = path
|
|
64
|
+
db = SQLite3::Database.new(path.to_s, readonly: true)
|
|
65
|
+
db.results_as_hash = true
|
|
66
|
+
statement = stream_statement(db)
|
|
67
|
+
statement.execute.each { |row| yield build_contact(db, row, path) }
|
|
68
|
+
ensure
|
|
69
|
+
statement&.close
|
|
70
|
+
db&.close
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def stream_statement(db)
|
|
74
|
+
db.prepare('SELECT * FROM ZABCDRECORD WHERE Z_ENT != 15')
|
|
75
|
+
rescue SQLite3::Exception
|
|
76
|
+
fail_required_schema(@active_db_path)
|
|
77
|
+
end
|
|
78
|
+
|
|
54
79
|
def parse_db(db_path)
|
|
55
80
|
@active_db_path = db_path
|
|
56
81
|
db = SQLite3::Database.new(db_path.to_s, readonly: @readonly)
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
# lib/abbu/snapshot_diff.rb
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require 'pathname'
|
|
5
|
+
require 'time'
|
|
6
|
+
|
|
7
|
+
require_relative 'utils/deduplicator'
|
|
8
|
+
|
|
9
|
+
module Abbu
|
|
10
|
+
# Snapshot observations, not assertions of identity or events between captures.
|
|
11
|
+
class SnapshotDiff
|
|
12
|
+
FIELDS = %i[first_name middle_name last_name emails phones company addresses groups group_memberships
|
|
13
|
+
nickname prefix suffix job_title department maiden_name phonetic_first_name phonetic_middle_name
|
|
14
|
+
phonetic_last_name phonetic_company pronouns ringtone texttone urls notes related_names
|
|
15
|
+
social_profiles birthday anniversary dates instant_messages verification_code lunar_birthday
|
|
16
|
+
image_uri image_path created_at modified_at source].freeze
|
|
17
|
+
|
|
18
|
+
attr_reader :result
|
|
19
|
+
|
|
20
|
+
def initialize(before, after, max_pairs: 100_000)
|
|
21
|
+
raise ArgumentError, 'max_pairs must be a positive Integer' unless max_pairs.is_a?(Integer) && max_pairs.positive?
|
|
22
|
+
|
|
23
|
+
left = before.respond_to?(:contacts) ? before.contacts : before.to_a
|
|
24
|
+
right = after.respond_to?(:contacts) ? after.contacts : after.to_a
|
|
25
|
+
raise ArgumentError, 'snapshot comparison exceeds max_pairs' if left.length * right.length > max_pairs
|
|
26
|
+
|
|
27
|
+
@result = freeze_tree(compare(left, right))
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def to_h
|
|
31
|
+
result
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
private
|
|
35
|
+
|
|
36
|
+
def compare(left, right)
|
|
37
|
+
candidates = candidates_for(left, right)
|
|
38
|
+
@left_counts = candidates.map { |item| item[:before_index] }.tally
|
|
39
|
+
@right_counts = candidates.map { |item| item[:after_index] }.tally
|
|
40
|
+
@old_records = left.map { |contact| record(contact) }
|
|
41
|
+
@new_records = right.map { |contact| record(contact) }
|
|
42
|
+
classify(candidates)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def classify(candidates)
|
|
46
|
+
paired, ambiguous = candidates.partition do |item|
|
|
47
|
+
item[:score] >= Utils::Deduplicator::PROBABLE_SCORE &&
|
|
48
|
+
@left_counts[item[:before_index]] == 1 && @right_counts[item[:after_index]] == 1
|
|
49
|
+
end
|
|
50
|
+
build_result(paired, ambiguous)
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def candidates_for(left, right)
|
|
54
|
+
left.each_with_index.flat_map do |old_contact, old_index|
|
|
55
|
+
right.each_with_index.filter_map do |new_contact, new_index|
|
|
56
|
+
match = Utils::Deduplicator.new([old_contact, new_contact]).matches.first
|
|
57
|
+
next unless match
|
|
58
|
+
|
|
59
|
+
{ before_index: old_index, after_index: new_index, score: match.score, evidence: copy(match.evidence) }
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def build_result(paired, ambiguous)
|
|
65
|
+
changes = paired.map { |pair| comparison(pair) }
|
|
66
|
+
{
|
|
67
|
+
schema_version: 1,
|
|
68
|
+
added: indexed_unmatched(@new_records, @right_counts),
|
|
69
|
+
removed: indexed_unmatched(@old_records, @left_counts),
|
|
70
|
+
changed: changes.reject { |item| item[:fields].empty? },
|
|
71
|
+
unchanged: changes.select { |item| item[:fields].empty? },
|
|
72
|
+
ambiguous: ambiguous.map { |pair| with_records(pair) }
|
|
73
|
+
}
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def with_records(pair)
|
|
77
|
+
pair.merge(before: @old_records.fetch(pair[:before_index]), after: @new_records.fetch(pair[:after_index]))
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def comparison(pair)
|
|
81
|
+
records = with_records(pair)
|
|
82
|
+
fields = FIELDS.filter_map do |field|
|
|
83
|
+
next if records[:before][field] == records[:after][field]
|
|
84
|
+
|
|
85
|
+
[field, { before: records[:before][field], after: records[:after][field] }]
|
|
86
|
+
end.to_h
|
|
87
|
+
records.merge(fields: fields)
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def indexed_unmatched(records, counts)
|
|
91
|
+
records.each_with_index.filter_map do |contact, index|
|
|
92
|
+
{ index: index, contact: contact } unless counts.key?(index)
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def record(contact)
|
|
97
|
+
FIELDS.to_h { |field| [field, copy(contact.public_send(field))] }
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
def copy(value)
|
|
101
|
+
case value
|
|
102
|
+
when Hash then value.to_h { |key, item| [key, copy(item)] }
|
|
103
|
+
when Array then value.map { |item| copy(item) }
|
|
104
|
+
else copy_scalar(value)
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def copy_scalar(value)
|
|
109
|
+
case value
|
|
110
|
+
when Time then value.iso8601(9)
|
|
111
|
+
when Pathname then value.to_s
|
|
112
|
+
when String then value.dup
|
|
113
|
+
else value
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
def freeze_tree(value)
|
|
118
|
+
case value
|
|
119
|
+
when Hash then value.each { |key, item| [key, item].each { |part| freeze_tree(part) } }
|
|
120
|
+
when Array then value.each { |item| freeze_tree(item) }
|
|
121
|
+
end
|
|
122
|
+
value.freeze
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
end
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
# lib/abbu/snapshot_history.rb
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require_relative 'archive'
|
|
5
|
+
require_relative 'history_tracker'
|
|
6
|
+
|
|
7
|
+
module Abbu
|
|
8
|
+
# Stream snapshot observations; IDs are local to this ordered analysis only.
|
|
9
|
+
class SnapshotHistory
|
|
10
|
+
include Enumerable
|
|
11
|
+
|
|
12
|
+
DEFAULT_LIMITS = { snapshots: 100, contacts: 10_000, pairs: 100_000 }.freeze
|
|
13
|
+
|
|
14
|
+
def initialize(paths, strict: false, retention: 2, limits: {})
|
|
15
|
+
limits = validated_limits(limits)
|
|
16
|
+
raise ArgumentError, 'retention must be a nonnegative integer' unless retention.is_a?(Integer) && retention >= 0
|
|
17
|
+
|
|
18
|
+
@paths = snapshot_paths(paths, limits[:snapshots])
|
|
19
|
+
@strict = strict
|
|
20
|
+
@options = { retention: retention, max_contacts: limits[:contacts], max_pairs: limits[:pairs] }.freeze
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def self.from_directory(path, **)
|
|
24
|
+
raise ArgumentError, 'snapshot directory is unavailable' unless File.directory?(path)
|
|
25
|
+
|
|
26
|
+
paths = Dir.children(path).sort.filter_map do |name|
|
|
27
|
+
candidate = File.join(path, name)
|
|
28
|
+
candidate if name.end_with?('.abbu') && File.directory?(candidate)
|
|
29
|
+
end
|
|
30
|
+
new(paths, **)
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def each
|
|
34
|
+
return enum_for(:each) unless block_given?
|
|
35
|
+
|
|
36
|
+
tracker = HistoryTracker.new(**@options)
|
|
37
|
+
@paths.each_with_index do |path, index|
|
|
38
|
+
archive = Archive.new(path, strict: @strict)
|
|
39
|
+
snapshot = { index: index, path: path }.freeze
|
|
40
|
+
transition = tracker.advance(archive.contacts, snapshot)
|
|
41
|
+
yield transition.merge(diagnostics: archive.diagnostics.map(&:to_h).freeze).freeze
|
|
42
|
+
end
|
|
43
|
+
self
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
private
|
|
47
|
+
|
|
48
|
+
def validated_limits(limits)
|
|
49
|
+
raise ArgumentError, 'unknown limit' unless (limits.keys - DEFAULT_LIMITS.keys).empty?
|
|
50
|
+
|
|
51
|
+
DEFAULT_LIMITS.merge(limits).tap do |settings|
|
|
52
|
+
settings.each_value do |bound|
|
|
53
|
+
raise ArgumentError, 'limits must be positive integers' unless bound.is_a?(Integer) && bound.positive?
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def snapshot_paths(paths, limit)
|
|
59
|
+
result = paths.take(limit + 1).map { |path| File.expand_path(path).freeze }.freeze
|
|
60
|
+
raise ArgumentError, 'snapshot count exceeds max_snapshots' if result.length > limit
|
|
61
|
+
raise ArgumentError, 'snapshot paths must be unique' if result.uniq.length != result.length
|
|
62
|
+
|
|
63
|
+
result
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
end
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
# lib/abbu/utils/fuzzy_matcher.rb
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require_relative 'deduplicator'
|
|
5
|
+
|
|
6
|
+
module Abbu
|
|
7
|
+
module Utils
|
|
8
|
+
# Suggestions, not identity assertions. Bounds fail explicitly rather than
|
|
9
|
+
# silently truncating candidates; source contacts are never modified.
|
|
10
|
+
class FuzzyMatcher
|
|
11
|
+
FUZZY_WEIGHT = 0.15
|
|
12
|
+
|
|
13
|
+
def initialize(contact, candidates, threshold: 0.85, max_candidates: 10_000, max_name_length: 128)
|
|
14
|
+
validate_options(threshold, max_candidates, max_name_length)
|
|
15
|
+
@contact = contact
|
|
16
|
+
@candidates = candidates.take(max_candidates + 1)
|
|
17
|
+
raise ArgumentError, 'candidate limit exceeded' if @candidates.length > max_candidates
|
|
18
|
+
|
|
19
|
+
@threshold = threshold
|
|
20
|
+
@max_name_length = max_name_length
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def matches
|
|
24
|
+
left_name = normalized_name(@contact)
|
|
25
|
+
results = @candidates.filter_map do |candidate|
|
|
26
|
+
next if candidate.equal?(@contact)
|
|
27
|
+
|
|
28
|
+
suggestion(candidate, left_name, normalized_name(candidate))
|
|
29
|
+
end
|
|
30
|
+
return results unless results.count { |match| match.score >= Deduplicator::MINIMUM_SCORE } > 1
|
|
31
|
+
|
|
32
|
+
results.map do |match|
|
|
33
|
+
Deduplicator::Match.new(left: match.left, right: match.right, evidence: match.evidence,
|
|
34
|
+
score: match.score, status: :ambiguous)
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
private
|
|
39
|
+
|
|
40
|
+
def validate_options(threshold, *bounds)
|
|
41
|
+
raise ArgumentError, 'threshold must be finite and between 0 and 1' unless
|
|
42
|
+
threshold.is_a?(Numeric) && threshold.real? && threshold.finite? && threshold.between?(0, 1)
|
|
43
|
+
|
|
44
|
+
bounds.each { |bound| validate_bound(bound) }
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def validate_bound(bound)
|
|
48
|
+
raise ArgumentError, 'bounds must be positive integers' unless bound.is_a?(Integer) && bound.positive?
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def normalized_name(contact)
|
|
52
|
+
name = contact.full_name.unicode_normalize(:nfc).downcase.strip.gsub(/\s+/, ' ')
|
|
53
|
+
raise ArgumentError, 'name length limit exceeded' if name.length > @max_name_length
|
|
54
|
+
|
|
55
|
+
name
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def suggestion(candidate, left_name, right_name)
|
|
59
|
+
exact = Deduplicator.new([@contact, candidate]).matches.first
|
|
60
|
+
fuzzy = fuzzy_evidence(candidate, left_name, right_name)
|
|
61
|
+
return unless exact || fuzzy
|
|
62
|
+
|
|
63
|
+
evidence = combined_evidence(exact, fuzzy)
|
|
64
|
+
score = exact ? exact.score : fuzzy.fetch(:contribution)
|
|
65
|
+
Deduplicator::Match.new(left: @contact, right: candidate, evidence: evidence,
|
|
66
|
+
score: score, status: exact ? exact.status : :ambiguous)
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def combined_evidence(exact, fuzzy)
|
|
70
|
+
evidence = exact ? exact_evidence(exact) : []
|
|
71
|
+
evidence << fuzzy.merge(contribution: exact ? 0.0 : fuzzy.fetch(:contribution)).freeze if fuzzy
|
|
72
|
+
evidence
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def exact_evidence(exact)
|
|
76
|
+
remaining = exact.score
|
|
77
|
+
seen = []
|
|
78
|
+
exact.evidence.map do |item|
|
|
79
|
+
contribution = seen.include?(item[:type]) ? 0.0 : [Deduplicator::WEIGHTS.fetch(item[:type]), remaining].min
|
|
80
|
+
remaining = (remaining - contribution).round(2)
|
|
81
|
+
seen << item[:type]
|
|
82
|
+
item.merge(contribution: contribution).freeze
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def fuzzy_evidence(candidate, left_name, right_name)
|
|
87
|
+
return if left_name.empty? || right_name.empty?
|
|
88
|
+
|
|
89
|
+
distance = edit_distance(left_name.codepoints, right_name.codepoints)
|
|
90
|
+
similarity = 1.0 - (distance.to_f / [left_name.length, right_name.length].max)
|
|
91
|
+
return if similarity < @threshold
|
|
92
|
+
|
|
93
|
+
{ type: :fuzzy_name, algorithm: :levenshtein, left_raw: @contact.full_name,
|
|
94
|
+
right_raw: candidate.full_name, left_normalized: left_name, right_normalized: right_name,
|
|
95
|
+
distance: distance, similarity: similarity, contribution: (similarity * FUZZY_WEIGHT).round(4) }.freeze
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def edit_distance(left, right)
|
|
99
|
+
row = (0..right.length).to_a
|
|
100
|
+
left.each_with_index do |character, index|
|
|
101
|
+
row = distance_row(row, right, character, index)
|
|
102
|
+
end
|
|
103
|
+
row.last
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def distance_row(row, right, character, index)
|
|
107
|
+
next_row = [index + 1]
|
|
108
|
+
right.each_with_index do |other, column|
|
|
109
|
+
substitution = row[column] + (character == other ? 0 : 1)
|
|
110
|
+
next_row << [next_row[column] + 1, row[column + 1] + 1, substitution].min
|
|
111
|
+
end
|
|
112
|
+
next_row
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
end
|
data/lib/abbu/version.rb
CHANGED
data/lib/abbu.rb
CHANGED
|
@@ -7,17 +7,26 @@ require_relative 'abbu/diagnostic'
|
|
|
7
7
|
require_relative 'abbu/parse_error'
|
|
8
8
|
require_relative 'abbu/archive'
|
|
9
9
|
require_relative 'abbu/live_store'
|
|
10
|
+
require_relative 'abbu/machine_command'
|
|
11
|
+
require_relative 'abbu/merge_plan'
|
|
12
|
+
require_relative 'abbu/merge_preview'
|
|
10
13
|
require_relative 'abbu/query'
|
|
11
14
|
require_relative 'abbu/schema_inspector'
|
|
15
|
+
require_relative 'abbu/snapshot_diff'
|
|
16
|
+
require_relative 'abbu/snapshot_history'
|
|
12
17
|
require_relative 'abbu/source'
|
|
13
18
|
require_relative 'abbu/parsers/sqlite_parser'
|
|
14
19
|
require_relative 'abbu/parsers/plist_parser'
|
|
15
20
|
require_relative 'abbu/exporters/csv_exporter'
|
|
21
|
+
require_relative 'abbu/exporters/icalendar_exporter'
|
|
16
22
|
require_relative 'abbu/exporters/json_exporter'
|
|
23
|
+
require_relative 'abbu/exporters/jsonl_exporter'
|
|
24
|
+
require_relative 'abbu/exporters/sqlite_exporter'
|
|
17
25
|
require_relative 'abbu/exporters/vcard_exporter'
|
|
18
26
|
require_relative 'abbu/image_extractor'
|
|
19
27
|
require_relative 'abbu/utils/contact_identity'
|
|
20
28
|
require_relative 'abbu/utils/deduplicator'
|
|
29
|
+
require_relative 'abbu/utils/fuzzy_matcher'
|
|
21
30
|
require_relative 'abbu/utils/label_normalizer'
|
|
22
31
|
|
|
23
32
|
module Abbu
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# sig/fuzzy_matcher.rbs
|
|
2
|
+
module Abbu
|
|
3
|
+
module Utils
|
|
4
|
+
class Deduplicator
|
|
5
|
+
class Match
|
|
6
|
+
attr_reader left: Contact
|
|
7
|
+
attr_reader right: Contact
|
|
8
|
+
attr_reader evidence: Array[Hash[Symbol, untyped]]
|
|
9
|
+
attr_reader score: Float
|
|
10
|
+
attr_reader confidence: Symbol
|
|
11
|
+
attr_reader status: Symbol
|
|
12
|
+
def sources: () -> Array[untyped]
|
|
13
|
+
def ambiguous?: () -> bool
|
|
14
|
+
def merge: (?policy: untyped) -> untyped
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
class FuzzyMatcher
|
|
18
|
+
def initialize: (Contact contact, Enumerable[Contact] candidates, ?threshold: Numeric, ?max_candidates: Integer, ?max_name_length: Integer) -> void
|
|
19
|
+
def matches: () -> Array[Deduplicator::Match]
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# sig/icalendar_exporter.rbs
|
|
2
|
+
module Abbu
|
|
3
|
+
module Exporters
|
|
4
|
+
class IcalendarExporter
|
|
5
|
+
type diagnostic = { category: Symbol, contact_index: Integer, field: Symbol }
|
|
6
|
+
attr_reader diagnostics: Array[diagnostic]
|
|
7
|
+
def initialize: (Enumerable[Contact] contacts, year: Integer, generated_at: Time, calendar_id: String) -> void
|
|
8
|
+
def to_ical: () -> String
|
|
9
|
+
def to_file: (String | Pathname path) -> Integer
|
|
10
|
+
def to_stdout: () -> nil
|
|
11
|
+
end
|
|
12
|
+
end
|
|
13
|
+
end
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# sig/machine_output.rbs
|
|
2
|
+
module Abbu
|
|
3
|
+
module Exporters
|
|
4
|
+
class JsonExporter
|
|
5
|
+
def payload: () -> Array[Hash[Symbol, untyped]]
|
|
6
|
+
end
|
|
7
|
+
end
|
|
8
|
+
|
|
9
|
+
class MachineOutput
|
|
10
|
+
def initialize: (untyped input) -> void
|
|
11
|
+
def contacts: (?Enumerable[Contact] records) -> Array[Hash[Symbol, untyped]]
|
|
12
|
+
def stats: () -> Hash[Symbol, Integer]
|
|
13
|
+
def diagnostics: () -> Array[Hash[Symbol, untyped]]
|
|
14
|
+
def duplicates: () -> Array[Hash[Symbol, untyped]]
|
|
15
|
+
def matches: () -> Array[Hash[Symbol, untyped]]
|
|
16
|
+
def images: (String destination) -> Hash[Symbol, untyped]
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
class MachineCommand
|
|
20
|
+
def initialize: (Hash[Symbol, untyped] options, Array[String] arguments, ?stdout: untyped, ?stderr: untyped) -> void
|
|
21
|
+
def run: () -> Integer
|
|
22
|
+
end
|
|
23
|
+
end
|
data/sig/mcp.rbs
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# sig/mcp.rbs
|
|
2
|
+
module Abbu
|
|
3
|
+
module Mcp
|
|
4
|
+
class ReadSession
|
|
5
|
+
def initialize: (untyped input) -> void
|
|
6
|
+
def query: (mode: String, value: String, ?offset: Integer, ?limit: Integer) -> Hash[Symbol, untyped]
|
|
7
|
+
def stats: () -> Hash[Symbol, Integer]
|
|
8
|
+
def diagnostics: () -> Array[Hash[Symbol, untyped]]
|
|
9
|
+
def sources: () -> Array[Hash[Symbol, untyped]]
|
|
10
|
+
def groups: () -> Array[Hash[Symbol, untyped]]
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
class Server
|
|
14
|
+
attr_reader server: untyped
|
|
15
|
+
def initialize: (path: String, ?live: bool, ?strict: bool) -> void
|
|
16
|
+
def open: () -> untyped
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
end
|