active_sanction 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +538 -0
- data/CODE_OF_CONDUCT.md +74 -0
- data/CONTRIBUTING.md +312 -0
- data/LICENSE.txt +21 -0
- data/README.md +616 -0
- data/SECURITY.md +97 -0
- data/active_sanction.gemspec +98 -0
- data/docs/adding_a_source.md +1083 -0
- data/docs/api_stability.md +357 -0
- data/docs/bundle_format.md +407 -0
- data/lib/active_sanction/address.rb +170 -0
- data/lib/active_sanction/client.rb +341 -0
- data/lib/active_sanction/configuration.rb +721 -0
- data/lib/active_sanction/countries.txt +292 -0
- data/lib/active_sanction/country.rb +164 -0
- data/lib/active_sanction/deprecation.rb +168 -0
- data/lib/active_sanction/diff/change.rb +208 -0
- data/lib/active_sanction/diff.rb +348 -0
- data/lib/active_sanction/doctor/checkup.rb +356 -0
- data/lib/active_sanction/doctor/diagnosis.rb +245 -0
- data/lib/active_sanction/doctor/finding.rb +183 -0
- data/lib/active_sanction/doctor/profile.rb +384 -0
- data/lib/active_sanction/doctor/report.rb +227 -0
- data/lib/active_sanction/doctor.rb +347 -0
- data/lib/active_sanction/entity.rb +293 -0
- data/lib/active_sanction/error.rb +332 -0
- data/lib/active_sanction/fetcher/result.rb +116 -0
- data/lib/active_sanction/fetcher.rb +242 -0
- data/lib/active_sanction/http_client/errors.rb +94 -0
- data/lib/active_sanction/http_client/response.rb +129 -0
- data/lib/active_sanction/http_client.rb +324 -0
- data/lib/active_sanction/identifier.rb +212 -0
- data/lib/active_sanction/index/builder.rb +89 -0
- data/lib/active_sanction/index/candidate.rb +63 -0
- data/lib/active_sanction/index/entry.rb +66 -0
- data/lib/active_sanction/index/features.rb +112 -0
- data/lib/active_sanction/index.rb +381 -0
- data/lib/active_sanction/match_result.rb +345 -0
- data/lib/active_sanction/matcher.rb +388 -0
- data/lib/active_sanction/name.rb +183 -0
- data/lib/active_sanction/normalizer/cache.rb +82 -0
- data/lib/active_sanction/normalizer/dictionaries/honorifics.txt +30 -0
- data/lib/active_sanction/normalizer/dictionaries/legal_forms.txt +72 -0
- data/lib/active_sanction/normalizer/dictionaries/organization_stopwords.txt +20 -0
- data/lib/active_sanction/normalizer/dictionaries/particles.txt +35 -0
- data/lib/active_sanction/normalizer/dictionary/stoplist.rb +121 -0
- data/lib/active_sanction/normalizer/dictionary.rb +243 -0
- data/lib/active_sanction/normalizer/form.rb +279 -0
- data/lib/active_sanction/normalizer.rb +137 -0
- data/lib/active_sanction/parsers/column_shape.rb +274 -0
- data/lib/active_sanction/parsers/delimited_table/reader.rb +194 -0
- data/lib/active_sanction/parsers/delimited_table/row.rb +81 -0
- data/lib/active_sanction/parsers/delimited_table.rb +151 -0
- data/lib/active_sanction/parsers/format.rb +107 -0
- data/lib/active_sanction/parsers/join.rb +149 -0
- data/lib/active_sanction/parsers/spreadsheet/archive.rb +258 -0
- data/lib/active_sanction/parsers/spreadsheet/reader.rb +213 -0
- data/lib/active_sanction/parsers/spreadsheet/row.rb +88 -0
- data/lib/active_sanction/parsers/spreadsheet/workbook.rb +314 -0
- data/lib/active_sanction/parsers/spreadsheet.rb +168 -0
- data/lib/active_sanction/parsers/xml_records/backends/nokogiri.rb +134 -0
- data/lib/active_sanction/parsers/xml_records/backends/rexml.rb +157 -0
- data/lib/active_sanction/parsers/xml_records/backends.rb +103 -0
- data/lib/active_sanction/parsers/xml_records/builder.rb +90 -0
- data/lib/active_sanction/parsers/xml_records/reader.rb +134 -0
- data/lib/active_sanction/parsers/xml_records/record.rb +174 -0
- data/lib/active_sanction/parsers/xml_records.rb +140 -0
- data/lib/active_sanction/parsers.rb +94 -0
- data/lib/active_sanction/partial_date/parser.rb +124 -0
- data/lib/active_sanction/partial_date.rb +303 -0
- data/lib/active_sanction/payload_cache/checksum.rb +66 -0
- data/lib/active_sanction/payload_cache/entry.rb +304 -0
- data/lib/active_sanction/payload_cache.rb +366 -0
- data/lib/active_sanction/phonetics/double_metaphone.rb +662 -0
- data/lib/active_sanction/phonetics.rb +40 -0
- data/lib/active_sanction/query.rb +290 -0
- data/lib/active_sanction/rescreen/alert.rb +314 -0
- data/lib/active_sanction/rescreen.rb +332 -0
- data/lib/active_sanction/scorer/adjustments.rb +241 -0
- data/lib/active_sanction/scorer/name_score.rb +279 -0
- data/lib/active_sanction/scorer/reason.rb +136 -0
- data/lib/active_sanction/scorer/result.rb +129 -0
- data/lib/active_sanction/scorer/subject.rb +235 -0
- data/lib/active_sanction/scorer/weights.rb +306 -0
- data/lib/active_sanction/scorer.rb +303 -0
- data/lib/active_sanction/similarity/jaro_winkler.rb +210 -0
- data/lib/active_sanction/similarity/levenshtein.rb +179 -0
- data/lib/active_sanction/similarity/token_set.rb +159 -0
- data/lib/active_sanction/similarity/token_sort.rb +120 -0
- data/lib/active_sanction/similarity.rb +183 -0
- data/lib/active_sanction/snapshot/bundle/header.rb +302 -0
- data/lib/active_sanction/snapshot/bundle/payload.rb +185 -0
- data/lib/active_sanction/snapshot/bundle/signature.rb +201 -0
- data/lib/active_sanction/snapshot/bundle.rb +381 -0
- data/lib/active_sanction/snapshot.rb +305 -0
- data/lib/active_sanction/sources/australia_dfat/published_date.rb +166 -0
- data/lib/active_sanction/sources/australia_dfat/record.rb +319 -0
- data/lib/active_sanction/sources/australia_dfat.rb +291 -0
- data/lib/active_sanction/sources/base.rb +295 -0
- data/lib/active_sanction/sources/canada_sema/record.rb +282 -0
- data/lib/active_sanction/sources/canada_sema/source_ref.rb +89 -0
- data/lib/active_sanction/sources/canada_sema.rb +153 -0
- data/lib/active_sanction/sources/definition.rb +360 -0
- data/lib/active_sanction/sources/eu_fsf/record.rb +467 -0
- data/lib/active_sanction/sources/eu_fsf.rb +203 -0
- data/lib/active_sanction/sources/ofac/record.rb +251 -0
- data/lib/active_sanction/sources/ofac/remarks_parser/coverage.rb +120 -0
- data/lib/active_sanction/sources/ofac/remarks_parser/vocabulary.rb +121 -0
- data/lib/active_sanction/sources/ofac/remarks_parser.rb +312 -0
- data/lib/active_sanction/sources/ofac.rb +248 -0
- data/lib/active_sanction/sources/ofac_consolidated/record.rb +37 -0
- data/lib/active_sanction/sources/ofac_consolidated.rb +203 -0
- data/lib/active_sanction/sources/ofac_sdn.rb +43 -0
- data/lib/active_sanction/sources/remarks.rb +84 -0
- data/lib/active_sanction/sources/uk_sanctions_list/published_date.rb +104 -0
- data/lib/active_sanction/sources/uk_sanctions_list/record.rb +412 -0
- data/lib/active_sanction/sources/uk_sanctions_list.rb +214 -0
- data/lib/active_sanction/sources/un_consolidated/record.rb +279 -0
- data/lib/active_sanction/sources/un_consolidated.rb +140 -0
- data/lib/active_sanction/sources.rb +196 -0
- data/lib/active_sanction/storage/active_record/reader.rb +124 -0
- data/lib/active_sanction/storage/active_record/row.rb +175 -0
- data/lib/active_sanction/storage/active_record/writer.rb +141 -0
- data/lib/active_sanction/storage/active_record.rb +277 -0
- data/lib/active_sanction/storage/base.rb +216 -0
- data/lib/active_sanction/storage/file_system.rb +373 -0
- data/lib/active_sanction/storage/memory.rb +67 -0
- data/lib/active_sanction/storage/meta.rb +188 -0
- data/lib/active_sanction/storage.rb +74 -0
- data/lib/active_sanction/subject.rb +271 -0
- data/lib/active_sanction/sync/report.rb +245 -0
- data/lib/active_sanction/sync/result.rb +296 -0
- data/lib/active_sanction/sync.rb +390 -0
- data/lib/active_sanction/validator_store/file_system.rb +103 -0
- data/lib/active_sanction/validator_store/memory.rb +41 -0
- data/lib/active_sanction/validator_store.rb +119 -0
- data/lib/active_sanction/validators.rb +195 -0
- data/lib/active_sanction/version.rb +23 -0
- data/lib/active_sanction.rb +331 -0
- data/lib/generators/active_sanction/install/install_generator.rb +60 -0
- data/lib/generators/active_sanction/install/templates/create_active_sanction_tables.rb.tt +142 -0
- metadata +236 -0
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
# typed: strict
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "sorbet-runtime"
|
|
5
|
+
|
|
6
|
+
require "set"
|
|
7
|
+
require "active_sanction/parsers/format"
|
|
8
|
+
require "active_sanction/parsers/xml_records/record"
|
|
9
|
+
require "active_sanction/parsers/xml_records/builder"
|
|
10
|
+
require "active_sanction/parsers/xml_records/backends"
|
|
11
|
+
require "active_sanction/parsers/xml_records/reader"
|
|
12
|
+
|
|
13
|
+
module ActiveSanction
|
|
14
|
+
module Parsers
|
|
15
|
+
# Reads a record-oriented XML list -- the UN, Canada, the EU and the UK all
|
|
16
|
+
# publish one -- into records an adapter can map onto Entities.
|
|
17
|
+
#
|
|
18
|
+
# A table is a description of the document, built once and reused for every
|
|
19
|
+
# sync; a Reader is one pass over one payload.
|
|
20
|
+
#
|
|
21
|
+
# UN = ActiveSanction::Parsers::XmlRecords.new(records: %w[INDIVIDUAL ENTITY])
|
|
22
|
+
#
|
|
23
|
+
# reader = UN.read(bytes)
|
|
24
|
+
# reader.each do |record|
|
|
25
|
+
# record.name # => "INDIVIDUAL"
|
|
26
|
+
# record["FIRST_NAME"] # => "ERIC"
|
|
27
|
+
# record.values("NATIONALITY/VALUE") # => ["Chad"]
|
|
28
|
+
# record.nodes("INDIVIDUAL_ALIAS") # => [Record, ...]
|
|
29
|
+
# end
|
|
30
|
+
# reader.root["dateGenerated"] # the publisher's own version marker
|
|
31
|
+
#
|
|
32
|
+
# ### Streaming from day one, before anything needs it
|
|
33
|
+
#
|
|
34
|
+
# Every list this gem launches with is small: the UN is 2.2 MB, Canada is
|
|
35
|
+
# 2.9 MB, and either would load into a DOM without anyone noticing. The one
|
|
36
|
+
# that is coming does not. OFAC's `SDN_ADVANCED.XML` is 126 MB, and a DOM
|
|
37
|
+
# design would meet it by being rewritten.
|
|
38
|
+
#
|
|
39
|
+
# So the interface is record-at-a-time now, while it is free to be: the
|
|
40
|
+
# parser holds one record's depth on a stack and drops it as soon as the
|
|
41
|
+
# adapter is done with it. What that buys is not speed, it is that the
|
|
42
|
+
# adapter written against this today is the adapter that reads a 126 MB
|
|
43
|
+
# file later, unchanged.
|
|
44
|
+
#
|
|
45
|
+
# ### Naming the records, and only the records
|
|
46
|
+
#
|
|
47
|
+
# A document's scaffolding -- `<CONSOLIDATED_LIST>`, `<INDIVIDUALS>` -- is
|
|
48
|
+
# skipped entirely rather than being built into nodes nobody asked for.
|
|
49
|
+
# Naming several record elements is normal: the UN files people under
|
|
50
|
+
# `<INDIVIDUAL>` and organizations under `<ENTITY>`, in one document, and
|
|
51
|
+
# an adapter wants a single pass over both.
|
|
52
|
+
#
|
|
53
|
+
# ### Namespaces
|
|
54
|
+
#
|
|
55
|
+
# Element and attribute names are matched with any prefix removed, so a
|
|
56
|
+
# publisher adding an `xmlns` next quarter does not silently stop matching.
|
|
57
|
+
# See Backends.local_name for why the prefix is dropped rather than
|
|
58
|
+
# resolved.
|
|
59
|
+
class XmlRecords
|
|
60
|
+
extend T::Sig
|
|
61
|
+
include Format
|
|
62
|
+
|
|
63
|
+
# Raised by a backend when the payload stops being XML. Caught by Reader,
|
|
64
|
+
# which decides between salvaging the records already read and refusing
|
|
65
|
+
# the payload outright; it escapes as a ParseError either way, so a
|
|
66
|
+
# caller rescuing the toolkit's errors does not have to know about it.
|
|
67
|
+
#
|
|
68
|
+
# `line` is ParseError's, and is nil where the backend reports no
|
|
69
|
+
# position -- libxml2 does not always.
|
|
70
|
+
class MalformedDocument < ParseError; end
|
|
71
|
+
|
|
72
|
+
# A Set: `record?` is asked once per element in the document, which for
|
|
73
|
+
# the UN is roughly 30,000 times a pass.
|
|
74
|
+
sig { returns(T::Set[String]) }
|
|
75
|
+
attr_reader :records
|
|
76
|
+
|
|
77
|
+
sig { override.returns(T::Array[String]) }
|
|
78
|
+
attr_reader :nulls
|
|
79
|
+
|
|
80
|
+
sig { override.returns(Encoding) }
|
|
81
|
+
attr_reader :encoding
|
|
82
|
+
|
|
83
|
+
# `null:` is here for the same reason DelimitedTable has it -- a
|
|
84
|
+
# publisher that writes a sentinel where it means nothing -- though the
|
|
85
|
+
# XML lists mostly use an empty element instead, which is already nil.
|
|
86
|
+
#
|
|
87
|
+
# `backend:` overrides the configured default for this table alone. Most
|
|
88
|
+
# adapters should not pass it: which XML library parses a list is an
|
|
89
|
+
# installation's decision, not a list's. See Backends.
|
|
90
|
+
sig { params(records: T.untyped, null: T.untyped, encoding: T.untyped, backend: T.untyped).void }
|
|
91
|
+
def initialize(records:, null: nil, encoding: DEFAULT_ENCODING, backend: nil)
|
|
92
|
+
@records = T.let(records!(records), T::Set[String])
|
|
93
|
+
@nulls = T.let(nulls!(null), T::Array[String])
|
|
94
|
+
@encoding = T.let(encoding!(encoding), Encoding)
|
|
95
|
+
@backend = T.let(backend&.to_sym, T.nilable(Symbol))
|
|
96
|
+
freeze
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# A pass over one payload. Takes the bytes as a String, which is what
|
|
100
|
+
# Sources::Base hands #parse.
|
|
101
|
+
sig { params(payload: T.untyped).returns(Reader) }
|
|
102
|
+
def read(payload) = Reader.new(table: self, payload: payload)
|
|
103
|
+
|
|
104
|
+
# Whether an element name starts a record. Asked by every backend for
|
|
105
|
+
# every element outside a record, so it stays a Set lookup.
|
|
106
|
+
sig { params(name: String).returns(T::Boolean) }
|
|
107
|
+
def record?(name) = records.include?(name)
|
|
108
|
+
|
|
109
|
+
# The record element names in declaration order, for a message or an
|
|
110
|
+
# inspect -- Set is the right shape to ask `record?` of and the wrong
|
|
111
|
+
# shape to print.
|
|
112
|
+
sig { returns(T::Array[String]) }
|
|
113
|
+
def record_names = records.to_a
|
|
114
|
+
|
|
115
|
+
# Resolved per call rather than at construction, so a table built at
|
|
116
|
+
# class-definition time -- which is where an adapter builds it -- still
|
|
117
|
+
# honours an `xml_backend` set later in an initializer.
|
|
118
|
+
sig { returns(T.untyped) }
|
|
119
|
+
def backend = Backends.resolve(@backend || ActiveSanction.config.xml_backend)
|
|
120
|
+
|
|
121
|
+
sig { returns(String) }
|
|
122
|
+
def inspect
|
|
123
|
+
"#<#{self.class} records=#{record_names.join(", ")}#{" null=#{nulls.first.inspect}" if nulls.any?}>"
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
private
|
|
127
|
+
|
|
128
|
+
# Accepts one element name or several. Stored as a Set: a document with
|
|
129
|
+
# 1,011 records asks `record?` once per element in the file, which for
|
|
130
|
+
# the UN is roughly 30,000 times.
|
|
131
|
+
sig { params(value: T.untyped).returns(T::Set[String]) }
|
|
132
|
+
def records!(value)
|
|
133
|
+
names = Array(value).map { |name| -name.to_s.strip }.reject(&:empty?).uniq
|
|
134
|
+
raise InvalidArgument, "records must name at least one element, e.g. records: \"INDIVIDUAL\"" if names.empty?
|
|
135
|
+
|
|
136
|
+
Set.new(names).freeze
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
end
|
|
140
|
+
end
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
# typed: strict
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "sorbet-runtime"
|
|
5
|
+
|
|
6
|
+
require "active_sanction/error"
|
|
7
|
+
|
|
8
|
+
module ActiveSanction
|
|
9
|
+
# The format toolkits adapters parse with. A source declares what its file
|
|
10
|
+
# looks like -- delimiter, column names, null sentinel -- and gets rows back;
|
|
11
|
+
# what those rows *mean* stays in the adapter, because that is the part no
|
|
12
|
+
# two publishers agree on.
|
|
13
|
+
#
|
|
14
|
+
# Nothing here knows about Entity. These are file readers, and keeping them
|
|
15
|
+
# ignorant of the canonical model is what lets an adapter for a list nobody
|
|
16
|
+
# here has seen reuse them.
|
|
17
|
+
module Parsers
|
|
18
|
+
# A payload that could not be read at all: the wrong format, a truncated
|
|
19
|
+
# download, an encoding that cannot be decoded. Distinct from a Warning,
|
|
20
|
+
# which is a *row* that could not be read while the rest of the file could.
|
|
21
|
+
#
|
|
22
|
+
# The same class as ActiveSanction::ParseError, under the name the toolkits
|
|
23
|
+
# raise it by. It is declared with the rest of the public hierarchy rather
|
|
24
|
+
# than here because a caller rescuing "this list could not be read" should
|
|
25
|
+
# not have to know which of these toolkits read it -- and because a source
|
|
26
|
+
# adapter parsing a format none of them covers raises the same thing.
|
|
27
|
+
ParseError = ActiveSanction::ParseError
|
|
28
|
+
|
|
29
|
+
# One row the parser could not use, kept rather than raised.
|
|
30
|
+
#
|
|
31
|
+
# A sanctions list is not a file we control. OFAC ships 19,321 rows and a
|
|
32
|
+
# single unbalanced quote somewhere in the middle must not cost the other
|
|
33
|
+
# 19,320 -- refusing to load a list because one record is malformed fails
|
|
34
|
+
# exactly when the list is most needed. So a bad row is recorded here and
|
|
35
|
+
# skipped, and the caller decides whether the count is tolerable.
|
|
36
|
+
#
|
|
37
|
+
# `line` is the line number within the file, which is what makes a warning
|
|
38
|
+
# actionable: a 5.6 MB CSV is only debuggable if the complaint says where.
|
|
39
|
+
# It is nil when the parser cannot say -- libxml2 reports no position for a
|
|
40
|
+
# record -- and a warning that cannot point at a line still says what went
|
|
41
|
+
# wrong rather than pointing at the wrong one.
|
|
42
|
+
class Warning
|
|
43
|
+
extend T::Sig
|
|
44
|
+
|
|
45
|
+
# nil when the parser cannot say which line -- see above.
|
|
46
|
+
sig { returns(T.nilable(Integer)) }
|
|
47
|
+
attr_reader :line
|
|
48
|
+
|
|
49
|
+
sig { returns(String) }
|
|
50
|
+
attr_reader :message
|
|
51
|
+
|
|
52
|
+
sig { returns(T.nilable(String)) }
|
|
53
|
+
attr_reader :snippet
|
|
54
|
+
|
|
55
|
+
sig { params(line: T.nilable(Integer), message: String, snippet: T.untyped).void }
|
|
56
|
+
def initialize(line:, message:, snippet: nil)
|
|
57
|
+
@line = T.let(line, T.nilable(Integer))
|
|
58
|
+
@message = T.let(message, String)
|
|
59
|
+
@snippet = T.let(snippet.nil? ? nil : truncate(snippet), T.nilable(String))
|
|
60
|
+
freeze
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
sig { returns(String) }
|
|
64
|
+
def to_s
|
|
65
|
+
"#{"line #{line}: " if line}#{message}#{" -- #{snippet.inspect}" if snippet}"
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
sig { returns(T::Hash[Symbol, T.untyped]) }
|
|
69
|
+
def to_h = { line: line, message: message, snippet: snippet }
|
|
70
|
+
|
|
71
|
+
sig { returns(String) }
|
|
72
|
+
def inspect = "#<#{self.class} #{self}>"
|
|
73
|
+
|
|
74
|
+
private
|
|
75
|
+
|
|
76
|
+
# A malformed row is frequently malformed because it is enormous -- an
|
|
77
|
+
# unclosed quote swallows everything after it -- so the evidence is
|
|
78
|
+
# trimmed before it is kept.
|
|
79
|
+
sig { params(text: T.untyped).returns(String) }
|
|
80
|
+
def truncate(text)
|
|
81
|
+
string = text.to_s
|
|
82
|
+
-(string.length > 120 ? "#{string[0, 120]}..." : string)
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
require "active_sanction/parsers/column_shape"
|
|
89
|
+
require "active_sanction/parsers/delimited_table"
|
|
90
|
+
require "active_sanction/parsers/spreadsheet"
|
|
91
|
+
require "active_sanction/parsers/xml_records"
|
|
92
|
+
require "active_sanction/parsers/xml_records/backends/rexml"
|
|
93
|
+
require "active_sanction/parsers/xml_records/backends/nokogiri"
|
|
94
|
+
require "active_sanction/parsers/join"
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
# typed: strict
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "sorbet-runtime"
|
|
5
|
+
|
|
6
|
+
require "date"
|
|
7
|
+
|
|
8
|
+
module ActiveSanction
|
|
9
|
+
class PartialDate
|
|
10
|
+
# Reads the date expressions the lists publish in free text. OFAC ships
|
|
11
|
+
# dates inside a remarks blob ("DOB circa 1962"), so the vocabulary here is
|
|
12
|
+
# whatever a government typist actually wrote, not a format we chose.
|
|
13
|
+
#
|
|
14
|
+
# Returns nil on anything it cannot read, per the issue: a birth date is one
|
|
15
|
+
# field on a record, and an unanticipated string should not abort the import
|
|
16
|
+
# of the entity around it. Callers decide whether that is worth reporting.
|
|
17
|
+
#
|
|
18
|
+
# @api private
|
|
19
|
+
module Parser
|
|
20
|
+
extend T::Sig
|
|
21
|
+
|
|
22
|
+
# "Sept" is not in Date::ABBR_MONTHNAMES but appears in OFAC free text.
|
|
23
|
+
MONTHS = T.let(
|
|
24
|
+
[Date::MONTHNAMES, Date::ABBR_MONTHNAMES].each_with_object({ "sept" => 9 }) do |names, months|
|
|
25
|
+
names.each_with_index { |name, number| months[name.downcase] = number if name }
|
|
26
|
+
end.freeze,
|
|
27
|
+
T::Hash[String, Integer]
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
ISO = T.let(/\A(\d{4})(?:-(\d{1,2})(?:-(\d{1,2}))?)?\z/, Regexp)
|
|
31
|
+
DAY_MONTH_YEAR = T.let(/\A(\d{1,2})\s+([a-z]+)\.?,?\s+(\d{4})\z/i, Regexp)
|
|
32
|
+
MONTH_DAY_YEAR = T.let(/\A([a-z]+)\.?\s+(\d{1,2})(?:st|nd|rd|th)?,?\s+(\d{4})\z/i, Regexp)
|
|
33
|
+
MONTH_YEAR = T.let(/\A([a-z]+)\.?,?\s+(\d{4})\z/i, Regexp)
|
|
34
|
+
APPROXIMATE = T.let(/\A(?:circa|approximately|approx|about|ca|c)\.?\s+|\A~\s*/i, Regexp)
|
|
35
|
+
BETWEEN = T.let(/\Abetween\s+/i, Regexp)
|
|
36
|
+
WORD_RANGE = T.let(/\A(.+?)\s+(?:to|and|through|until)\s+(.+)\z/i, Regexp)
|
|
37
|
+
# A bare dash separates a span only between two four-digit years. Anything
|
|
38
|
+
# looser would read the "-" in 1972-04 as a span from 1972 to April.
|
|
39
|
+
YEAR_RANGE = T.let(/\A(\d{4})\s*[-–—]\s*(\d{4})\z/, Regexp)
|
|
40
|
+
|
|
41
|
+
# An XML Schema `xs:date` may carry a UTC offset, and the UN publishes
|
|
42
|
+
# nine of its listing dates that way: "2015-07-01-04:00". The offset
|
|
43
|
+
# records which midnight a clerk was working against, not which day the
|
|
44
|
+
# listing is, so it is trimmed rather than applied -- shifting a listing
|
|
45
|
+
# date across a day boundary to honour a timezone the publisher never
|
|
46
|
+
# meant would be a worse answer than ignoring it. Without this the whole
|
|
47
|
+
# date reads as nil and nine listings silently lose their date.
|
|
48
|
+
ZONE = T.let(/(?<=\d)(?:Z|[+-]\d{2}:\d{2})\z/, Regexp)
|
|
49
|
+
|
|
50
|
+
module_function
|
|
51
|
+
|
|
52
|
+
sig { params(text: T.untyped).returns(T.nilable(PartialDate)) }
|
|
53
|
+
def call(text)
|
|
54
|
+
string = text.to_s.strip.squeeze(" ").sub(ZONE, "")
|
|
55
|
+
return nil if string.empty?
|
|
56
|
+
|
|
57
|
+
approximate = APPROXIMATE.match?(string)
|
|
58
|
+
string = string.sub(APPROXIMATE, "").sub(BETWEEN, "")
|
|
59
|
+
span(string, approximate) || point(string, approximate)
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# Endpoints go back through .call, so "1971 to circa 1973" reads. A nested
|
|
63
|
+
# span raises in the constructor and comes back as nil.
|
|
64
|
+
sig { params(string: String, approximate: T::Boolean).returns(T.nilable(PartialDate)) }
|
|
65
|
+
def span(string, approximate)
|
|
66
|
+
match = WORD_RANGE.match(string) || YEAR_RANGE.match(string)
|
|
67
|
+
return nil unless match
|
|
68
|
+
|
|
69
|
+
from = call(match[1])
|
|
70
|
+
to = call(match[2])
|
|
71
|
+
return nil if from.nil? || to.nil?
|
|
72
|
+
|
|
73
|
+
build(from: from, to: to, approximate: approximate)
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
sig { params(string: String, approximate: T::Boolean).returns(T.nilable(PartialDate)) }
|
|
77
|
+
def point(string, approximate)
|
|
78
|
+
attributes = point_attributes(string)
|
|
79
|
+
return nil if attributes.nil?
|
|
80
|
+
|
|
81
|
+
build(**attributes, approximate: approximate)
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# A shape the regexes accept can still be an impossible date -- 31
|
|
85
|
+
# February 1972 -- and that is the constructor's judgment, not ours.
|
|
86
|
+
sig { params(attributes: T.untyped).returns(T.nilable(PartialDate)) }
|
|
87
|
+
def build(**attributes)
|
|
88
|
+
PartialDate.new(**attributes)
|
|
89
|
+
rescue ArgumentError
|
|
90
|
+
nil
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
sig { params(string: String).returns(T.nilable(T::Hash[Symbol, T.nilable(Integer)])) }
|
|
94
|
+
def point_attributes(string)
|
|
95
|
+
match = ISO.match(string)
|
|
96
|
+
return { year: match[1].to_i, month: match[2]&.to_i, day: match[3]&.to_i } if match
|
|
97
|
+
|
|
98
|
+
worded_attributes(string)
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
sig { params(string: String).returns(T.nilable(T::Hash[Symbol, T.nilable(Integer)])) }
|
|
102
|
+
def worded_attributes(string)
|
|
103
|
+
if (match = DAY_MONTH_YEAR.match(string))
|
|
104
|
+
named_month(year: match[3], month: match[2], day: match[1])
|
|
105
|
+
elsif (match = MONTH_DAY_YEAR.match(string))
|
|
106
|
+
named_month(year: match[3], month: match[1], day: match[2])
|
|
107
|
+
elsif (match = MONTH_YEAR.match(string))
|
|
108
|
+
named_month(year: match[2], month: match[1])
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
sig do
|
|
113
|
+
params(year: T.untyped, month: T.untyped, day: T.untyped)
|
|
114
|
+
.returns(T.nilable(T::Hash[Symbol, T.nilable(Integer)]))
|
|
115
|
+
end
|
|
116
|
+
def named_month(year:, month:, day: nil)
|
|
117
|
+
number = MONTHS[month.downcase]
|
|
118
|
+
return nil if number.nil?
|
|
119
|
+
|
|
120
|
+
{ year: year.to_i, month: number, day: day&.to_i }
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
end
|
|
@@ -0,0 +1,303 @@
|
|
|
1
|
+
# typed: strict
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "sorbet-runtime"
|
|
5
|
+
|
|
6
|
+
require "date"
|
|
7
|
+
require "active_sanction/partial_date/parser"
|
|
8
|
+
|
|
9
|
+
module ActiveSanction
|
|
10
|
+
# A date a sanctions list published imprecisely. `Date` cannot hold one:
|
|
11
|
+
# collapsing "1972" to 1972-01-01 invents a precision the publisher never
|
|
12
|
+
# claimed, and a scorer that believes it will call 1972 and 1972-04-29 a
|
|
13
|
+
# conflict when they are in fact a match.
|
|
14
|
+
#
|
|
15
|
+
# PartialDate.parse("1972") # year only
|
|
16
|
+
# PartialDate.parse("circa 1962") # approximate
|
|
17
|
+
# PartialDate.parse("between 1971 and 1973") # a span
|
|
18
|
+
# PartialDate.new(year: 1965, month: 4, day: 29)
|
|
19
|
+
#
|
|
20
|
+
# Every instance carries a first and last possible date, which is what makes
|
|
21
|
+
# #overlaps? and #conflicts_with? exact regardless of how precise either side
|
|
22
|
+
# is. Instances are frozen on construction and compare by value.
|
|
23
|
+
class PartialDate
|
|
24
|
+
extend T::Sig
|
|
25
|
+
|
|
26
|
+
# :range is a precision in the sense the scorer (#32) cares about -- how
|
|
27
|
+
# much of the calendar a date could be -- not a grammatical one.
|
|
28
|
+
PRECISIONS = T.let(%i[year month day range].freeze, T::Array[Symbol])
|
|
29
|
+
|
|
30
|
+
# Canonical member order. Snapshot (#8) checksums the serialized form, so
|
|
31
|
+
# #to_h must lay its keys out the same way every time. `precision` is
|
|
32
|
+
# derived rather than stored: a stored copy can disagree with the fields it
|
|
33
|
+
# describes, and then two records that mean the same thing checksum apart.
|
|
34
|
+
#
|
|
35
|
+
# @api private
|
|
36
|
+
MEMBERS = T.let(%i[year month day from to approximate].freeze, T::Array[Symbol])
|
|
37
|
+
|
|
38
|
+
# "Circa 1962" and "1963" are the same claim about a person, made by two
|
|
39
|
+
# governments with different sources. Comparing an approximate date on its
|
|
40
|
+
# literal bounds would call that a conflict and penalize a true match, so
|
|
41
|
+
# comparison -- and only comparison, never #to_h or #to_s -- widens an
|
|
42
|
+
# approximate date by a year on each side.
|
|
43
|
+
#
|
|
44
|
+
# @api private
|
|
45
|
+
APPROXIMATE_SLACK_YEARS = 1
|
|
46
|
+
|
|
47
|
+
# A point date carries year/month/day and no endpoints; a range carries its
|
|
48
|
+
# endpoints and no year of its own. Which is which is #range?.
|
|
49
|
+
sig { returns(T.nilable(Integer)) }
|
|
50
|
+
attr_reader :year
|
|
51
|
+
|
|
52
|
+
sig { returns(T.nilable(Integer)) }
|
|
53
|
+
attr_reader :month
|
|
54
|
+
|
|
55
|
+
sig { returns(T.nilable(Integer)) }
|
|
56
|
+
attr_reader :day
|
|
57
|
+
|
|
58
|
+
sig { returns(T.nilable(PartialDate)) }
|
|
59
|
+
attr_reader :from
|
|
60
|
+
|
|
61
|
+
sig { returns(T.nilable(PartialDate)) }
|
|
62
|
+
attr_reader :to
|
|
63
|
+
|
|
64
|
+
sig { returns(T::Boolean) }
|
|
65
|
+
attr_reader :approximate
|
|
66
|
+
|
|
67
|
+
# The two dates that bound this one, whatever shape it is -- which is what
|
|
68
|
+
# makes #overlaps? exact regardless of how precise either side is. Never
|
|
69
|
+
# nil: #initialize derives both for every instance it will build.
|
|
70
|
+
sig { returns(Date) }
|
|
71
|
+
attr_reader :first_date
|
|
72
|
+
|
|
73
|
+
sig { returns(Date) }
|
|
74
|
+
attr_reader :last_date
|
|
75
|
+
|
|
76
|
+
# Reads a date expression from free text, returning nil on anything it
|
|
77
|
+
# cannot read. The vocabulary lives in Parser, which is where new source
|
|
78
|
+
# spellings get added.
|
|
79
|
+
sig { params(text: T.untyped).returns(T.nilable(PartialDate)) }
|
|
80
|
+
def self.parse(text)
|
|
81
|
+
Parser.call(text)
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# Rebuilds a date from #to_h output. Accepts string keys, so a record that
|
|
85
|
+
# has been through JSON round-trips without a separate coercion step.
|
|
86
|
+
sig { params(hash: T.untyped).returns(T.attached_class) }
|
|
87
|
+
def self.from_h(hash)
|
|
88
|
+
attributes = hash.to_h.transform_keys(&:to_sym)
|
|
89
|
+
unknown = attributes.keys - MEMBERS
|
|
90
|
+
raise InvalidArgument, "unknown PartialDate attribute(s): #{unknown.join(", ")}" if unknown.any?
|
|
91
|
+
|
|
92
|
+
new(**attributes)
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# A span, from the UN's `TYPE_OF_DATE = BETWEEN`. Endpoints may be
|
|
96
|
+
# PartialDates, #to_h hashes, or strings this class can parse.
|
|
97
|
+
sig { params(from: T.untyped, to: T.untyped, approximate: T::Boolean).returns(T.attached_class) }
|
|
98
|
+
def self.range(from, to, approximate: false)
|
|
99
|
+
new(from: from, to: to, approximate: approximate)
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# Untyped on purpose, and the same choice Entity makes: these arrive as
|
|
103
|
+
# whatever a publisher wrote and a parser made of it. What comes back out
|
|
104
|
+
# is typed -- see the readers above.
|
|
105
|
+
sig do
|
|
106
|
+
params(year: T.untyped, month: T.untyped, day: T.untyped, from: T.untyped, to: T.untyped,
|
|
107
|
+
approximate: T.untyped).void
|
|
108
|
+
end
|
|
109
|
+
def initialize(year: nil, month: nil, day: nil, from: nil, to: nil, approximate: false)
|
|
110
|
+
@approximate = T.let(approximate ? true : false, T::Boolean)
|
|
111
|
+
@year = T.let(nil, T.nilable(Integer))
|
|
112
|
+
@month = T.let(nil, T.nilable(Integer))
|
|
113
|
+
@day = T.let(nil, T.nilable(Integer))
|
|
114
|
+
@from = T.let(nil, T.nilable(PartialDate))
|
|
115
|
+
@to = T.let(nil, T.nilable(PartialDate))
|
|
116
|
+
# Both branches assign the members they own and hand back the pair of
|
|
117
|
+
# dates that bound them, which is what makes those two non-nil for every
|
|
118
|
+
# instance rather than for most of them.
|
|
119
|
+
first, last =
|
|
120
|
+
if from.nil? && to.nil?
|
|
121
|
+
assign_point(year, month, day)
|
|
122
|
+
else
|
|
123
|
+
reject_mixed_shape(year, month, day)
|
|
124
|
+
assign_range(from, to)
|
|
125
|
+
end
|
|
126
|
+
@first_date = T.let(first, Date)
|
|
127
|
+
@last_date = T.let(last, Date)
|
|
128
|
+
freeze
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
sig { returns(T::Boolean) }
|
|
132
|
+
def range? = !from.nil?
|
|
133
|
+
|
|
134
|
+
sig { returns(T::Boolean) }
|
|
135
|
+
def approximate? = approximate
|
|
136
|
+
|
|
137
|
+
sig { returns(Symbol) }
|
|
138
|
+
def precision
|
|
139
|
+
return :range if range?
|
|
140
|
+
return :day if day
|
|
141
|
+
return :month if month
|
|
142
|
+
|
|
143
|
+
:year
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
# Every date this could be, which is the whole point of the type.
|
|
147
|
+
sig { returns(T::Range[Date]) }
|
|
148
|
+
def to_range = first_date..last_date
|
|
149
|
+
|
|
150
|
+
# True when the two dates could describe the same day. Precision does not
|
|
151
|
+
# have to match: a year-only date overlaps every full date inside it, which
|
|
152
|
+
# is what lets #32 treat 1972 against 1972-04-29 as a moderate boost rather
|
|
153
|
+
# than a miss.
|
|
154
|
+
sig { params(other: T.untyped).returns(T::Boolean) }
|
|
155
|
+
def overlaps?(other)
|
|
156
|
+
return false if other.nil?
|
|
157
|
+
|
|
158
|
+
mine = comparison_range
|
|
159
|
+
theirs = comparable!(other).comparison_range
|
|
160
|
+
mine.first <= theirs.last && theirs.first <= mine.last
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
# The strict complement of #overlaps? for two known dates. A missing date
|
|
164
|
+
# is not a conflict -- nobody claimed anything to contradict -- so nil
|
|
165
|
+
# answers false to both questions.
|
|
166
|
+
sig { params(other: T.untyped).returns(T::Boolean) }
|
|
167
|
+
def conflicts_with?(other)
|
|
168
|
+
return false if other.nil?
|
|
169
|
+
|
|
170
|
+
!overlaps?(other)
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
# Widened by APPROXIMATE_SLACK_YEARS when the publisher said circa.
|
|
174
|
+
sig { returns(T::Range[Date]) }
|
|
175
|
+
def comparison_range
|
|
176
|
+
return to_range unless approximate?
|
|
177
|
+
|
|
178
|
+
first_date.prev_year(APPROXIMATE_SLACK_YEARS)..last_date.next_year(APPROXIMATE_SLACK_YEARS)
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
sig { returns(T::Hash[Symbol, T.untyped]) }
|
|
182
|
+
def to_h
|
|
183
|
+
{ year: year, month: month, day: day, from: from&.to_h, to: to&.to_h, approximate: approximate }
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
# Renders in a form .parse reads back, so a date survives a trip through
|
|
187
|
+
# free text -- which is how OFAC publishes them in the first place.
|
|
188
|
+
sig { returns(String) }
|
|
189
|
+
def to_s
|
|
190
|
+
text = range? ? "#{from} to #{to}" : point_to_s
|
|
191
|
+
approximate? ? "circa #{text}" : text
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
sig { params(other: T.untyped).returns(T::Boolean) }
|
|
195
|
+
def ==(other)
|
|
196
|
+
return false unless other.instance_of?(self.class)
|
|
197
|
+
|
|
198
|
+
to_h == other.to_h
|
|
199
|
+
end
|
|
200
|
+
alias eql? ==
|
|
201
|
+
|
|
202
|
+
sig { returns(Integer) }
|
|
203
|
+
def hash
|
|
204
|
+
[self.class, to_h].hash
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
sig { returns(String) }
|
|
208
|
+
def inspect
|
|
209
|
+
"#<#{self.class} #{self} precision=#{precision.inspect}>"
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
private
|
|
213
|
+
|
|
214
|
+
sig { params(year: T.untyped, month: T.untyped, day: T.untyped).returns([Date, Date]) }
|
|
215
|
+
def assign_point(year, month, day)
|
|
216
|
+
raise InvalidArgument, "year is required" if year.nil?
|
|
217
|
+
|
|
218
|
+
number = integer!(:year, year)
|
|
219
|
+
@year = number
|
|
220
|
+
@month = optional_integer(:month, month)
|
|
221
|
+
@day = optional_integer(:day, day)
|
|
222
|
+
validate_point!(number)
|
|
223
|
+
first = Date.new(number, @month || 1, @day || 1).freeze
|
|
224
|
+
# A year-only date runs to 31 December, a month to its own real last day.
|
|
225
|
+
[first, @day ? first : Date.new(number, @month || 12, -1).freeze]
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
sig { params(year: Integer).void }
|
|
229
|
+
def validate_point!(year)
|
|
230
|
+
raise InvalidArgument, "day given without a month" if @day && @month.nil?
|
|
231
|
+
raise InvalidArgument, "not a real date: #{to_s.inspect}" unless Date.valid_date?(year, @month || 1, @day || 1)
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
sig { params(from: T.untyped, to: T.untyped).returns([Date, Date]) }
|
|
235
|
+
def assign_range(from, to)
|
|
236
|
+
raise InvalidArgument, "a range needs both from and to" if from.nil? || to.nil?
|
|
237
|
+
|
|
238
|
+
first = endpoint!(:from, from)
|
|
239
|
+
last = endpoint!(:to, to)
|
|
240
|
+
@from = first
|
|
241
|
+
@to = last
|
|
242
|
+
raise InvalidArgument, "range runs backwards: #{first} to #{last}" if first.first_date > last.last_date
|
|
243
|
+
|
|
244
|
+
[first.first_date, last.last_date]
|
|
245
|
+
end
|
|
246
|
+
|
|
247
|
+
sig { params(year: T.untyped, month: T.untyped, day: T.untyped).void }
|
|
248
|
+
def reject_mixed_shape(year, month, day)
|
|
249
|
+
return if [year, month, day].all?(&:nil?)
|
|
250
|
+
|
|
251
|
+
raise InvalidArgument, "a range carries its year in its endpoints, not alongside them"
|
|
252
|
+
end
|
|
253
|
+
|
|
254
|
+
# Endpoints are themselves PartialDates so a span between two year-only
|
|
255
|
+
# dates keeps both years, and they may not nest: "between (1971 to 1972)
|
|
256
|
+
# and 1973" is not something any list publishes.
|
|
257
|
+
sig { params(member: Symbol, value: T.untyped).returns(PartialDate) }
|
|
258
|
+
def endpoint!(member, value)
|
|
259
|
+
date = coerce_endpoint(value)
|
|
260
|
+
raise InvalidArgument, "#{member} is not a date: #{value.inspect}" if date.nil?
|
|
261
|
+
raise InvalidArgument, "#{member} cannot itself be a range" if date.range?
|
|
262
|
+
|
|
263
|
+
date
|
|
264
|
+
end
|
|
265
|
+
|
|
266
|
+
sig { params(value: T.untyped).returns(T.nilable(PartialDate)) }
|
|
267
|
+
def coerce_endpoint(value)
|
|
268
|
+
case value
|
|
269
|
+
when PartialDate then value
|
|
270
|
+
when Hash then PartialDate.from_h(value)
|
|
271
|
+
else PartialDate.parse(value)
|
|
272
|
+
end
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
sig { params(other: T.untyped).returns(PartialDate) }
|
|
276
|
+
def comparable!(other)
|
|
277
|
+
return other if other.is_a?(PartialDate)
|
|
278
|
+
|
|
279
|
+
raise InvalidArgument, "expected a #{self.class}, got #{other.class}"
|
|
280
|
+
end
|
|
281
|
+
|
|
282
|
+
sig { params(member: Symbol, value: T.untyped).returns(T.nilable(Integer)) }
|
|
283
|
+
def optional_integer(member, value)
|
|
284
|
+
value.nil? ? nil : integer!(member, value)
|
|
285
|
+
end
|
|
286
|
+
|
|
287
|
+
sig { params(member: Symbol, value: T.untyped).returns(Integer) }
|
|
288
|
+
def integer!(member, value)
|
|
289
|
+
Integer(value.to_s, 10)
|
|
290
|
+
rescue TypeError, ArgumentError
|
|
291
|
+
raise InvalidArgument, "#{member} is not a number: #{value.inspect}"
|
|
292
|
+
end
|
|
293
|
+
|
|
294
|
+
sig { returns(String) }
|
|
295
|
+
def point_to_s
|
|
296
|
+
case precision
|
|
297
|
+
when :day then format("%<year>04d-%<month>02d-%<day>02d", year: year, month: month, day: day)
|
|
298
|
+
when :month then format("%<year>04d-%<month>02d", year: year, month: month)
|
|
299
|
+
else format("%<year>04d", year: year)
|
|
300
|
+
end
|
|
301
|
+
end
|
|
302
|
+
end
|
|
303
|
+
end
|