active_sanction 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +538 -0
  3. data/CODE_OF_CONDUCT.md +74 -0
  4. data/CONTRIBUTING.md +312 -0
  5. data/LICENSE.txt +21 -0
  6. data/README.md +616 -0
  7. data/SECURITY.md +97 -0
  8. data/active_sanction.gemspec +98 -0
  9. data/docs/adding_a_source.md +1083 -0
  10. data/docs/api_stability.md +357 -0
  11. data/docs/bundle_format.md +407 -0
  12. data/lib/active_sanction/address.rb +170 -0
  13. data/lib/active_sanction/client.rb +341 -0
  14. data/lib/active_sanction/configuration.rb +721 -0
  15. data/lib/active_sanction/countries.txt +292 -0
  16. data/lib/active_sanction/country.rb +164 -0
  17. data/lib/active_sanction/deprecation.rb +168 -0
  18. data/lib/active_sanction/diff/change.rb +208 -0
  19. data/lib/active_sanction/diff.rb +348 -0
  20. data/lib/active_sanction/doctor/checkup.rb +356 -0
  21. data/lib/active_sanction/doctor/diagnosis.rb +245 -0
  22. data/lib/active_sanction/doctor/finding.rb +183 -0
  23. data/lib/active_sanction/doctor/profile.rb +384 -0
  24. data/lib/active_sanction/doctor/report.rb +227 -0
  25. data/lib/active_sanction/doctor.rb +347 -0
  26. data/lib/active_sanction/entity.rb +293 -0
  27. data/lib/active_sanction/error.rb +332 -0
  28. data/lib/active_sanction/fetcher/result.rb +116 -0
  29. data/lib/active_sanction/fetcher.rb +242 -0
  30. data/lib/active_sanction/http_client/errors.rb +94 -0
  31. data/lib/active_sanction/http_client/response.rb +129 -0
  32. data/lib/active_sanction/http_client.rb +324 -0
  33. data/lib/active_sanction/identifier.rb +212 -0
  34. data/lib/active_sanction/index/builder.rb +89 -0
  35. data/lib/active_sanction/index/candidate.rb +63 -0
  36. data/lib/active_sanction/index/entry.rb +66 -0
  37. data/lib/active_sanction/index/features.rb +112 -0
  38. data/lib/active_sanction/index.rb +381 -0
  39. data/lib/active_sanction/match_result.rb +345 -0
  40. data/lib/active_sanction/matcher.rb +388 -0
  41. data/lib/active_sanction/name.rb +183 -0
  42. data/lib/active_sanction/normalizer/cache.rb +82 -0
  43. data/lib/active_sanction/normalizer/dictionaries/honorifics.txt +30 -0
  44. data/lib/active_sanction/normalizer/dictionaries/legal_forms.txt +72 -0
  45. data/lib/active_sanction/normalizer/dictionaries/organization_stopwords.txt +20 -0
  46. data/lib/active_sanction/normalizer/dictionaries/particles.txt +35 -0
  47. data/lib/active_sanction/normalizer/dictionary/stoplist.rb +121 -0
  48. data/lib/active_sanction/normalizer/dictionary.rb +243 -0
  49. data/lib/active_sanction/normalizer/form.rb +279 -0
  50. data/lib/active_sanction/normalizer.rb +137 -0
  51. data/lib/active_sanction/parsers/column_shape.rb +274 -0
  52. data/lib/active_sanction/parsers/delimited_table/reader.rb +194 -0
  53. data/lib/active_sanction/parsers/delimited_table/row.rb +81 -0
  54. data/lib/active_sanction/parsers/delimited_table.rb +151 -0
  55. data/lib/active_sanction/parsers/format.rb +107 -0
  56. data/lib/active_sanction/parsers/join.rb +149 -0
  57. data/lib/active_sanction/parsers/spreadsheet/archive.rb +258 -0
  58. data/lib/active_sanction/parsers/spreadsheet/reader.rb +213 -0
  59. data/lib/active_sanction/parsers/spreadsheet/row.rb +88 -0
  60. data/lib/active_sanction/parsers/spreadsheet/workbook.rb +314 -0
  61. data/lib/active_sanction/parsers/spreadsheet.rb +168 -0
  62. data/lib/active_sanction/parsers/xml_records/backends/nokogiri.rb +134 -0
  63. data/lib/active_sanction/parsers/xml_records/backends/rexml.rb +157 -0
  64. data/lib/active_sanction/parsers/xml_records/backends.rb +103 -0
  65. data/lib/active_sanction/parsers/xml_records/builder.rb +90 -0
  66. data/lib/active_sanction/parsers/xml_records/reader.rb +134 -0
  67. data/lib/active_sanction/parsers/xml_records/record.rb +174 -0
  68. data/lib/active_sanction/parsers/xml_records.rb +140 -0
  69. data/lib/active_sanction/parsers.rb +94 -0
  70. data/lib/active_sanction/partial_date/parser.rb +124 -0
  71. data/lib/active_sanction/partial_date.rb +303 -0
  72. data/lib/active_sanction/payload_cache/checksum.rb +66 -0
  73. data/lib/active_sanction/payload_cache/entry.rb +304 -0
  74. data/lib/active_sanction/payload_cache.rb +366 -0
  75. data/lib/active_sanction/phonetics/double_metaphone.rb +662 -0
  76. data/lib/active_sanction/phonetics.rb +40 -0
  77. data/lib/active_sanction/query.rb +290 -0
  78. data/lib/active_sanction/rescreen/alert.rb +314 -0
  79. data/lib/active_sanction/rescreen.rb +332 -0
  80. data/lib/active_sanction/scorer/adjustments.rb +241 -0
  81. data/lib/active_sanction/scorer/name_score.rb +279 -0
  82. data/lib/active_sanction/scorer/reason.rb +136 -0
  83. data/lib/active_sanction/scorer/result.rb +129 -0
  84. data/lib/active_sanction/scorer/subject.rb +235 -0
  85. data/lib/active_sanction/scorer/weights.rb +306 -0
  86. data/lib/active_sanction/scorer.rb +303 -0
  87. data/lib/active_sanction/similarity/jaro_winkler.rb +210 -0
  88. data/lib/active_sanction/similarity/levenshtein.rb +179 -0
  89. data/lib/active_sanction/similarity/token_set.rb +159 -0
  90. data/lib/active_sanction/similarity/token_sort.rb +120 -0
  91. data/lib/active_sanction/similarity.rb +183 -0
  92. data/lib/active_sanction/snapshot/bundle/header.rb +302 -0
  93. data/lib/active_sanction/snapshot/bundle/payload.rb +185 -0
  94. data/lib/active_sanction/snapshot/bundle/signature.rb +201 -0
  95. data/lib/active_sanction/snapshot/bundle.rb +381 -0
  96. data/lib/active_sanction/snapshot.rb +305 -0
  97. data/lib/active_sanction/sources/australia_dfat/published_date.rb +166 -0
  98. data/lib/active_sanction/sources/australia_dfat/record.rb +319 -0
  99. data/lib/active_sanction/sources/australia_dfat.rb +291 -0
  100. data/lib/active_sanction/sources/base.rb +295 -0
  101. data/lib/active_sanction/sources/canada_sema/record.rb +282 -0
  102. data/lib/active_sanction/sources/canada_sema/source_ref.rb +89 -0
  103. data/lib/active_sanction/sources/canada_sema.rb +153 -0
  104. data/lib/active_sanction/sources/definition.rb +360 -0
  105. data/lib/active_sanction/sources/eu_fsf/record.rb +467 -0
  106. data/lib/active_sanction/sources/eu_fsf.rb +203 -0
  107. data/lib/active_sanction/sources/ofac/record.rb +251 -0
  108. data/lib/active_sanction/sources/ofac/remarks_parser/coverage.rb +120 -0
  109. data/lib/active_sanction/sources/ofac/remarks_parser/vocabulary.rb +121 -0
  110. data/lib/active_sanction/sources/ofac/remarks_parser.rb +312 -0
  111. data/lib/active_sanction/sources/ofac.rb +248 -0
  112. data/lib/active_sanction/sources/ofac_consolidated/record.rb +37 -0
  113. data/lib/active_sanction/sources/ofac_consolidated.rb +203 -0
  114. data/lib/active_sanction/sources/ofac_sdn.rb +43 -0
  115. data/lib/active_sanction/sources/remarks.rb +84 -0
  116. data/lib/active_sanction/sources/uk_sanctions_list/published_date.rb +104 -0
  117. data/lib/active_sanction/sources/uk_sanctions_list/record.rb +412 -0
  118. data/lib/active_sanction/sources/uk_sanctions_list.rb +214 -0
  119. data/lib/active_sanction/sources/un_consolidated/record.rb +279 -0
  120. data/lib/active_sanction/sources/un_consolidated.rb +140 -0
  121. data/lib/active_sanction/sources.rb +196 -0
  122. data/lib/active_sanction/storage/active_record/reader.rb +124 -0
  123. data/lib/active_sanction/storage/active_record/row.rb +175 -0
  124. data/lib/active_sanction/storage/active_record/writer.rb +141 -0
  125. data/lib/active_sanction/storage/active_record.rb +277 -0
  126. data/lib/active_sanction/storage/base.rb +216 -0
  127. data/lib/active_sanction/storage/file_system.rb +373 -0
  128. data/lib/active_sanction/storage/memory.rb +67 -0
  129. data/lib/active_sanction/storage/meta.rb +188 -0
  130. data/lib/active_sanction/storage.rb +74 -0
  131. data/lib/active_sanction/subject.rb +271 -0
  132. data/lib/active_sanction/sync/report.rb +245 -0
  133. data/lib/active_sanction/sync/result.rb +296 -0
  134. data/lib/active_sanction/sync.rb +390 -0
  135. data/lib/active_sanction/validator_store/file_system.rb +103 -0
  136. data/lib/active_sanction/validator_store/memory.rb +41 -0
  137. data/lib/active_sanction/validator_store.rb +119 -0
  138. data/lib/active_sanction/validators.rb +195 -0
  139. data/lib/active_sanction/version.rb +23 -0
  140. data/lib/active_sanction.rb +331 -0
  141. data/lib/generators/active_sanction/install/install_generator.rb +60 -0
  142. data/lib/generators/active_sanction/install/templates/create_active_sanction_tables.rb.tt +142 -0
  143. metadata +236 -0
@@ -0,0 +1,140 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ require "sorbet-runtime"
5
+
6
+ require "set"
7
+ require "active_sanction/parsers/format"
8
+ require "active_sanction/parsers/xml_records/record"
9
+ require "active_sanction/parsers/xml_records/builder"
10
+ require "active_sanction/parsers/xml_records/backends"
11
+ require "active_sanction/parsers/xml_records/reader"
12
+
13
+ module ActiveSanction
14
+ module Parsers
15
+ # Reads a record-oriented XML list -- the UN, Canada, the EU and the UK all
16
+ # publish one -- into records an adapter can map onto Entities.
17
+ #
18
+ # A table is a description of the document, built once and reused for every
19
+ # sync; a Reader is one pass over one payload.
20
+ #
21
+ # UN = ActiveSanction::Parsers::XmlRecords.new(records: %w[INDIVIDUAL ENTITY])
22
+ #
23
+ # reader = UN.read(bytes)
24
+ # reader.each do |record|
25
+ # record.name # => "INDIVIDUAL"
26
+ # record["FIRST_NAME"] # => "ERIC"
27
+ # record.values("NATIONALITY/VALUE") # => ["Chad"]
28
+ # record.nodes("INDIVIDUAL_ALIAS") # => [Record, ...]
29
+ # end
30
+ # reader.root["dateGenerated"] # the publisher's own version marker
31
+ #
32
+ # ### Streaming from day one, before anything needs it
33
+ #
34
+ # Every list this gem launches with is small: the UN is 2.2 MB, Canada is
35
+ # 2.9 MB, and either would load into a DOM without anyone noticing. The one
36
+ # that is coming does not. OFAC's `SDN_ADVANCED.XML` is 126 MB, and a DOM
37
+ # design would meet it by being rewritten.
38
+ #
39
+ # So the interface is record-at-a-time now, while it is free to be: the
40
+ # parser holds one record's depth on a stack and drops it as soon as the
41
+ # adapter is done with it. What that buys is not speed, it is that the
42
+ # adapter written against this today is the adapter that reads a 126 MB
43
+ # file later, unchanged.
44
+ #
45
+ # ### Naming the records, and only the records
46
+ #
47
+ # A document's scaffolding -- `<CONSOLIDATED_LIST>`, `<INDIVIDUALS>` -- is
48
+ # skipped entirely rather than being built into nodes nobody asked for.
49
+ # Naming several record elements is normal: the UN files people under
50
+ # `<INDIVIDUAL>` and organizations under `<ENTITY>`, in one document, and
51
+ # an adapter wants a single pass over both.
52
+ #
53
+ # ### Namespaces
54
+ #
55
+ # Element and attribute names are matched with any prefix removed, so a
56
+ # publisher adding an `xmlns` next quarter does not silently stop matching.
57
+ # See Backends.local_name for why the prefix is dropped rather than
58
+ # resolved.
59
+ class XmlRecords
60
+ extend T::Sig
61
+ include Format
62
+
63
+ # Raised by a backend when the payload stops being XML. Caught by Reader,
64
+ # which decides between salvaging the records already read and refusing
65
+ # the payload outright; it escapes as a ParseError either way, so a
66
+ # caller rescuing the toolkit's errors does not have to know about it.
67
+ #
68
+ # `line` is ParseError's, and is nil where the backend reports no
69
+ # position -- libxml2 does not always.
70
+ class MalformedDocument < ParseError; end
71
+
72
+ # A Set: `record?` is asked once per element in the document, which for
73
+ # the UN is roughly 30,000 times a pass.
74
+ sig { returns(T::Set[String]) }
75
+ attr_reader :records
76
+
77
+ sig { override.returns(T::Array[String]) }
78
+ attr_reader :nulls
79
+
80
+ sig { override.returns(Encoding) }
81
+ attr_reader :encoding
82
+
83
+ # `null:` is here for the same reason DelimitedTable has it -- a
84
+ # publisher that writes a sentinel where it means nothing -- though the
85
+ # XML lists mostly use an empty element instead, which is already nil.
86
+ #
87
+ # `backend:` overrides the configured default for this table alone. Most
88
+ # adapters should not pass it: which XML library parses a list is an
89
+ # installation's decision, not a list's. See Backends.
90
+ sig { params(records: T.untyped, null: T.untyped, encoding: T.untyped, backend: T.untyped).void }
91
+ def initialize(records:, null: nil, encoding: DEFAULT_ENCODING, backend: nil)
92
+ @records = T.let(records!(records), T::Set[String])
93
+ @nulls = T.let(nulls!(null), T::Array[String])
94
+ @encoding = T.let(encoding!(encoding), Encoding)
95
+ @backend = T.let(backend&.to_sym, T.nilable(Symbol))
96
+ freeze
97
+ end
98
+
99
+ # A pass over one payload. Takes the bytes as a String, which is what
100
+ # Sources::Base hands #parse.
101
+ sig { params(payload: T.untyped).returns(Reader) }
102
+ def read(payload) = Reader.new(table: self, payload: payload)
103
+
104
+ # Whether an element name starts a record. Asked by every backend for
105
+ # every element outside a record, so it stays a Set lookup.
106
+ sig { params(name: String).returns(T::Boolean) }
107
+ def record?(name) = records.include?(name)
108
+
109
+ # The record element names in declaration order, for a message or an
110
+ # inspect -- Set is the right shape to ask `record?` of and the wrong
111
+ # shape to print.
112
+ sig { returns(T::Array[String]) }
113
+ def record_names = records.to_a
114
+
115
+ # Resolved per call rather than at construction, so a table built at
116
+ # class-definition time -- which is where an adapter builds it -- still
117
+ # honours an `xml_backend` set later in an initializer.
118
+ sig { returns(T.untyped) }
119
+ def backend = Backends.resolve(@backend || ActiveSanction.config.xml_backend)
120
+
121
+ sig { returns(String) }
122
+ def inspect
123
+ "#<#{self.class} records=#{record_names.join(", ")}#{" null=#{nulls.first.inspect}" if nulls.any?}>"
124
+ end
125
+
126
+ private
127
+
128
+ # Accepts one element name or several. Stored as a Set: a document with
129
+ # 1,011 records asks `record?` once per element in the file, which for
130
+ # the UN is roughly 30,000 times.
131
+ sig { params(value: T.untyped).returns(T::Set[String]) }
132
+ def records!(value)
133
+ names = Array(value).map { |name| -name.to_s.strip }.reject(&:empty?).uniq
134
+ raise InvalidArgument, "records must name at least one element, e.g. records: \"INDIVIDUAL\"" if names.empty?
135
+
136
+ Set.new(names).freeze
137
+ end
138
+ end
139
+ end
140
+ end
@@ -0,0 +1,94 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ require "sorbet-runtime"
5
+
6
+ require "active_sanction/error"
7
+
8
+ module ActiveSanction
9
+ # The format toolkits adapters parse with. A source declares what its file
10
+ # looks like -- delimiter, column names, null sentinel -- and gets rows back;
11
+ # what those rows *mean* stays in the adapter, because that is the part no
12
+ # two publishers agree on.
13
+ #
14
+ # Nothing here knows about Entity. These are file readers, and keeping them
15
+ # ignorant of the canonical model is what lets an adapter for a list nobody
16
+ # here has seen reuse them.
17
+ module Parsers
18
+ # A payload that could not be read at all: the wrong format, a truncated
19
+ # download, an encoding that cannot be decoded. Distinct from a Warning,
20
+ # which is a *row* that could not be read while the rest of the file could.
21
+ #
22
+ # The same class as ActiveSanction::ParseError, under the name the toolkits
23
+ # raise it by. It is declared with the rest of the public hierarchy rather
24
+ # than here because a caller rescuing "this list could not be read" should
25
+ # not have to know which of these toolkits read it -- and because a source
26
+ # adapter parsing a format none of them covers raises the same thing.
27
+ ParseError = ActiveSanction::ParseError
28
+
29
+ # One row the parser could not use, kept rather than raised.
30
+ #
31
+ # A sanctions list is not a file we control. OFAC ships 19,321 rows and a
32
+ # single unbalanced quote somewhere in the middle must not cost the other
33
+ # 19,320 -- refusing to load a list because one record is malformed fails
34
+ # exactly when the list is most needed. So a bad row is recorded here and
35
+ # skipped, and the caller decides whether the count is tolerable.
36
+ #
37
+ # `line` is the line number within the file, which is what makes a warning
38
+ # actionable: a 5.6 MB CSV is only debuggable if the complaint says where.
39
+ # It is nil when the parser cannot say -- libxml2 reports no position for a
40
+ # record -- and a warning that cannot point at a line still says what went
41
+ # wrong rather than pointing at the wrong one.
42
+ class Warning
43
+ extend T::Sig
44
+
45
+ # nil when the parser cannot say which line -- see above.
46
+ sig { returns(T.nilable(Integer)) }
47
+ attr_reader :line
48
+
49
+ sig { returns(String) }
50
+ attr_reader :message
51
+
52
+ sig { returns(T.nilable(String)) }
53
+ attr_reader :snippet
54
+
55
+ sig { params(line: T.nilable(Integer), message: String, snippet: T.untyped).void }
56
+ def initialize(line:, message:, snippet: nil)
57
+ @line = T.let(line, T.nilable(Integer))
58
+ @message = T.let(message, String)
59
+ @snippet = T.let(snippet.nil? ? nil : truncate(snippet), T.nilable(String))
60
+ freeze
61
+ end
62
+
63
+ sig { returns(String) }
64
+ def to_s
65
+ "#{"line #{line}: " if line}#{message}#{" -- #{snippet.inspect}" if snippet}"
66
+ end
67
+
68
+ sig { returns(T::Hash[Symbol, T.untyped]) }
69
+ def to_h = { line: line, message: message, snippet: snippet }
70
+
71
+ sig { returns(String) }
72
+ def inspect = "#<#{self.class} #{self}>"
73
+
74
+ private
75
+
76
+ # A malformed row is frequently malformed because it is enormous -- an
77
+ # unclosed quote swallows everything after it -- so the evidence is
78
+ # trimmed before it is kept.
79
+ sig { params(text: T.untyped).returns(String) }
80
+ def truncate(text)
81
+ string = text.to_s
82
+ -(string.length > 120 ? "#{string[0, 120]}..." : string)
83
+ end
84
+ end
85
+ end
86
+ end
87
+
88
+ require "active_sanction/parsers/column_shape"
89
+ require "active_sanction/parsers/delimited_table"
90
+ require "active_sanction/parsers/spreadsheet"
91
+ require "active_sanction/parsers/xml_records"
92
+ require "active_sanction/parsers/xml_records/backends/rexml"
93
+ require "active_sanction/parsers/xml_records/backends/nokogiri"
94
+ require "active_sanction/parsers/join"
@@ -0,0 +1,124 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ require "sorbet-runtime"
5
+
6
+ require "date"
7
+
8
+ module ActiveSanction
9
+ class PartialDate
10
+ # Reads the date expressions the lists publish in free text. OFAC ships
11
+ # dates inside a remarks blob ("DOB circa 1962"), so the vocabulary here is
12
+ # whatever a government typist actually wrote, not a format we chose.
13
+ #
14
+ # Returns nil on anything it cannot read, per the issue: a birth date is one
15
+ # field on a record, and an unanticipated string should not abort the import
16
+ # of the entity around it. Callers decide whether that is worth reporting.
17
+ #
18
+ # @api private
19
+ module Parser
20
+ extend T::Sig
21
+
22
+ # "Sept" is not in Date::ABBR_MONTHNAMES but appears in OFAC free text.
23
+ MONTHS = T.let(
24
+ [Date::MONTHNAMES, Date::ABBR_MONTHNAMES].each_with_object({ "sept" => 9 }) do |names, months|
25
+ names.each_with_index { |name, number| months[name.downcase] = number if name }
26
+ end.freeze,
27
+ T::Hash[String, Integer]
28
+ )
29
+
30
+ ISO = T.let(/\A(\d{4})(?:-(\d{1,2})(?:-(\d{1,2}))?)?\z/, Regexp)
31
+ DAY_MONTH_YEAR = T.let(/\A(\d{1,2})\s+([a-z]+)\.?,?\s+(\d{4})\z/i, Regexp)
32
+ MONTH_DAY_YEAR = T.let(/\A([a-z]+)\.?\s+(\d{1,2})(?:st|nd|rd|th)?,?\s+(\d{4})\z/i, Regexp)
33
+ MONTH_YEAR = T.let(/\A([a-z]+)\.?,?\s+(\d{4})\z/i, Regexp)
34
+ APPROXIMATE = T.let(/\A(?:circa|approximately|approx|about|ca|c)\.?\s+|\A~\s*/i, Regexp)
35
+ BETWEEN = T.let(/\Abetween\s+/i, Regexp)
36
+ WORD_RANGE = T.let(/\A(.+?)\s+(?:to|and|through|until)\s+(.+)\z/i, Regexp)
37
+ # A bare dash separates a span only between two four-digit years. Anything
38
+ # looser would read the "-" in 1972-04 as a span from 1972 to April.
39
+ YEAR_RANGE = T.let(/\A(\d{4})\s*[-–—]\s*(\d{4})\z/, Regexp)
40
+
41
+ # An XML Schema `xs:date` may carry a UTC offset, and the UN publishes
42
+ # nine of its listing dates that way: "2015-07-01-04:00". The offset
43
+ # records which midnight a clerk was working against, not which day the
44
+ # listing is, so it is trimmed rather than applied -- shifting a listing
45
+ # date across a day boundary to honour a timezone the publisher never
46
+ # meant would be a worse answer than ignoring it. Without this the whole
47
+ # date reads as nil and nine listings silently lose their date.
48
+ ZONE = T.let(/(?<=\d)(?:Z|[+-]\d{2}:\d{2})\z/, Regexp)
49
+
50
+ module_function
51
+
52
+ sig { params(text: T.untyped).returns(T.nilable(PartialDate)) }
53
+ def call(text)
54
+ string = text.to_s.strip.squeeze(" ").sub(ZONE, "")
55
+ return nil if string.empty?
56
+
57
+ approximate = APPROXIMATE.match?(string)
58
+ string = string.sub(APPROXIMATE, "").sub(BETWEEN, "")
59
+ span(string, approximate) || point(string, approximate)
60
+ end
61
+
62
+ # Endpoints go back through .call, so "1971 to circa 1973" reads. A nested
63
+ # span raises in the constructor and comes back as nil.
64
+ sig { params(string: String, approximate: T::Boolean).returns(T.nilable(PartialDate)) }
65
+ def span(string, approximate)
66
+ match = WORD_RANGE.match(string) || YEAR_RANGE.match(string)
67
+ return nil unless match
68
+
69
+ from = call(match[1])
70
+ to = call(match[2])
71
+ return nil if from.nil? || to.nil?
72
+
73
+ build(from: from, to: to, approximate: approximate)
74
+ end
75
+
76
+ sig { params(string: String, approximate: T::Boolean).returns(T.nilable(PartialDate)) }
77
+ def point(string, approximate)
78
+ attributes = point_attributes(string)
79
+ return nil if attributes.nil?
80
+
81
+ build(**attributes, approximate: approximate)
82
+ end
83
+
84
+ # A shape the regexes accept can still be an impossible date -- 31
85
+ # February 1972 -- and that is the constructor's judgment, not ours.
86
+ sig { params(attributes: T.untyped).returns(T.nilable(PartialDate)) }
87
+ def build(**attributes)
88
+ PartialDate.new(**attributes)
89
+ rescue ArgumentError
90
+ nil
91
+ end
92
+
93
+ sig { params(string: String).returns(T.nilable(T::Hash[Symbol, T.nilable(Integer)])) }
94
+ def point_attributes(string)
95
+ match = ISO.match(string)
96
+ return { year: match[1].to_i, month: match[2]&.to_i, day: match[3]&.to_i } if match
97
+
98
+ worded_attributes(string)
99
+ end
100
+
101
+ sig { params(string: String).returns(T.nilable(T::Hash[Symbol, T.nilable(Integer)])) }
102
+ def worded_attributes(string)
103
+ if (match = DAY_MONTH_YEAR.match(string))
104
+ named_month(year: match[3], month: match[2], day: match[1])
105
+ elsif (match = MONTH_DAY_YEAR.match(string))
106
+ named_month(year: match[3], month: match[1], day: match[2])
107
+ elsif (match = MONTH_YEAR.match(string))
108
+ named_month(year: match[2], month: match[1])
109
+ end
110
+ end
111
+
112
+ sig do
113
+ params(year: T.untyped, month: T.untyped, day: T.untyped)
114
+ .returns(T.nilable(T::Hash[Symbol, T.nilable(Integer)]))
115
+ end
116
+ def named_month(year:, month:, day: nil)
117
+ number = MONTHS[month.downcase]
118
+ return nil if number.nil?
119
+
120
+ { year: year.to_i, month: number, day: day&.to_i }
121
+ end
122
+ end
123
+ end
124
+ end
@@ -0,0 +1,303 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ require "sorbet-runtime"
5
+
6
+ require "date"
7
+ require "active_sanction/partial_date/parser"
8
+
9
+ module ActiveSanction
10
+ # A date a sanctions list published imprecisely. `Date` cannot hold one:
11
+ # collapsing "1972" to 1972-01-01 invents a precision the publisher never
12
+ # claimed, and a scorer that believes it will call 1972 and 1972-04-29 a
13
+ # conflict when they are in fact a match.
14
+ #
15
+ # PartialDate.parse("1972") # year only
16
+ # PartialDate.parse("circa 1962") # approximate
17
+ # PartialDate.parse("between 1971 and 1973") # a span
18
+ # PartialDate.new(year: 1965, month: 4, day: 29)
19
+ #
20
+ # Every instance carries a first and last possible date, which is what makes
21
+ # #overlaps? and #conflicts_with? exact regardless of how precise either side
22
+ # is. Instances are frozen on construction and compare by value.
23
+ class PartialDate
24
+ extend T::Sig
25
+
26
+ # :range is a precision in the sense the scorer (#32) cares about -- how
27
+ # much of the calendar a date could be -- not a grammatical one.
28
+ PRECISIONS = T.let(%i[year month day range].freeze, T::Array[Symbol])
29
+
30
+ # Canonical member order. Snapshot (#8) checksums the serialized form, so
31
+ # #to_h must lay its keys out the same way every time. `precision` is
32
+ # derived rather than stored: a stored copy can disagree with the fields it
33
+ # describes, and then two records that mean the same thing checksum apart.
34
+ #
35
+ # @api private
36
+ MEMBERS = T.let(%i[year month day from to approximate].freeze, T::Array[Symbol])
37
+
38
+ # "Circa 1962" and "1963" are the same claim about a person, made by two
39
+ # governments with different sources. Comparing an approximate date on its
40
+ # literal bounds would call that a conflict and penalize a true match, so
41
+ # comparison -- and only comparison, never #to_h or #to_s -- widens an
42
+ # approximate date by a year on each side.
43
+ #
44
+ # @api private
45
+ APPROXIMATE_SLACK_YEARS = 1
46
+
47
+ # A point date carries year/month/day and no endpoints; a range carries its
48
+ # endpoints and no year of its own. Which is which is #range?.
49
+ sig { returns(T.nilable(Integer)) }
50
+ attr_reader :year
51
+
52
+ sig { returns(T.nilable(Integer)) }
53
+ attr_reader :month
54
+
55
+ sig { returns(T.nilable(Integer)) }
56
+ attr_reader :day
57
+
58
+ sig { returns(T.nilable(PartialDate)) }
59
+ attr_reader :from
60
+
61
+ sig { returns(T.nilable(PartialDate)) }
62
+ attr_reader :to
63
+
64
+ sig { returns(T::Boolean) }
65
+ attr_reader :approximate
66
+
67
+ # The two dates that bound this one, whatever shape it is -- which is what
68
+ # makes #overlaps? exact regardless of how precise either side is. Never
69
+ # nil: #initialize derives both for every instance it will build.
70
+ sig { returns(Date) }
71
+ attr_reader :first_date
72
+
73
+ sig { returns(Date) }
74
+ attr_reader :last_date
75
+
76
+ # Reads a date expression from free text, returning nil on anything it
77
+ # cannot read. The vocabulary lives in Parser, which is where new source
78
+ # spellings get added.
79
+ sig { params(text: T.untyped).returns(T.nilable(PartialDate)) }
80
+ def self.parse(text)
81
+ Parser.call(text)
82
+ end
83
+
84
+ # Rebuilds a date from #to_h output. Accepts string keys, so a record that
85
+ # has been through JSON round-trips without a separate coercion step.
86
+ sig { params(hash: T.untyped).returns(T.attached_class) }
87
+ def self.from_h(hash)
88
+ attributes = hash.to_h.transform_keys(&:to_sym)
89
+ unknown = attributes.keys - MEMBERS
90
+ raise InvalidArgument, "unknown PartialDate attribute(s): #{unknown.join(", ")}" if unknown.any?
91
+
92
+ new(**attributes)
93
+ end
94
+
95
+ # A span, from the UN's `TYPE_OF_DATE = BETWEEN`. Endpoints may be
96
+ # PartialDates, #to_h hashes, or strings this class can parse.
97
+ sig { params(from: T.untyped, to: T.untyped, approximate: T::Boolean).returns(T.attached_class) }
98
+ def self.range(from, to, approximate: false)
99
+ new(from: from, to: to, approximate: approximate)
100
+ end
101
+
102
+ # Untyped on purpose, and the same choice Entity makes: these arrive as
103
+ # whatever a publisher wrote and a parser made of it. What comes back out
104
+ # is typed -- see the readers above.
105
+ sig do
106
+ params(year: T.untyped, month: T.untyped, day: T.untyped, from: T.untyped, to: T.untyped,
107
+ approximate: T.untyped).void
108
+ end
109
+ def initialize(year: nil, month: nil, day: nil, from: nil, to: nil, approximate: false)
110
+ @approximate = T.let(approximate ? true : false, T::Boolean)
111
+ @year = T.let(nil, T.nilable(Integer))
112
+ @month = T.let(nil, T.nilable(Integer))
113
+ @day = T.let(nil, T.nilable(Integer))
114
+ @from = T.let(nil, T.nilable(PartialDate))
115
+ @to = T.let(nil, T.nilable(PartialDate))
116
+ # Both branches assign the members they own and hand back the pair of
117
+ # dates that bound them, which is what makes those two non-nil for every
118
+ # instance rather than for most of them.
119
+ first, last =
120
+ if from.nil? && to.nil?
121
+ assign_point(year, month, day)
122
+ else
123
+ reject_mixed_shape(year, month, day)
124
+ assign_range(from, to)
125
+ end
126
+ @first_date = T.let(first, Date)
127
+ @last_date = T.let(last, Date)
128
+ freeze
129
+ end
130
+
131
+ sig { returns(T::Boolean) }
132
+ def range? = !from.nil?
133
+
134
+ sig { returns(T::Boolean) }
135
+ def approximate? = approximate
136
+
137
+ sig { returns(Symbol) }
138
+ def precision
139
+ return :range if range?
140
+ return :day if day
141
+ return :month if month
142
+
143
+ :year
144
+ end
145
+
146
+ # Every date this could be, which is the whole point of the type.
147
+ sig { returns(T::Range[Date]) }
148
+ def to_range = first_date..last_date
149
+
150
+ # True when the two dates could describe the same day. Precision does not
151
+ # have to match: a year-only date overlaps every full date inside it, which
152
+ # is what lets #32 treat 1972 against 1972-04-29 as a moderate boost rather
153
+ # than a miss.
154
+ sig { params(other: T.untyped).returns(T::Boolean) }
155
+ def overlaps?(other)
156
+ return false if other.nil?
157
+
158
+ mine = comparison_range
159
+ theirs = comparable!(other).comparison_range
160
+ mine.first <= theirs.last && theirs.first <= mine.last
161
+ end
162
+
163
+ # The strict complement of #overlaps? for two known dates. A missing date
164
+ # is not a conflict -- nobody claimed anything to contradict -- so nil
165
+ # answers false to both questions.
166
+ sig { params(other: T.untyped).returns(T::Boolean) }
167
+ def conflicts_with?(other)
168
+ return false if other.nil?
169
+
170
+ !overlaps?(other)
171
+ end
172
+
173
+ # Widened by APPROXIMATE_SLACK_YEARS when the publisher said circa.
174
+ sig { returns(T::Range[Date]) }
175
+ def comparison_range
176
+ return to_range unless approximate?
177
+
178
+ first_date.prev_year(APPROXIMATE_SLACK_YEARS)..last_date.next_year(APPROXIMATE_SLACK_YEARS)
179
+ end
180
+
181
+ sig { returns(T::Hash[Symbol, T.untyped]) }
182
+ def to_h
183
+ { year: year, month: month, day: day, from: from&.to_h, to: to&.to_h, approximate: approximate }
184
+ end
185
+
186
+ # Renders in a form .parse reads back, so a date survives a trip through
187
+ # free text -- which is how OFAC publishes them in the first place.
188
+ sig { returns(String) }
189
+ def to_s
190
+ text = range? ? "#{from} to #{to}" : point_to_s
191
+ approximate? ? "circa #{text}" : text
192
+ end
193
+
194
+ sig { params(other: T.untyped).returns(T::Boolean) }
195
+ def ==(other)
196
+ return false unless other.instance_of?(self.class)
197
+
198
+ to_h == other.to_h
199
+ end
200
+ alias eql? ==
201
+
202
+ sig { returns(Integer) }
203
+ def hash
204
+ [self.class, to_h].hash
205
+ end
206
+
207
+ sig { returns(String) }
208
+ def inspect
209
+ "#<#{self.class} #{self} precision=#{precision.inspect}>"
210
+ end
211
+
212
+ private
213
+
214
+ sig { params(year: T.untyped, month: T.untyped, day: T.untyped).returns([Date, Date]) }
215
+ def assign_point(year, month, day)
216
+ raise InvalidArgument, "year is required" if year.nil?
217
+
218
+ number = integer!(:year, year)
219
+ @year = number
220
+ @month = optional_integer(:month, month)
221
+ @day = optional_integer(:day, day)
222
+ validate_point!(number)
223
+ first = Date.new(number, @month || 1, @day || 1).freeze
224
+ # A year-only date runs to 31 December, a month to its own real last day.
225
+ [first, @day ? first : Date.new(number, @month || 12, -1).freeze]
226
+ end
227
+
228
+ sig { params(year: Integer).void }
229
+ def validate_point!(year)
230
+ raise InvalidArgument, "day given without a month" if @day && @month.nil?
231
+ raise InvalidArgument, "not a real date: #{to_s.inspect}" unless Date.valid_date?(year, @month || 1, @day || 1)
232
+ end
233
+
234
+ sig { params(from: T.untyped, to: T.untyped).returns([Date, Date]) }
235
+ def assign_range(from, to)
236
+ raise InvalidArgument, "a range needs both from and to" if from.nil? || to.nil?
237
+
238
+ first = endpoint!(:from, from)
239
+ last = endpoint!(:to, to)
240
+ @from = first
241
+ @to = last
242
+ raise InvalidArgument, "range runs backwards: #{first} to #{last}" if first.first_date > last.last_date
243
+
244
+ [first.first_date, last.last_date]
245
+ end
246
+
247
+ sig { params(year: T.untyped, month: T.untyped, day: T.untyped).void }
248
+ def reject_mixed_shape(year, month, day)
249
+ return if [year, month, day].all?(&:nil?)
250
+
251
+ raise InvalidArgument, "a range carries its year in its endpoints, not alongside them"
252
+ end
253
+
254
+ # Endpoints are themselves PartialDates so a span between two year-only
255
+ # dates keeps both years, and they may not nest: "between (1971 to 1972)
256
+ # and 1973" is not something any list publishes.
257
+ sig { params(member: Symbol, value: T.untyped).returns(PartialDate) }
258
+ def endpoint!(member, value)
259
+ date = coerce_endpoint(value)
260
+ raise InvalidArgument, "#{member} is not a date: #{value.inspect}" if date.nil?
261
+ raise InvalidArgument, "#{member} cannot itself be a range" if date.range?
262
+
263
+ date
264
+ end
265
+
266
+ sig { params(value: T.untyped).returns(T.nilable(PartialDate)) }
267
+ def coerce_endpoint(value)
268
+ case value
269
+ when PartialDate then value
270
+ when Hash then PartialDate.from_h(value)
271
+ else PartialDate.parse(value)
272
+ end
273
+ end
274
+
275
+ sig { params(other: T.untyped).returns(PartialDate) }
276
+ def comparable!(other)
277
+ return other if other.is_a?(PartialDate)
278
+
279
+ raise InvalidArgument, "expected a #{self.class}, got #{other.class}"
280
+ end
281
+
282
+ sig { params(member: Symbol, value: T.untyped).returns(T.nilable(Integer)) }
283
+ def optional_integer(member, value)
284
+ value.nil? ? nil : integer!(member, value)
285
+ end
286
+
287
+ sig { params(member: Symbol, value: T.untyped).returns(Integer) }
288
+ def integer!(member, value)
289
+ Integer(value.to_s, 10)
290
+ rescue TypeError, ArgumentError
291
+ raise InvalidArgument, "#{member} is not a number: #{value.inspect}"
292
+ end
293
+
294
+ sig { returns(String) }
295
+ def point_to_s
296
+ case precision
297
+ when :day then format("%<year>04d-%<month>02d-%<day>02d", year: year, month: month, day: day)
298
+ when :month then format("%<year>04d-%<month>02d", year: year, month: month)
299
+ else format("%<year>04d", year: year)
300
+ end
301
+ end
302
+ end
303
+ end