active_sanction 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +538 -0
  3. data/CODE_OF_CONDUCT.md +74 -0
  4. data/CONTRIBUTING.md +312 -0
  5. data/LICENSE.txt +21 -0
  6. data/README.md +616 -0
  7. data/SECURITY.md +97 -0
  8. data/active_sanction.gemspec +98 -0
  9. data/docs/adding_a_source.md +1083 -0
  10. data/docs/api_stability.md +357 -0
  11. data/docs/bundle_format.md +407 -0
  12. data/lib/active_sanction/address.rb +170 -0
  13. data/lib/active_sanction/client.rb +341 -0
  14. data/lib/active_sanction/configuration.rb +721 -0
  15. data/lib/active_sanction/countries.txt +292 -0
  16. data/lib/active_sanction/country.rb +164 -0
  17. data/lib/active_sanction/deprecation.rb +168 -0
  18. data/lib/active_sanction/diff/change.rb +208 -0
  19. data/lib/active_sanction/diff.rb +348 -0
  20. data/lib/active_sanction/doctor/checkup.rb +356 -0
  21. data/lib/active_sanction/doctor/diagnosis.rb +245 -0
  22. data/lib/active_sanction/doctor/finding.rb +183 -0
  23. data/lib/active_sanction/doctor/profile.rb +384 -0
  24. data/lib/active_sanction/doctor/report.rb +227 -0
  25. data/lib/active_sanction/doctor.rb +347 -0
  26. data/lib/active_sanction/entity.rb +293 -0
  27. data/lib/active_sanction/error.rb +332 -0
  28. data/lib/active_sanction/fetcher/result.rb +116 -0
  29. data/lib/active_sanction/fetcher.rb +242 -0
  30. data/lib/active_sanction/http_client/errors.rb +94 -0
  31. data/lib/active_sanction/http_client/response.rb +129 -0
  32. data/lib/active_sanction/http_client.rb +324 -0
  33. data/lib/active_sanction/identifier.rb +212 -0
  34. data/lib/active_sanction/index/builder.rb +89 -0
  35. data/lib/active_sanction/index/candidate.rb +63 -0
  36. data/lib/active_sanction/index/entry.rb +66 -0
  37. data/lib/active_sanction/index/features.rb +112 -0
  38. data/lib/active_sanction/index.rb +381 -0
  39. data/lib/active_sanction/match_result.rb +345 -0
  40. data/lib/active_sanction/matcher.rb +388 -0
  41. data/lib/active_sanction/name.rb +183 -0
  42. data/lib/active_sanction/normalizer/cache.rb +82 -0
  43. data/lib/active_sanction/normalizer/dictionaries/honorifics.txt +30 -0
  44. data/lib/active_sanction/normalizer/dictionaries/legal_forms.txt +72 -0
  45. data/lib/active_sanction/normalizer/dictionaries/organization_stopwords.txt +20 -0
  46. data/lib/active_sanction/normalizer/dictionaries/particles.txt +35 -0
  47. data/lib/active_sanction/normalizer/dictionary/stoplist.rb +121 -0
  48. data/lib/active_sanction/normalizer/dictionary.rb +243 -0
  49. data/lib/active_sanction/normalizer/form.rb +279 -0
  50. data/lib/active_sanction/normalizer.rb +137 -0
  51. data/lib/active_sanction/parsers/column_shape.rb +274 -0
  52. data/lib/active_sanction/parsers/delimited_table/reader.rb +194 -0
  53. data/lib/active_sanction/parsers/delimited_table/row.rb +81 -0
  54. data/lib/active_sanction/parsers/delimited_table.rb +151 -0
  55. data/lib/active_sanction/parsers/format.rb +107 -0
  56. data/lib/active_sanction/parsers/join.rb +149 -0
  57. data/lib/active_sanction/parsers/spreadsheet/archive.rb +258 -0
  58. data/lib/active_sanction/parsers/spreadsheet/reader.rb +213 -0
  59. data/lib/active_sanction/parsers/spreadsheet/row.rb +88 -0
  60. data/lib/active_sanction/parsers/spreadsheet/workbook.rb +314 -0
  61. data/lib/active_sanction/parsers/spreadsheet.rb +168 -0
  62. data/lib/active_sanction/parsers/xml_records/backends/nokogiri.rb +134 -0
  63. data/lib/active_sanction/parsers/xml_records/backends/rexml.rb +157 -0
  64. data/lib/active_sanction/parsers/xml_records/backends.rb +103 -0
  65. data/lib/active_sanction/parsers/xml_records/builder.rb +90 -0
  66. data/lib/active_sanction/parsers/xml_records/reader.rb +134 -0
  67. data/lib/active_sanction/parsers/xml_records/record.rb +174 -0
  68. data/lib/active_sanction/parsers/xml_records.rb +140 -0
  69. data/lib/active_sanction/parsers.rb +94 -0
  70. data/lib/active_sanction/partial_date/parser.rb +124 -0
  71. data/lib/active_sanction/partial_date.rb +303 -0
  72. data/lib/active_sanction/payload_cache/checksum.rb +66 -0
  73. data/lib/active_sanction/payload_cache/entry.rb +304 -0
  74. data/lib/active_sanction/payload_cache.rb +366 -0
  75. data/lib/active_sanction/phonetics/double_metaphone.rb +662 -0
  76. data/lib/active_sanction/phonetics.rb +40 -0
  77. data/lib/active_sanction/query.rb +290 -0
  78. data/lib/active_sanction/rescreen/alert.rb +314 -0
  79. data/lib/active_sanction/rescreen.rb +332 -0
  80. data/lib/active_sanction/scorer/adjustments.rb +241 -0
  81. data/lib/active_sanction/scorer/name_score.rb +279 -0
  82. data/lib/active_sanction/scorer/reason.rb +136 -0
  83. data/lib/active_sanction/scorer/result.rb +129 -0
  84. data/lib/active_sanction/scorer/subject.rb +235 -0
  85. data/lib/active_sanction/scorer/weights.rb +306 -0
  86. data/lib/active_sanction/scorer.rb +303 -0
  87. data/lib/active_sanction/similarity/jaro_winkler.rb +210 -0
  88. data/lib/active_sanction/similarity/levenshtein.rb +179 -0
  89. data/lib/active_sanction/similarity/token_set.rb +159 -0
  90. data/lib/active_sanction/similarity/token_sort.rb +120 -0
  91. data/lib/active_sanction/similarity.rb +183 -0
  92. data/lib/active_sanction/snapshot/bundle/header.rb +302 -0
  93. data/lib/active_sanction/snapshot/bundle/payload.rb +185 -0
  94. data/lib/active_sanction/snapshot/bundle/signature.rb +201 -0
  95. data/lib/active_sanction/snapshot/bundle.rb +381 -0
  96. data/lib/active_sanction/snapshot.rb +305 -0
  97. data/lib/active_sanction/sources/australia_dfat/published_date.rb +166 -0
  98. data/lib/active_sanction/sources/australia_dfat/record.rb +319 -0
  99. data/lib/active_sanction/sources/australia_dfat.rb +291 -0
  100. data/lib/active_sanction/sources/base.rb +295 -0
  101. data/lib/active_sanction/sources/canada_sema/record.rb +282 -0
  102. data/lib/active_sanction/sources/canada_sema/source_ref.rb +89 -0
  103. data/lib/active_sanction/sources/canada_sema.rb +153 -0
  104. data/lib/active_sanction/sources/definition.rb +360 -0
  105. data/lib/active_sanction/sources/eu_fsf/record.rb +467 -0
  106. data/lib/active_sanction/sources/eu_fsf.rb +203 -0
  107. data/lib/active_sanction/sources/ofac/record.rb +251 -0
  108. data/lib/active_sanction/sources/ofac/remarks_parser/coverage.rb +120 -0
  109. data/lib/active_sanction/sources/ofac/remarks_parser/vocabulary.rb +121 -0
  110. data/lib/active_sanction/sources/ofac/remarks_parser.rb +312 -0
  111. data/lib/active_sanction/sources/ofac.rb +248 -0
  112. data/lib/active_sanction/sources/ofac_consolidated/record.rb +37 -0
  113. data/lib/active_sanction/sources/ofac_consolidated.rb +203 -0
  114. data/lib/active_sanction/sources/ofac_sdn.rb +43 -0
  115. data/lib/active_sanction/sources/remarks.rb +84 -0
  116. data/lib/active_sanction/sources/uk_sanctions_list/published_date.rb +104 -0
  117. data/lib/active_sanction/sources/uk_sanctions_list/record.rb +412 -0
  118. data/lib/active_sanction/sources/uk_sanctions_list.rb +214 -0
  119. data/lib/active_sanction/sources/un_consolidated/record.rb +279 -0
  120. data/lib/active_sanction/sources/un_consolidated.rb +140 -0
  121. data/lib/active_sanction/sources.rb +196 -0
  122. data/lib/active_sanction/storage/active_record/reader.rb +124 -0
  123. data/lib/active_sanction/storage/active_record/row.rb +175 -0
  124. data/lib/active_sanction/storage/active_record/writer.rb +141 -0
  125. data/lib/active_sanction/storage/active_record.rb +277 -0
  126. data/lib/active_sanction/storage/base.rb +216 -0
  127. data/lib/active_sanction/storage/file_system.rb +373 -0
  128. data/lib/active_sanction/storage/memory.rb +67 -0
  129. data/lib/active_sanction/storage/meta.rb +188 -0
  130. data/lib/active_sanction/storage.rb +74 -0
  131. data/lib/active_sanction/subject.rb +271 -0
  132. data/lib/active_sanction/sync/report.rb +245 -0
  133. data/lib/active_sanction/sync/result.rb +296 -0
  134. data/lib/active_sanction/sync.rb +390 -0
  135. data/lib/active_sanction/validator_store/file_system.rb +103 -0
  136. data/lib/active_sanction/validator_store/memory.rb +41 -0
  137. data/lib/active_sanction/validator_store.rb +119 -0
  138. data/lib/active_sanction/validators.rb +195 -0
  139. data/lib/active_sanction/version.rb +23 -0
  140. data/lib/active_sanction.rb +331 -0
  141. data/lib/generators/active_sanction/install/install_generator.rb +60 -0
  142. data/lib/generators/active_sanction/install/templates/create_active_sanction_tables.rb.tt +142 -0
  143. metadata +236 -0
@@ -0,0 +1,314 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ require "sorbet-runtime"
5
+
6
+ require "date"
7
+ require "active_sanction/parsers/xml_records"
8
+
9
+ module ActiveSanction
10
+ module Parsers
11
+ class Spreadsheet
12
+ # The three parts of a workbook that have to be read before a single cell
13
+ # means anything: which part holds the sheet, what the shared strings are,
14
+ # and which cell styles are dates.
15
+ #
16
+ # workbook = Workbook.new(table: table, archive: archive)
17
+ # workbook.sheet_names # => ["Consolidated List"]
18
+ # workbook.strings[19] # => "MOHAMMAD HASSAN AKHUND"
19
+ # workbook.precision(5) # => :day
20
+ #
21
+ # ### A spreadsheet does not store text, or dates, in its cells
22
+ #
23
+ # Two indirections stand between a cell and its value, and both are here.
24
+ #
25
+ # **Text lives in a separate part.** A cell of type `s` holds an offset
26
+ # into `xl/sharedStrings.xml`, which is how one workbook stores 109,264
27
+ # cells as 24,467 distinct strings. A reader that skipped that part would
28
+ # produce a list of integers.
29
+ #
30
+ # **A date is a number plus a display format.** `18798` is a date if the
31
+ # cell's style formats it as one and the year 18798 if it does not, and
32
+ # the file says which only in `xl/styles.xml`. The Australian list turns
33
+ # on exactly this: 4,194 of its birth dates are serial numbers whose style
34
+ # is `m/d/yyyy`, and 2,711 are the *year* the person was born written as a
35
+ # plain number under the General format. Read without the styles they are
36
+ # the same thing, and one of the two readings is wrong for every row.
37
+ #
38
+ # @api private
39
+ class Workbook
40
+ extend T::Sig
41
+
42
+ WORKBOOK = T.let("xl/workbook.xml", String)
43
+ RELATIONSHIPS = T.let("xl/_rels/workbook.xml.rels", String)
44
+ CORE = T.let("docProps/core.xml", String)
45
+ BASE = T.let("xl/", String)
46
+
47
+ SHEETS = T.let(Parsers::XmlRecords.new(records: %w[sheet workbookPr]), Parsers::XmlRecords)
48
+ LINKS = T.let(Parsers::XmlRecords.new(records: %w[Relationship]), Parsers::XmlRecords)
49
+ STRINGS = T.let(Parsers::XmlRecords.new(records: %w[si]), Parsers::XmlRecords)
50
+ STYLES = T.let(Parsers::XmlRecords.new(records: %w[numFmt cellXfs]), Parsers::XmlRecords)
51
+ CORE_PROPERTIES = T.let(Parsers::XmlRecords.new(records: %w[modified]), Parsers::XmlRecords)
52
+
53
+ # The date formats every spreadsheet writer has without declaring them,
54
+ # from ECMA-376 -- and only the ones that carry a date. 18 to 21, 45, 46
55
+ # and 47 are times, whose serial fraction says nothing about a day.
56
+ BUILTIN_FORMATS = T.let(
57
+ { 14 => "mm-dd-yy", 15 => "d-mmm-yy", 16 => "d-mmm", 17 => "mmm-yy", 22 => "m/d/yy h:mm" }.freeze,
58
+ T::Hash[Integer, String]
59
+ )
60
+
61
+ # What a format code says once its literal text is out of the way: an
62
+ # `m` means a month beside a `y` and minutes beside an `h`, so year is
63
+ # what makes the difference and `d` settles the precision on its own.
64
+ LITERALS = T.let(%r{"[^"]*"|\[[^\]]*\]|\\.|AM/PM|A/P}i, Regexp)
65
+
66
+ # Serial 0 is 1900-01-00 and serial 60 is 1900-02-29, neither of which
67
+ # exists: Lotus 1-2-3 treated 1900 as a leap year and every spreadsheet
68
+ # since has kept the bug for compatibility. So the epoch that makes the
69
+ # arithmetic come out right is two days before 1900-01-01 for serials
70
+ # past the phantom day, and one day before it for the 59 below it.
71
+ EPOCH_1900 = T.let(Date.new(1899, 12, 30), Date)
72
+ LEAP_BUG_EPOCH = T.let(Date.new(1899, 12, 31), Date)
73
+ LEAP_BUG_SERIAL = T.let(60, Integer)
74
+
75
+ # The other date system, which Excel for Mac wrote until 2011 and which
76
+ # a workbook declares on `<workbookPr date1904="1">`.
77
+ EPOCH_1904 = T.let(Date.new(1904, 1, 1), Date)
78
+
79
+ # Serials outside this are not dates anybody typed. The low end rejects
80
+ # a plain small number that happens to sit in a date-formatted cell; the
81
+ # high end is the year 9999, past which Date arithmetic is answering a
82
+ # question nobody asked.
83
+ SERIALS = T.let(1..2_958_465, T::Range[Integer])
84
+
85
+ sig { returns(Archive) }
86
+ attr_reader :archive
87
+
88
+ sig { params(table: Spreadsheet, archive: Archive).void }
89
+ def initialize(table:, archive:)
90
+ @table = T.let(table, Spreadsheet)
91
+ @archive = T.let(archive, Archive)
92
+ @sheets = T.let(nil, T.nilable(T::Array[[String, String]]))
93
+ @strings = T.let(nil, T.nilable(T::Array[String]))
94
+ @precisions = T.let(nil, T.nilable(T::Array[T.nilable(Symbol)]))
95
+ @date1904 = T.let(nil, T.nilable(T::Boolean))
96
+ end
97
+
98
+ # The sheet names, in the order the workbook lists them.
99
+ sig { returns(T::Array[String]) }
100
+ def sheet_names = sheets.map(&:first)
101
+
102
+ # The bytes of the sheet a caller asked for by name, by zero-based
103
+ # index, or -- passing nil -- of the first one, which is the whole
104
+ # workbook for every list that publishes as a spreadsheet.
105
+ sig { params(wanted: T.untyped).returns(String) }
106
+ def sheet(wanted = nil)
107
+ archive.fetch(sheet_part(wanted))
108
+ end
109
+
110
+ # The shared string table, indexed the way a cell of type `s` indexes
111
+ # it. Empty for a workbook that has no such part, which is legal and
112
+ # means every string in it is inline.
113
+ sig { returns(T::Array[String]) }
114
+ def strings
115
+ @strings ||= read_strings
116
+ end
117
+
118
+ # What a cell carrying this style index means by a number: :day, :month
119
+ # or :year for the date formats, nil for everything else -- which is
120
+ # every General, numeric and text format, and every time-only one.
121
+ sig { params(style: T.nilable(Integer)).returns(T.nilable(Symbol)) }
122
+ def precision(style)
123
+ return nil if style.nil?
124
+
125
+ precisions[style]
126
+ end
127
+
128
+ # A serial number as the date its workbook means by it, or nil for one
129
+ # outside the range any real date occupies. The fractional part is the
130
+ # time of day and is dropped: a spreadsheet's date cell carries one
131
+ # whether or not anybody typed one.
132
+ sig { params(serial: T.untyped).returns(T.nilable(Date)) }
133
+ def date(serial)
134
+ number = Float(serial, exception: false)
135
+ return nil if number.nil?
136
+
137
+ days = number.floor
138
+ return nil unless SERIALS.cover?(days)
139
+ return EPOCH_1904 + days if date1904?
140
+
141
+ days > LEAP_BUG_SERIAL ? EPOCH_1900 + days : LEAP_BUG_EPOCH + days
142
+ end
143
+
144
+ # When the workbook was last saved, as its own core properties record it
145
+ # -- `2026-09-04T05:37:12Z`. A publisher who exports a fresh spreadsheet
146
+ # on every update stamps the export here, which makes it a version
147
+ # marker from inside the document rather than from the HTTP response.
148
+ # nil for a workbook that carries no core properties, which is legal.
149
+ sig { returns(T.nilable(String)) }
150
+ def modified
151
+ part = archive[CORE]
152
+ return nil if part.nil?
153
+
154
+ CORE_PROPERTIES.read(part).first&.text
155
+ end
156
+
157
+ # Whether the workbook counts its days from 1904 rather than from 1900.
158
+ sig { returns(T::Boolean) }
159
+ def date1904?
160
+ read_sheets if @date1904.nil?
161
+ @date1904 || false
162
+ end
163
+
164
+ sig { returns(String) }
165
+ def inspect = "#<#{self.class} #{sheet_names.join(", ")}>"
166
+
167
+ private
168
+
169
+ sig { returns(Spreadsheet) }
170
+ attr_reader :table
171
+
172
+ sig { params(wanted: T.untyped).returns(String) }
173
+ def sheet_part(wanted)
174
+ return by_index(Integer(wanted)) if wanted.is_a?(Integer)
175
+ return by_name(wanted.to_s) unless wanted.nil?
176
+
177
+ by_index(0)
178
+ end
179
+
180
+ sig { params(index: Integer).returns(String) }
181
+ def by_index(index)
182
+ found = sheets[index]
183
+ return found.last if found
184
+
185
+ raise ParseError,
186
+ "this workbook has #{sheets.size} sheet(s), so there is no sheet #{index}: #{sheet_names.join(", ")}"
187
+ end
188
+
189
+ sig { params(name: String).returns(String) }
190
+ def by_name(name)
191
+ found = sheets.find { |sheet_name, _part| sheet_name == name }
192
+ return found.last if found
193
+
194
+ raise ParseError, "this workbook has no sheet named #{name.inspect}. It has: #{sheet_names.join(", ")}"
195
+ end
196
+
197
+ sig { returns(T::Array[[String, String]]) }
198
+ def sheets
199
+ @sheets ||= read_sheets
200
+ end
201
+
202
+ # `<sheet name="Consolidated List" r:id="rId1"/>` says which sheet is
203
+ # which, and the relationship that `rId1` names says which part holds
204
+ # it. Neither half is optional: a workbook is free to store its first
205
+ # sheet in `sheet3.xml`, and several do.
206
+ sig { returns(T::Array[[String, String]]) }
207
+ def read_sheets
208
+ @date1904 = false
209
+ targets = relationships
210
+ named = SHEETS.read(archive.fetch(WORKBOOK)).filter_map { |node| declared_sheet(node, targets) }
211
+ return named if named.any?
212
+
213
+ raise ParseError, "#{WORKBOOK} declares no sheets, so this payload is not a workbook this can read"
214
+ end
215
+
216
+ sig do
217
+ params(node: Parsers::XmlRecords::Record, targets: T::Hash[String, String])
218
+ .returns(T.nilable([String, String]))
219
+ end
220
+ def declared_sheet(node, targets)
221
+ if node.name == "workbookPr"
222
+ @date1904 = %w[1 true].include?(node.attribute("date1904").to_s)
223
+ return nil
224
+ end
225
+
226
+ part = targets[node.attribute("id").to_s]
227
+ part.nil? ? nil : [node.attribute("name").to_s, part]
228
+ end
229
+
230
+ # Relationship id to part name, with the target resolved against `xl/`,
231
+ # which is the directory the workbook part lives in. A writer may state
232
+ # the target absolutely instead, and OpenOffice does.
233
+ sig { returns(T::Hash[String, String]) }
234
+ def relationships
235
+ LINKS.read(archive.fetch(RELATIONSHIPS)).to_h do |node|
236
+ target = node.attribute("Target").to_s
237
+ [node.attribute("Id").to_s, target.start_with?("/") ? target.delete_prefix("/") : "#{BASE}#{target}"]
238
+ end
239
+ end
240
+
241
+ # ### What a rich-text string loses here
242
+ #
243
+ # A shared string may be split into runs -- `<si><r><t>` -- when parts
244
+ # of it are styled differently, and whitespace at a run boundary does
245
+ # not survive being read element by element. No sanctions list has ever
246
+ # published one: this workbook's 24,467 strings contain not a single
247
+ # `<r>`, and a publisher who started styling half a name would be doing
248
+ # something no reader of the list wants. The phonetic guides in `<rPh>`
249
+ # are skipped, being a Japanese reading aid rather than part of the
250
+ # string.
251
+ sig { returns(T::Array[String]) }
252
+ def read_strings
253
+ part = archive["xl/sharedStrings.xml"]
254
+ return [] if part.nil?
255
+
256
+ STRINGS.read(part).map { |node| table.unescape(runs(node)).to_s }
257
+ end
258
+
259
+ sig { params(node: Parsers::XmlRecords::Record).returns(String) }
260
+ def runs(node)
261
+ return node.text.to_s if node.name == "t"
262
+
263
+ node.children.reject { |child| child.name == "rPh" }.map { |child| runs(child) }.join
264
+ end
265
+
266
+ # `<cellXfs>` is a list of cell formats, and a cell's `s` attribute is
267
+ # an index into it; each entry names a number format by id. Read once
268
+ # into a flat array, because it is asked of every numeric cell in the
269
+ # sheet.
270
+ sig { returns(T::Array[T.nilable(Symbol)]) }
271
+ def precisions
272
+ @precisions ||= read_precisions
273
+ end
274
+
275
+ # `<numFmt>` and `<cellXfs>` arrive in one pass, and the order the file
276
+ # puts them in is the order they are needed: the schema requires
277
+ # `<numFmts>` before `<cellXfs>`.
278
+ #
279
+ # It has to be `<cellXfs>` and not every `<xf>` in the part. `<xf>` also
280
+ # appears inside `<cellStyleXfs>`, which precedes it and holds the named
281
+ # styles a cell format inherits from -- collecting both would shift every
282
+ # index by the number of named styles, and a cell would be read against
283
+ # some other cell's number format.
284
+ sig { returns(T::Array[T.nilable(Symbol)]) }
285
+ def read_precisions
286
+ part = archive["xl/styles.xml"]
287
+ return [] if part.nil?
288
+
289
+ codes = BUILTIN_FORMATS.dup
290
+ formats = T.let([], T::Array[T.nilable(Symbol)])
291
+ STYLES.read(part).each do |node|
292
+ if node.name == "numFmt"
293
+ codes[node.attribute("numFmtId").to_i] = node.attribute("formatCode").to_s
294
+ else
295
+ formats = node.nodes("xf").map { |style| date_precision(codes[style.attribute("numFmtId").to_i]) }
296
+ end
297
+ end
298
+ formats
299
+ end
300
+
301
+ sig { params(code: T.nilable(String)).returns(T.nilable(Symbol)) }
302
+ def date_precision(code)
303
+ return nil if code.nil?
304
+
305
+ tokens = code.gsub(LITERALS, "").downcase
306
+ return :day if tokens.include?("d")
307
+ return nil unless tokens.include?("y")
308
+
309
+ tokens.include?("m") ? :month : :year
310
+ end
311
+ end
312
+ end
313
+ end
314
+ end
@@ -0,0 +1,168 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ require "sorbet-runtime"
5
+
6
+ require "active_sanction/parsers/format"
7
+ require "active_sanction/parsers/spreadsheet/archive"
8
+ require "active_sanction/parsers/spreadsheet/workbook"
9
+ require "active_sanction/parsers/spreadsheet/row"
10
+ require "active_sanction/parsers/spreadsheet/reader"
11
+
12
+ module ActiveSanction
13
+ module Parsers
14
+ # Reads an Office Open XML workbook -- an `.xlsx` file -- into rows an
15
+ # adapter can map onto Entities.
16
+ #
17
+ # A table is a description of the file, built once and reused for every
18
+ # sync; a Reader is one pass over one payload.
19
+ #
20
+ # LIST = ActiveSanction::Parsers::Spreadsheet.new(sheet: "Consolidated List")
21
+ #
22
+ # LIST.read(bytes).each { |row| row[:name_of_individual_or_entity] }
23
+ #
24
+ # ### With no dependency, which was the point
25
+ #
26
+ # Australia publishes its Consolidated List as a spreadsheet and as nothing
27
+ # else -- no CSV, no XML, no JSON -- so reading it is the price of screening
28
+ # against Australian sanctions at all. The alternative was a spreadsheet
29
+ # gem, which would have been this library's first third-party dependency
30
+ # taken on one publisher's behalf, in a gem whose stated rule is that a
31
+ # compliance library should not be the reason a deployment installs
32
+ # something.
33
+ #
34
+ # It turned out not to cost much. An `.xlsx` is a ZIP of XML parts; `zlib`
35
+ # is in the standard library and this gem already reads XML, so what was
36
+ # actually missing was a ZIP header unpacker (Archive) and the two lookups
37
+ # that make a cell mean something (Workbook). Everything below that is the
38
+ # XML toolkit the other five adapters use.
39
+ #
40
+ # ### What it reads, and what it does not
41
+ #
42
+ # One sheet of cell values, as strings. Dates are rendered ISO 8601 at the
43
+ # precision the cell's own format displays -- see Workbook -- so that
44
+ # `PartialDate::Parser` reads them without an adapter writing a format.
45
+ #
46
+ # Formulas are not evaluated: a formula cell is read as the value last
47
+ # cached in it, which is what a publisher's export contains and what the
48
+ # file displays. Merged cells, comments, charts, styling and every other
49
+ # thing a spreadsheet can hold are ignored, because none of them is data on
50
+ # a sanctions list. Only `.xlsx` is read, not the older binary `.xls` --
51
+ # they share a file extension in conversation and nothing at all in format.
52
+ #
53
+ # ### Columns, and why declaring them is optional here
54
+ #
55
+ # A published spreadsheet has a header row, unlike OFAC's CSVs, so the first
56
+ # row of the sheet is always the header and never a record. By default its
57
+ # cells are what the columns are named. Declaring `columns:` instead renames
58
+ # them by position, which pins the sheet's shape for a publisher who has
59
+ # form for re-labelling things -- the header is still consumed, because it
60
+ # is still a header.
61
+ class Spreadsheet
62
+ extend T::Sig
63
+ include Format
64
+
65
+ # Excel escapes a character XML cannot carry as `_x000D_`, and escapes a
66
+ # literal `_x000D_` somebody typed as `_x005F_x000D_`. Both are matched
67
+ # here, the doubled form first, so unescaping does not itself turn one
68
+ # into the other. The Australian list carries them in 205 cells, all
69
+ # carriage returns inside a birth date, an address or a place of birth.
70
+ #
71
+ # @api private
72
+ ESCAPE = T.let(/_x005F_(_x[0-9A-Fa-f]{4}_)|_x([0-9A-Fa-f]{4})_/, Regexp)
73
+
74
+ # nil where the sheet names its own columns -- see #headers?.
75
+ sig { returns(T.nilable(T::Array[Symbol])) }
76
+ attr_reader :columns
77
+
78
+ # The sheet to read: a name, a zero-based index, or nil for the first one.
79
+ sig { returns(T.untyped) }
80
+ attr_reader :sheet
81
+
82
+ sig { override.returns(T::Array[String]) }
83
+ attr_reader :nulls
84
+
85
+ # Always UTF-8, and not a caller's choice: the parts of a workbook are XML
86
+ # documents that declare their own encoding, and every writer emits UTF-8.
87
+ sig { override.returns(Encoding) }
88
+ attr_reader :encoding
89
+
90
+ sig { params(columns: T.untyped, null: T.untyped, sheet: T.untyped).void }
91
+ def initialize(columns: nil, null: nil, sheet: nil)
92
+ @columns = T.let(columns!(columns), T.nilable(T::Array[Symbol]))
93
+ @nulls = T.let(nulls!(null), T::Array[String])
94
+ @sheet = T.let(sheet, T.untyped)
95
+ @encoding = T.let(DEFAULT_ENCODING, Encoding)
96
+ freeze
97
+ end
98
+
99
+ # A pass over one payload. Takes the bytes as a String, which is what
100
+ # Sources::Base hands #parse.
101
+ sig { params(payload: T.untyped).returns(Reader) }
102
+ def read(payload) = Reader.new(table: self, payload: payload)
103
+
104
+ # A cell's text with Excel's escapes resolved. Applied to every string a
105
+ # sheet holds, shared or inline, because a name carrying a literal
106
+ # `_x000D_` is a name nothing will match.
107
+ sig { params(text: T.nilable(String)).returns(T.nilable(String)) }
108
+ def unescape(text)
109
+ return text if text.nil? || !text.include?("_x")
110
+
111
+ text.gsub(ESCAPE) { ::Regexp.last_match(1) || [::Regexp.last_match(2).to_s.hex].pack("U") }
112
+ end
113
+
114
+ # Whether the sheet names its own columns.
115
+ sig { returns(T::Boolean) }
116
+ def headers? = columns.nil?
117
+
118
+ # What to call the sheet being read, for a message: the name or index the
119
+ # caller asked for, or what "the first one" means when they asked for
120
+ # nothing.
121
+ sig { returns(String) }
122
+ def sheet_name
123
+ return "the first sheet" if sheet.nil?
124
+
125
+ sheet.is_a?(Integer) ? "sheet #{sheet}" : sheet.to_s.inspect
126
+ end
127
+
128
+ # Zips a row's cells against the column names by position. A column the
129
+ # row left empty is nil, and a cell past the last named column is dropped
130
+ # -- the Reader has already warned about the second.
131
+ sig do
132
+ params(names: T::Array[Symbol], cells: T::Hash[Integer, String])
133
+ .returns(T::Hash[Symbol, T.nilable(String)])
134
+ end
135
+ def coerce(names, cells)
136
+ names.each_with_index.to_h { |name, index| [name, cells[index]] }.freeze
137
+ end
138
+
139
+ sig { returns(String) }
140
+ def inspect
141
+ declared = columns
142
+ shape = declared.nil? ? "headers from the sheet" : "#{declared.size} columns"
143
+ "#<#{self.class} #{sheet_name}, #{shape}#{" null=#{nulls.first.inspect}" if nulls.any?}>"
144
+ end
145
+
146
+ private
147
+
148
+ sig { params(value: T.untyped).returns(T.nilable(T::Array[Symbol])) }
149
+ def columns!(value)
150
+ return nil if value.nil?
151
+
152
+ names = column_names!(value)
153
+ duplicated = names.tally.select { |_, count| count > 1 }.keys
154
+ raise InvalidArgument, "duplicate column name(s): #{duplicated.join(", ")}" if duplicated.any?
155
+
156
+ names.freeze
157
+ end
158
+
159
+ sig { params(value: T.untyped).returns(T::Array[Symbol]) }
160
+ def column_names!(value)
161
+ raise InvalidArgument, "columns must be an Array of names, got #{value.inspect}" unless value.is_a?(Array)
162
+ raise InvalidArgument, "columns cannot be empty; pass nil to read them from the sheet's header" if value.empty?
163
+
164
+ value.map { |name| name.to_s.strip.to_sym }
165
+ end
166
+ end
167
+ end
168
+ end
@@ -0,0 +1,134 @@
1
+ # typed: strict
2
+ # frozen_string_literal: true
3
+
4
+ require "sorbet-runtime"
5
+
6
+ require "active_sanction/parsers/xml_records/backends"
7
+ require "active_sanction/parsers/xml_records/builder"
8
+
9
+ module ActiveSanction
10
+ module Parsers
11
+ class XmlRecords
12
+ module Backends
13
+ # libxml2 through Nokogiri::XML::Reader, which is a pull parser and so
14
+ # streams on the same terms REXML does.
15
+ #
16
+ # Opt-in, never automatic:
17
+ #
18
+ # ActiveSanction.configure { |c| c.xml_backend = :nokogiri }
19
+ #
20
+ # Nokogiri is not a dependency of this gem and is not required here.
21
+ # It is used only if the host application has already loaded it, which
22
+ # keeps a compliance library off the list of things that make a
23
+ # deployment build native extensions it did not ask for.
24
+ #
25
+ # ### What it costs
26
+ #
27
+ # `Nokogiri::XML::Reader` exposes no position, so records parsed
28
+ # through this backend carry `line: nil` and a warning about one says
29
+ # only what went wrong, not where. A malformed *document* still reports
30
+ # its line, because libxml2's SyntaxError carries one -- so the failure
31
+ # that actually needs locating is located either way.
32
+ #
33
+ # The second difference is worth knowing before switching: given the
34
+ # whole payload in memory, libxml2 checks the document's structure
35
+ # before it yields anything, so a mismatched tag in the middle of a
36
+ # file is refused whole. REXML finds the same error where it sits and
37
+ # keeps every record parsed before it. A list that arrives mangled
38
+ # mid-file needs investigating under either backend, but an operator
39
+ # comparing two installations should know why one reported 400 records
40
+ # and a warning where the other reported a failure. A download that
41
+ # merely stops early -- the commoner accident -- streams and salvages
42
+ # identically on both.
43
+ class Nokogiri
44
+ extend T::Sig
45
+
46
+ # Deliberately `defined?` rather than a require: see above.
47
+ sig { returns(T::Boolean) }
48
+ def self.available? = defined?(::Nokogiri::XML::Reader) ? true : false
49
+
50
+ sig { returns(T.nilable(String)) }
51
+ def self.unavailable_reason
52
+ "nokogiri is not loaded. Add `gem \"nokogiri\"` to your Gemfile and require it, or leave " \
53
+ "`xml_backend` at :rexml"
54
+ end
55
+
56
+ # The document element's attributes, which is where these publishers
57
+ # put the version of the list.
58
+ sig { returns(T.nilable(T::Hash[String, String])) }
59
+ attr_reader :root
60
+
61
+ sig { params(table: XmlRecords, xml: String).void }
62
+ def initialize(table:, xml:)
63
+ @table = T.let(table, XmlRecords)
64
+ @xml = T.let(xml, String)
65
+ @root = T.let(nil, T.nilable(T::Hash[String, String]))
66
+ @builder = T.let(Builder.new(table: table), Builder)
67
+ @text_types = T.let(nil, T.nilable(T::Array[T.untyped]))
68
+ end
69
+
70
+ sig { params(block: T.proc.params(record: Record).void).void }
71
+ def each_record(&block)
72
+ reader = ::Nokogiri::XML::Reader(@xml)
73
+ reader.each { |node| handle(node, &block) }
74
+ truncated! if @builder.open?
75
+ rescue ::Nokogiri::XML::SyntaxError => e
76
+ raise MalformedDocument.new(e.message.to_s.strip, line: e.line)
77
+ end
78
+
79
+ private
80
+
81
+ sig { params(node: T.untyped, block: T.proc.params(record: Record).void).void }
82
+ def handle(node, &block)
83
+ case node.node_type
84
+ when ::Nokogiri::XML::Reader::TYPE_ELEMENT then start(node, &block)
85
+ when ::Nokogiri::XML::Reader::TYPE_END_ELEMENT then finish(&block)
86
+ when *text_types then @builder.text(node.value.to_s) if @builder.open?
87
+ end
88
+ end
89
+
90
+ # Resolved on first use rather than into a constant: this file is
91
+ # loaded whether or not the host has Nokogiri, and naming its
92
+ # constants at load time would make merely requiring the gem fail.
93
+ sig { returns(T::Array[T.untyped]) }
94
+ def text_types
95
+ @text_types ||= [::Nokogiri::XML::Reader::TYPE_TEXT,
96
+ ::Nokogiri::XML::Reader::TYPE_CDATA,
97
+ ::Nokogiri::XML::Reader::TYPE_WHITESPACE,
98
+ ::Nokogiri::XML::Reader::TYPE_SIGNIFICANT_WHITESPACE].freeze
99
+ end
100
+
101
+ # libxml2 reports `<QUALITY/>` as a start with no matching end, so a
102
+ # self-closing element is opened and closed here rather than waiting
103
+ # for an end event that never arrives. The UN's placeholder aliases
104
+ # are made entirely of these.
105
+ sig { params(node: T.untyped, block: T.proc.params(record: Record).void).void }
106
+ def start(node, &block)
107
+ local = Backends.local_name(node.name)
108
+ attrs = Backends.local_attributes(node.attributes)
109
+ @root ||= attrs
110
+ return unless @builder.open? || @table.record?(local)
111
+
112
+ @builder.enter(local, attrs)
113
+ finish(&block) if node.self_closing?
114
+ end
115
+
116
+ sig { params(block: T.proc.params(record: Record).void).void }
117
+ def finish(&block)
118
+ return unless @builder.open?
119
+
120
+ record = @builder.leave
121
+ block.call(record) if record
122
+ end
123
+
124
+ sig { void }
125
+ def truncated!
126
+ raise MalformedDocument.new("the document ended inside an unclosed element", line: nil)
127
+ end
128
+ end
129
+
130
+ register(:nokogiri, Nokogiri)
131
+ end
132
+ end
133
+ end
134
+ end