active_sanction 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +538 -0
- data/CODE_OF_CONDUCT.md +74 -0
- data/CONTRIBUTING.md +312 -0
- data/LICENSE.txt +21 -0
- data/README.md +616 -0
- data/SECURITY.md +97 -0
- data/active_sanction.gemspec +98 -0
- data/docs/adding_a_source.md +1083 -0
- data/docs/api_stability.md +357 -0
- data/docs/bundle_format.md +407 -0
- data/lib/active_sanction/address.rb +170 -0
- data/lib/active_sanction/client.rb +341 -0
- data/lib/active_sanction/configuration.rb +721 -0
- data/lib/active_sanction/countries.txt +292 -0
- data/lib/active_sanction/country.rb +164 -0
- data/lib/active_sanction/deprecation.rb +168 -0
- data/lib/active_sanction/diff/change.rb +208 -0
- data/lib/active_sanction/diff.rb +348 -0
- data/lib/active_sanction/doctor/checkup.rb +356 -0
- data/lib/active_sanction/doctor/diagnosis.rb +245 -0
- data/lib/active_sanction/doctor/finding.rb +183 -0
- data/lib/active_sanction/doctor/profile.rb +384 -0
- data/lib/active_sanction/doctor/report.rb +227 -0
- data/lib/active_sanction/doctor.rb +347 -0
- data/lib/active_sanction/entity.rb +293 -0
- data/lib/active_sanction/error.rb +332 -0
- data/lib/active_sanction/fetcher/result.rb +116 -0
- data/lib/active_sanction/fetcher.rb +242 -0
- data/lib/active_sanction/http_client/errors.rb +94 -0
- data/lib/active_sanction/http_client/response.rb +129 -0
- data/lib/active_sanction/http_client.rb +324 -0
- data/lib/active_sanction/identifier.rb +212 -0
- data/lib/active_sanction/index/builder.rb +89 -0
- data/lib/active_sanction/index/candidate.rb +63 -0
- data/lib/active_sanction/index/entry.rb +66 -0
- data/lib/active_sanction/index/features.rb +112 -0
- data/lib/active_sanction/index.rb +381 -0
- data/lib/active_sanction/match_result.rb +345 -0
- data/lib/active_sanction/matcher.rb +388 -0
- data/lib/active_sanction/name.rb +183 -0
- data/lib/active_sanction/normalizer/cache.rb +82 -0
- data/lib/active_sanction/normalizer/dictionaries/honorifics.txt +30 -0
- data/lib/active_sanction/normalizer/dictionaries/legal_forms.txt +72 -0
- data/lib/active_sanction/normalizer/dictionaries/organization_stopwords.txt +20 -0
- data/lib/active_sanction/normalizer/dictionaries/particles.txt +35 -0
- data/lib/active_sanction/normalizer/dictionary/stoplist.rb +121 -0
- data/lib/active_sanction/normalizer/dictionary.rb +243 -0
- data/lib/active_sanction/normalizer/form.rb +279 -0
- data/lib/active_sanction/normalizer.rb +137 -0
- data/lib/active_sanction/parsers/column_shape.rb +274 -0
- data/lib/active_sanction/parsers/delimited_table/reader.rb +194 -0
- data/lib/active_sanction/parsers/delimited_table/row.rb +81 -0
- data/lib/active_sanction/parsers/delimited_table.rb +151 -0
- data/lib/active_sanction/parsers/format.rb +107 -0
- data/lib/active_sanction/parsers/join.rb +149 -0
- data/lib/active_sanction/parsers/spreadsheet/archive.rb +258 -0
- data/lib/active_sanction/parsers/spreadsheet/reader.rb +213 -0
- data/lib/active_sanction/parsers/spreadsheet/row.rb +88 -0
- data/lib/active_sanction/parsers/spreadsheet/workbook.rb +314 -0
- data/lib/active_sanction/parsers/spreadsheet.rb +168 -0
- data/lib/active_sanction/parsers/xml_records/backends/nokogiri.rb +134 -0
- data/lib/active_sanction/parsers/xml_records/backends/rexml.rb +157 -0
- data/lib/active_sanction/parsers/xml_records/backends.rb +103 -0
- data/lib/active_sanction/parsers/xml_records/builder.rb +90 -0
- data/lib/active_sanction/parsers/xml_records/reader.rb +134 -0
- data/lib/active_sanction/parsers/xml_records/record.rb +174 -0
- data/lib/active_sanction/parsers/xml_records.rb +140 -0
- data/lib/active_sanction/parsers.rb +94 -0
- data/lib/active_sanction/partial_date/parser.rb +124 -0
- data/lib/active_sanction/partial_date.rb +303 -0
- data/lib/active_sanction/payload_cache/checksum.rb +66 -0
- data/lib/active_sanction/payload_cache/entry.rb +304 -0
- data/lib/active_sanction/payload_cache.rb +366 -0
- data/lib/active_sanction/phonetics/double_metaphone.rb +662 -0
- data/lib/active_sanction/phonetics.rb +40 -0
- data/lib/active_sanction/query.rb +290 -0
- data/lib/active_sanction/rescreen/alert.rb +314 -0
- data/lib/active_sanction/rescreen.rb +332 -0
- data/lib/active_sanction/scorer/adjustments.rb +241 -0
- data/lib/active_sanction/scorer/name_score.rb +279 -0
- data/lib/active_sanction/scorer/reason.rb +136 -0
- data/lib/active_sanction/scorer/result.rb +129 -0
- data/lib/active_sanction/scorer/subject.rb +235 -0
- data/lib/active_sanction/scorer/weights.rb +306 -0
- data/lib/active_sanction/scorer.rb +303 -0
- data/lib/active_sanction/similarity/jaro_winkler.rb +210 -0
- data/lib/active_sanction/similarity/levenshtein.rb +179 -0
- data/lib/active_sanction/similarity/token_set.rb +159 -0
- data/lib/active_sanction/similarity/token_sort.rb +120 -0
- data/lib/active_sanction/similarity.rb +183 -0
- data/lib/active_sanction/snapshot/bundle/header.rb +302 -0
- data/lib/active_sanction/snapshot/bundle/payload.rb +185 -0
- data/lib/active_sanction/snapshot/bundle/signature.rb +201 -0
- data/lib/active_sanction/snapshot/bundle.rb +381 -0
- data/lib/active_sanction/snapshot.rb +305 -0
- data/lib/active_sanction/sources/australia_dfat/published_date.rb +166 -0
- data/lib/active_sanction/sources/australia_dfat/record.rb +319 -0
- data/lib/active_sanction/sources/australia_dfat.rb +291 -0
- data/lib/active_sanction/sources/base.rb +295 -0
- data/lib/active_sanction/sources/canada_sema/record.rb +282 -0
- data/lib/active_sanction/sources/canada_sema/source_ref.rb +89 -0
- data/lib/active_sanction/sources/canada_sema.rb +153 -0
- data/lib/active_sanction/sources/definition.rb +360 -0
- data/lib/active_sanction/sources/eu_fsf/record.rb +467 -0
- data/lib/active_sanction/sources/eu_fsf.rb +203 -0
- data/lib/active_sanction/sources/ofac/record.rb +251 -0
- data/lib/active_sanction/sources/ofac/remarks_parser/coverage.rb +120 -0
- data/lib/active_sanction/sources/ofac/remarks_parser/vocabulary.rb +121 -0
- data/lib/active_sanction/sources/ofac/remarks_parser.rb +312 -0
- data/lib/active_sanction/sources/ofac.rb +248 -0
- data/lib/active_sanction/sources/ofac_consolidated/record.rb +37 -0
- data/lib/active_sanction/sources/ofac_consolidated.rb +203 -0
- data/lib/active_sanction/sources/ofac_sdn.rb +43 -0
- data/lib/active_sanction/sources/remarks.rb +84 -0
- data/lib/active_sanction/sources/uk_sanctions_list/published_date.rb +104 -0
- data/lib/active_sanction/sources/uk_sanctions_list/record.rb +412 -0
- data/lib/active_sanction/sources/uk_sanctions_list.rb +214 -0
- data/lib/active_sanction/sources/un_consolidated/record.rb +279 -0
- data/lib/active_sanction/sources/un_consolidated.rb +140 -0
- data/lib/active_sanction/sources.rb +196 -0
- data/lib/active_sanction/storage/active_record/reader.rb +124 -0
- data/lib/active_sanction/storage/active_record/row.rb +175 -0
- data/lib/active_sanction/storage/active_record/writer.rb +141 -0
- data/lib/active_sanction/storage/active_record.rb +277 -0
- data/lib/active_sanction/storage/base.rb +216 -0
- data/lib/active_sanction/storage/file_system.rb +373 -0
- data/lib/active_sanction/storage/memory.rb +67 -0
- data/lib/active_sanction/storage/meta.rb +188 -0
- data/lib/active_sanction/storage.rb +74 -0
- data/lib/active_sanction/subject.rb +271 -0
- data/lib/active_sanction/sync/report.rb +245 -0
- data/lib/active_sanction/sync/result.rb +296 -0
- data/lib/active_sanction/sync.rb +390 -0
- data/lib/active_sanction/validator_store/file_system.rb +103 -0
- data/lib/active_sanction/validator_store/memory.rb +41 -0
- data/lib/active_sanction/validator_store.rb +119 -0
- data/lib/active_sanction/validators.rb +195 -0
- data/lib/active_sanction/version.rb +23 -0
- data/lib/active_sanction.rb +331 -0
- data/lib/generators/active_sanction/install/install_generator.rb +60 -0
- data/lib/generators/active_sanction/install/templates/create_active_sanction_tables.rb.tt +142 -0
- metadata +236 -0
|
@@ -0,0 +1,314 @@
|
|
|
1
|
+
# typed: strict
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "sorbet-runtime"
|
|
5
|
+
|
|
6
|
+
require "date"
|
|
7
|
+
require "active_sanction/parsers/xml_records"
|
|
8
|
+
|
|
9
|
+
module ActiveSanction
|
|
10
|
+
module Parsers
|
|
11
|
+
class Spreadsheet
|
|
12
|
+
# The three parts of a workbook that have to be read before a single cell
|
|
13
|
+
# means anything: which part holds the sheet, what the shared strings are,
|
|
14
|
+
# and which cell styles are dates.
|
|
15
|
+
#
|
|
16
|
+
# workbook = Workbook.new(table: table, archive: archive)
|
|
17
|
+
# workbook.sheet_names # => ["Consolidated List"]
|
|
18
|
+
# workbook.strings[19] # => "MOHAMMAD HASSAN AKHUND"
|
|
19
|
+
# workbook.precision(5) # => :day
|
|
20
|
+
#
|
|
21
|
+
# ### A spreadsheet does not store text, or dates, in its cells
|
|
22
|
+
#
|
|
23
|
+
# Two indirections stand between a cell and its value, and both are here.
|
|
24
|
+
#
|
|
25
|
+
# **Text lives in a separate part.** A cell of type `s` holds an offset
|
|
26
|
+
# into `xl/sharedStrings.xml`, which is how one workbook stores 109,264
|
|
27
|
+
# cells as 24,467 distinct strings. A reader that skipped that part would
|
|
28
|
+
# produce a list of integers.
|
|
29
|
+
#
|
|
30
|
+
# **A date is a number plus a display format.** `18798` is a date if the
|
|
31
|
+
# cell's style formats it as one and the year 18798 if it does not, and
|
|
32
|
+
# the file says which only in `xl/styles.xml`. The Australian list turns
|
|
33
|
+
# on exactly this: 4,194 of its birth dates are serial numbers whose style
|
|
34
|
+
# is `m/d/yyyy`, and 2,711 are the *year* the person was born written as a
|
|
35
|
+
# plain number under the General format. Read without the styles they are
|
|
36
|
+
# the same thing, and one of the two readings is wrong for every row.
|
|
37
|
+
#
|
|
38
|
+
# @api private
|
|
39
|
+
class Workbook
|
|
40
|
+
extend T::Sig
|
|
41
|
+
|
|
42
|
+
WORKBOOK = T.let("xl/workbook.xml", String)
|
|
43
|
+
RELATIONSHIPS = T.let("xl/_rels/workbook.xml.rels", String)
|
|
44
|
+
CORE = T.let("docProps/core.xml", String)
|
|
45
|
+
BASE = T.let("xl/", String)
|
|
46
|
+
|
|
47
|
+
SHEETS = T.let(Parsers::XmlRecords.new(records: %w[sheet workbookPr]), Parsers::XmlRecords)
|
|
48
|
+
LINKS = T.let(Parsers::XmlRecords.new(records: %w[Relationship]), Parsers::XmlRecords)
|
|
49
|
+
STRINGS = T.let(Parsers::XmlRecords.new(records: %w[si]), Parsers::XmlRecords)
|
|
50
|
+
STYLES = T.let(Parsers::XmlRecords.new(records: %w[numFmt cellXfs]), Parsers::XmlRecords)
|
|
51
|
+
CORE_PROPERTIES = T.let(Parsers::XmlRecords.new(records: %w[modified]), Parsers::XmlRecords)
|
|
52
|
+
|
|
53
|
+
# The date formats every spreadsheet writer has without declaring them,
|
|
54
|
+
# from ECMA-376 -- and only the ones that carry a date. 18 to 21, 45, 46
|
|
55
|
+
# and 47 are times, whose serial fraction says nothing about a day.
|
|
56
|
+
BUILTIN_FORMATS = T.let(
|
|
57
|
+
{ 14 => "mm-dd-yy", 15 => "d-mmm-yy", 16 => "d-mmm", 17 => "mmm-yy", 22 => "m/d/yy h:mm" }.freeze,
|
|
58
|
+
T::Hash[Integer, String]
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
# What a format code says once its literal text is out of the way: an
|
|
62
|
+
# `m` means a month beside a `y` and minutes beside an `h`, so year is
|
|
63
|
+
# what makes the difference and `d` settles the precision on its own.
|
|
64
|
+
LITERALS = T.let(%r{"[^"]*"|\[[^\]]*\]|\\.|AM/PM|A/P}i, Regexp)
|
|
65
|
+
|
|
66
|
+
# Serial 0 is 1900-01-00 and serial 60 is 1900-02-29, neither of which
|
|
67
|
+
# exists: Lotus 1-2-3 treated 1900 as a leap year and every spreadsheet
|
|
68
|
+
# since has kept the bug for compatibility. So the epoch that makes the
|
|
69
|
+
# arithmetic come out right is two days before 1900-01-01 for serials
|
|
70
|
+
# past the phantom day, and one day before it for the 59 below it.
|
|
71
|
+
EPOCH_1900 = T.let(Date.new(1899, 12, 30), Date)
|
|
72
|
+
LEAP_BUG_EPOCH = T.let(Date.new(1899, 12, 31), Date)
|
|
73
|
+
LEAP_BUG_SERIAL = T.let(60, Integer)
|
|
74
|
+
|
|
75
|
+
# The other date system, which Excel for Mac wrote until 2011 and which
|
|
76
|
+
# a workbook declares on `<workbookPr date1904="1">`.
|
|
77
|
+
EPOCH_1904 = T.let(Date.new(1904, 1, 1), Date)
|
|
78
|
+
|
|
79
|
+
# Serials outside this are not dates anybody typed. The low end rejects
|
|
80
|
+
# a plain small number that happens to sit in a date-formatted cell; the
|
|
81
|
+
# high end is the year 9999, past which Date arithmetic is answering a
|
|
82
|
+
# question nobody asked.
|
|
83
|
+
SERIALS = T.let(1..2_958_465, T::Range[Integer])
|
|
84
|
+
|
|
85
|
+
sig { returns(Archive) }
|
|
86
|
+
attr_reader :archive
|
|
87
|
+
|
|
88
|
+
sig { params(table: Spreadsheet, archive: Archive).void }
|
|
89
|
+
def initialize(table:, archive:)
|
|
90
|
+
@table = T.let(table, Spreadsheet)
|
|
91
|
+
@archive = T.let(archive, Archive)
|
|
92
|
+
@sheets = T.let(nil, T.nilable(T::Array[[String, String]]))
|
|
93
|
+
@strings = T.let(nil, T.nilable(T::Array[String]))
|
|
94
|
+
@precisions = T.let(nil, T.nilable(T::Array[T.nilable(Symbol)]))
|
|
95
|
+
@date1904 = T.let(nil, T.nilable(T::Boolean))
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# The sheet names, in the order the workbook lists them.
|
|
99
|
+
sig { returns(T::Array[String]) }
|
|
100
|
+
def sheet_names = sheets.map(&:first)
|
|
101
|
+
|
|
102
|
+
# The bytes of the sheet a caller asked for by name, by zero-based
|
|
103
|
+
# index, or -- passing nil -- of the first one, which is the whole
|
|
104
|
+
# workbook for every list that publishes as a spreadsheet.
|
|
105
|
+
sig { params(wanted: T.untyped).returns(String) }
|
|
106
|
+
def sheet(wanted = nil)
|
|
107
|
+
archive.fetch(sheet_part(wanted))
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# The shared string table, indexed the way a cell of type `s` indexes
|
|
111
|
+
# it. Empty for a workbook that has no such part, which is legal and
|
|
112
|
+
# means every string in it is inline.
|
|
113
|
+
sig { returns(T::Array[String]) }
|
|
114
|
+
def strings
|
|
115
|
+
@strings ||= read_strings
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# What a cell carrying this style index means by a number: :day, :month
|
|
119
|
+
# or :year for the date formats, nil for everything else -- which is
|
|
120
|
+
# every General, numeric and text format, and every time-only one.
|
|
121
|
+
sig { params(style: T.nilable(Integer)).returns(T.nilable(Symbol)) }
|
|
122
|
+
def precision(style)
|
|
123
|
+
return nil if style.nil?
|
|
124
|
+
|
|
125
|
+
precisions[style]
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
# A serial number as the date its workbook means by it, or nil for one
|
|
129
|
+
# outside the range any real date occupies. The fractional part is the
|
|
130
|
+
# time of day and is dropped: a spreadsheet's date cell carries one
|
|
131
|
+
# whether or not anybody typed one.
|
|
132
|
+
sig { params(serial: T.untyped).returns(T.nilable(Date)) }
|
|
133
|
+
def date(serial)
|
|
134
|
+
number = Float(serial, exception: false)
|
|
135
|
+
return nil if number.nil?
|
|
136
|
+
|
|
137
|
+
days = number.floor
|
|
138
|
+
return nil unless SERIALS.cover?(days)
|
|
139
|
+
return EPOCH_1904 + days if date1904?
|
|
140
|
+
|
|
141
|
+
days > LEAP_BUG_SERIAL ? EPOCH_1900 + days : LEAP_BUG_EPOCH + days
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
# When the workbook was last saved, as its own core properties record it
|
|
145
|
+
# -- `2026-09-04T05:37:12Z`. A publisher who exports a fresh spreadsheet
|
|
146
|
+
# on every update stamps the export here, which makes it a version
|
|
147
|
+
# marker from inside the document rather than from the HTTP response.
|
|
148
|
+
# nil for a workbook that carries no core properties, which is legal.
|
|
149
|
+
sig { returns(T.nilable(String)) }
|
|
150
|
+
def modified
|
|
151
|
+
part = archive[CORE]
|
|
152
|
+
return nil if part.nil?
|
|
153
|
+
|
|
154
|
+
CORE_PROPERTIES.read(part).first&.text
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
# Whether the workbook counts its days from 1904 rather than from 1900.
|
|
158
|
+
sig { returns(T::Boolean) }
|
|
159
|
+
def date1904?
|
|
160
|
+
read_sheets if @date1904.nil?
|
|
161
|
+
@date1904 || false
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
sig { returns(String) }
|
|
165
|
+
def inspect = "#<#{self.class} #{sheet_names.join(", ")}>"
|
|
166
|
+
|
|
167
|
+
private
|
|
168
|
+
|
|
169
|
+
sig { returns(Spreadsheet) }
|
|
170
|
+
attr_reader :table
|
|
171
|
+
|
|
172
|
+
sig { params(wanted: T.untyped).returns(String) }
|
|
173
|
+
def sheet_part(wanted)
|
|
174
|
+
return by_index(Integer(wanted)) if wanted.is_a?(Integer)
|
|
175
|
+
return by_name(wanted.to_s) unless wanted.nil?
|
|
176
|
+
|
|
177
|
+
by_index(0)
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
sig { params(index: Integer).returns(String) }
|
|
181
|
+
def by_index(index)
|
|
182
|
+
found = sheets[index]
|
|
183
|
+
return found.last if found
|
|
184
|
+
|
|
185
|
+
raise ParseError,
|
|
186
|
+
"this workbook has #{sheets.size} sheet(s), so there is no sheet #{index}: #{sheet_names.join(", ")}"
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
sig { params(name: String).returns(String) }
|
|
190
|
+
def by_name(name)
|
|
191
|
+
found = sheets.find { |sheet_name, _part| sheet_name == name }
|
|
192
|
+
return found.last if found
|
|
193
|
+
|
|
194
|
+
raise ParseError, "this workbook has no sheet named #{name.inspect}. It has: #{sheet_names.join(", ")}"
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
sig { returns(T::Array[[String, String]]) }
|
|
198
|
+
def sheets
|
|
199
|
+
@sheets ||= read_sheets
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
# `<sheet name="Consolidated List" r:id="rId1"/>` says which sheet is
|
|
203
|
+
# which, and the relationship that `rId1` names says which part holds
|
|
204
|
+
# it. Neither half is optional: a workbook is free to store its first
|
|
205
|
+
# sheet in `sheet3.xml`, and several do.
|
|
206
|
+
sig { returns(T::Array[[String, String]]) }
|
|
207
|
+
def read_sheets
|
|
208
|
+
@date1904 = false
|
|
209
|
+
targets = relationships
|
|
210
|
+
named = SHEETS.read(archive.fetch(WORKBOOK)).filter_map { |node| declared_sheet(node, targets) }
|
|
211
|
+
return named if named.any?
|
|
212
|
+
|
|
213
|
+
raise ParseError, "#{WORKBOOK} declares no sheets, so this payload is not a workbook this can read"
|
|
214
|
+
end
|
|
215
|
+
|
|
216
|
+
sig do
|
|
217
|
+
params(node: Parsers::XmlRecords::Record, targets: T::Hash[String, String])
|
|
218
|
+
.returns(T.nilable([String, String]))
|
|
219
|
+
end
|
|
220
|
+
def declared_sheet(node, targets)
|
|
221
|
+
if node.name == "workbookPr"
|
|
222
|
+
@date1904 = %w[1 true].include?(node.attribute("date1904").to_s)
|
|
223
|
+
return nil
|
|
224
|
+
end
|
|
225
|
+
|
|
226
|
+
part = targets[node.attribute("id").to_s]
|
|
227
|
+
part.nil? ? nil : [node.attribute("name").to_s, part]
|
|
228
|
+
end
|
|
229
|
+
|
|
230
|
+
# Relationship id to part name, with the target resolved against `xl/`,
|
|
231
|
+
# which is the directory the workbook part lives in. A writer may state
|
|
232
|
+
# the target absolutely instead, and OpenOffice does.
|
|
233
|
+
sig { returns(T::Hash[String, String]) }
|
|
234
|
+
def relationships
|
|
235
|
+
LINKS.read(archive.fetch(RELATIONSHIPS)).to_h do |node|
|
|
236
|
+
target = node.attribute("Target").to_s
|
|
237
|
+
[node.attribute("Id").to_s, target.start_with?("/") ? target.delete_prefix("/") : "#{BASE}#{target}"]
|
|
238
|
+
end
|
|
239
|
+
end
|
|
240
|
+
|
|
241
|
+
# ### What a rich-text string loses here
|
|
242
|
+
#
|
|
243
|
+
# A shared string may be split into runs -- `<si><r><t>` -- when parts
|
|
244
|
+
# of it are styled differently, and whitespace at a run boundary does
|
|
245
|
+
# not survive being read element by element. No sanctions list has ever
|
|
246
|
+
# published one: this workbook's 24,467 strings contain not a single
|
|
247
|
+
# `<r>`, and a publisher who started styling half a name would be doing
|
|
248
|
+
# something no reader of the list wants. The phonetic guides in `<rPh>`
|
|
249
|
+
# are skipped, being a Japanese reading aid rather than part of the
|
|
250
|
+
# string.
|
|
251
|
+
sig { returns(T::Array[String]) }
|
|
252
|
+
def read_strings
|
|
253
|
+
part = archive["xl/sharedStrings.xml"]
|
|
254
|
+
return [] if part.nil?
|
|
255
|
+
|
|
256
|
+
STRINGS.read(part).map { |node| table.unescape(runs(node)).to_s }
|
|
257
|
+
end
|
|
258
|
+
|
|
259
|
+
sig { params(node: Parsers::XmlRecords::Record).returns(String) }
|
|
260
|
+
def runs(node)
|
|
261
|
+
return node.text.to_s if node.name == "t"
|
|
262
|
+
|
|
263
|
+
node.children.reject { |child| child.name == "rPh" }.map { |child| runs(child) }.join
|
|
264
|
+
end
|
|
265
|
+
|
|
266
|
+
# `<cellXfs>` is a list of cell formats, and a cell's `s` attribute is
|
|
267
|
+
# an index into it; each entry names a number format by id. Read once
|
|
268
|
+
# into a flat array, because it is asked of every numeric cell in the
|
|
269
|
+
# sheet.
|
|
270
|
+
sig { returns(T::Array[T.nilable(Symbol)]) }
|
|
271
|
+
def precisions
|
|
272
|
+
@precisions ||= read_precisions
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
# `<numFmt>` and `<cellXfs>` arrive in one pass, and the order the file
|
|
276
|
+
# puts them in is the order they are needed: the schema requires
|
|
277
|
+
# `<numFmts>` before `<cellXfs>`.
|
|
278
|
+
#
|
|
279
|
+
# It has to be `<cellXfs>` and not every `<xf>` in the part. `<xf>` also
|
|
280
|
+
# appears inside `<cellStyleXfs>`, which precedes it and holds the named
|
|
281
|
+
# styles a cell format inherits from -- collecting both would shift every
|
|
282
|
+
# index by the number of named styles, and a cell would be read against
|
|
283
|
+
# some other cell's number format.
|
|
284
|
+
sig { returns(T::Array[T.nilable(Symbol)]) }
|
|
285
|
+
def read_precisions
|
|
286
|
+
part = archive["xl/styles.xml"]
|
|
287
|
+
return [] if part.nil?
|
|
288
|
+
|
|
289
|
+
codes = BUILTIN_FORMATS.dup
|
|
290
|
+
formats = T.let([], T::Array[T.nilable(Symbol)])
|
|
291
|
+
STYLES.read(part).each do |node|
|
|
292
|
+
if node.name == "numFmt"
|
|
293
|
+
codes[node.attribute("numFmtId").to_i] = node.attribute("formatCode").to_s
|
|
294
|
+
else
|
|
295
|
+
formats = node.nodes("xf").map { |style| date_precision(codes[style.attribute("numFmtId").to_i]) }
|
|
296
|
+
end
|
|
297
|
+
end
|
|
298
|
+
formats
|
|
299
|
+
end
|
|
300
|
+
|
|
301
|
+
sig { params(code: T.nilable(String)).returns(T.nilable(Symbol)) }
|
|
302
|
+
def date_precision(code)
|
|
303
|
+
return nil if code.nil?
|
|
304
|
+
|
|
305
|
+
tokens = code.gsub(LITERALS, "").downcase
|
|
306
|
+
return :day if tokens.include?("d")
|
|
307
|
+
return nil unless tokens.include?("y")
|
|
308
|
+
|
|
309
|
+
tokens.include?("m") ? :month : :year
|
|
310
|
+
end
|
|
311
|
+
end
|
|
312
|
+
end
|
|
313
|
+
end
|
|
314
|
+
end
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
# typed: strict
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "sorbet-runtime"
|
|
5
|
+
|
|
6
|
+
require "active_sanction/parsers/format"
|
|
7
|
+
require "active_sanction/parsers/spreadsheet/archive"
|
|
8
|
+
require "active_sanction/parsers/spreadsheet/workbook"
|
|
9
|
+
require "active_sanction/parsers/spreadsheet/row"
|
|
10
|
+
require "active_sanction/parsers/spreadsheet/reader"
|
|
11
|
+
|
|
12
|
+
module ActiveSanction
|
|
13
|
+
module Parsers
|
|
14
|
+
# Reads an Office Open XML workbook -- an `.xlsx` file -- into rows an
|
|
15
|
+
# adapter can map onto Entities.
|
|
16
|
+
#
|
|
17
|
+
# A table is a description of the file, built once and reused for every
|
|
18
|
+
# sync; a Reader is one pass over one payload.
|
|
19
|
+
#
|
|
20
|
+
# LIST = ActiveSanction::Parsers::Spreadsheet.new(sheet: "Consolidated List")
|
|
21
|
+
#
|
|
22
|
+
# LIST.read(bytes).each { |row| row[:name_of_individual_or_entity] }
|
|
23
|
+
#
|
|
24
|
+
# ### With no dependency, which was the point
|
|
25
|
+
#
|
|
26
|
+
# Australia publishes its Consolidated List as a spreadsheet and as nothing
|
|
27
|
+
# else -- no CSV, no XML, no JSON -- so reading it is the price of screening
|
|
28
|
+
# against Australian sanctions at all. The alternative was a spreadsheet
|
|
29
|
+
# gem, which would have been this library's first third-party dependency
|
|
30
|
+
# taken on one publisher's behalf, in a gem whose stated rule is that a
|
|
31
|
+
# compliance library should not be the reason a deployment installs
|
|
32
|
+
# something.
|
|
33
|
+
#
|
|
34
|
+
# It turned out not to cost much. An `.xlsx` is a ZIP of XML parts; `zlib`
|
|
35
|
+
# is in the standard library and this gem already reads XML, so what was
|
|
36
|
+
# actually missing was a ZIP header unpacker (Archive) and the two lookups
|
|
37
|
+
# that make a cell mean something (Workbook). Everything below that is the
|
|
38
|
+
# XML toolkit the other five adapters use.
|
|
39
|
+
#
|
|
40
|
+
# ### What it reads, and what it does not
|
|
41
|
+
#
|
|
42
|
+
# One sheet of cell values, as strings. Dates are rendered ISO 8601 at the
|
|
43
|
+
# precision the cell's own format displays -- see Workbook -- so that
|
|
44
|
+
# `PartialDate::Parser` reads them without an adapter writing a format.
|
|
45
|
+
#
|
|
46
|
+
# Formulas are not evaluated: a formula cell is read as the value last
|
|
47
|
+
# cached in it, which is what a publisher's export contains and what the
|
|
48
|
+
# file displays. Merged cells, comments, charts, styling and every other
|
|
49
|
+
# thing a spreadsheet can hold are ignored, because none of them is data on
|
|
50
|
+
# a sanctions list. Only `.xlsx` is read, not the older binary `.xls` --
|
|
51
|
+
# they share a file extension in conversation and nothing at all in format.
|
|
52
|
+
#
|
|
53
|
+
# ### Columns, and why declaring them is optional here
|
|
54
|
+
#
|
|
55
|
+
# A published spreadsheet has a header row, unlike OFAC's CSVs, so the first
|
|
56
|
+
# row of the sheet is always the header and never a record. By default its
|
|
57
|
+
# cells are what the columns are named. Declaring `columns:` instead renames
|
|
58
|
+
# them by position, which pins the sheet's shape for a publisher who has
|
|
59
|
+
# form for re-labelling things -- the header is still consumed, because it
|
|
60
|
+
# is still a header.
|
|
61
|
+
class Spreadsheet
|
|
62
|
+
extend T::Sig
|
|
63
|
+
include Format
|
|
64
|
+
|
|
65
|
+
# Excel escapes a character XML cannot carry as `_x000D_`, and escapes a
|
|
66
|
+
# literal `_x000D_` somebody typed as `_x005F_x000D_`. Both are matched
|
|
67
|
+
# here, the doubled form first, so unescaping does not itself turn one
|
|
68
|
+
# into the other. The Australian list carries them in 205 cells, all
|
|
69
|
+
# carriage returns inside a birth date, an address or a place of birth.
|
|
70
|
+
#
|
|
71
|
+
# @api private
|
|
72
|
+
ESCAPE = T.let(/_x005F_(_x[0-9A-Fa-f]{4}_)|_x([0-9A-Fa-f]{4})_/, Regexp)
|
|
73
|
+
|
|
74
|
+
# nil where the sheet names its own columns -- see #headers?.
|
|
75
|
+
sig { returns(T.nilable(T::Array[Symbol])) }
|
|
76
|
+
attr_reader :columns
|
|
77
|
+
|
|
78
|
+
# The sheet to read: a name, a zero-based index, or nil for the first one.
|
|
79
|
+
sig { returns(T.untyped) }
|
|
80
|
+
attr_reader :sheet
|
|
81
|
+
|
|
82
|
+
sig { override.returns(T::Array[String]) }
|
|
83
|
+
attr_reader :nulls
|
|
84
|
+
|
|
85
|
+
# Always UTF-8, and not a caller's choice: the parts of a workbook are XML
|
|
86
|
+
# documents that declare their own encoding, and every writer emits UTF-8.
|
|
87
|
+
sig { override.returns(Encoding) }
|
|
88
|
+
attr_reader :encoding
|
|
89
|
+
|
|
90
|
+
sig { params(columns: T.untyped, null: T.untyped, sheet: T.untyped).void }
|
|
91
|
+
def initialize(columns: nil, null: nil, sheet: nil)
|
|
92
|
+
@columns = T.let(columns!(columns), T.nilable(T::Array[Symbol]))
|
|
93
|
+
@nulls = T.let(nulls!(null), T::Array[String])
|
|
94
|
+
@sheet = T.let(sheet, T.untyped)
|
|
95
|
+
@encoding = T.let(DEFAULT_ENCODING, Encoding)
|
|
96
|
+
freeze
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# A pass over one payload. Takes the bytes as a String, which is what
|
|
100
|
+
# Sources::Base hands #parse.
|
|
101
|
+
sig { params(payload: T.untyped).returns(Reader) }
|
|
102
|
+
def read(payload) = Reader.new(table: self, payload: payload)
|
|
103
|
+
|
|
104
|
+
# A cell's text with Excel's escapes resolved. Applied to every string a
|
|
105
|
+
# sheet holds, shared or inline, because a name carrying a literal
|
|
106
|
+
# `_x000D_` is a name nothing will match.
|
|
107
|
+
sig { params(text: T.nilable(String)).returns(T.nilable(String)) }
|
|
108
|
+
def unescape(text)
|
|
109
|
+
return text if text.nil? || !text.include?("_x")
|
|
110
|
+
|
|
111
|
+
text.gsub(ESCAPE) { ::Regexp.last_match(1) || [::Regexp.last_match(2).to_s.hex].pack("U") }
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# Whether the sheet names its own columns.
|
|
115
|
+
sig { returns(T::Boolean) }
|
|
116
|
+
def headers? = columns.nil?
|
|
117
|
+
|
|
118
|
+
# What to call the sheet being read, for a message: the name or index the
|
|
119
|
+
# caller asked for, or what "the first one" means when they asked for
|
|
120
|
+
# nothing.
|
|
121
|
+
sig { returns(String) }
|
|
122
|
+
def sheet_name
|
|
123
|
+
return "the first sheet" if sheet.nil?
|
|
124
|
+
|
|
125
|
+
sheet.is_a?(Integer) ? "sheet #{sheet}" : sheet.to_s.inspect
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
# Zips a row's cells against the column names by position. A column the
|
|
129
|
+
# row left empty is nil, and a cell past the last named column is dropped
|
|
130
|
+
# -- the Reader has already warned about the second.
|
|
131
|
+
sig do
|
|
132
|
+
params(names: T::Array[Symbol], cells: T::Hash[Integer, String])
|
|
133
|
+
.returns(T::Hash[Symbol, T.nilable(String)])
|
|
134
|
+
end
|
|
135
|
+
def coerce(names, cells)
|
|
136
|
+
names.each_with_index.to_h { |name, index| [name, cells[index]] }.freeze
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
sig { returns(String) }
|
|
140
|
+
def inspect
|
|
141
|
+
declared = columns
|
|
142
|
+
shape = declared.nil? ? "headers from the sheet" : "#{declared.size} columns"
|
|
143
|
+
"#<#{self.class} #{sheet_name}, #{shape}#{" null=#{nulls.first.inspect}" if nulls.any?}>"
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
private
|
|
147
|
+
|
|
148
|
+
sig { params(value: T.untyped).returns(T.nilable(T::Array[Symbol])) }
|
|
149
|
+
def columns!(value)
|
|
150
|
+
return nil if value.nil?
|
|
151
|
+
|
|
152
|
+
names = column_names!(value)
|
|
153
|
+
duplicated = names.tally.select { |_, count| count > 1 }.keys
|
|
154
|
+
raise InvalidArgument, "duplicate column name(s): #{duplicated.join(", ")}" if duplicated.any?
|
|
155
|
+
|
|
156
|
+
names.freeze
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
sig { params(value: T.untyped).returns(T::Array[Symbol]) }
|
|
160
|
+
def column_names!(value)
|
|
161
|
+
raise InvalidArgument, "columns must be an Array of names, got #{value.inspect}" unless value.is_a?(Array)
|
|
162
|
+
raise InvalidArgument, "columns cannot be empty; pass nil to read them from the sheet's header" if value.empty?
|
|
163
|
+
|
|
164
|
+
value.map { |name| name.to_s.strip.to_sym }
|
|
165
|
+
end
|
|
166
|
+
end
|
|
167
|
+
end
|
|
168
|
+
end
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
# typed: strict
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "sorbet-runtime"
|
|
5
|
+
|
|
6
|
+
require "active_sanction/parsers/xml_records/backends"
|
|
7
|
+
require "active_sanction/parsers/xml_records/builder"
|
|
8
|
+
|
|
9
|
+
module ActiveSanction
|
|
10
|
+
module Parsers
|
|
11
|
+
class XmlRecords
|
|
12
|
+
module Backends
|
|
13
|
+
# libxml2 through Nokogiri::XML::Reader, which is a pull parser and so
|
|
14
|
+
# streams on the same terms REXML does.
|
|
15
|
+
#
|
|
16
|
+
# Opt-in, never automatic:
|
|
17
|
+
#
|
|
18
|
+
# ActiveSanction.configure { |c| c.xml_backend = :nokogiri }
|
|
19
|
+
#
|
|
20
|
+
# Nokogiri is not a dependency of this gem and is not required here.
|
|
21
|
+
# It is used only if the host application has already loaded it, which
|
|
22
|
+
# keeps a compliance library off the list of things that make a
|
|
23
|
+
# deployment build native extensions it did not ask for.
|
|
24
|
+
#
|
|
25
|
+
# ### What it costs
|
|
26
|
+
#
|
|
27
|
+
# `Nokogiri::XML::Reader` exposes no position, so records parsed
|
|
28
|
+
# through this backend carry `line: nil` and a warning about one says
|
|
29
|
+
# only what went wrong, not where. A malformed *document* still reports
|
|
30
|
+
# its line, because libxml2's SyntaxError carries one -- so the failure
|
|
31
|
+
# that actually needs locating is located either way.
|
|
32
|
+
#
|
|
33
|
+
# The second difference is worth knowing before switching: given the
|
|
34
|
+
# whole payload in memory, libxml2 checks the document's structure
|
|
35
|
+
# before it yields anything, so a mismatched tag in the middle of a
|
|
36
|
+
# file is refused whole. REXML finds the same error where it sits and
|
|
37
|
+
# keeps every record parsed before it. A list that arrives mangled
|
|
38
|
+
# mid-file needs investigating under either backend, but an operator
|
|
39
|
+
# comparing two installations should know why one reported 400 records
|
|
40
|
+
# and a warning where the other reported a failure. A download that
|
|
41
|
+
# merely stops early -- the commoner accident -- streams and salvages
|
|
42
|
+
# identically on both.
|
|
43
|
+
class Nokogiri
|
|
44
|
+
extend T::Sig
|
|
45
|
+
|
|
46
|
+
# Deliberately `defined?` rather than a require: see above.
|
|
47
|
+
sig { returns(T::Boolean) }
|
|
48
|
+
def self.available? = defined?(::Nokogiri::XML::Reader) ? true : false
|
|
49
|
+
|
|
50
|
+
sig { returns(T.nilable(String)) }
|
|
51
|
+
def self.unavailable_reason
|
|
52
|
+
"nokogiri is not loaded. Add `gem \"nokogiri\"` to your Gemfile and require it, or leave " \
|
|
53
|
+
"`xml_backend` at :rexml"
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# The document element's attributes, which is where these publishers
|
|
57
|
+
# put the version of the list.
|
|
58
|
+
sig { returns(T.nilable(T::Hash[String, String])) }
|
|
59
|
+
attr_reader :root
|
|
60
|
+
|
|
61
|
+
sig { params(table: XmlRecords, xml: String).void }
|
|
62
|
+
def initialize(table:, xml:)
|
|
63
|
+
@table = T.let(table, XmlRecords)
|
|
64
|
+
@xml = T.let(xml, String)
|
|
65
|
+
@root = T.let(nil, T.nilable(T::Hash[String, String]))
|
|
66
|
+
@builder = T.let(Builder.new(table: table), Builder)
|
|
67
|
+
@text_types = T.let(nil, T.nilable(T::Array[T.untyped]))
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
sig { params(block: T.proc.params(record: Record).void).void }
|
|
71
|
+
def each_record(&block)
|
|
72
|
+
reader = ::Nokogiri::XML::Reader(@xml)
|
|
73
|
+
reader.each { |node| handle(node, &block) }
|
|
74
|
+
truncated! if @builder.open?
|
|
75
|
+
rescue ::Nokogiri::XML::SyntaxError => e
|
|
76
|
+
raise MalformedDocument.new(e.message.to_s.strip, line: e.line)
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
private
|
|
80
|
+
|
|
81
|
+
sig { params(node: T.untyped, block: T.proc.params(record: Record).void).void }
|
|
82
|
+
def handle(node, &block)
|
|
83
|
+
case node.node_type
|
|
84
|
+
when ::Nokogiri::XML::Reader::TYPE_ELEMENT then start(node, &block)
|
|
85
|
+
when ::Nokogiri::XML::Reader::TYPE_END_ELEMENT then finish(&block)
|
|
86
|
+
when *text_types then @builder.text(node.value.to_s) if @builder.open?
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# Resolved on first use rather than into a constant: this file is
|
|
91
|
+
# loaded whether or not the host has Nokogiri, and naming its
|
|
92
|
+
# constants at load time would make merely requiring the gem fail.
|
|
93
|
+
sig { returns(T::Array[T.untyped]) }
|
|
94
|
+
def text_types
|
|
95
|
+
@text_types ||= [::Nokogiri::XML::Reader::TYPE_TEXT,
|
|
96
|
+
::Nokogiri::XML::Reader::TYPE_CDATA,
|
|
97
|
+
::Nokogiri::XML::Reader::TYPE_WHITESPACE,
|
|
98
|
+
::Nokogiri::XML::Reader::TYPE_SIGNIFICANT_WHITESPACE].freeze
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# libxml2 reports `<QUALITY/>` as a start with no matching end, so a
|
|
102
|
+
# self-closing element is opened and closed here rather than waiting
|
|
103
|
+
# for an end event that never arrives. The UN's placeholder aliases
|
|
104
|
+
# are made entirely of these.
|
|
105
|
+
sig { params(node: T.untyped, block: T.proc.params(record: Record).void).void }
|
|
106
|
+
def start(node, &block)
|
|
107
|
+
local = Backends.local_name(node.name)
|
|
108
|
+
attrs = Backends.local_attributes(node.attributes)
|
|
109
|
+
@root ||= attrs
|
|
110
|
+
return unless @builder.open? || @table.record?(local)
|
|
111
|
+
|
|
112
|
+
@builder.enter(local, attrs)
|
|
113
|
+
finish(&block) if node.self_closing?
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
sig { params(block: T.proc.params(record: Record).void).void }
|
|
117
|
+
def finish(&block)
|
|
118
|
+
return unless @builder.open?
|
|
119
|
+
|
|
120
|
+
record = @builder.leave
|
|
121
|
+
block.call(record) if record
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
sig { void }
|
|
125
|
+
def truncated!
|
|
126
|
+
raise MalformedDocument.new("the document ended inside an unclosed element", line: nil)
|
|
127
|
+
end
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
register(:nokogiri, Nokogiri)
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
end
|