relaton 3.0.0.pre.alpha.18 → 3.0.0.pre.alpha.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/data/equivalence/adoption-chains.yaml +21 -0
- data/lib/relaton/bib/model/formattedref.rb +3 -8
- data/lib/relaton/bib/model/item_shared.rb +10 -1
- data/lib/relaton/bsi/CLAUDE.md +1 -1
- data/lib/relaton/cen/CLAUDE.md +1 -1
- data/lib/relaton/core/data_fetcher.rb +93 -1
- data/lib/relaton/db.rb +6 -4
- data/lib/relaton/doi/CLAUDE.md +1 -1
- data/lib/relaton/equivalence.rb +28 -0
- data/lib/relaton/index/CLAUDE.md +4 -3
- data/lib/relaton/index/type.rb +9 -2
- data/lib/relaton/isbn/CLAUDE.md +1 -1
- data/lib/relaton/iso/CLAUDE.md +1 -1
- data/lib/relaton/itu/CLAUDE.md +3 -3
- data/lib/relaton/nist/CLAUDE.md +10 -13
- data/lib/relaton/ogc/data_fetcher.rb +9 -3
- data/lib/relaton/version.rb +1 -1
- data/lib/relaton/w3c/CLAUDE.md +1 -1
- metadata +4 -4
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 3e828566f371c377f356fb6ebf21ffb4e67d3763f2955b9fe747bb2ea6a09889
|
|
4
|
+
data.tar.gz: 194fb89c438cad7fc5a34a134dc1d5399240f8b95261f993e9b7b3fa78b81211
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 27a73f7fc5743b541a18abaa8e893e40495d542653b8f085f7df0d4761691cc6d8f258f791fdafab65b104f6115aadddfc0c606fbaea212f3ad336b29767751d
|
|
7
|
+
data.tar.gz: 196b153ec562bf1fd8486dbff09f3f8c067c1506fd1f3c5415355140b90f4e2d41ac04d769f73fdb07667a275d3ac7967758485f9e3fe97d78bc825dd34f5451
|
|
@@ -27,6 +27,18 @@ chains:
|
|
|
27
27
|
- 'EN \1'
|
|
28
28
|
- '\1'
|
|
29
29
|
|
|
30
|
+
# Adoption ladders: the reverse edges. A code matching `pattern` is a base
|
|
31
|
+
# document; `forms` are its national/regional adoptions, ordered
|
|
32
|
+
# outermost-last, so the resolver can walk a chain from either end
|
|
33
|
+
# (TODO.sota: adoption chains resolvable end-to-end, bidirectional).
|
|
34
|
+
adoptions:
|
|
35
|
+
- pattern: '^(ISO[ /].+)$'
|
|
36
|
+
forms:
|
|
37
|
+
- 'EN \1'
|
|
38
|
+
- 'BS EN \1'
|
|
39
|
+
- 'DIN EN \1'
|
|
40
|
+
- 'NF EN \1'
|
|
41
|
+
|
|
30
42
|
# Explicit equivalences: every form in `equivalents` resolves to
|
|
31
43
|
# `canonical` (and the canonical resolves to itself). Used where no pattern
|
|
32
44
|
# expresses the relationship (joint publications, historical adoptions).
|
|
@@ -36,3 +48,12 @@ equivalences:
|
|
|
36
48
|
- EN ISO/IEC 17025
|
|
37
49
|
- BS EN ISO/IEC 17025
|
|
38
50
|
- DIN EN ISO/IEC 17025
|
|
51
|
+
# Joint publication: the same document under a different publisher
|
|
52
|
+
# ordering (Law 3.1) — the direction each corpus prints is the data.
|
|
53
|
+
- canonical: ISO/IEC 14515-1
|
|
54
|
+
equivalents:
|
|
55
|
+
- IEC/ISO 14515-1
|
|
56
|
+
# Variant spelling: the prefixed form of a bare reference (Law 3.3).
|
|
57
|
+
- canonical: RFC 7200
|
|
58
|
+
equivalents:
|
|
59
|
+
- IETF RFC 7200
|
|
@@ -1,15 +1,10 @@
|
|
|
1
1
|
module Relaton
|
|
2
2
|
module Bib
|
|
3
3
|
class Formattedref < LocalizedMarkedUpString
|
|
4
|
-
|
|
5
|
-
|
|
4
|
+
# No `format` attribute: the bibdata grammar dropped it (relaton-bib
|
|
5
|
+
# #94) — an ingested format= is silently dropped, never re-emitted.
|
|
6
6
|
xml do
|
|
7
7
|
root "formattedref"
|
|
8
|
-
map_attribute "format", to: :format
|
|
9
|
-
end
|
|
10
|
-
|
|
11
|
-
key_value do
|
|
12
|
-
map "format", to: :format
|
|
13
8
|
end
|
|
14
9
|
|
|
15
10
|
# Handle plain string input from YAML/JSON deserialization
|
|
@@ -35,7 +30,7 @@ module Relaton
|
|
|
35
30
|
def self.simple_content?(instance, format)
|
|
36
31
|
return false if format == :xml
|
|
37
32
|
|
|
38
|
-
instance.
|
|
33
|
+
instance.language.nil? &&
|
|
39
34
|
instance.script.nil? && instance.locale.nil?
|
|
40
35
|
end
|
|
41
36
|
end
|
|
@@ -23,7 +23,16 @@ module Relaton
|
|
|
23
23
|
attribute :docnumber, :string
|
|
24
24
|
attribute :date, Date, collection: true, initialize_empty: true
|
|
25
25
|
attribute :contributor, Contributor, collection: true, initialize_empty: true
|
|
26
|
-
|
|
26
|
+
# THE LAW: a bibitem has exactly one edition. A second <edition>
|
|
27
|
+
# element is always wrong output — in the one known case, the
|
|
28
|
+
# iso-10303 collection render fused the SMRL library bibdata (a
|
|
29
|
+
# different document that contains the parts) into part bibitems
|
|
30
|
+
# (relaton-db#149). The tolerance is a crash guard against that
|
|
31
|
+
# wrong output: it keeps the bibitem's own (leading) edition instead
|
|
32
|
+
# of aborting the round-trip with CollectionTrueMissingError
|
|
33
|
+
# (lutaml-model#971). The emitter fix is the real fix; this must
|
|
34
|
+
# never fire on correct data.
|
|
35
|
+
attribute :edition, Edition, singular_tolerance: :first
|
|
27
36
|
attribute :version, Version, collection: true, initialize_empty: true
|
|
28
37
|
attribute :note, Note, collection: true, initialize_empty: true
|
|
29
38
|
attribute :language, :string, collection: true, initialize_empty: true
|
data/lib/relaton/bsi/CLAUDE.md
CHANGED
|
@@ -32,4 +32,4 @@ Namespace: `Relaton::Bsi`. Retrieval flow:
|
|
|
32
32
|
|
|
33
33
|
## Testing
|
|
34
34
|
|
|
35
|
-
RSpec with WebMock + VCR (cassettes in `spec/vcr_cassettes/`, record `:once`,
|
|
35
|
+
RSpec with WebMock + VCR (cassettes in `spec/vcr_cassettes/`, record `:once`, never auto-rerecorded). Network is blocked via WebMock. No index fixture — BSI has no curated index; `spec/fixtures/` holds XML/YAML round-trip data.
|
data/lib/relaton/cen/CLAUDE.md
CHANGED
|
@@ -193,7 +193,7 @@ The `:supplement_year` exclude key is a different thing and stays.
|
|
|
193
193
|
|
|
194
194
|
## Testing
|
|
195
195
|
|
|
196
|
-
RSpec with WebMock + VCR (cassettes in `spec/vcr_cassettes/`, record `:once`,
|
|
196
|
+
RSpec with WebMock + VCR (cassettes in `spec/vcr_cassettes/`, record `:once`, never auto-rerecorded). Network is blocked via WebMock. No index fixture — CEN is scraped, not indexed; `spec/fixtures/` holds XML/YAML round-trip data. The scraper is inherently brittle to portal HTML changes.
|
|
197
197
|
|
|
198
198
|
- `spec/cen/relaton/cen/docidentifier_spec.rb` unit-tests the three mutators over
|
|
199
199
|
every identifier form the portal serves, including the two rows above that the
|
|
@@ -36,6 +36,73 @@ module Relaton
|
|
|
36
36
|
# @return [Boolean] whether this run prunes the existing output tree
|
|
37
37
|
attr_accessor :prune
|
|
38
38
|
|
|
39
|
+
# @return [Boolean] whether this run consults and maintains the
|
|
40
|
+
# content-digest ledger (see #unchanged?). Off by default: a flavor
|
|
41
|
+
# turns it on only where its enumeration entry determines the record
|
|
42
|
+
# (the TODO.sota crawl-audit rule).
|
|
43
|
+
attr_accessor :digest_ledger
|
|
44
|
+
|
|
45
|
+
# -- Content-digest ledger -----------------------------------------
|
|
46
|
+
#
|
|
47
|
+
# A per-output sidecar (`.digests.yaml`: key -> digest + file) lets a
|
|
48
|
+
# run skip the fetch+parse+write of a record whose enumeration entry is
|
|
49
|
+
# unchanged, where the entry determines the record. The sidecar is
|
|
50
|
+
# tracked in the data repo so a fresh CI clone still skips. Skipped
|
|
51
|
+
# files are recorded as written, so #prune_unwritten keeps them; the
|
|
52
|
+
# saved ledger keeps only files this run owned, so deleted records
|
|
53
|
+
# drop out with their file.
|
|
54
|
+
#
|
|
55
|
+
# Sound only where the entry bounds the record: a per-item detail
|
|
56
|
+
# fetch whose content the entry does not determine, or relation output
|
|
57
|
+
# discovered across records, must stay outside the skip (TODO.sota,
|
|
58
|
+
# crawl audit 2026-10-23).
|
|
59
|
+
|
|
60
|
+
LEDGER_FILE = ".digests.yaml"
|
|
61
|
+
|
|
62
|
+
# True when the ledger holds `key` with the entry's digest - the
|
|
63
|
+
# record is unchanged and the caller may skip it. Records the
|
|
64
|
+
# ledger's file as written, so prune keeps a skipped record.
|
|
65
|
+
#
|
|
66
|
+
# @param key [String] the record's stable identity (its docid)
|
|
67
|
+
# @param entry [Object] the enumeration entry; digested as JSON
|
|
68
|
+
# @return [Boolean]
|
|
69
|
+
def unchanged?(key, entry)
|
|
70
|
+
return false unless @digest_ledger
|
|
71
|
+
|
|
72
|
+
row = ledger[key]
|
|
73
|
+
return false unless row.is_a?(Hash) && row["digest"] == entry_digest(entry)
|
|
74
|
+
|
|
75
|
+
@written << row["file"] if row["file"]
|
|
76
|
+
warn "Skipping unchanged `#{key}`"
|
|
77
|
+
true
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# Record the entry's digest for `key`, with the file the record was
|
|
81
|
+
# written to (nil when this run wrote no file for it - the entry
|
|
82
|
+
# drops out of the ledger at save time).
|
|
83
|
+
#
|
|
84
|
+
# @param key [String]
|
|
85
|
+
# @param entry [Object]
|
|
86
|
+
# @param file [String, nil]
|
|
87
|
+
# @return [void]
|
|
88
|
+
def note_digest(key, entry, file: nil)
|
|
89
|
+
return unless @digest_ledger
|
|
90
|
+
|
|
91
|
+
ledger_new[key] = { "digest" => entry_digest(entry), "file" => file }
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# Persist the ledger: this run's noted entries over the loaded ones,
|
|
95
|
+
# keeping only entries whose file this run recorded as written.
|
|
96
|
+
#
|
|
97
|
+
# @return [void]
|
|
98
|
+
def save_ledger
|
|
99
|
+
kept = ledger.merge(ledger_new).select do |_, row|
|
|
100
|
+
row.is_a?(Hash) && row["file"] && @written.include?(row["file"])
|
|
101
|
+
end
|
|
102
|
+
File.write ledger_path, YAML.dump(kept), encoding: "UTF-8"
|
|
103
|
+
@ledger = kept
|
|
104
|
+
end
|
|
105
|
+
|
|
39
106
|
# API method for external service
|
|
40
107
|
#
|
|
41
108
|
# @param prune [Boolean] delete output files this run did not write. The
|
|
@@ -43,7 +110,8 @@ module Relaton
|
|
|
43
110
|
# ratio guard stops a partial crawl from wiping the dataset.
|
|
44
111
|
# @return the value returned by the instance `#fetch`, so callers can act
|
|
45
112
|
# on the outcome (e.g. relaton-iso returns whether it rebuilt).
|
|
46
|
-
def self.fetch(source = nil, output: "data", format: "yaml", prune: false
|
|
113
|
+
def self.fetch(source = nil, output: "data", format: "yaml", prune: false,
|
|
114
|
+
digest_ledger: false)
|
|
47
115
|
t1 = Time.now
|
|
48
116
|
puts "Started at: #{t1}"
|
|
49
117
|
FileUtils.mkdir_p output
|
|
@@ -51,8 +119,10 @@ module Relaton
|
|
|
51
119
|
# Only touched in prune mode: the default path stays exactly what it
|
|
52
120
|
# was, and existing ::fetch callers (and their specs) see no new call.
|
|
53
121
|
fetcher.prune = true if prune
|
|
122
|
+
fetcher.digest_ledger = digest_ledger
|
|
54
123
|
result = fetcher.fetch(source)
|
|
55
124
|
fetcher.prune_unwritten if prune
|
|
125
|
+
fetcher.save_ledger if digest_ledger
|
|
56
126
|
t2 = Time.now
|
|
57
127
|
puts "Stopped at: #{t2}"
|
|
58
128
|
puts "Done in: #{(t2 - t1).round} sec."
|
|
@@ -251,6 +321,28 @@ module Relaton
|
|
|
251
321
|
|
|
252
322
|
private
|
|
253
323
|
|
|
324
|
+
def ledger_path
|
|
325
|
+
File.join @output, LEDGER_FILE
|
|
326
|
+
end
|
|
327
|
+
|
|
328
|
+
def ledger
|
|
329
|
+
@ledger ||= if File.exist?(ledger_path)
|
|
330
|
+
YAML.safe_load_file(ledger_path) || {}
|
|
331
|
+
else
|
|
332
|
+
{}
|
|
333
|
+
end
|
|
334
|
+
end
|
|
335
|
+
|
|
336
|
+
def ledger_new
|
|
337
|
+
@ledger_new ||= {}
|
|
338
|
+
end
|
|
339
|
+
|
|
340
|
+
def entry_digest(entry)
|
|
341
|
+
require "digest"
|
|
342
|
+
require "json"
|
|
343
|
+
Digest::SHA256.hexdigest JSON.generate(entry)
|
|
344
|
+
end
|
|
345
|
+
|
|
254
346
|
# Write, clobbering whatever is there. The caller has already established
|
|
255
347
|
# that the path is ours to take. Identical content is left untouched:
|
|
256
348
|
# an incremental crawl (one that keeps the data tree) must not churn
|
data/lib/relaton/db.rb
CHANGED
|
@@ -158,13 +158,15 @@ module Relaton
|
|
|
158
158
|
end
|
|
159
159
|
|
|
160
160
|
# Cross-SDO equivalence as data: on a total miss, walk the equivalence
|
|
161
|
-
# tables from relaton/equivalence.rb — a national
|
|
162
|
-
# the EN, the EN to the ISO
|
|
163
|
-
#
|
|
161
|
+
# tables from relaton/equivalence.rb in both directions — a national
|
|
162
|
+
# adoption falls back to the EN, the EN to the ISO, and a base document
|
|
163
|
+
# falls forward to its declared adoptions (.related_for). Each candidate
|
|
164
|
+
# goes through the full routed fetch (cache, co-publishers); an
|
|
165
|
+
# unrouteable candidate is skipped.
|
|
164
166
|
#
|
|
165
167
|
# @return [Relaton::Bib::ItemData, nil]
|
|
166
168
|
def fetch_equivalent(code, year, opts)
|
|
167
|
-
Equivalence.
|
|
169
|
+
Equivalence.related_for(code).each do |candidate|
|
|
168
170
|
Util.info "Not found; trying equivalent `#{candidate}`", key: code
|
|
169
171
|
std, pubid = @registry.route(candidate)
|
|
170
172
|
bib = fetch_routed(candidate, year, opts, std, pubid) and return bib
|
data/lib/relaton/doi/CLAUDE.md
CHANGED
|
@@ -36,7 +36,7 @@ Key classes in `lib/relaton/doi/`:
|
|
|
36
36
|
## Test Setup
|
|
37
37
|
|
|
38
38
|
- **RSpec** with `expect` syntax only (monkey patching disabled)
|
|
39
|
-
- **VCR** cassettes in `spec/vcr_cassettes/` record Crossref HTTP responses
|
|
39
|
+
- **VCR** cassettes in `spec/vcr_cassettes/` record Crossref HTTP responses and are never auto-rerecorded (see root `CLAUDE.md`). Because Crossref rate-limits, **re-record this suite alone** — under parallel `rake spec` it answers 429 and the empty body gets recorded as if it were data.
|
|
40
40
|
- **XML fixtures** in `spec/fixtures/` — expected output XML files. The `read_fixture` helper auto-substitutes today's date into `<fetched>` tags.
|
|
41
41
|
- **equivalent-xml** gem for XML comparison in integration tests
|
|
42
42
|
- Integration tests in `spec/relaton/doi_spec.rb` cover 40+ document types via VCR cassettes
|
data/lib/relaton/equivalence.rb
CHANGED
|
@@ -84,6 +84,34 @@ module Relaton
|
|
|
84
84
|
end
|
|
85
85
|
out
|
|
86
86
|
end
|
|
87
|
+
|
|
88
|
+
# Every edge of `code`'s equivalence set: the canonical walk
|
|
89
|
+
# (.equivalents_for — a national adoption falls back to the EN, the EN
|
|
90
|
+
# to the ISO) PLUS the reverse edges, generated from the tables'
|
|
91
|
+
# `adoptions` ladders (the ISO resolves to its EN, BS EN, DIN EN and
|
|
92
|
+
# NF EN adoptions). The code itself is excluded; the walk order is
|
|
93
|
+
# canonical-first, ladder outermost-last.
|
|
94
|
+
#
|
|
95
|
+
# @param [String] code
|
|
96
|
+
# @return [Array<String>]
|
|
97
|
+
def related_for(code)
|
|
98
|
+
out = equivalents_for(code)
|
|
99
|
+
load_seed
|
|
100
|
+
([code] + out.uniq).each do |candidate|
|
|
101
|
+
tables.each do |table|
|
|
102
|
+
(table["adoptions"] || []).each do |ladder|
|
|
103
|
+
m = Regexp.new(ladder["pattern"]).match(candidate)
|
|
104
|
+
next unless m
|
|
105
|
+
|
|
106
|
+
Array(ladder["forms"]).each do |template|
|
|
107
|
+
form = template.gsub(/\\(\d+)/) { m[Regexp.last_match(1).to_i] }
|
|
108
|
+
out << form unless form == code || out.include?(form)
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
out
|
|
114
|
+
end
|
|
87
115
|
end
|
|
88
116
|
end
|
|
89
117
|
end
|
data/lib/relaton/index/CLAUDE.md
CHANGED
|
@@ -54,7 +54,8 @@ bin/console
|
|
|
54
54
|
4. Search matches against `:id` field (string comparison via `include?` or custom block)
|
|
55
55
|
5. `save` writes index as YAML to local file
|
|
56
56
|
|
|
57
|
-
With `pages_url:` (
|
|
57
|
+
With `pages_url:` (#189; W3C piloted it, and every indexed flavor's consumer
|
|
58
|
+
now uses it except JIS, NIST and Plateau, which still read the zip) steps 2–3 change: a non-String query
|
|
58
59
|
goes to `ShardSource#rows` and reads one shard, and `Type#index` (a String query,
|
|
59
60
|
a block-only search) is `ShardSource#whole_index`, the `<index>.zip` that the
|
|
60
61
|
manifest names, read from the Pages site. No `FileIO#read`, no disk cache.
|
|
@@ -172,8 +173,8 @@ the same arguments:
|
|
|
172
173
|
| call site | `url:` | `pubid_class:` |
|
|
173
174
|
|---|---|---|
|
|
174
175
|
| producer — `DataFetcher#index` | none (`file:` is CWD-relative) | **yes** — `FileIO#save` calls `to_hash` only for instances of it |
|
|
175
|
-
| consumer — `Bibliography`/`HitCollection#index` | the published `.zip` | **yes** — without it the rows stay raw hashes and `Type#search` stops narrowing |
|
|
176
|
-
| consumer on the Pages shards (
|
|
176
|
+
| consumer on the zip (JIS, NIST, Plateau) — `Bibliography`/`HitCollection#index` | the published `.zip` | **yes** — without it the rows stay raw hashes and `Type#search` stops narrowing |
|
|
177
|
+
| consumer on the Pages shards (all other indexed flavors) | none; **`pages_url:`** | **yes** — a shard row is deserialized with it |
|
|
177
178
|
| delete — `Processor#remove_index_file` | **`true`** | **no** |
|
|
178
179
|
|
|
179
180
|
The delete path is `Type#remove_file` → `FileIO#remove` →
|
data/lib/relaton/index/type.rb
CHANGED
|
@@ -308,14 +308,21 @@ module Relaton
|
|
|
308
308
|
end
|
|
309
309
|
|
|
310
310
|
# The query is the reference, so two identifiers take the subset match.
|
|
311
|
-
# See #search for the exact-equality escape hatch.
|
|
311
|
+
# See #search for the exact-equality escape hatch. A dated base query
|
|
312
|
+
# also resolves a wrapped record of exactly that edition (pubid#580's
|
|
313
|
+
# ruling): the undated query is the family wildcard, so the dated one
|
|
314
|
+
# names one edition, and the record embodying it answers through
|
|
315
|
+
# pubid's wrapped_edition_of? even when its printed identity is one
|
|
316
|
+
# level down ("IEC CA 01:2020" → the CSV wrapper).
|
|
312
317
|
def match_item(item, id)
|
|
313
318
|
if item[:id].is_a?(String)
|
|
314
319
|
item[:id].include?(id.is_a?(String) ? id : id.to_s)
|
|
315
320
|
elsif id.is_a?(String)
|
|
316
321
|
item[:id].to_s.include?(id)
|
|
317
322
|
else
|
|
318
|
-
id === item[:id]
|
|
323
|
+
id === item[:id] ||
|
|
324
|
+
(item[:id].is_a?(::Pubid::Identifier) &&
|
|
325
|
+
item[:id].wrapped_edition_of?(id))
|
|
319
326
|
end
|
|
320
327
|
end
|
|
321
328
|
end
|
data/lib/relaton/isbn/CLAUDE.md
CHANGED
|
@@ -36,7 +36,7 @@ Uses `Relaton::Bib::` namespace (not the old `RelatonBib::`). Key classes: `Item
|
|
|
36
36
|
|
|
37
37
|
## Testing
|
|
38
38
|
|
|
39
|
-
- **VCR** records HTTP interactions in `spec/vcr_cassettes/` (re-
|
|
39
|
+
- **VCR** records HTTP interactions in `spec/vcr_cassettes/` (never auto-rerecorded; re-record deliberately)
|
|
40
40
|
- **WebMock** disables real HTTP by default
|
|
41
41
|
- **SimpleCov** tracks coverage
|
|
42
42
|
- Fixtures in `spec/fixtures/`
|
data/lib/relaton/iso/CLAUDE.md
CHANGED
|
@@ -35,4 +35,4 @@ Key dependency: `pubid-iso` gem handles ISO publication identifier parsing and c
|
|
|
35
35
|
- **Framework:** RSpec with VCR cassettes and WebMock
|
|
36
36
|
- **Network access:** fully blocked via `WebMock.disable_net_connect!`
|
|
37
37
|
- **Index fixture:** `spec/fixtures/index-v1.zip` is pre-loaded into `Relaton::Index` pool in `before(:suite)` (configured in `spec/support/webmock.rb`). Run `rake spec:update_index` to refresh from relaton-data-iso.
|
|
38
|
-
- **VCR:** cassettes in `spec/vcr_cassettes/`, record mode `:once`,
|
|
38
|
+
- **VCR:** cassettes in `spec/vcr_cassettes/`, record mode `:once`, never auto-rerecorded (`re_record_interval: nil`). Index download requests are ignored by VCR (handled by fixture instead).
|
data/lib/relaton/itu/CLAUDE.md
CHANGED
|
@@ -646,8 +646,8 @@ is gone (see **Runtime lookup**).
|
|
|
646
646
|
`spec/itu/vcr_cassettes/itu_r_re*` cassettes back `data_crawler_r_spec.rb`, and
|
|
647
647
|
each is owned by **exactly one example**. That is not stylistic: VCR re-records
|
|
648
648
|
per cassette *insertion*, so a cassette shared across several examples is
|
|
649
|
-
truncated to the first example's requests the moment
|
|
650
|
-
|
|
649
|
+
truncated to the first example's requests the moment it is re-recorded
|
|
650
|
+
(verified — `itu_r_rec_bo` went 10 interactions → 1
|
|
651
651
|
and took 15 examples down with it). If you add a crawler example, give it its
|
|
652
652
|
own cassette rather than reusing one.
|
|
653
653
|
|
|
@@ -667,7 +667,7 @@ is gone (see **Runtime lookup**).
|
|
|
667
667
|
responses. VCR ignores only `index-v2.zip` (the index comes from the fixture
|
|
668
668
|
above), so the per-record `raw.githubusercontent.com/…/data/*.yaml` GETs **are**
|
|
669
669
|
recorded. Re-record by deleting the file and running the suite with network
|
|
670
|
-
access
|
|
670
|
+
access. `re_record_interval` is nil, so a cassette never refreshes by itself.
|
|
671
671
|
- **Coverage:** SimpleCov, target near 100%
|
|
672
672
|
|
|
673
673
|
Round-trip tests (serialize → deserialize → compare) are the primary pattern for model classes.
|
data/lib/relaton/nist/CLAUDE.md
CHANGED
|
@@ -135,19 +135,16 @@ errors = schema.validate(file)
|
|
|
135
135
|
### HTTP Recording
|
|
136
136
|
|
|
137
137
|
Tests use VCR with WebMock. Cassettes are stored in `spec/vcr_cassettes/` and
|
|
138
|
-
`spec/support/vcr.rb` sets `record: :once`
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
(delete the stale cassette, run the spec) and, for the integration fixtures that
|
|
149
|
-
compare full XML (`get.xml`, `hit.xml`, `hit_bibitem.xml`), regenerate them
|
|
150
|
-
(delete → the spec rewrites them via `File.write … unless File.exist?`).
|
|
138
|
+
`spec/support/vcr.rb` sets `record: :once` with `re_record_interval: nil`, so a
|
|
139
|
+
cassette replays as committed and never re-records by itself (relaton#360).
|
|
140
|
+
|
|
141
|
+
**When you re-record deliberately:** the NIST GitHub source data can drift (an
|
|
142
|
+
author renamed, a `loc.gov` `identifier` dropped), so the round-tripped XML no
|
|
143
|
+
longer matches the on-disk `spec/fixtures/*.xml` fixtures; and an upstream that
|
|
144
|
+
throttles the recorder records a bad response. Delete the stale cassette, run
|
|
145
|
+
the spec, and, for the integration fixtures that compare full XML (`get.xml`,
|
|
146
|
+
`hit.xml`, `hit_bibitem.xml`), regenerate them (delete → the spec rewrites them
|
|
147
|
+
via `File.write … unless File.exist?`) and read the diff.
|
|
151
148
|
|
|
152
149
|
### Test Data Stubbing
|
|
153
150
|
|
|
@@ -46,9 +46,13 @@ module Relaton
|
|
|
46
46
|
|
|
47
47
|
def fetch_doc(hit)
|
|
48
48
|
return if hit["type"] == "CC"
|
|
49
|
+
# The list entry carries the whole record payload, so an unchanged
|
|
50
|
+
# entry skips the parse and the write (the digest ledger).
|
|
51
|
+
return if unchanged?(hit["identifier"], hit)
|
|
49
52
|
|
|
50
53
|
bib = Scraper.parse_page hit, @errors
|
|
51
|
-
write_document bib
|
|
54
|
+
file = write_document bib
|
|
55
|
+
note_digest(hit["identifier"], hit, file: file) if file
|
|
52
56
|
true
|
|
53
57
|
rescue StandardError => e
|
|
54
58
|
Util.error "Fetching document: #{hit['identifier']}\n" \
|
|
@@ -56,17 +60,19 @@ module Relaton
|
|
|
56
60
|
false
|
|
57
61
|
end
|
|
58
62
|
|
|
63
|
+
# @return [String, nil] the file written, or nil for a duplicate
|
|
59
64
|
def write_document(bib) # rubocop:disable Metrics/AbcSize
|
|
60
65
|
docid = bib.docidentifier[0].content
|
|
61
66
|
if @docids.include?(docid)
|
|
62
67
|
@dupids << docid
|
|
63
|
-
return
|
|
68
|
+
return nil
|
|
64
69
|
end
|
|
65
70
|
|
|
66
71
|
@docids << docid
|
|
67
72
|
file = file_name bib
|
|
68
73
|
index.add_or_update index_id(docid), file
|
|
69
|
-
|
|
74
|
+
force_write file, serialize(bib)
|
|
75
|
+
file
|
|
70
76
|
end
|
|
71
77
|
|
|
72
78
|
#
|
data/lib/relaton/version.rb
CHANGED
data/lib/relaton/w3c/CLAUDE.md
CHANGED
|
@@ -323,7 +323,7 @@ Tests use [Jing](https://github.com/jing-trang/jing-trang) for RelaxNG validatio
|
|
|
323
323
|
Tests use RSpec with:
|
|
324
324
|
- **Round-trip tests** — YAML/XML → object → YAML/XML, verifying lossless serialization
|
|
325
325
|
- **Schema validation** — XML output validated against `grammars/relaton-w3c-compile.rng`
|
|
326
|
-
- **VCR** — recorded HTTP cassettes in `spec/vcr_cassettes/` (
|
|
326
|
+
- **VCR** — recorded HTTP cassettes in `spec/vcr_cassettes/` (never auto-rerecorded; re-record deliberately)
|
|
327
327
|
- **WebMock** — disables external HTTP in tests
|
|
328
328
|
|
|
329
329
|
Test fixtures live in `spec/fixtures/` (YAML and XML files).
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: relaton
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 3.0.0.pre.alpha.
|
|
4
|
+
version: 3.0.0.pre.alpha.20
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Ribose Inc.
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-10-
|
|
11
|
+
date: 2026-10-26 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: addressable
|
|
@@ -442,14 +442,14 @@ dependencies:
|
|
|
442
442
|
requirements:
|
|
443
443
|
- - "~>"
|
|
444
444
|
- !ruby/object:Gem::Version
|
|
445
|
-
version: 2.0.0.pre.alpha.
|
|
445
|
+
version: 2.0.0.pre.alpha.62
|
|
446
446
|
type: :runtime
|
|
447
447
|
prerelease: false
|
|
448
448
|
version_requirements: !ruby/object:Gem::Requirement
|
|
449
449
|
requirements:
|
|
450
450
|
- - "~>"
|
|
451
451
|
- !ruby/object:Gem::Version
|
|
452
|
-
version: 2.0.0.pre.alpha.
|
|
452
|
+
version: 2.0.0.pre.alpha.62
|
|
453
453
|
- !ruby/object:Gem::Dependency
|
|
454
454
|
name: yeptris
|
|
455
455
|
requirement: !ruby/object:Gem::Requirement
|