iev 0.4.6 → 0.4.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.claude/scheduled_tasks.lock +1 -0
- data/.rubocop_todo.yml +5 -47
- data/CLAUDE.md +11 -1
- data/Gemfile +4 -2
- data/TODO.reconcile/01-design.md +57 -0
- data/TODO.reconcile/02-change-models.md +44 -0
- data/TODO.reconcile/02-termbase-loader.md +48 -0
- data/TODO.reconcile/03-live-loader.md +28 -0
- data/TODO.reconcile/03-status-mapper.md +24 -0
- data/TODO.reconcile/04-concept-merger.md +45 -0
- data/TODO.reconcile/04-termbase-loader.md +28 -0
- data/TODO.reconcile/05-content-diff.md +45 -0
- data/TODO.reconcile/05-live-loader.md +20 -0
- data/TODO.reconcile/06-content-differ.md +32 -0
- data/TODO.reconcile/06-v3-serializer.md +42 -0
- data/TODO.reconcile/07-cli-and-run.md +27 -0
- data/TODO.reconcile/07-concept-merger.md +32 -0
- data/TODO.reconcile/08-report.md +53 -0
- data/TODO.reconcile/09-pipeline.md +23 -0
- data/TODO.reconcile/10-script-and-run.md +15 -0
- data/data/locales/glossarist_enums.yml +705 -0
- data/iev.gemspec +4 -2
- data/lib/iev/bibliography_builder.rb +87 -0
- data/lib/iev/cli/command.rb +29 -0
- data/lib/iev/cli/command_helper.rb +1 -6
- data/lib/iev/config.rb +10 -1
- data/lib/iev/exporter.rb +37 -4
- data/lib/iev/figure_builder.rb +186 -0
- data/lib/iev/multi_doc_yaml.rb +60 -0
- data/lib/iev/reconciler/change.rb +29 -0
- data/lib/iev/reconciler/change_set.rb +48 -0
- data/lib/iev/reconciler/concept_merger.rb +130 -0
- data/lib/iev/reconciler/content_differ.rb +157 -0
- data/lib/iev/reconciler/live_loader.rb +163 -0
- data/lib/iev/reconciler/pipeline.rb +104 -0
- data/lib/iev/reconciler/reconciled_concept.rb +14 -0
- data/lib/iev/reconciler/report.rb +118 -0
- data/lib/iev/reconciler/status_mapper.rb +29 -0
- data/lib/iev/reconciler/term_marker_parser.rb +196 -0
- data/lib/iev/reconciler/termbase_loader.rb +141 -0
- data/lib/iev/reconciler.rb +17 -0
- data/lib/iev/scraper/browser.rb +155 -50
- data/lib/iev/scraper/page_parser.rb +116 -24
- data/lib/iev/source_parser.rb +12 -1
- data/lib/iev/subject_area_concepts.rb +4 -2
- data/lib/iev/utilities.rb +7 -5
- data/lib/iev/version.rb +1 -1
- data/lib/iev.rb +4 -0
- data/scripts/find_unparseable.rb +22 -0
- metadata +54 -6
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: a016c3e0df9de5b20b48d9354f3a61ebbc604ac94e07b116a07ce2023b244eea
|
|
4
|
+
data.tar.gz: badb0152246599074ab74131274b70972f8ab8fa3dd1e4b49b06bd38dde81a33
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 6a1527631c2f47363372dca4f12f9c3b4026a3d936ecfabf8336dc061c3a188fb79d1e4797aa028c11d13936ef6164d09c2e396b50315cc532e01cda65f30d08
|
|
7
|
+
data.tar.gz: 7d1c71fcb8b1f87bbc7787688b8937780cb775fc43a6e7058d59f41a269cc2f4bdcd725ce303bc160ae5317505465ba88bd0c453d4cf311063f3207ed8e99bd3
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"sessionId":"7e802925-11d5-47e6-9ff2-7e151d3b2dfb","pid":23315,"procStart":"Wed Jul 8 10:33:56 2026","acquiredAt":1783506837511}
|
data/.rubocop_todo.yml
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# This configuration was generated by
|
|
2
2
|
# `rubocop --auto-gen-config`
|
|
3
|
-
# on 2026-06-
|
|
3
|
+
# on 2026-06-22 10:01:24 UTC using RuboCop version 1.86.1.
|
|
4
4
|
# The point is for the user to remove these configuration records
|
|
5
5
|
# one by one as the offenses are removed from the code base.
|
|
6
6
|
# Note that changes in the inspected code, or installation of new
|
|
@@ -11,37 +11,7 @@ Gemspec/RequiredRubyVersion:
|
|
|
11
11
|
Exclude:
|
|
12
12
|
- 'iev.gemspec'
|
|
13
13
|
|
|
14
|
-
# Offense count:
|
|
15
|
-
# This cop supports safe autocorrection (--autocorrect).
|
|
16
|
-
# Configuration parameters: EnforcedStyle, IndentationWidth.
|
|
17
|
-
# SupportedStyles: with_first_argument, with_fixed_indentation
|
|
18
|
-
Layout/ArgumentAlignment:
|
|
19
|
-
Exclude:
|
|
20
|
-
- 'spec/iev/exporter_spec.rb'
|
|
21
|
-
|
|
22
|
-
# Offense count: 1
|
|
23
|
-
# This cop supports safe autocorrection (--autocorrect).
|
|
24
|
-
# Configuration parameters: EnforcedStyleAlignWith.
|
|
25
|
-
# SupportedStylesAlignWith: either, start_of_block, start_of_line
|
|
26
|
-
Layout/BlockAlignment:
|
|
27
|
-
Exclude:
|
|
28
|
-
- 'spec/iev/exporter_spec.rb'
|
|
29
|
-
|
|
30
|
-
# Offense count: 1
|
|
31
|
-
# This cop supports safe autocorrection (--autocorrect).
|
|
32
|
-
Layout/BlockEndNewline:
|
|
33
|
-
Exclude:
|
|
34
|
-
- 'spec/iev/exporter_spec.rb'
|
|
35
|
-
|
|
36
|
-
# Offense count: 2
|
|
37
|
-
# This cop supports safe autocorrection (--autocorrect).
|
|
38
|
-
# Configuration parameters: Width, EnforcedStyleAlignWith, AllowedPatterns.
|
|
39
|
-
# SupportedStylesAlignWith: start_of_line, relative_to_receiver
|
|
40
|
-
Layout/IndentationWidth:
|
|
41
|
-
Exclude:
|
|
42
|
-
- 'spec/iev/exporter_spec.rb'
|
|
43
|
-
|
|
44
|
-
# Offense count: 65
|
|
14
|
+
# Offense count: 72
|
|
45
15
|
# This cop supports safe autocorrection (--autocorrect).
|
|
46
16
|
# Configuration parameters: Max, AllowHeredoc, AllowURI, AllowQualifiedName, URISchemes, AllowRBSInlineAnnotation, AllowCopDirectives, AllowedPatterns, SplitStrings.
|
|
47
17
|
# URISchemes: http, https
|
|
@@ -65,7 +35,7 @@ Lint/ShadowedException:
|
|
|
65
35
|
Exclude:
|
|
66
36
|
- 'lib/iev/source_parser.rb'
|
|
67
37
|
|
|
68
|
-
# Offense count:
|
|
38
|
+
# Offense count: 29
|
|
69
39
|
# Configuration parameters: AllowedMethods, AllowedPatterns, CountRepeatedAttributes, Max.
|
|
70
40
|
Metrics/AbcSize:
|
|
71
41
|
Exclude:
|
|
@@ -94,12 +64,12 @@ Metrics/CyclomaticComplexity:
|
|
|
94
64
|
- 'lib/iev/term_builder.rb'
|
|
95
65
|
- 'lib/iev/utilities.rb'
|
|
96
66
|
|
|
97
|
-
# Offense count:
|
|
67
|
+
# Offense count: 38
|
|
98
68
|
# Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns.
|
|
99
69
|
Metrics/MethodLength:
|
|
100
70
|
Max: 73
|
|
101
71
|
|
|
102
|
-
# Offense count:
|
|
72
|
+
# Offense count: 2
|
|
103
73
|
# Configuration parameters: CountKeywordArgs, MaxOptionalParameters.
|
|
104
74
|
Metrics/ParameterLists:
|
|
105
75
|
Max: 7
|
|
@@ -128,15 +98,3 @@ Naming/MethodParameterName:
|
|
|
128
98
|
Naming/VariableNumber:
|
|
129
99
|
Exclude:
|
|
130
100
|
- 'spec/iev/exporter_spec.rb'
|
|
131
|
-
|
|
132
|
-
# Offense count: 5
|
|
133
|
-
# This cop supports safe autocorrection (--autocorrect).
|
|
134
|
-
# Configuration parameters: EnforcedStyle, ProceduralMethods, FunctionalMethods, AllowedMethods, AllowedPatterns, AllowBracesOnProceduralOneLiners, BracesRequiredMethods.
|
|
135
|
-
# SupportedStyles: line_count_based, semantic, braces_for_chaining, always_braces
|
|
136
|
-
# ProceduralMethods: benchmark, bm, bmbm, create, each_with_object, measure, new, realtime, tap, with_object
|
|
137
|
-
# FunctionalMethods: let, let!, subject, watch
|
|
138
|
-
# AllowedMethods: lambda, proc, it
|
|
139
|
-
Style/BlockDelimiters:
|
|
140
|
-
Exclude:
|
|
141
|
-
- 'lib/iev/exporter.rb'
|
|
142
|
-
- 'spec/iev/exporter_spec.rb'
|
data/CLAUDE.md
CHANGED
|
@@ -33,7 +33,9 @@ This is a Ruby gem (`iev`) for working with the International Electrotechnical V
|
|
|
33
33
|
- `SupersessionParser` — parses the REPLACES field for deprecated term relationships.
|
|
34
34
|
- `SubjectAreas` — manages the IEV subject area/section hierarchy. Bundled `data/subject_areas.yaml` contains the area/section tree. URI scheme: `area-{code}` and `section-{code}`.
|
|
35
35
|
- `SubjectAreaConcepts` — builds area and section hierarchy concepts. Uses `ConceptReference` with proper `ref_type` per `ConceptReferenceType`: `"domain"` for thematic area classification, `"section"` for structural section membership. Sets `ConceptData#domain` to area title text.
|
|
36
|
-
- `Exporter` — full export pipeline (Excel/SQLite → Glossarist YAML). Assigns domain and section `ConceptReference` objects via `domain_references_for`. Uses `Glossarist::DatasetRegister` model for `register.yaml`. Sets `schema_version: "3"` on all exported concepts.
|
|
36
|
+
- `Exporter` — full export pipeline (Excel/SQLite → Glossarist YAML). Assigns domain and section `ConceptReference` objects via `domain_references_for`. Uses `Glossarist::DatasetRegister` model for `register.yaml`. Sets `schema_version: "3"` on all exported concepts. Pipeline order: build → subject areas → section relations → figure extraction → reference enrichment → save concepts → save figures → save bibliography → save register.
|
|
37
|
+
- `FigureBuilder` — destructive extraction pass that hoists AsciiDoc image macros (emitted from SIMG tags by `Utilities`) into dataset-shared `Glossarist::Figure` entities. Rewrites inline text to `{{fig:id, display}}` mentions and adds `FigureReference` entries to `ManagedConceptData#figures`. One Figure entity per unique image file; captions merge across languages.
|
|
38
|
+
- `BibliographyBuilder` — collects unique `(source, id)` pairs from every concept's sources (localized and managed) into a `Glossarist::BibliographyData`. Entry ids are normalized with the same rules as `Glossarist::Validation::BibliographyIndex` so consumers can resolve anchors.
|
|
37
39
|
- `Converter::MathmlToAsciimath` — converts MathML markup to AsciiMath using Plurimath.
|
|
38
40
|
- `Utilities` — HTML processing: converts IEV cross-references (`<a href=IEV...>`) to `{{URN, term}}` format (ID first, display text last), handles figures, images, bold tags, and newline normalization.
|
|
39
41
|
|
|
@@ -45,6 +47,14 @@ Per the concept model's `ConceptReferenceType`:
|
|
|
45
47
|
|
|
46
48
|
Each concept's `ManagedConceptData#domains` contains both refs. `ConceptData#domain` (a `LocalizedString`) holds the section/area title text. The `ManagedConcept#related` array holds `broader`/`narrower` relationships for the hierarchy tree.
|
|
47
49
|
|
|
50
|
+
### V3 Output Artifacts
|
|
51
|
+
|
|
52
|
+
An export produces these files alongside the concepts/ directory:
|
|
53
|
+
- `register.yaml` — `Glossarist::DatasetRegister` with section tree, languages, owner, URN.
|
|
54
|
+
- `bibliography.yaml` — single `bibliography:` key wrapping an array of `BibliographyEntry` objects. Entry `id` is the normalized anchor that `Glossarist::Validation::BibliographyIndex` will resolve against.
|
|
55
|
+
- `figures/{fig-id}.yaml` — one `Glossarist::Figure` per unique image. Each concept carries a `FigureReference` on `ManagedConceptData#figures` and an inline `{{fig:id, display}}` mention in the text where the figure appeared.
|
|
56
|
+
- References on localized concepts are populated by `Glossarist::ConceptEnricher#inject_references`, which scans text for `{{urn:...}}`, `<<xref>>`, and `image::` patterns.
|
|
57
|
+
|
|
48
58
|
### Configuration
|
|
49
59
|
|
|
50
60
|
`Iev.configure` yields a `Config` object with:
|
data/Gemfile
CHANGED
|
@@ -3,16 +3,18 @@
|
|
|
3
3
|
source "https://rubygems.org"
|
|
4
4
|
|
|
5
5
|
# Use local glossarist-ruby when available for development.
|
|
6
|
-
# Otherwise falls back to released gem (requires >= 2.8.
|
|
6
|
+
# Otherwise falls back to released gem (requires >= 2.8.15 for
|
|
7
|
+
# BibliographyData, Figure/NonVerbRep, and ConceptEnricher support).
|
|
7
8
|
if File.directory?(File.expand_path("../glossarist-ruby", __dir__))
|
|
8
9
|
gem "glossarist", path: "../glossarist-ruby"
|
|
9
10
|
else
|
|
10
|
-
gem "glossarist", ">= 2.8.
|
|
11
|
+
gem "glossarist", ">= 2.8.15"
|
|
11
12
|
end
|
|
12
13
|
|
|
13
14
|
gem "benchmark"
|
|
14
15
|
gem "canon"
|
|
15
16
|
gem "openssl"
|
|
17
|
+
gem "playwright-ruby-client", require: false
|
|
16
18
|
gem "rake"
|
|
17
19
|
gem "rspec"
|
|
18
20
|
gem "rubocop"
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
# 01 — Architecture & autoloads
|
|
2
|
+
|
|
3
|
+
## Audit of current approach (scripts/reconcile.rb)
|
|
4
|
+
|
|
5
|
+
Problems:
|
|
6
|
+
1. **Monolithic** — 400-line script, module of static methods, no classes
|
|
7
|
+
2. **Not model-driven** — builds raw Hashes instead of Glossarist model objects
|
|
8
|
+
3. **No field-level diff** — knows THAT content changed, not WHAT changed
|
|
9
|
+
4. **No before/after values** — loses the historical content
|
|
10
|
+
5. **No machine-readable change log** — no way to audit changes
|
|
11
|
+
6. **No specs**
|
|
12
|
+
7. **Not MECE** — merge, diff, serialize all mixed together
|
|
13
|
+
8. **Not autoloaded** — everything in a script file
|
|
14
|
+
|
|
15
|
+
## New architecture
|
|
16
|
+
|
|
17
|
+
```
|
|
18
|
+
lib/iev/reconciler.rb module + autoloads
|
|
19
|
+
lib/iev/reconciler/change.rb field-level change (field, lang, old, new)
|
|
20
|
+
lib/iev/reconciler/change_set.rb collection of Change per concept
|
|
21
|
+
lib/iev/reconciler/status_mapper.rb termbase status -> V3 ConceptStatus
|
|
22
|
+
lib/iev/reconciler/termbase_loader.rb termbase.yaml -> {code => ManagedConcept}
|
|
23
|
+
lib/iev/reconciler/live_loader.rb HTML pages -> {code => ManagedConcept}
|
|
24
|
+
lib/iev/reconciler/content_differ.rb diff two ManagedConcepts -> ChangeSet
|
|
25
|
+
lib/iev/reconciler/concept_merger.rb merge termbase + live -> ReconciledConcept
|
|
26
|
+
lib/iev/reconciler/reconciled_concept.rb ManagedConcept + ChangeSet + source
|
|
27
|
+
lib/iev/reconciler/report.rb dataset-level change reporting
|
|
28
|
+
lib/iev/reconciler/pipeline.rb orchestration
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
Each class has ONE responsibility (MECE). The Pipeline composes them.
|
|
32
|
+
|
|
33
|
+
## History reporting
|
|
34
|
+
|
|
35
|
+
Three layers:
|
|
36
|
+
|
|
37
|
+
1. **Model-level** — `ManagedConcept#dates` (accepted/amended/retired),
|
|
38
|
+
`ManagedConcept#status` (valid/draft/superseded/retired),
|
|
39
|
+
`ManagedConcept#related` (supersedes/superseded_by).
|
|
40
|
+
|
|
41
|
+
2. **Concept-level annotations** — Each amended concept gets a
|
|
42
|
+
`ConceptData#annotations` entry describing what field changed, with
|
|
43
|
+
before/after text.
|
|
44
|
+
|
|
45
|
+
3. **Dataset-level report** — Summary statistics + machine-readable
|
|
46
|
+
change log written to `report/`:
|
|
47
|
+
- `summary.yaml` — aggregate counts by change type, section, language
|
|
48
|
+
- `changes.csv` — one row per field-level change
|
|
49
|
+
- `retired.yaml` — concepts retired since termbase snapshot
|
|
50
|
+
- `new_concepts.yaml` — concepts added since termbase snapshot
|
|
51
|
+
|
|
52
|
+
## Checklist
|
|
53
|
+
|
|
54
|
+
- [x] Audit current approach
|
|
55
|
+
- [x] Design new architecture
|
|
56
|
+
- [x] Create lib/iev/reconciler.rb with autoloads
|
|
57
|
+
- [x] Register in lib/iev.rb
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# 02 — Change + ChangeSet models
|
|
2
|
+
|
|
3
|
+
## Goal
|
|
4
|
+
|
|
5
|
+
The core of history reporting. A `Change` captures a single field-level
|
|
6
|
+
difference between the termbase snapshot and the live snapshot. A
|
|
7
|
+
`ChangeSet` groups all changes for one concept.
|
|
8
|
+
|
|
9
|
+
## Change
|
|
10
|
+
|
|
11
|
+
```ruby
|
|
12
|
+
Change = Struct.new(:code, :field, :language, :old_value, :new_value,
|
|
13
|
+
:detected_at, keyword_init: true)
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
Fields:
|
|
17
|
+
- `code`: IEV code (e.g. "102-01-01")
|
|
18
|
+
- `field`: `:designation` | `:definition` | `:notes` | `:examples` | `:status` | `:language_added` | `:language_removed`
|
|
19
|
+
- `language`: ISO 639-2 code ("eng", "fra", nil for concept-level)
|
|
20
|
+
- `old_value`: the termbase value (nil for additions)
|
|
21
|
+
- `new_value`: the live value (nil for removals)
|
|
22
|
+
- `detected_at`: date the change was observed (mirror date)
|
|
23
|
+
|
|
24
|
+
## ChangeSet
|
|
25
|
+
|
|
26
|
+
```ruby
|
|
27
|
+
class ChangeSet
|
|
28
|
+
include Enumerable
|
|
29
|
+
def initialize(code)
|
|
30
|
+
def add(change)
|
|
31
|
+
def each
|
|
32
|
+
def empty?
|
|
33
|
+
def by_field(field)
|
|
34
|
+
def by_language(lang)
|
|
35
|
+
def summary # hash of field => count
|
|
36
|
+
end
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
## Checklist
|
|
40
|
+
|
|
41
|
+
- [x] Implement Change (Struct)
|
|
42
|
+
- [x] Implement ChangeSet
|
|
43
|
+
- [x] Spec: ChangeSet query methods
|
|
44
|
+
- [x] Spec: ChangeSet#summary
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# 02 — Termbase loader
|
|
2
|
+
|
|
3
|
+
## Goal
|
|
4
|
+
|
|
5
|
+
Load `iev-data/termbase.yaml` into a uniform per-code hash keyed by IEV
|
|
6
|
+
code. The termbase is the authoritative source for lifecycle dates and
|
|
7
|
+
historical content.
|
|
8
|
+
|
|
9
|
+
## What termbase.yaml contains
|
|
10
|
+
|
|
11
|
+
22,228 concepts in the old ISO/IEV termbase format:
|
|
12
|
+
|
|
13
|
+
```yaml
|
|
14
|
+
102-01-01:
|
|
15
|
+
termid: 102-01-01
|
|
16
|
+
term: equality
|
|
17
|
+
eng:
|
|
18
|
+
id: 102-01-01
|
|
19
|
+
terms: [...]
|
|
20
|
+
definition: "..."
|
|
21
|
+
language_code: eng
|
|
22
|
+
notes: [...]
|
|
23
|
+
examples: [...]
|
|
24
|
+
entry_status: Standard
|
|
25
|
+
date_accepted: '2008-08-01T00:00:00+00:00'
|
|
26
|
+
date_amended: '2008-08-01T00:00:00+00:00'
|
|
27
|
+
review_date: '2008-08-01T00:00:00+00:00'
|
|
28
|
+
review_decision_date: '2008-08-01T00:00:00+00:00'
|
|
29
|
+
review_decision_event: published
|
|
30
|
+
fra:
|
|
31
|
+
...
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## Deliverable
|
|
35
|
+
|
|
36
|
+
Class: `Iev::Reconciler::TermbaseLoader`
|
|
37
|
+
|
|
38
|
+
- Loads termbase.yaml (22,228 concepts).
|
|
39
|
+
- Returns a Hash keyed by IEV code string → concept hash.
|
|
40
|
+
- Each concept hash normalizes per-language data into a consistent
|
|
41
|
+
structure so downstream stages don't need to handle the old format
|
|
42
|
+
ad-hoc.
|
|
43
|
+
|
|
44
|
+
## Checklist
|
|
45
|
+
|
|
46
|
+
- [x] Define TermbaseLoader interface
|
|
47
|
+
- [x] Implement
|
|
48
|
+
- [x] Test: loads all 22,228 codes; sample code has expected fields
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# 03 — Live HTML loader
|
|
2
|
+
|
|
3
|
+
## Goal
|
|
4
|
+
|
|
5
|
+
Load the mirrored HTML pages from `iev-data-latest/pages/` using the
|
|
6
|
+
existing `Iev::Scraper::PageParser` to extract structured concept data.
|
|
7
|
+
|
|
8
|
+
## Source
|
|
9
|
+
|
|
10
|
+
17,473 HTML pages at `../iev-data-latest/pages/*.html`. Each page is
|
|
11
|
+
the current Electropedia display page for one IEV code.
|
|
12
|
+
|
|
13
|
+
## Deliverable
|
|
14
|
+
|
|
15
|
+
Class: `Iev::Reconciler::LiveLoader`
|
|
16
|
+
|
|
17
|
+
- Walks `Iev::Fetcher::PageStore#each_concept`.
|
|
18
|
+
- For each page, uses `Iev::Scraper::PageParser` to parse into the raw
|
|
19
|
+
hash that `build_concept_from_raw` expects.
|
|
20
|
+
- Returns a Hash keyed by IEV code → parsed raw hash.
|
|
21
|
+
- Extracts the publication date from the HTML (e.g. "Publication date:
|
|
22
|
+
2008-08") for concepts not in termbase.
|
|
23
|
+
|
|
24
|
+
## Checklist
|
|
25
|
+
|
|
26
|
+
- [x] Define LiveLoader interface
|
|
27
|
+
- [x] Implement
|
|
28
|
+
- [x] Test: sample page parses with expected localized_concepts
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# 03 — StatusMapper
|
|
2
|
+
|
|
3
|
+
## Goal
|
|
4
|
+
|
|
5
|
+
Map termbase `entry_status` strings to Glossarist V3 `ConceptStatus`
|
|
6
|
+
enum values. Stateless, single responsibility.
|
|
7
|
+
|
|
8
|
+
## Mapping
|
|
9
|
+
|
|
10
|
+
| Termbase | V3 |
|
|
11
|
+
|----------|-----|
|
|
12
|
+
| Standard / Published / Effective | valid |
|
|
13
|
+
| Draft | draft |
|
|
14
|
+
| Not Valid | notValid |
|
|
15
|
+
| Superseded | superseded |
|
|
16
|
+
| Retired | retired |
|
|
17
|
+
|
|
18
|
+
Unknown values default to `valid`.
|
|
19
|
+
|
|
20
|
+
## Checklist
|
|
21
|
+
|
|
22
|
+
- [x] Implement StatusMapper
|
|
23
|
+
- [x] Spec: all mapped values
|
|
24
|
+
- [x] Spec: unknown defaults to valid
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# 04 — Concept merger
|
|
2
|
+
|
|
3
|
+
## Goal
|
|
4
|
+
|
|
5
|
+
For each code in the union of termbase + live sources, merge the two
|
|
6
|
+
into a single V3 `ManagedConcept` with full lifecycle data.
|
|
7
|
+
|
|
8
|
+
## Three populations
|
|
9
|
+
|
|
10
|
+
1. **In both** (~16,643 codes): termbase provides dates/status,
|
|
11
|
+
live provides current content. If content differs, record an
|
|
12
|
+
amendment.
|
|
13
|
+
2. **Termbase only** (~5,585 codes): not on live Electropedia → mark
|
|
14
|
+
`status: retired`, add `ConceptDate(type: retired)`.
|
|
15
|
+
3. **Live only** (~830 codes): not in 2020 termbase → new concept,
|
|
16
|
+
`ConceptDate(type: accepted)` from HTML publication date.
|
|
17
|
+
|
|
18
|
+
## Merge rules
|
|
19
|
+
|
|
20
|
+
| Field | Source |
|
|
21
|
+
|-------|--------|
|
|
22
|
+
| `identifier` | either (same code) |
|
|
23
|
+
| `status` | termbase `entry_status` → mapped to `ConceptStatus` enum |
|
|
24
|
+
| `dates` | termbase `date_accepted`/`date_amended`/`review_date`/`review_decision_date` → `ConceptDate` array |
|
|
25
|
+
| `localized_concepts.designations` | **live** if available, else termbase |
|
|
26
|
+
| `localized_concepts.definition` | **live** if available, else termbase |
|
|
27
|
+
| `localized_concepts.notes/examples` | **live** if available, else termbase |
|
|
28
|
+
| `sources` | live parsed sources (richer) |
|
|
29
|
+
| `related` | termbase supersession data + any extracted from live |
|
|
30
|
+
|
|
31
|
+
## Status mapping
|
|
32
|
+
|
|
33
|
+
| Termbase `entry_status` | V3 `ConceptStatus` |
|
|
34
|
+
|--------------------------|---------------------|
|
|
35
|
+
| `Standard` / `Published` | `valid` |
|
|
36
|
+
| `Draft` | `draft` |
|
|
37
|
+
| `Not Valid` | `notValid` |
|
|
38
|
+
| `Superseded` | `superseded` |
|
|
39
|
+
| `Retired` / absent from live | `retired` |
|
|
40
|
+
|
|
41
|
+
## Checklist
|
|
42
|
+
|
|
43
|
+
- [x] Define merge rules
|
|
44
|
+
- [x] Implement ConceptMerger
|
|
45
|
+
- [x] Test: in-both, termbase-only, live-only cases
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# 04 — TermbaseLoader
|
|
2
|
+
|
|
3
|
+
## Goal
|
|
4
|
+
|
|
5
|
+
Load `termbase.yaml` and build `Glossarist::ManagedConcept` objects
|
|
6
|
+
with full lifecycle dates from the old termbase format.
|
|
7
|
+
|
|
8
|
+
## Input
|
|
9
|
+
|
|
10
|
+
`iev-data/termbase.yaml` — 22,228 concepts in old ISO/IEV format.
|
|
11
|
+
|
|
12
|
+
## Output
|
|
13
|
+
|
|
14
|
+
`Hash<String, Glossarist::ManagedConcept>` keyed by IEV code.
|
|
15
|
+
|
|
16
|
+
Each ManagedConcept has:
|
|
17
|
+
- `id` = code
|
|
18
|
+
- `status` from StatusMapper
|
|
19
|
+
- `dates` from `date_accepted` + `date_amended` (ConceptDate objects)
|
|
20
|
+
- `localized_concepts` hash with one LocalizedConcept per language
|
|
21
|
+
- Each LocalizedConcept has ConceptData with terms, definition, notes, examples
|
|
22
|
+
|
|
23
|
+
## Checklist
|
|
24
|
+
|
|
25
|
+
- [x] Implement TermbaseLoader
|
|
26
|
+
- [x] Spec: loads sample termbase entry correctly
|
|
27
|
+
- [x] Spec: dates extracted properly
|
|
28
|
+
- [x] Spec: multi-language concepts
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# 05 — Content diff detector
|
|
2
|
+
|
|
3
|
+
## Goal
|
|
4
|
+
|
|
5
|
+
Detect content changes between the termbase snapshot and the live HTML
|
|
6
|
+
for concepts that exist in both sources. Each detected change becomes
|
|
7
|
+
an amendment record.
|
|
8
|
+
|
|
9
|
+
## What to diff
|
|
10
|
+
|
|
11
|
+
Per localized concept (per language):
|
|
12
|
+
|
|
13
|
+
1. **Designations** (terms): added, removed, or changed text
|
|
14
|
+
2. **Definition**: text content changed
|
|
15
|
+
3. **Notes**: added, removed, or changed
|
|
16
|
+
4. **Examples**: added, removed, or changed
|
|
17
|
+
|
|
18
|
+
## Output
|
|
19
|
+
|
|
20
|
+
For each changed field, record:
|
|
21
|
+
|
|
22
|
+
```ruby
|
|
23
|
+
{
|
|
24
|
+
field: :definition,
|
|
25
|
+
language: :eng,
|
|
26
|
+
old_value: "...", # from termbase
|
|
27
|
+
new_value: "...", # from live
|
|
28
|
+
changed_at: MIRROR_DATE # the date we captured the live page
|
|
29
|
+
}
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
These feed into the merger to append `ConceptDate(type: amended)` and
|
|
33
|
+
optionally an annotation describing the change.
|
|
34
|
+
|
|
35
|
+
## Approach
|
|
36
|
+
|
|
37
|
+
String normalization before comparison: strip whitespace, normalize
|
|
38
|
+
unicode, ignore case-only differences in designation. Real semantic
|
|
39
|
+
diffs (definition rewrite) vs cosmetic diffs (encoding fix).
|
|
40
|
+
|
|
41
|
+
## Checklist
|
|
42
|
+
|
|
43
|
+
- [x] Define diff strategy
|
|
44
|
+
- [x] implement ContentDiff
|
|
45
|
+
- [x] Test: identical → no change; modified definition → one diff
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# 05 — LiveLoader
|
|
2
|
+
|
|
3
|
+
## Goal
|
|
4
|
+
|
|
5
|
+
Load live HTML pages and build `Glossarist::ManagedConcept` objects
|
|
6
|
+
using the existing `Iev::Scraper::PageParser`.
|
|
7
|
+
|
|
8
|
+
## Input
|
|
9
|
+
|
|
10
|
+
`iev-data-latest/pages/*.html` — 17,473 current Electropedia pages.
|
|
11
|
+
|
|
12
|
+
## Output
|
|
13
|
+
|
|
14
|
+
`Hash<String, Glossarist::ManagedConcept>` keyed by IEV code.
|
|
15
|
+
|
|
16
|
+
## Checklist
|
|
17
|
+
|
|
18
|
+
- [x] Implement LiveLoader
|
|
19
|
+
- [x] Spec: parses sample HTML page
|
|
20
|
+
- [x] Spec: skips placeholder pages (no localized_concepts)
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# 06 — ContentDiffer
|
|
2
|
+
|
|
3
|
+
## Goal
|
|
4
|
+
|
|
5
|
+
Compare two `ManagedConcept` objects and produce a `ChangeSet` with
|
|
6
|
+
field-level differences. This is the core of change reporting.
|
|
7
|
+
|
|
8
|
+
## What to diff
|
|
9
|
+
|
|
10
|
+
Per localized concept (per language):
|
|
11
|
+
- `designation` (term text)
|
|
12
|
+
- `definition` (text content)
|
|
13
|
+
- `notes` (added/removed/changed)
|
|
14
|
+
- `examples` (added/removed/changed)
|
|
15
|
+
|
|
16
|
+
At concept level:
|
|
17
|
+
- `status` (e.g. valid → retired)
|
|
18
|
+
- language added (in live but not termbase)
|
|
19
|
+
- language removed (in termbase but not live)
|
|
20
|
+
|
|
21
|
+
## Normalization
|
|
22
|
+
|
|
23
|
+
Before comparison: strip HTML tags, normalize whitespace, normalize
|
|
24
|
+
unicode quotes. Avoid false positives from encoding differences.
|
|
25
|
+
|
|
26
|
+
## Checklist
|
|
27
|
+
|
|
28
|
+
- [x] Implement ContentDiffer
|
|
29
|
+
- [x] Spec: identical concepts → empty ChangeSet
|
|
30
|
+
- [x] Spec: definition changed → one Change
|
|
31
|
+
- [x] Spec: language added → one Change
|
|
32
|
+
- [x] Spec: status changed → one Change
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# 06 — V3 serializer
|
|
2
|
+
|
|
3
|
+
## Goal
|
|
4
|
+
|
|
5
|
+
Serialize the merged `ManagedConcept` objects into Glossarist V3 YAML
|
|
6
|
+
files using the existing `Glossarist::ManagedConceptCollection` API.
|
|
7
|
+
|
|
8
|
+
## Output
|
|
9
|
+
|
|
10
|
+
```
|
|
11
|
+
output/
|
|
12
|
+
concepts/
|
|
13
|
+
concept-102-01-01.yaml
|
|
14
|
+
concept-102-01-02.yaml
|
|
15
|
+
...
|
|
16
|
+
register.yaml # DatasetRegister
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
Each concept file uses `schema_version: "3"` and the V3 field names
|
|
20
|
+
from the concept model:
|
|
21
|
+
|
|
22
|
+
```yaml
|
|
23
|
+
---
|
|
24
|
+
schema_version: "3"
|
|
25
|
+
identifier: "102-01-01"
|
|
26
|
+
status: valid
|
|
27
|
+
dates:
|
|
28
|
+
- type: accepted
|
|
29
|
+
date: "2008-08-01"
|
|
30
|
+
- type: amended
|
|
31
|
+
date: "2008-08-01"
|
|
32
|
+
localized_concepts:
|
|
33
|
+
eng: ...
|
|
34
|
+
fra: ...
|
|
35
|
+
related: ...
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
## Checklist
|
|
39
|
+
|
|
40
|
+
- [x] Define output layout
|
|
41
|
+
- [x] Implement V3 serializer (reuse ManagedConceptCollection#save)
|
|
42
|
+
- [x] Test: round-trip — load output back, verify fields
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# 07 — CLI command + end-to-end run
|
|
2
|
+
|
|
3
|
+
## Goal
|
|
4
|
+
|
|
5
|
+
Wire the reconciliation pipeline into a runnable command and execute
|
|
6
|
+
it to produce the final dataset.
|
|
7
|
+
|
|
8
|
+
## Deliverable
|
|
9
|
+
|
|
10
|
+
Either:
|
|
11
|
+
- A new Thor CLI command: `iev reconcile --termbase <path> --output <dir>`
|
|
12
|
+
- Or a standalone script: `scripts/reconcile.rb`
|
|
13
|
+
|
|
14
|
+
The command should:
|
|
15
|
+
1. Load termbase.yaml
|
|
16
|
+
2. Load live HTML from PageStore
|
|
17
|
+
3. Merge + diff
|
|
18
|
+
4. Serialize V3 YAML
|
|
19
|
+
5. Write register.yaml
|
|
20
|
+
6. Report: total concepts, in-both, termbase-only, live-only, amendments detected
|
|
21
|
+
|
|
22
|
+
## Checklist
|
|
23
|
+
|
|
24
|
+
- [x] Define command interface
|
|
25
|
+
- [x] Implement
|
|
26
|
+
- [x] Run end-to-end and verify output
|
|
27
|
+
- [x] Spot-check 5 concepts manually (in-both, termbase-only, live-only, with changes)
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# 07 — ConceptMerger + ReconciledConcept
|
|
2
|
+
|
|
3
|
+
## Goal
|
|
4
|
+
|
|
5
|
+
Merge termbase and live concepts into a single `ManagedConcept` with
|
|
6
|
+
full lifecycle data, and package it with its `ChangeSet` in a
|
|
7
|
+
`ReconciledConcept` value object.
|
|
8
|
+
|
|
9
|
+
## ReconciledConcept
|
|
10
|
+
|
|
11
|
+
```ruby
|
|
12
|
+
ReconciledConcept = Struct.new(:managed_concept, :change_set, :source,
|
|
13
|
+
keyword_init: true)
|
|
14
|
+
# source: :both | :termbase_only | :live_only
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
## Merge rules
|
|
18
|
+
|
|
19
|
+
| Source | Behavior |
|
|
20
|
+
|--------|----------|
|
|
21
|
+
| both | live content + termbase dates; if changes detected, add ConceptDate(amended) + annotations |
|
|
22
|
+
| termbase only | mark retired, add ConceptDate(retired), preserve full content |
|
|
23
|
+
| live only | new concept, add ConceptDate(accepted) from publication date |
|
|
24
|
+
|
|
25
|
+
## Checklist
|
|
26
|
+
|
|
27
|
+
- [x] Implement ReconciledConcept (Struct)
|
|
28
|
+
- [x] Implement ConceptMerger
|
|
29
|
+
- [x] Spec: merge-both produces live content + termbase dates
|
|
30
|
+
- [x] Spec: termbase-only produces retired status
|
|
31
|
+
- [x] Spec: live-only produces accepted date
|
|
32
|
+
- [x] Spec: changes produce annotation entries
|