iev 0.4.6 → 0.4.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. checksums.yaml +4 -4
  2. data/.claude/scheduled_tasks.lock +1 -0
  3. data/.rubocop_todo.yml +5 -47
  4. data/CLAUDE.md +11 -1
  5. data/Gemfile +4 -2
  6. data/TODO.reconcile/01-design.md +57 -0
  7. data/TODO.reconcile/02-change-models.md +44 -0
  8. data/TODO.reconcile/02-termbase-loader.md +48 -0
  9. data/TODO.reconcile/03-live-loader.md +28 -0
  10. data/TODO.reconcile/03-status-mapper.md +24 -0
  11. data/TODO.reconcile/04-concept-merger.md +45 -0
  12. data/TODO.reconcile/04-termbase-loader.md +28 -0
  13. data/TODO.reconcile/05-content-diff.md +45 -0
  14. data/TODO.reconcile/05-live-loader.md +20 -0
  15. data/TODO.reconcile/06-content-differ.md +32 -0
  16. data/TODO.reconcile/06-v3-serializer.md +42 -0
  17. data/TODO.reconcile/07-cli-and-run.md +27 -0
  18. data/TODO.reconcile/07-concept-merger.md +32 -0
  19. data/TODO.reconcile/08-report.md +53 -0
  20. data/TODO.reconcile/09-pipeline.md +23 -0
  21. data/TODO.reconcile/10-script-and-run.md +15 -0
  22. data/data/locales/glossarist_enums.yml +705 -0
  23. data/iev.gemspec +4 -2
  24. data/lib/iev/bibliography_builder.rb +87 -0
  25. data/lib/iev/cli/command.rb +29 -0
  26. data/lib/iev/cli/command_helper.rb +1 -6
  27. data/lib/iev/config.rb +10 -1
  28. data/lib/iev/exporter.rb +37 -4
  29. data/lib/iev/figure_builder.rb +186 -0
  30. data/lib/iev/multi_doc_yaml.rb +60 -0
  31. data/lib/iev/reconciler/change.rb +29 -0
  32. data/lib/iev/reconciler/change_set.rb +48 -0
  33. data/lib/iev/reconciler/concept_merger.rb +130 -0
  34. data/lib/iev/reconciler/content_differ.rb +157 -0
  35. data/lib/iev/reconciler/live_loader.rb +163 -0
  36. data/lib/iev/reconciler/pipeline.rb +104 -0
  37. data/lib/iev/reconciler/reconciled_concept.rb +14 -0
  38. data/lib/iev/reconciler/report.rb +118 -0
  39. data/lib/iev/reconciler/status_mapper.rb +29 -0
  40. data/lib/iev/reconciler/term_marker_parser.rb +196 -0
  41. data/lib/iev/reconciler/termbase_loader.rb +141 -0
  42. data/lib/iev/reconciler.rb +17 -0
  43. data/lib/iev/scraper/browser.rb +155 -50
  44. data/lib/iev/scraper/page_parser.rb +116 -24
  45. data/lib/iev/source_parser.rb +12 -1
  46. data/lib/iev/subject_area_concepts.rb +4 -2
  47. data/lib/iev/utilities.rb +7 -5
  48. data/lib/iev/version.rb +1 -1
  49. data/lib/iev.rb +4 -0
  50. data/scripts/find_unparseable.rb +22 -0
  51. metadata +54 -6
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: f5d3397238922342cdb3fb6b5adcf8fa969d2b3d6510742485d7d2a20b1d1a12
4
- data.tar.gz: 6a82e64e1201a9a2f04cff66d024de1b23643030ce6e37f26ef09bb2ad334b10
3
+ metadata.gz: a016c3e0df9de5b20b48d9354f3a61ebbc604ac94e07b116a07ce2023b244eea
4
+ data.tar.gz: badb0152246599074ab74131274b70972f8ab8fa3dd1e4b49b06bd38dde81a33
5
5
  SHA512:
6
- metadata.gz: be3b984bd46c791580a73644e88dccca4b98b708ce0e28c50f72e822abb609573455201b633d0e4d791b9271cdecb20a555b11cc674dda46bde90fcf7eaccb74
7
- data.tar.gz: e56796a853c58aae578dd5c4fe3fbaf3e18115e1a08e22feb7d5f876d9521dee4d27fb734fb05baa12e297b1dc9ac89b67197cad0b51695b2224c3fdb622f3a5
6
+ metadata.gz: 6a1527631c2f47363372dca4f12f9c3b4026a3d936ecfabf8336dc061c3a188fb79d1e4797aa028c11d13936ef6164d09c2e396b50315cc532e01cda65f30d08
7
+ data.tar.gz: 7d1c71fcb8b1f87bbc7787688b8937780cb775fc43a6e7058d59f41a269cc2f4bdcd725ce303bc160ae5317505465ba88bd0c453d4cf311063f3207ed8e99bd3
@@ -0,0 +1 @@
1
+ {"sessionId":"7e802925-11d5-47e6-9ff2-7e151d3b2dfb","pid":23315,"procStart":"Wed Jul 8 10:33:56 2026","acquiredAt":1783506837511}
data/.rubocop_todo.yml CHANGED
@@ -1,6 +1,6 @@
1
1
  # This configuration was generated by
2
2
  # `rubocop --auto-gen-config`
3
- # on 2026-06-13 04:18:23 UTC using RuboCop version 1.86.1.
3
+ # on 2026-06-22 10:01:24 UTC using RuboCop version 1.86.1.
4
4
  # The point is for the user to remove these configuration records
5
5
  # one by one as the offenses are removed from the code base.
6
6
  # Note that changes in the inspected code, or installation of new
@@ -11,37 +11,7 @@ Gemspec/RequiredRubyVersion:
11
11
  Exclude:
12
12
  - 'iev.gemspec'
13
13
 
14
- # Offense count: 2
15
- # This cop supports safe autocorrection (--autocorrect).
16
- # Configuration parameters: EnforcedStyle, IndentationWidth.
17
- # SupportedStyles: with_first_argument, with_fixed_indentation
18
- Layout/ArgumentAlignment:
19
- Exclude:
20
- - 'spec/iev/exporter_spec.rb'
21
-
22
- # Offense count: 1
23
- # This cop supports safe autocorrection (--autocorrect).
24
- # Configuration parameters: EnforcedStyleAlignWith.
25
- # SupportedStylesAlignWith: either, start_of_block, start_of_line
26
- Layout/BlockAlignment:
27
- Exclude:
28
- - 'spec/iev/exporter_spec.rb'
29
-
30
- # Offense count: 1
31
- # This cop supports safe autocorrection (--autocorrect).
32
- Layout/BlockEndNewline:
33
- Exclude:
34
- - 'spec/iev/exporter_spec.rb'
35
-
36
- # Offense count: 2
37
- # This cop supports safe autocorrection (--autocorrect).
38
- # Configuration parameters: Width, EnforcedStyleAlignWith, AllowedPatterns.
39
- # SupportedStylesAlignWith: start_of_line, relative_to_receiver
40
- Layout/IndentationWidth:
41
- Exclude:
42
- - 'spec/iev/exporter_spec.rb'
43
-
44
- # Offense count: 65
14
+ # Offense count: 72
45
15
  # This cop supports safe autocorrection (--autocorrect).
46
16
  # Configuration parameters: Max, AllowHeredoc, AllowURI, AllowQualifiedName, URISchemes, AllowRBSInlineAnnotation, AllowCopDirectives, AllowedPatterns, SplitStrings.
47
17
  # URISchemes: http, https
@@ -65,7 +35,7 @@ Lint/ShadowedException:
65
35
  Exclude:
66
36
  - 'lib/iev/source_parser.rb'
67
37
 
68
- # Offense count: 26
38
+ # Offense count: 29
69
39
  # Configuration parameters: AllowedMethods, AllowedPatterns, CountRepeatedAttributes, Max.
70
40
  Metrics/AbcSize:
71
41
  Exclude:
@@ -94,12 +64,12 @@ Metrics/CyclomaticComplexity:
94
64
  - 'lib/iev/term_builder.rb'
95
65
  - 'lib/iev/utilities.rb'
96
66
 
97
- # Offense count: 36
67
+ # Offense count: 38
98
68
  # Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns.
99
69
  Metrics/MethodLength:
100
70
  Max: 73
101
71
 
102
- # Offense count: 1
72
+ # Offense count: 2
103
73
  # Configuration parameters: CountKeywordArgs, MaxOptionalParameters.
104
74
  Metrics/ParameterLists:
105
75
  Max: 7
@@ -128,15 +98,3 @@ Naming/MethodParameterName:
128
98
  Naming/VariableNumber:
129
99
  Exclude:
130
100
  - 'spec/iev/exporter_spec.rb'
131
-
132
- # Offense count: 5
133
- # This cop supports safe autocorrection (--autocorrect).
134
- # Configuration parameters: EnforcedStyle, ProceduralMethods, FunctionalMethods, AllowedMethods, AllowedPatterns, AllowBracesOnProceduralOneLiners, BracesRequiredMethods.
135
- # SupportedStyles: line_count_based, semantic, braces_for_chaining, always_braces
136
- # ProceduralMethods: benchmark, bm, bmbm, create, each_with_object, measure, new, realtime, tap, with_object
137
- # FunctionalMethods: let, let!, subject, watch
138
- # AllowedMethods: lambda, proc, it
139
- Style/BlockDelimiters:
140
- Exclude:
141
- - 'lib/iev/exporter.rb'
142
- - 'spec/iev/exporter_spec.rb'
data/CLAUDE.md CHANGED
@@ -33,7 +33,9 @@ This is a Ruby gem (`iev`) for working with the International Electrotechnical V
33
33
  - `SupersessionParser` — parses the REPLACES field for deprecated term relationships.
34
34
  - `SubjectAreas` — manages the IEV subject area/section hierarchy. Bundled `data/subject_areas.yaml` contains the area/section tree. URI scheme: `area-{code}` and `section-{code}`.
35
35
  - `SubjectAreaConcepts` — builds area and section hierarchy concepts. Uses `ConceptReference` with proper `ref_type` per `ConceptReferenceType`: `"domain"` for thematic area classification, `"section"` for structural section membership. Sets `ConceptData#domain` to area title text.
36
- - `Exporter` — full export pipeline (Excel/SQLite → Glossarist YAML). Assigns domain and section `ConceptReference` objects via `domain_references_for`. Uses `Glossarist::DatasetRegister` model for `register.yaml`. Sets `schema_version: "3"` on all exported concepts.
36
+ - `Exporter` — full export pipeline (Excel/SQLite → Glossarist YAML). Assigns domain and section `ConceptReference` objects via `domain_references_for`. Uses `Glossarist::DatasetRegister` model for `register.yaml`. Sets `schema_version: "3"` on all exported concepts. Pipeline order: build → subject areas → section relations → figure extraction → reference enrichment → save concepts → save figures → save bibliography → save register.
37
+ - `FigureBuilder` — destructive extraction pass that hoists AsciiDoc image macros (emitted from SIMG tags by `Utilities`) into dataset-shared `Glossarist::Figure` entities. Rewrites inline text to `{{fig:id, display}}` mentions and adds `FigureReference` entries to `ManagedConceptData#figures`. One Figure entity per unique image file; captions merge across languages.
38
+ - `BibliographyBuilder` — collects unique `(source, id)` pairs from every concept's sources (localized and managed) into a `Glossarist::BibliographyData`. Entry ids are normalized with the same rules as `Glossarist::Validation::BibliographyIndex` so consumers can resolve anchors.
37
39
  - `Converter::MathmlToAsciimath` — converts MathML markup to AsciiMath using Plurimath.
38
40
  - `Utilities` — HTML processing: converts IEV cross-references (`<a href=IEV...>`) to `{{URN, term}}` format (ID first, display text last), handles figures, images, bold tags, and newline normalization.
39
41
 
@@ -45,6 +47,14 @@ Per the concept model's `ConceptReferenceType`:
45
47
 
46
48
  Each concept's `ManagedConceptData#domains` contains both refs. `ConceptData#domain` (a `LocalizedString`) holds the section/area title text. The `ManagedConcept#related` array holds `broader`/`narrower` relationships for the hierarchy tree.
47
49
 
50
+ ### V3 Output Artifacts
51
+
52
+ An export produces these files alongside the concepts/ directory:
53
+ - `register.yaml` — `Glossarist::DatasetRegister` with section tree, languages, owner, URN.
54
+ - `bibliography.yaml` — single `bibliography:` key wrapping an array of `BibliographyEntry` objects. Entry `id` is the normalized anchor that `Glossarist::Validation::BibliographyIndex` will resolve against.
55
+ - `figures/{fig-id}.yaml` — one `Glossarist::Figure` per unique image. Each concept carries a `FigureReference` on `ManagedConceptData#figures` and an inline `{{fig:id, display}}` mention in the text where the figure appeared.
56
+ - References on localized concepts are populated by `Glossarist::ConceptEnricher#inject_references`, which scans text for `{{urn:...}}`, `<<xref>>`, and `image::` patterns.
57
+
48
58
  ### Configuration
49
59
 
50
60
  `Iev.configure` yields a `Config` object with:
data/Gemfile CHANGED
@@ -3,16 +3,18 @@
3
3
  source "https://rubygems.org"
4
4
 
5
5
  # Use local glossarist-ruby when available for development.
6
- # Otherwise falls back to released gem (requires >= 2.8.2 for tags support).
6
+ # Otherwise falls back to released gem (requires >= 2.8.15 for
7
+ # BibliographyData, Figure/NonVerbRep, and ConceptEnricher support).
7
8
  if File.directory?(File.expand_path("../glossarist-ruby", __dir__))
8
9
  gem "glossarist", path: "../glossarist-ruby"
9
10
  else
10
- gem "glossarist", ">= 2.8.2"
11
+ gem "glossarist", ">= 2.8.15"
11
12
  end
12
13
 
13
14
  gem "benchmark"
14
15
  gem "canon"
15
16
  gem "openssl"
17
+ gem "playwright-ruby-client", require: false
16
18
  gem "rake"
17
19
  gem "rspec"
18
20
  gem "rubocop"
@@ -0,0 +1,57 @@
1
+ # 01 — Architecture & autoloads
2
+
3
+ ## Audit of current approach (scripts/reconcile.rb)
4
+
5
+ Problems:
6
+ 1. **Monolithic** — 400-line script, module of static methods, no classes
7
+ 2. **Not model-driven** — builds raw Hashes instead of Glossarist model objects
8
+ 3. **No field-level diff** — knows THAT content changed, not WHAT changed
9
+ 4. **No before/after values** — loses the historical content
10
+ 5. **No machine-readable change log** — no way to audit changes
11
+ 6. **No specs**
12
+ 7. **Not MECE** — merge, diff, serialize all mixed together
13
+ 8. **Not autoloaded** — everything in a script file
14
+
15
+ ## New architecture
16
+
17
+ ```
18
+ lib/iev/reconciler.rb module + autoloads
19
+ lib/iev/reconciler/change.rb field-level change (field, lang, old, new)
20
+ lib/iev/reconciler/change_set.rb collection of Change per concept
21
+ lib/iev/reconciler/status_mapper.rb termbase status -> V3 ConceptStatus
22
+ lib/iev/reconciler/termbase_loader.rb termbase.yaml -> {code => ManagedConcept}
23
+ lib/iev/reconciler/live_loader.rb HTML pages -> {code => ManagedConcept}
24
+ lib/iev/reconciler/content_differ.rb diff two ManagedConcepts -> ChangeSet
25
+ lib/iev/reconciler/concept_merger.rb merge termbase + live -> ReconciledConcept
26
+ lib/iev/reconciler/reconciled_concept.rb ManagedConcept + ChangeSet + source
27
+ lib/iev/reconciler/report.rb dataset-level change reporting
28
+ lib/iev/reconciler/pipeline.rb orchestration
29
+ ```
30
+
31
+ Each class has ONE responsibility (MECE). The Pipeline composes them.
32
+
33
+ ## History reporting
34
+
35
+ Three layers:
36
+
37
+ 1. **Model-level** — `ManagedConcept#dates` (accepted/amended/retired),
38
+ `ManagedConcept#status` (valid/draft/superseded/retired),
39
+ `ManagedConcept#related` (supersedes/superseded_by).
40
+
41
+ 2. **Concept-level annotations** — Each amended concept gets a
42
+ `ConceptData#annotations` entry describing what field changed, with
43
+ before/after text.
44
+
45
+ 3. **Dataset-level report** — Summary statistics + machine-readable
46
+ change log written to `report/`:
47
+ - `summary.yaml` — aggregate counts by change type, section, language
48
+ - `changes.csv` — one row per field-level change
49
+ - `retired.yaml` — concepts retired since termbase snapshot
50
+ - `new_concepts.yaml` — concepts added since termbase snapshot
51
+
52
+ ## Checklist
53
+
54
+ - [x] Audit current approach
55
+ - [x] Design new architecture
56
+ - [x] Create lib/iev/reconciler.rb with autoloads
57
+ - [x] Register in lib/iev.rb
@@ -0,0 +1,44 @@
1
+ # 02 — Change + ChangeSet models
2
+
3
+ ## Goal
4
+
5
+ The core of history reporting. A `Change` captures a single field-level
6
+ difference between the termbase snapshot and the live snapshot. A
7
+ `ChangeSet` groups all changes for one concept.
8
+
9
+ ## Change
10
+
11
+ ```ruby
12
+ Change = Struct.new(:code, :field, :language, :old_value, :new_value,
13
+ :detected_at, keyword_init: true)
14
+ ```
15
+
16
+ Fields:
17
+ - `code`: IEV code (e.g. "102-01-01")
18
+ - `field`: `:designation` | `:definition` | `:notes` | `:examples` | `:status` | `:language_added` | `:language_removed`
19
+ - `language`: ISO 639-2 code ("eng", "fra", nil for concept-level)
20
+ - `old_value`: the termbase value (nil for additions)
21
+ - `new_value`: the live value (nil for removals)
22
+ - `detected_at`: date the change was observed (mirror date)
23
+
24
+ ## ChangeSet
25
+
26
+ ```ruby
27
+ class ChangeSet
28
+ include Enumerable
29
+ def initialize(code)
30
+ def add(change)
31
+ def each
32
+ def empty?
33
+ def by_field(field)
34
+ def by_language(lang)
35
+ def summary # hash of field => count
36
+ end
37
+ ```
38
+
39
+ ## Checklist
40
+
41
+ - [x] Implement Change (Struct)
42
+ - [x] Implement ChangeSet
43
+ - [x] Spec: ChangeSet query methods
44
+ - [x] Spec: ChangeSet#summary
@@ -0,0 +1,48 @@
1
+ # 02 — Termbase loader
2
+
3
+ ## Goal
4
+
5
+ Load `iev-data/termbase.yaml` into a uniform per-code hash keyed by IEV
6
+ code. The termbase is the authoritative source for lifecycle dates and
7
+ historical content.
8
+
9
+ ## What termbase.yaml contains
10
+
11
+ 22,228 concepts in the old ISO/IEV termbase format:
12
+
13
+ ```yaml
14
+ 102-01-01:
15
+ termid: 102-01-01
16
+ term: equality
17
+ eng:
18
+ id: 102-01-01
19
+ terms: [...]
20
+ definition: "..."
21
+ language_code: eng
22
+ notes: [...]
23
+ examples: [...]
24
+ entry_status: Standard
25
+ date_accepted: '2008-08-01T00:00:00+00:00'
26
+ date_amended: '2008-08-01T00:00:00+00:00'
27
+ review_date: '2008-08-01T00:00:00+00:00'
28
+ review_decision_date: '2008-08-01T00:00:00+00:00'
29
+ review_decision_event: published
30
+ fra:
31
+ ...
32
+ ```
33
+
34
+ ## Deliverable
35
+
36
+ Class: `Iev::Reconciler::TermbaseLoader`
37
+
38
+ - Loads termbase.yaml (22,228 concepts).
39
+ - Returns a Hash keyed by IEV code string → concept hash.
40
+ - Each concept hash normalizes per-language data into a consistent
41
+ structure so downstream stages don't need to handle the old format
42
+ ad-hoc.
43
+
44
+ ## Checklist
45
+
46
+ - [x] Define TermbaseLoader interface
47
+ - [x] Implement
48
+ - [x] Test: loads all 22,228 codes; sample code has expected fields
@@ -0,0 +1,28 @@
1
+ # 03 — Live HTML loader
2
+
3
+ ## Goal
4
+
5
+ Load the mirrored HTML pages from `iev-data-latest/pages/` using the
6
+ existing `Iev::Scraper::PageParser` to extract structured concept data.
7
+
8
+ ## Source
9
+
10
+ 17,473 HTML pages at `../iev-data-latest/pages/*.html`. Each page is
11
+ the current Electropedia display page for one IEV code.
12
+
13
+ ## Deliverable
14
+
15
+ Class: `Iev::Reconciler::LiveLoader`
16
+
17
+ - Walks `Iev::Fetcher::PageStore#each_concept`.
18
+ - For each page, uses `Iev::Scraper::PageParser` to parse into the raw
19
+ hash that `build_concept_from_raw` expects.
20
+ - Returns a Hash keyed by IEV code → parsed raw hash.
21
+ - Extracts the publication date from the HTML (e.g. "Publication date:
22
+ 2008-08") for concepts not in termbase.
23
+
24
+ ## Checklist
25
+
26
+ - [x] Define LiveLoader interface
27
+ - [x] Implement
28
+ - [x] Test: sample page parses with expected localized_concepts
@@ -0,0 +1,24 @@
1
+ # 03 — StatusMapper
2
+
3
+ ## Goal
4
+
5
+ Map termbase `entry_status` strings to Glossarist V3 `ConceptStatus`
6
+ enum values. Stateless, single responsibility.
7
+
8
+ ## Mapping
9
+
10
+ | Termbase | V3 |
11
+ |----------|-----|
12
+ | Standard / Published / Effective | valid |
13
+ | Draft | draft |
14
+ | Not Valid | notValid |
15
+ | Superseded | superseded |
16
+ | Retired | retired |
17
+
18
+ Unknown values default to `valid`.
19
+
20
+ ## Checklist
21
+
22
+ - [x] Implement StatusMapper
23
+ - [x] Spec: all mapped values
24
+ - [x] Spec: unknown defaults to valid
@@ -0,0 +1,45 @@
1
+ # 04 — Concept merger
2
+
3
+ ## Goal
4
+
5
+ For each code in the union of termbase + live sources, merge the two
6
+ into a single V3 `ManagedConcept` with full lifecycle data.
7
+
8
+ ## Three populations
9
+
10
+ 1. **In both** (~16,643 codes): termbase provides dates/status,
11
+ live provides current content. If content differs, record an
12
+ amendment.
13
+ 2. **Termbase only** (~5,585 codes): not on live Electropedia → mark
14
+ `status: retired`, add `ConceptDate(type: retired)`.
15
+ 3. **Live only** (~830 codes): not in 2020 termbase → new concept,
16
+ `ConceptDate(type: accepted)` from HTML publication date.
17
+
18
+ ## Merge rules
19
+
20
+ | Field | Source |
21
+ |-------|--------|
22
+ | `identifier` | either (same code) |
23
+ | `status` | termbase `entry_status` → mapped to `ConceptStatus` enum |
24
+ | `dates` | termbase `date_accepted`/`date_amended`/`review_date`/`review_decision_date` → `ConceptDate` array |
25
+ | `localized_concepts.designations` | **live** if available, else termbase |
26
+ | `localized_concepts.definition` | **live** if available, else termbase |
27
+ | `localized_concepts.notes/examples` | **live** if available, else termbase |
28
+ | `sources` | live parsed sources (richer) |
29
+ | `related` | termbase supersession data + any extracted from live |
30
+
31
+ ## Status mapping
32
+
33
+ | Termbase `entry_status` | V3 `ConceptStatus` |
34
+ |--------------------------|---------------------|
35
+ | `Standard` / `Published` | `valid` |
36
+ | `Draft` | `draft` |
37
+ | `Not Valid` | `notValid` |
38
+ | `Superseded` | `superseded` |
39
+ | `Retired` / absent from live | `retired` |
40
+
41
+ ## Checklist
42
+
43
+ - [x] Define merge rules
44
+ - [x] Implement ConceptMerger
45
+ - [x] Test: in-both, termbase-only, live-only cases
@@ -0,0 +1,28 @@
1
+ # 04 — TermbaseLoader
2
+
3
+ ## Goal
4
+
5
+ Load `termbase.yaml` and build `Glossarist::ManagedConcept` objects
6
+ with full lifecycle dates from the old termbase format.
7
+
8
+ ## Input
9
+
10
+ `iev-data/termbase.yaml` — 22,228 concepts in old ISO/IEV format.
11
+
12
+ ## Output
13
+
14
+ `Hash<String, Glossarist::ManagedConcept>` keyed by IEV code.
15
+
16
+ Each ManagedConcept has:
17
+ - `id` = code
18
+ - `status` from StatusMapper
19
+ - `dates` from `date_accepted` + `date_amended` (ConceptDate objects)
20
+ - `localized_concepts` hash with one LocalizedConcept per language
21
+ - Each LocalizedConcept has ConceptData with terms, definition, notes, examples
22
+
23
+ ## Checklist
24
+
25
+ - [x] Implement TermbaseLoader
26
+ - [x] Spec: loads sample termbase entry correctly
27
+ - [x] Spec: dates extracted properly
28
+ - [x] Spec: multi-language concepts
@@ -0,0 +1,45 @@
1
+ # 05 — Content diff detector
2
+
3
+ ## Goal
4
+
5
+ Detect content changes between the termbase snapshot and the live HTML
6
+ for concepts that exist in both sources. Each detected change becomes
7
+ an amendment record.
8
+
9
+ ## What to diff
10
+
11
+ Per localized concept (per language):
12
+
13
+ 1. **Designations** (terms): added, removed, or changed text
14
+ 2. **Definition**: text content changed
15
+ 3. **Notes**: added, removed, or changed
16
+ 4. **Examples**: added, removed, or changed
17
+
18
+ ## Output
19
+
20
+ For each changed field, record:
21
+
22
+ ```ruby
23
+ {
24
+ field: :definition,
25
+ language: :eng,
26
+ old_value: "...", # from termbase
27
+ new_value: "...", # from live
28
+ changed_at: MIRROR_DATE # the date we captured the live page
29
+ }
30
+ ```
31
+
32
+ These feed into the merger to append `ConceptDate(type: amended)` and
33
+ optionally an annotation describing the change.
34
+
35
+ ## Approach
36
+
37
+ String normalization before comparison: strip whitespace, normalize
38
+ unicode, ignore case-only differences in designation. Real semantic
39
+ diffs (definition rewrite) vs cosmetic diffs (encoding fix).
40
+
41
+ ## Checklist
42
+
43
+ - [x] Define diff strategy
44
+ - [x] implement ContentDiff
45
+ - [x] Test: identical → no change; modified definition → one diff
@@ -0,0 +1,20 @@
1
+ # 05 — LiveLoader
2
+
3
+ ## Goal
4
+
5
+ Load live HTML pages and build `Glossarist::ManagedConcept` objects
6
+ using the existing `Iev::Scraper::PageParser`.
7
+
8
+ ## Input
9
+
10
+ `iev-data-latest/pages/*.html` — 17,473 current Electropedia pages.
11
+
12
+ ## Output
13
+
14
+ `Hash<String, Glossarist::ManagedConcept>` keyed by IEV code.
15
+
16
+ ## Checklist
17
+
18
+ - [x] Implement LiveLoader
19
+ - [x] Spec: parses sample HTML page
20
+ - [x] Spec: skips placeholder pages (no localized_concepts)
@@ -0,0 +1,32 @@
1
+ # 06 — ContentDiffer
2
+
3
+ ## Goal
4
+
5
+ Compare two `ManagedConcept` objects and produce a `ChangeSet` with
6
+ field-level differences. This is the core of change reporting.
7
+
8
+ ## What to diff
9
+
10
+ Per localized concept (per language):
11
+ - `designation` (term text)
12
+ - `definition` (text content)
13
+ - `notes` (added/removed/changed)
14
+ - `examples` (added/removed/changed)
15
+
16
+ At concept level:
17
+ - `status` (e.g. valid → retired)
18
+ - language added (in live but not termbase)
19
+ - language removed (in termbase but not live)
20
+
21
+ ## Normalization
22
+
23
+ Before comparison: strip HTML tags, normalize whitespace, normalize
24
+ unicode quotes. Avoid false positives from encoding differences.
25
+
26
+ ## Checklist
27
+
28
+ - [x] Implement ContentDiffer
29
+ - [x] Spec: identical concepts → empty ChangeSet
30
+ - [x] Spec: definition changed → one Change
31
+ - [x] Spec: language added → one Change
32
+ - [x] Spec: status changed → one Change
@@ -0,0 +1,42 @@
1
+ # 06 — V3 serializer
2
+
3
+ ## Goal
4
+
5
+ Serialize the merged `ManagedConcept` objects into Glossarist V3 YAML
6
+ files using the existing `Glossarist::ManagedConceptCollection` API.
7
+
8
+ ## Output
9
+
10
+ ```
11
+ output/
12
+ concepts/
13
+ concept-102-01-01.yaml
14
+ concept-102-01-02.yaml
15
+ ...
16
+ register.yaml # DatasetRegister
17
+ ```
18
+
19
+ Each concept file uses `schema_version: "3"` and the V3 field names
20
+ from the concept model:
21
+
22
+ ```yaml
23
+ ---
24
+ schema_version: "3"
25
+ identifier: "102-01-01"
26
+ status: valid
27
+ dates:
28
+ - type: accepted
29
+ date: "2008-08-01"
30
+ - type: amended
31
+ date: "2008-08-01"
32
+ localized_concepts:
33
+ eng: ...
34
+ fra: ...
35
+ related: ...
36
+ ```
37
+
38
+ ## Checklist
39
+
40
+ - [x] Define output layout
41
+ - [x] Implement V3 serializer (reuse ManagedConceptCollection#save)
42
+ - [x] Test: round-trip — load output back, verify fields
@@ -0,0 +1,27 @@
1
+ # 07 — CLI command + end-to-end run
2
+
3
+ ## Goal
4
+
5
+ Wire the reconciliation pipeline into a runnable command and execute
6
+ it to produce the final dataset.
7
+
8
+ ## Deliverable
9
+
10
+ Either:
11
+ - A new Thor CLI command: `iev reconcile --termbase <path> --output <dir>`
12
+ - Or a standalone script: `scripts/reconcile.rb`
13
+
14
+ The command should:
15
+ 1. Load termbase.yaml
16
+ 2. Load live HTML from PageStore
17
+ 3. Merge + diff
18
+ 4. Serialize V3 YAML
19
+ 5. Write register.yaml
20
+ 6. Report: total concepts, in-both, termbase-only, live-only, amendments detected
21
+
22
+ ## Checklist
23
+
24
+ - [x] Define command interface
25
+ - [x] Implement
26
+ - [x] Run end-to-end and verify output
27
+ - [x] Spot-check 5 concepts manually (in-both, termbase-only, live-only, with changes)
@@ -0,0 +1,32 @@
1
+ # 07 — ConceptMerger + ReconciledConcept
2
+
3
+ ## Goal
4
+
5
+ Merge termbase and live concepts into a single `ManagedConcept` with
6
+ full lifecycle data, and package it with its `ChangeSet` in a
7
+ `ReconciledConcept` value object.
8
+
9
+ ## ReconciledConcept
10
+
11
+ ```ruby
12
+ ReconciledConcept = Struct.new(:managed_concept, :change_set, :source,
13
+ keyword_init: true)
14
+ # source: :both | :termbase_only | :live_only
15
+ ```
16
+
17
+ ## Merge rules
18
+
19
+ | Source | Behavior |
20
+ |--------|----------|
21
+ | both | live content + termbase dates; if changes detected, add ConceptDate(amended) + annotations |
22
+ | termbase only | mark retired, add ConceptDate(retired), preserve full content |
23
+ | live only | new concept, add ConceptDate(accepted) from publication date |
24
+
25
+ ## Checklist
26
+
27
+ - [x] Implement ReconciledConcept (Struct)
28
+ - [x] Implement ConceptMerger
29
+ - [x] Spec: merge-both produces live content + termbase dates
30
+ - [x] Spec: termbase-only produces retired status
31
+ - [x] Spec: live-only produces accepted date
32
+ - [x] Spec: changes produce annotation entries