iriq 0.30.2 → 0.33.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/Gemfile.lock DELETED
@@ -1,103 +0,0 @@
1
- PATH
2
- remote: .
3
- specs:
4
- iriq (0.30.2)
5
-
6
- GEM
7
- remote: https://rubygems.org/
8
- specs:
9
- date (3.5.1)
10
- debug (1.11.1)
11
- irb (~> 1.10)
12
- reline (>= 0.3.8)
13
- diff-lcs (1.6.2)
14
- docile (1.4.1)
15
- erb (6.0.4)
16
- io-console (0.8.2)
17
- irb (1.17.0)
18
- pp (>= 0.6.0)
19
- prism (>= 1.3.0)
20
- rdoc (>= 4.0.0)
21
- reline (>= 0.4.2)
22
- mini_portile2 (2.8.9)
23
- pp (0.6.3)
24
- prettyprint
25
- prettyprint (0.2.0)
26
- prism (1.9.0)
27
- psych (5.3.1)
28
- date
29
- stringio
30
- rdoc (7.2.0)
31
- erb
32
- psych (>= 4.0.0)
33
- tsort
34
- reline (0.6.3)
35
- io-console (~> 0.5)
36
- rspec (3.13.2)
37
- rspec-core (~> 3.13.0)
38
- rspec-expectations (~> 3.13.0)
39
- rspec-mocks (~> 3.13.0)
40
- rspec-core (3.13.6)
41
- rspec-support (~> 3.13.0)
42
- rspec-debugging (0.0.4)
43
- rspec-expectations (>= 3)
44
- rspec-expectations (3.13.5)
45
- diff-lcs (>= 1.2.0, < 2.0)
46
- rspec-support (~> 3.13.0)
47
- rspec-mocks (3.13.8)
48
- diff-lcs (>= 1.2.0, < 2.0)
49
- rspec-support (~> 3.13.0)
50
- rspec-support (3.13.7)
51
- simplecov (0.22.0)
52
- docile (~> 1.1)
53
- simplecov-html (~> 0.11)
54
- simplecov_json_formatter (~> 0.1)
55
- simplecov-html (0.13.2)
56
- simplecov_json_formatter (0.1.4)
57
- sqlite3 (2.9.5)
58
- mini_portile2 (~> 2.8.0)
59
- stringio (3.2.0)
60
- tsort (0.2.0)
61
-
62
- PLATFORMS
63
- ruby
64
-
65
- DEPENDENCIES
66
- debug (>= 1)
67
- iriq!
68
- rspec (>= 3.10)
69
- rspec-debugging
70
- simplecov (>= 0.22)
71
- sqlite3 (>= 1.6)
72
-
73
- CHECKSUMS
74
- date (3.5.1) sha256=750d06384d7b9c15d562c76291407d89e368dda4d4fff957eb94962d325a0dc0
75
- debug (1.11.1) sha256=2e0b0ac6119f2207a6f8ac7d4a73ca8eb4e440f64da0a3136c30343146e952b6
76
- diff-lcs (1.6.2) sha256=9ae0d2cba7d4df3075fe8cd8602a8604993efc0dfa934cff568969efb1909962
77
- docile (1.4.1) sha256=96159be799bfa73cdb721b840e9802126e4e03dfc26863db73647204c727f21e
78
- erb (6.0.4) sha256=38e3803694be357fe2bfe312487c74beaf9fb4e5beb3e22498952fe1645b95d9
79
- io-console (0.8.2) sha256=d6e3ae7a7cc7574f4b8893b4fca2162e57a825b223a177b7afa236c5ef9814cc
80
- irb (1.17.0) sha256=168c4ddb93d8a361a045c41d92b2952c7a118fa73f23fe14e55609eb7a863aae
81
- iriq (0.30.2)
82
- mini_portile2 (2.8.9) sha256=0cd7c7f824e010c072e33f68bc02d85a00aeb6fce05bb4819c03dfd3c140c289
83
- pp (0.6.3) sha256=2951d514450b93ccfeb1df7d021cae0da16e0a7f95ee1e2273719669d0ab9df6
84
- prettyprint (0.2.0) sha256=2bc9e15581a94742064a3cc8b0fb9d45aae3d03a1baa6ef80922627a0766f193
85
- prism (1.9.0) sha256=7b530c6a9f92c24300014919c9dcbc055bf4cdf51ec30aed099b06cd6674ef85
86
- psych (5.3.1) sha256=eb7a57cef10c9d70173ff74e739d843ac3b2c019a003de48447b2963d81b1974
87
- rdoc (7.2.0) sha256=8650f76cd4009c3b54955eb5d7e3a075c60a57276766ebf36f9085e8c9f23192
88
- reline (0.6.3) sha256=1198b04973565b36ec0f11542ab3f5cfeeec34823f4e54cebde90968092b1835
89
- rspec (3.13.2) sha256=206284a08ad798e61f86d7ca3e376718d52c0bc944626b2349266f239f820587
90
- rspec-core (3.13.6) sha256=a8823c6411667b60a8bca135364351dda34cd55e44ff94c4be4633b37d828b2d
91
- rspec-debugging (0.0.4) sha256=7a8e2dc240c140f0ed27b452a5661a56474ee8cf7b84c5bcbefd827ad36f0a6f
92
- rspec-expectations (3.13.5) sha256=33a4d3a1d95060aea4c94e9f237030a8f9eae5615e9bd85718fe3a09e4b58836
93
- rspec-mocks (3.13.8) sha256=086ad3d3d17533f4237643de0b5c42f04b66348c28bf6b9c2d3f4a3b01af1d47
94
- rspec-support (3.13.7) sha256=0640e5570872aafefd79867901deeeeb40b0c9875a36b983d85f54fb7381c47c
95
- simplecov (0.22.0) sha256=fe2622c7834ff23b98066bb0a854284b2729a569ac659f82621fc22ef36213a5
96
- simplecov-html (0.13.2) sha256=bd0b8e54e7c2d7685927e8d6286466359b6f16b18cb0df47b508e8d73c777246
97
- simplecov_json_formatter (0.1.4) sha256=529418fbe8de1713ac2b2d612aa3daa56d316975d307244399fa4838c601b428
98
- sqlite3 (2.9.5) sha256=04572973a3f943ad50a8adfffc8dd752a5f06e4c3db2026f71838fed8a982606
99
- stringio (3.2.0) sha256=c37cb2e58b4ffbd33fe5cd948c05934af997b36e0b6ca6fdf43afa234cf222e1
100
- tsort (0.2.0) sha256=9650a793f6859a43b6641671278f79cfead60ac714148aabe4e3f0060480089f
101
-
102
- BUNDLED WITH
103
- 4.0.9
data/Makefile DELETED
@@ -1,113 +0,0 @@
1
- # Iriq Go binary — build/install/clean/uninstall helpers.
2
- #
3
- # make - same as `make help`
4
- # make build - dev build into ./bin/iriq (no SQLite, debug info)
5
- # make build-sqlite - dev build with SQLite backend included
6
- # make release - stripped + trimpath build (no SQLite)
7
- # make release-sqlite - stripped + trimpath build with SQLite
8
- # make install - go install into $GOBIN
9
- # make test - go test ./... (both tag states)
10
- # make clean - remove ./bin/
11
- # make uninstall - remove the binary from $GOBIN
12
- #
13
- # The default build excludes the SQLite backend to keep the binary lean.
14
- # Pass `-tags sqlite` (or use the *-sqlite targets) to compile it in. The
15
- # CLI's `--version` output tells you which backends are baked in.
16
- #
17
- # Ruby gem build/install is handled by Bundler/RubyGems; see CLAUDE.md.
18
-
19
- GO ?= go
20
- GO_DIR := go
21
- BIN_DIR := bin
22
- BIN := $(BIN_DIR)/iriq
23
- # Absolute output path: builds run inside $(GO_DIR) via `go -C`, so a
24
- # relative -o would land under go/. Keep the binary at the repo-root bin/.
25
- ABS_BIN := $(CURDIR)/$(BIN)
26
- PKG := ./cmd/iriq
27
-
28
- # Rust crate lives under rust/; CI gates fmt + clippy + tests there.
29
- CARGO ?= cargo
30
- RUST_DIR := rust
31
-
32
- # Release flags strip the symbol table (-s), debug info (-w), and bake
33
- # reproducible paths (-trimpath). Drops binary size ~30% with no
34
- # functional impact; stack-trace function names are gone but file:line
35
- # resolution still works.
36
- RELEASE_FLAGS := -ldflags "-s -w" -trimpath
37
-
38
- # Resolve $GOBIN, falling back to $GOPATH/bin (Go's default install location).
39
- GOBIN := $(shell $(GO) env GOBIN)
40
- ifeq ($(GOBIN),)
41
- GOBIN := $(shell $(GO) env GOPATH)/bin
42
- endif
43
- INSTALLED := $(GOBIN)/iriq
44
-
45
- .DEFAULT_GOAL := help
46
- .PHONY: help build build-sqlite release release-sqlite install test clean uninstall check fmt hooks
47
-
48
- help:
49
- @echo "Iriq Go targets:"
50
- @echo " make build slim dev build into $(BIN)"
51
- @echo " make build-sqlite dev build with SQLite backend"
52
- @echo " make release stripped slim build into $(BIN)"
53
- @echo " make release-sqlite stripped build with SQLite backend"
54
- @echo " make install go install into $(GOBIN)"
55
- @echo " make test run go test ./... in both tag states"
56
- @echo " make check Rust gate: cargo fmt --check + clippy + test (run before merging)"
57
- @echo " make fmt cargo fmt the Rust crate"
58
- @echo " make hooks enable the committed git hooks (pre-push runs 'make check')"
59
- @echo " make clean remove $(BIN_DIR)/"
60
- @echo " make uninstall remove $(INSTALLED)"
61
-
62
- build:
63
- @mkdir -p $(BIN_DIR)
64
- $(GO) -C $(GO_DIR) build -o $(ABS_BIN) $(PKG)
65
- @echo "built $(BIN) (slim, debug)"
66
-
67
- build-sqlite:
68
- @mkdir -p $(BIN_DIR)
69
- $(GO) -C $(GO_DIR) build -tags sqlite -o $(ABS_BIN) $(PKG)
70
- @echo "built $(BIN) (sqlite, debug)"
71
-
72
- release:
73
- @mkdir -p $(BIN_DIR)
74
- $(GO) -C $(GO_DIR) build $(RELEASE_FLAGS) -o $(ABS_BIN) $(PKG)
75
- @echo "built $(BIN) (slim, stripped)"
76
-
77
- release-sqlite:
78
- @mkdir -p $(BIN_DIR)
79
- $(GO) -C $(GO_DIR) build -tags sqlite $(RELEASE_FLAGS) -o $(ABS_BIN) $(PKG)
80
- @echo "built $(BIN) (sqlite, stripped)"
81
-
82
- install:
83
- $(GO) -C $(GO_DIR) install $(PKG)
84
- @echo "installed $(INSTALLED)"
85
-
86
- test:
87
- $(GO) -C $(GO_DIR) test ./...
88
- $(GO) -C $(GO_DIR) test -tags sqlite ./...
89
-
90
- # The Rust gate — mirrors CI's Rust job. Run before merging/pushing (the
91
- # pre-push hook runs this for you once `make hooks` is enabled).
92
- check:
93
- cd $(RUST_DIR) && $(CARGO) fmt --check
94
- cd $(RUST_DIR) && $(CARGO) clippy --workspace --all-targets -- -D warnings
95
- cd $(RUST_DIR) && $(CARGO) test --workspace
96
-
97
- fmt:
98
- cd $(RUST_DIR) && $(CARGO) fmt
99
-
100
- hooks:
101
- git config core.hooksPath .githooks
102
- @echo "git hooks enabled (.githooks) — pre-push now runs 'make check'"
103
-
104
- clean:
105
- rm -rf $(BIN_DIR)
106
- @echo "removed $(BIN_DIR)/"
107
-
108
- uninstall:
109
- @if [ -f "$(INSTALLED)" ]; then \
110
- rm "$(INSTALLED)" && echo "removed $(INSTALLED)"; \
111
- else \
112
- echo "not installed at $(INSTALLED)"; \
113
- fi
data/docs/ARCHITECTURE.md DELETED
@@ -1,223 +0,0 @@
1
- # iriq architecture
2
-
3
- As of v0.28.0, this describes the system that's actually in the repo —
4
- Phase 1 of the rearchitecture roadmap (target model below) is complete,
5
- and Phase 2 has added the learning layer on top. Originally written as
6
- the *target* model at the start of Phase 1; the implementation has
7
- caught up.
8
-
9
- `ROADMAP.md` tracks what's still pending (inter-position correlations,
10
- near-shape clustering, and Phase 3 productize items).
11
-
12
- For the inner details, read the source: `lib/iriq/{recognizer,position,
13
- shape,evidence,event,reducer,recognizer_proposal,synthesized_recognizer,
14
- cross_host_shape,corpus}.rb` and their Go counterparts at the repo root.
15
-
16
- ## Core principle
17
-
18
- The system pivots around **Position** (a slot in a host's structure) and
19
- **Evidence** (what we know about a Position). Strings are observations;
20
- types are inferences. Everything composable hangs off these two nouns.
21
-
22
- ## Core types
23
-
24
- ### Identifier (existing, mostly unchanged)
25
-
26
- A parsed IRI: scheme, host, port, path_segments, query_params, fragment,
27
- plus `kind ∈ {:url, :urn}` and the urn `nss`. Produced by `Parser` from a
28
- candidate string; produced in bulk by `Extractor` from free text.
29
-
30
- ### Position (new)
31
-
32
- A stable identity for a slot in a host's structure.
33
-
34
- ```
35
- Position {
36
- host: string # normalized per host_strategy (full|registrable|none)
37
- scope: :path | :query
38
- locator: path_prefix # for :path — the typed-shape prefix that led here
39
- | param_name # for :query — the ?key= name
40
- }
41
- ```
42
-
43
- Two Identifiers occupy the *same* Position when their hosts agree and their
44
- typed prefix agrees. "Typed prefix" means the prefix rendered with inferred
45
- types (`/users/{integer}/...`), not raw values. This makes Position identity
46
- robust to the specific values seen.
47
-
48
- ### Recognizer (new)
49
-
50
- A pluggable classifier for a single (or small family of) types.
51
-
52
- ```
53
- Recognizer#try(string) -> { type:, confidence:, canonical:, notes: } | nil
54
- ```
55
-
56
- - `type` is a symbol from the recognized vocabulary (uuid, date, integer, …).
57
- - `confidence` is in `[0, 1]`. Calibrated against
58
- `spec/fixtures/calibration/`.
59
- - `canonical` is the canonical form (e.g. ISO date for `:date`, uppercased
60
- ISO 4217 for `:currency`). `nil` means "use the input as-is".
61
- - `notes` is an optional array of strings the Trace view may surface.
62
-
63
- Recognizers are independent. There is no global ordering. The
64
- **SegmentClassifier becomes an ensemble** that runs each Recognizer and
65
- picks the highest-confidence answer above a floor. `:literal` is the
66
- fallback when no Recognizer fires above the floor.
67
-
68
- ### Evidence (new)
69
-
70
- The universal substrate for explanation. A small fixed schema, emitted by
71
- both classification and inference.
72
-
73
- ```
74
- Evidence {
75
- position: Position
76
- kind: :lexical | :corpus | :recognizer | :neighbor | :policy
77
- payload: { ... kind-specific fields ... }
78
- weight: float in [0, 1] # contribution to the final inference
79
- }
80
- ```
81
-
82
- Examples:
83
- - `:recognizer` payload: `{ name:, type:, lexical_confidence: }`
84
- - `:corpus` payload: `{ observations:, distinct_values:, cardinality_frac: }`
85
- - `:neighbor` payload: `{ prior_literal:, inferred_hint: }`
86
- - `:policy` payload: `{ rule:, applied: }` (e.g. "ipv4→ip umbrella collapse")
87
-
88
- `Trace` and `Explanation` are *views* over a list of Evidence. They render
89
- strings for human consumption but the data shape is shared.
90
-
91
- ### Shape (new — replaces PathShape strings as the identity)
92
-
93
- An ordered list of typed Positions for a single Identifier path:
94
-
95
- ```
96
- Shape { positions: [Position], inferred_types: [Symbol] }
97
- ```
98
-
99
- `Shape#render(:string)` produces the human-readable form (`/users/{user_id}`)
100
- that today's `PathShape` produces. Cluster identity is the structural Shape,
101
- not the string. String renderings are derived for display, not for keying.
102
-
103
- ### Cluster (existing; refactored)
104
-
105
- Same role: a group of Identifiers sharing (host, Shape). Internals shift to
106
- key on structural Shape; aggregation (`PositionStats`) is unchanged.
107
- Inference (variability promotion, enum detection, etc.) moves *out* of
108
- Cluster and into its own stage so it can be re-run over stored evidence
109
- without re-feeding observations.
110
-
111
- ### Reducer (new)
112
-
113
- A function over the event stream that maintains a materialized view.
114
-
115
- ```
116
- Reducer#apply(event, state) -> state
117
- ```
118
-
119
- Examples ship in v1:
120
- - `HostCountsReducer`
121
- - `PositionStatsReducer`
122
- - `ClusterReducer`
123
- - `FingerprintReducer`
124
-
125
- Storage backends store the event log (or, equivalently, the materialized
126
- views — backends choose). Adding a new metric is: write a Reducer, declare
127
- its state shape; no other module changes.
128
-
129
- ## Pipeline (target end-state)
130
-
131
- ```
132
- text
133
- └─► Extractor ─► [candidate strings]
134
- └─► Parser ─► Identifier
135
- └─► Recognition ─► annotated segments + per-segment Evidence
136
- └─► Events ─► Storage event log
137
- └─► Reducers ─► materialized views
138
- (PositionStats, Clusters, ...)
139
- ├─► Inference ─► per-Position type + Evidence
140
- └─► Rendering ─► normalized string / Shape view
141
- ```
142
-
143
- Each stage is a separate module with input/output contracts. Inference
144
- re-reads materialized views; it does not require re-running the pipeline.
145
-
146
- ## What this fixes (vs. the today code)
147
-
148
- - **No more string-fingerprint cluster keys.** Classifier tuning no longer
149
- fractures clusters.
150
- - **No more order-dependent first-match classifier.** Add a Recognizer
151
- without considering global ordering.
152
- - **`SegmentClassifier` stops being a god module.** Recognizers own their
153
- patterns and canonical forms; reference data (locales, currencies,
154
- countries) lives in `ReferenceData::*`; display names / variability
155
- predicate live in a small `Policy` / `Naming` module.
156
- - **One Normalizer.** Mechanical mode is "NullEvidence" — no special case.
157
- - **Two-axis type taxonomy.** Lexical shape and semantic role can grow
158
- independently. Cardinality character is a *property of Position*, not of
159
- type — `:year`, `:http_status`, `:enum` collapse into "integer-shape +
160
- bounded-cardinality" expressed via Evidence.
161
- - **Re-runnable inference.** Threshold tuning no longer requires
162
- re-observation.
163
- - **One explanation substrate.** Trace, Explanation, future schema export,
164
- future PR-diff annotator are all views over Evidence.
165
-
166
- ## What stays the same
167
-
168
- - `Identifier` as the parsed record.
169
- - `Parser` + `Extractor` — already cleanly separated.
170
- - `Storage::*` backends (Memory, Json, Sqlite). Internals change to persist
171
- events rather than ad-hoc counters, but the file-extension routing API
172
- (`Corpus.open(path)`) stays.
173
- - Public CLI surface (`exe/iriq`, the Go `cmd/iriq` binary).
174
- - The four `Iriq.*` module methods: `parse`, `normalize`, `explain`,
175
- `extract`.
176
-
177
- ## Extensibility (where it stands)
178
-
179
- - **Recognizer registry**: per-classifier and mutable as of v0.26.
180
- `SegmentClassifier#register_recognizer` appends to the instance's
181
- ensemble; `SegmentClassifier::DEFAULT` is the module-level singleton
182
- that fresh corpora share, and the first `Corpus#activate_proposal`
183
- call swaps to a private classifier so activations don't leak.
184
- External users registering their own Recognizer subclasses works
185
- today via the same API; an *external* registry / discovery surface
186
- (load Recognizers from a config file or env var) is still future
187
- work.
188
- - **ProposalStrategy**: pluggable via `Iriq::ProposalStrategy::DEFAULTS`.
189
- Adding a strategy = define a class with `#propose(storage, **opts)`
190
- and append. v1 ships one strategy (PrefixUnderscoreId); next-segment
191
- / cross-position correlation strategies are pending Phase 2 work.
192
- - **Reducer registry**: the dispatch table `Iriq::Reducer::DEFAULTS`
193
- maps `Event` subclasses to reducer lambdas. Adding a metric = define
194
- an `Event` subtype, write a Reducer, register it. The registry is
195
- exposed as a constant; safe to monkeypatch in user code, though no
196
- public registration API ships yet.
197
- - **Storage backends**: three ship (Memory, JSON, SQLite). Adding a
198
- fourth = implement the `Storage` interface (lib/iriq/storage/memory.rb
199
- is the canonical reference), wire it into `Storage.open` extension
200
- routing, add it to `script/cli_parity.sh`.
201
-
202
- ## Learning layer (Phase 2, on top of the substrate above)
203
-
204
- The Phase 1 substrate gives us typed observations, structured Shape,
205
- and re-runnable inference. Phase 2 builds the learning pipeline on top:
206
-
207
- - **Source-IRI log** (v0.21). Storage persists every observed canonical
208
- IRI alongside the materialized views. `Corpus#reinfer` drops the
209
- views and replays the log through the current classifier + reducers
210
- — lets us tune thresholds or swap the classifier without re-feeding.
211
- - **RecognizerProposal** (v0.23). A struct describing a learned
212
- pattern: prefix / suggested_type / positions / hosts / coverage /
213
- confidence / observation_count / sample_values / strategy.
214
- Emitted by ProposalStrategy implementations; not auto-applied.
215
- - **SynthesizedRecognizer** (v0.26). Built from a proposal's prefix;
216
- regex `^<prefix>[A-Za-z0-9]+$`, `Specificity::SEMANTIC`. Same
217
- Recognizer interface as the built-ins (UUID, Date, Integer) — the
218
- ensemble doesn't know the difference.
219
- - **CrossHostShape** (v0.27). Read-side: route shapes that recur
220
- across multiple hosts. Independent evidence of semantic pattern.
221
- - **Confidence formula** (v0.28). `min(1.0, coverage + 0.05 *
222
- (host_count - 1))`. Single-host proposals are unchanged; cross-host
223
- proposals get boosted. `--activate-above F` checks confidence.
data/docs/ROADMAP.md DELETED
@@ -1,190 +0,0 @@
1
- # iriq roadmap
2
-
3
- A multi-quarter plan to lift iriq from a single-pass string classifier into an
4
- event-driven inference system over typed Positions. Pre-release; we will break
5
- APIs freely and bump minor versions per shipped feature.
6
-
7
- Source of the plan: principal-engineer architectural review on 2026-05-30,
8
- captured in the project history.
9
-
10
- ## Where to pick up next time
11
-
12
- Last updated: 2026-05-31, current version **v0.28.0**.
13
-
14
- **Phase 1**: complete (v0.11 → v0.19). The structural foundation —
15
- Recognizer ensemble, Position, Shape, Evidence, single Normalizer,
16
- events + reducers — is all in place.
17
-
18
- **Phase 2**: 5 of 7 items shipped (v0.21 → v0.28).
19
- - ✅ Re-runnable inference (library + CLI)
20
- - ✅ Learned recognizers (library + propose CLI + auto-activate CLI)
21
- - ✅ Cross-host learning (catalog + confidence integration)
22
- - ⏭️ **Inter-position correlations** — next up. When slot A's value
23
- predicts slot B's type, surface that ("after `/orgs/{org_id}/` the
24
- next segment is almost always `users` or `repos`"). Read-side query
25
- over existing cluster + position stats; no storage changes needed.
26
- Estimated scope: similar to cross-host shape learning. One commit.
27
- - ⏭️ **Near-shape clustering** — edit-distance over Shapes catches
28
- near-duplicates that today require exact match. Bigger scope — needs
29
- a Shape distance metric, candidate-pair search, and CLI for
30
- inspection. Two or three commits.
31
-
32
- **Phase 3**: not started. Schema export, PII mode, streaming observatory,
33
- external recognizer registry.
34
-
35
- **Housekeeping done**: Ruby 3.3 dropped (4d49509), shell completion
36
- with brew auto-install (v0.25), homebrew skill updated, README has the
37
- full learning-loop documentation, all CLI flags reflected in `-h` and
38
- completion scripts. Homebrew formula tracks 0.28.0.
39
-
40
- **Recommended next commit**: inter-position correlations. The
41
- read-side seam is clean (walk clusters, group segments by their
42
- predecessor) and the payoff is concrete (next-segment hints for
43
- corpus-informed normalization). Falls out of the existing data with
44
- no schema changes.
45
-
46
- ## Operating ground rules (decisions on file)
47
-
48
- - **Backward compatibility:** none required while pre-1.0. Break what you must.
49
- Bump the minor version per feature.
50
- - **Public API surface:** narrow on purpose. The four `Iriq.*` module methods
51
- and the CLI are the contract; everything else may move freely. Dev speed
52
- beats API stability for now.
53
- - **Ruby ↔ Go parity:** still the goal. The current discipline (Ruby-first,
54
- fixture regen, Go-mirror, parity test) stays in force as the *acceptance
55
- gate*; ordering within a single feature commit is implementer's call.
56
- - **Performance:** no hard target. Don't be wasteful. Watch out for obvious
57
- regressions (per-segment allocation in hot paths, redundant classify calls,
58
- etc.) but no benchmark gates yet.
59
- - **Extensibility:** internal-only is fine for v1. Design the seams so an
60
- external recognizer registry is a small follow-up, not a redesign.
61
- - **Calibration data:** we generate and check in our own. See task #3.
62
- - **Commit discipline:** one feature, one commit, immediate push. Tests must
63
- pass: `bundle exec rspec && go test ./... && script/cli_parity.sh`.
64
-
65
- ## Phase 1 — Year 1: structural foundation
66
-
67
- Status: **complete**. All 9 tasks shipped in a single session, v0.11.0 → v0.19.0.
68
-
69
- Goal: stop being a 554-line `SegmentClassifier` doing five jobs. Land the
70
- substrate that the year-2 learning work compounds on.
71
-
72
- Ordered task list (each was a commit checkpoint, each bumped a minor version):
73
-
74
- 1. ✅ **Docs + plan** — this file and `ARCHITECTURE.md`. Shipped at 866525f.
75
- 2. ✅ **Recognizer interface (uuid, date, integer)** — carved three
76
- Recognizers out of `SegmentClassifier`. No behavior change; first-match
77
- equivalent. Shipped in v0.12.0 (b3cc889).
78
- 3. ✅ **Calibration corpus** — 202 labeled segments across 25 types at
79
- `spec/fixtures/calibration/segments.json`, generated by
80
- `script/build_calibration.rb`. Loaded by both Ruby (`calibration_spec`)
81
- and Go (`calibration_test.go`) as a regression suite + future
82
- calibration target. Shipped in v0.13.0 (f9eb980).
83
- 4. ✅ **Scored ensemble** — added Specificity bands (SEMANTIC, STRUCTURED,
84
- BOUNDED, TYPED, PATTERN, FALLBACK) on each Recognizer's Verdict and an
85
- Ensemble helper that picks max(specificity × confidence) with stable
86
- earlier-wins tie-break. Shipped in v0.14.0 (924f1ba).
87
- 5. ✅ **Position promoted to first-class** — Position is { host, scope ∈
88
- {:path,:query}, locator } — a typed slot in a host's URL structure.
89
- Storage contracts now take Position; SQLite schema gained a `scope`
90
- column (bumped to v2). Shipped in v0.15.0 (954e7a6).
91
- 6. ✅ **Structured Shape** — Shape is a value object: an ordered list of
92
- typed segment entries plus Render() / Equal() over the structured form.
93
- PathShape became a thin wrapper. Cluster gained `#shape_object`.
94
- Shipped in v0.16.0 (97a60c2).
95
- 7. ✅ **Evidence records + Trace as view** — Evidence is the structured
96
- substrate for explanation: three subject kinds (segment / position /
97
- cluster), five sources (lexical / recognizer / corpus / neighbor /
98
- policy). Ruby Trace was rewritten to build Evidence internally and
99
- render it into the same `{value, type, output, notes}` hash. Go ships
100
- the Evidence types but kept Trace internals hand-coded (output parity
101
- preserved). Shipped in v0.17.0 (9e1133e).
102
- 8. ✅ **One Normalizer** — folded mechanical Normalizer and
103
- corpus-informed Corpus#normalize into one entry point with an evidence
104
- source. NullEvidence provides classifier-only behavior; Corpus
105
- implements the same RenderPath / RenderQuery interface for
106
- corpus-informed rendering. Shipped in v0.18.0 (24ad168).
107
- 9. ✅ **Events + reducers** — Corpus.observe now builds an ordered Event
108
- list (HostSeen, PathLengthSeen, RawShapeSeen, FingerprintSeen,
109
- per-segment PositionSeen, ClusterAddition) and applies each through
110
- the Reducer registry. Adding a new metric = define a new Event +
111
- Reducer; no other module changes. Event list is transient today;
112
- future commit can persist it for re-runnable inference. Shipped in
113
- v0.19.0 (891fd12).
114
-
115
- End of Phase 1 deliverable: the same CLI surface, but the internals are an
116
- evidence-driven inference system over typed Positions with pluggable
117
- recognizers and re-runnable-inference-ready event emission.
118
-
119
- ## Phase 2 — Year 2: learning that compounds on the new substrate
120
-
121
- Each of these is a non-trivial multi-PR initiative.
122
-
123
- - ✅ **Re-runnable inference (library)** — source-IRI log persisted alongside
124
- materialized views; `Corpus#reinfer` drops the views and replays the log
125
- through events + reducers. Lets users tune thresholds or swap the
126
- classifier without re-feeding IRIs. Shipped in v0.21.0.
127
- - ✅ **Re-runnable inference (CLI)** — `iriq --corpus PATH --reinfer`
128
- replays the log and prints a before/after summary. Parity scenarios
129
- added for JSON + SQLite backends. Shipped in v0.22.0.
130
- - **Inter-position correlations** — when slot A's value predicts slot B's
131
- type, surface that. Catches things like "the segment after `/orgs/{org_id}/`
132
- is almost always `users` or `repos`".
133
- - ✅ **Learned recognizers** — full loop:
134
- * `Corpus#propose_recognizers` scans observed values via pluggable
135
- ProposalStrategy strategies (v0.23.0). v1 ships PrefixUnderscoreId
136
- (detects `ghp_…`, `cus_…`, `sk_…` shapes at slug/opaque_id
137
- positions).
138
- * `iriq --corpus PATH --propose-recognizers [--json]` CLI flag with
139
- tunable `--min-observations` / `--min-coverage` / `--min-hosts`
140
- thresholds (v0.24.0).
141
- * `Corpus#activate_proposal(p)` promotes a proposal into a live
142
- SynthesizedRecognizer on the corpus's classifier, persists it,
143
- and reinfers. Reopens re-apply via `Corpus.open`. Doesn't leak
144
- to the module-level DEFAULT classifier — corpora are isolated
145
- (v0.26.0).
146
- * `iriq --corpus PATH --propose-recognizers --activate-above F`
147
- auto-activates every proposal at or above coverage F (v0.26.0).
148
- - ✅ **Cross-host learning** — full integration:
149
- * `Corpus#cross_host_shapes(min_hosts:)` lists route shapes that
150
- recur across multiple hosts (independent evidence of semantic
151
- pattern, not host-local quirk). CLI `--cross-host-shapes`. v0.27.0.
152
- * Cross-host count wires into `RecognizerProposal#confidence`:
153
- each additional host beyond the first adds 0.05 (capped at 1.0).
154
- Single-host proposals get `confidence == coverage`; cross-host
155
- proposals get boosted. `--activate-above F` checks confidence
156
- (not raw coverage). Proposals are sorted by confidence desc.
157
- v0.28.0.
158
- - **Near-shape clustering** — edit-distance over Shapes catches
159
- near-duplicates that today require exact match.
160
-
161
- ## Phase 3 — Year 3: productize
162
-
163
- Speculative; revisit after Phase 2.
164
-
165
- - Exported learned schemas (OpenAPI-ish: routes per host with inferred
166
- types per Position).
167
- - Privacy/PII mode (recognizer-driven redaction policies).
168
- - Streaming observatory with shape-drift alerts.
169
- - External recognizer registry (the public surface for what Phase 1 keeps
170
- internal).
171
-
172
- ## Risks tracked
173
-
174
- - **Parity tax compounds.** Every refactor is 2× surface. If a refactor's Go
175
- port slips a release, mark it explicitly in CHANGELOG and treat it as tech
176
- debt — don't let it sit.
177
- - **Confidence calibration is theater without truth data.** Task #3 ships
178
- before #4 for that reason.
179
- - **Recognizer-set versioning.** Once we ship pluggable recognizers, stored
180
- corpora need to know which recognizer-set produced them. Plan: stamp
181
- `recognizer_set_id` into `PositionStats` at observation time. Defer
182
- implementation until Phase 2.
183
-
184
- ## How to use this document
185
-
186
- - Update it as decisions change. It is not a frozen artifact.
187
- - Each completed Phase 1 task: tick it off, add a one-line "shipped in vX.Y.Z"
188
- pointer. Don't delete history.
189
- - Bigger plan changes (reorder, scope cut, scope grow): leave the prior
190
- list intact and append a "Plan change YYYY-MM-DD" note explaining why.