iriq 0.2.0 → 0.33.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +110 -0
- data/README.md +259 -356
- data/completions/_iriq +54 -0
- data/completions/iriq.bash +70 -0
- data/iriq.gemspec +2 -2
- data/lib/iriq/cli.rb +502 -50
- data/lib/iriq/cluster.rb +332 -14
- data/lib/iriq/clusterer.rb +0 -11
- data/lib/iriq/corpus.rb +321 -38
- data/lib/iriq/cross_host_shape.rb +37 -0
- data/lib/iriq/event.rb +22 -0
- data/lib/iriq/evidence.rb +114 -0
- data/lib/iriq/explanation.rb +1 -1
- data/lib/iriq/identifier.rb +1 -1
- data/lib/iriq/normalizer.rb +71 -29
- data/lib/iriq/parser.rb +5 -1
- data/lib/iriq/path_shape.rb +30 -24
- data/lib/iriq/position.rb +75 -0
- data/lib/iriq/position_stats.rb +74 -8
- data/lib/iriq/recognizer.rb +54 -0
- data/lib/iriq/recognizer_proposal.rb +167 -0
- data/lib/iriq/recognizers/date.rb +53 -0
- data/lib/iriq/recognizers/integer.rb +37 -0
- data/lib/iriq/recognizers/uuid.rb +16 -0
- data/lib/iriq/reducer.rb +37 -0
- data/lib/iriq/registrable_domain.rb +56 -0
- data/lib/iriq/segment_classifier.rb +478 -23
- data/lib/iriq/segment_hints.rb +9 -0
- data/lib/iriq/shape.rb +106 -0
- data/lib/iriq/specificity.rb +35 -0
- data/lib/iriq/storage/memory.rb +83 -12
- data/lib/iriq/storage/sqlite.rb +279 -43
- data/lib/iriq/synthesized_recognizer.rb +56 -0
- data/lib/iriq/trace.rb +294 -0
- data/lib/iriq/version.rb +1 -1
- data/lib/iriq.rb +17 -0
- metadata +19 -6
- data/CLAUDE.md +0 -121
- data/Gemfile +0 -3
- data/Gemfile.lock +0 -103
- data/Makefile +0 -56
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
module Iriq
|
|
2
|
+
# Recognizer built dynamically from a learned-prefix pattern.
|
|
3
|
+
#
|
|
4
|
+
# Used by Corpus#activate_proposal to promote a RecognizerProposal
|
|
5
|
+
# into a live Recognizer that the classifier ensemble consults. Same
|
|
6
|
+
# shape as the built-in Recognizers — uuid, date, integer — but the
|
|
7
|
+
# pattern + type are supplied at construction instead of compiled-in.
|
|
8
|
+
#
|
|
9
|
+
# r = SynthesizedRecognizer.new(prefix: "ghp_", type: :ghp)
|
|
10
|
+
# r.try("ghp_abcdef123") # → {type: :ghp, confidence: 1.0, specificity: 1.0}
|
|
11
|
+
#
|
|
12
|
+
# Pattern: `<prefix><[A-Za-z0-9]+>` — anchored, alphanumeric suffix
|
|
13
|
+
# only. Matches the same shape PrefixUnderscoreId proposes from, so
|
|
14
|
+
# round-trip (propose → activate → reinfer) reclassifies the same
|
|
15
|
+
# values the proposal was derived from.
|
|
16
|
+
#
|
|
17
|
+
# Specificity defaults to SEMANTIC. A learned prefix is very specific
|
|
18
|
+
# by construction (a distinctive literal prefix that recurred enough
|
|
19
|
+
# to clear the proposal noise floor) — calling it as confident as a
|
|
20
|
+
# built-in UUID is reasonable.
|
|
21
|
+
class SynthesizedRecognizer < Recognizer
|
|
22
|
+
attr_reader :prefix, :type, :specificity
|
|
23
|
+
|
|
24
|
+
def self.from_proposal(proposal)
|
|
25
|
+
new(prefix: proposal.prefix, type: proposal.suggested_type)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def initialize(prefix:, type:, specificity: Specificity::SEMANTIC)
|
|
29
|
+
raise ArgumentError, "prefix must be a non-empty string" if prefix.nil? || prefix.empty?
|
|
30
|
+
raise ArgumentError, "type must be a symbol" unless type.is_a?(Symbol)
|
|
31
|
+
|
|
32
|
+
@prefix = prefix
|
|
33
|
+
@type = type
|
|
34
|
+
@specificity = specificity
|
|
35
|
+
@pattern = /\A#{Regexp.escape(prefix)}[A-Za-z0-9]+\z/.freeze
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def try(segment)
|
|
39
|
+
return nil unless segment.start_with?(@prefix) && @pattern.match?(segment)
|
|
40
|
+
|
|
41
|
+
{ type: @type, confidence: 1.0, specificity: @specificity }
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def to_dump
|
|
45
|
+
{ "prefix" => @prefix, "type" => @type.to_s, "specificity" => @specificity }
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def self.from_dump(h)
|
|
49
|
+
new(
|
|
50
|
+
prefix: h["prefix"],
|
|
51
|
+
type: h["type"].to_sym,
|
|
52
|
+
specificity: h.fetch("specificity", Specificity::SEMANTIC),
|
|
53
|
+
)
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
data/lib/iriq/trace.rb
ADDED
|
@@ -0,0 +1,294 @@
|
|
|
1
|
+
module Iriq
|
|
2
|
+
# Produces an annotated trace explaining how an identifier got
|
|
3
|
+
# normalized — segment by segment, with notes for each non-obvious
|
|
4
|
+
# transformation (currency upcase, IP umbrella, hint suppression,
|
|
5
|
+
# canonical date, param-name lift, etc.).
|
|
6
|
+
#
|
|
7
|
+
# Trace.for("https://shop.com/pricing/usd?currency=eur")
|
|
8
|
+
# # => {
|
|
9
|
+
# # input: "...",
|
|
10
|
+
# # normalized: "https://shop.com/pricing/USD?currency=EUR",
|
|
11
|
+
# # scheme: "https", host: "shop.com",
|
|
12
|
+
# # path: [...per-segment rows...],
|
|
13
|
+
# # query: [...per-param rows...],
|
|
14
|
+
# # }
|
|
15
|
+
#
|
|
16
|
+
# Each row is `{ value, type, output, notes }` for path entries and
|
|
17
|
+
# `{ name, value, type, output, notes }` for query entries. The string
|
|
18
|
+
# notes are rendered from structured Iriq::Evidence::Record values;
|
|
19
|
+
# callers that want the structured form can use Trace.evidence_for.
|
|
20
|
+
module Trace
|
|
21
|
+
module_function
|
|
22
|
+
|
|
23
|
+
HINT_NOTE_TEMPLATE = "semantic type — surfaced as {%s}, not {%s}".freeze
|
|
24
|
+
|
|
25
|
+
# Render-ready Trace output. The public format consumers depend on.
|
|
26
|
+
def for(input, classifier: SegmentClassifier::DEFAULT, hints: true)
|
|
27
|
+
iri = coerce(input)
|
|
28
|
+
normalized = Normalizer.normalize_identifier(iri, classifier: classifier, hints: hints)
|
|
29
|
+
|
|
30
|
+
out = {
|
|
31
|
+
input: iri.canonical,
|
|
32
|
+
normalized: normalized,
|
|
33
|
+
scheme: iri.scheme,
|
|
34
|
+
host: iri.host,
|
|
35
|
+
}
|
|
36
|
+
out[:port] = iri.port if iri.port
|
|
37
|
+
|
|
38
|
+
if iri.urn?
|
|
39
|
+
out[:path] = urn_rows(iri, classifier, hints)
|
|
40
|
+
else
|
|
41
|
+
out[:path] = path_rows(iri.path_segments, classifier, hints)
|
|
42
|
+
if iri.query_params && !iri.query_params.empty?
|
|
43
|
+
out[:query] = query_rows(iri.query_params, classifier)
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
out
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# Structured Evidence list for `input`. Each segment + query param
|
|
51
|
+
# contributes one classification Evidence plus zero or more
|
|
52
|
+
# transformation Evidence records (canonical date, IP umbrella
|
|
53
|
+
# collapse, param-name hint, hint suppression).
|
|
54
|
+
#
|
|
55
|
+
# Position + Cluster Evidence are not emitted here — they belong to
|
|
56
|
+
# corpus-informed trace (Corpus#trace), which a follow-up step lands.
|
|
57
|
+
def evidence_for(input, classifier: SegmentClassifier::DEFAULT, hints: true)
|
|
58
|
+
iri = coerce(input)
|
|
59
|
+
records = []
|
|
60
|
+
segments = iri.urn? ? urn_parts(iri) : (iri.path_segments || [])
|
|
61
|
+
entries = SegmentHints.derive(segments, classifier)
|
|
62
|
+
|
|
63
|
+
entries.each_with_index do |entry, i|
|
|
64
|
+
records.concat(segment_evidence(entry, segments, i, classifier, hints))
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
if !iri.urn? && iri.query_params && !iri.query_params.empty?
|
|
68
|
+
iri.query_params.keys.sort.each do |k|
|
|
69
|
+
records.concat(query_param_evidence(k, iri.query_params[k].to_s, classifier))
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
records
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# ── Evidence builders ────────────────────────────────────────────────
|
|
77
|
+
|
|
78
|
+
def segment_evidence(entry, segments, idx, classifier, hints)
|
|
79
|
+
records = []
|
|
80
|
+
|
|
81
|
+
records << Evidence.segment(
|
|
82
|
+
index: idx, value: entry[:value],
|
|
83
|
+
source: :recognizer,
|
|
84
|
+
payload: { type: entry[:type], variable: entry[:variable], hint: entry[:hint] },
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
if entry[:variable]
|
|
88
|
+
if entry[:type] == :date && (canon = SegmentClassifier.canonical_date(entry[:value]))
|
|
89
|
+
if canon != entry[:value]
|
|
90
|
+
records << Evidence.segment(
|
|
91
|
+
index: idx, value: entry[:value],
|
|
92
|
+
source: :policy,
|
|
93
|
+
payload: { rule: :canonical_date, before: entry[:value], after: canon },
|
|
94
|
+
notes: ["canonical date (#{entry[:value]} → #{canon})"],
|
|
95
|
+
)
|
|
96
|
+
end
|
|
97
|
+
elsif entry[:type] == :currency && (canon = SegmentClassifier.canonical_currency(entry[:value]))
|
|
98
|
+
if canon != entry[:value]
|
|
99
|
+
records << Evidence.segment(
|
|
100
|
+
index: idx, value: entry[:value],
|
|
101
|
+
source: :policy,
|
|
102
|
+
payload: { rule: :canonical_currency, before: entry[:value], after: canon },
|
|
103
|
+
notes: ["currency upcase (#{entry[:value]} → #{canon})"],
|
|
104
|
+
)
|
|
105
|
+
end
|
|
106
|
+
else
|
|
107
|
+
extra = placeholder_decoration_evidence(entry, segments, idx, classifier, hints)
|
|
108
|
+
records.concat(extra)
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
records
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def placeholder_decoration_evidence(entry, segments, idx, classifier, hints)
|
|
116
|
+
out = []
|
|
117
|
+
type = entry[:type]
|
|
118
|
+
|
|
119
|
+
if type == :ipv4 || type == :ipv6
|
|
120
|
+
out << Evidence.segment(
|
|
121
|
+
index: idx, value: entry[:value],
|
|
122
|
+
source: :policy,
|
|
123
|
+
payload: { rule: :ip_umbrella_collapse, from: type, to: :ip },
|
|
124
|
+
notes: ["ip umbrella collapse (#{type} → ip)"],
|
|
125
|
+
)
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
if hints && entry[:hint].nil? && !SegmentHints::HINT_ELIGIBLE_TYPES.include?(type)
|
|
129
|
+
if (would_be = would_be_hint(segments, idx, type, classifier))
|
|
130
|
+
display = SegmentClassifier.display_type(type)
|
|
131
|
+
out << Evidence.segment(
|
|
132
|
+
index: idx, value: entry[:value],
|
|
133
|
+
source: :neighbor,
|
|
134
|
+
payload: { rule: :hint_suppression, surfaced: display, would_be: would_be, semantic_type: type },
|
|
135
|
+
notes: [format(HINT_NOTE_TEMPLATE, display, would_be)],
|
|
136
|
+
)
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
out
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def query_param_evidence(name, value, classifier)
|
|
144
|
+
records = []
|
|
145
|
+
base_type = classifier.classify(value)
|
|
146
|
+
effective = base_type
|
|
147
|
+
|
|
148
|
+
if (hint = SegmentClassifier.param_name_hint(name, base_type))
|
|
149
|
+
effective = hint
|
|
150
|
+
records << Evidence.segment(
|
|
151
|
+
index: name, value: value,
|
|
152
|
+
source: :neighbor,
|
|
153
|
+
payload: { rule: :param_name_hint, name: name, before: base_type, after: hint },
|
|
154
|
+
notes: ["param-name hint (`#{name}=`) lifted #{base_type} → #{hint}"],
|
|
155
|
+
)
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
records << Evidence.segment(
|
|
159
|
+
index: name, value: value,
|
|
160
|
+
source: :recognizer,
|
|
161
|
+
payload: { type: effective, variable: SegmentClassifier::DEFAULT.variable?(effective) },
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
if effective == :date && (canon = SegmentClassifier.canonical_date(value))
|
|
165
|
+
if canon != value
|
|
166
|
+
records << Evidence.segment(
|
|
167
|
+
index: name, value: value,
|
|
168
|
+
source: :policy,
|
|
169
|
+
payload: { rule: :canonical_date, before: value, after: canon },
|
|
170
|
+
notes: ["canonical date (#{value} → #{canon})"],
|
|
171
|
+
)
|
|
172
|
+
end
|
|
173
|
+
elsif effective == :currency && (canon = SegmentClassifier.canonical_currency(value))
|
|
174
|
+
if canon != value
|
|
175
|
+
records << Evidence.segment(
|
|
176
|
+
index: name, value: value,
|
|
177
|
+
source: :policy,
|
|
178
|
+
payload: { rule: :canonical_currency, before: value, after: canon },
|
|
179
|
+
notes: ["currency upcase (#{value} → #{canon})"],
|
|
180
|
+
)
|
|
181
|
+
end
|
|
182
|
+
elsif effective == :ipv4 || effective == :ipv6
|
|
183
|
+
records << Evidence.segment(
|
|
184
|
+
index: name, value: value,
|
|
185
|
+
source: :policy,
|
|
186
|
+
payload: { rule: :ip_umbrella_collapse, from: effective, to: :ip },
|
|
187
|
+
notes: ["ip umbrella collapse (#{effective} → ip)"],
|
|
188
|
+
)
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
records
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
# ── View rendering (Evidence → Trace.for hash) ───────────────────────
|
|
195
|
+
|
|
196
|
+
def path_rows(segments, classifier, hints)
|
|
197
|
+
return [] if segments.nil? || segments.empty?
|
|
198
|
+
|
|
199
|
+
entries = SegmentHints.derive(segments, classifier)
|
|
200
|
+
entries.each_with_index.map do |entry, i|
|
|
201
|
+
ev = segment_evidence(entry, segments, i, classifier, hints)
|
|
202
|
+
render_segment_row(entry, ev, hints)
|
|
203
|
+
end
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
def urn_rows(iri, classifier, hints)
|
|
207
|
+
parts = urn_parts(iri)
|
|
208
|
+
return [] if parts.empty?
|
|
209
|
+
|
|
210
|
+
entries = SegmentHints.derive(parts, classifier)
|
|
211
|
+
entries.each_with_index.map do |entry, i|
|
|
212
|
+
ev = segment_evidence(entry, parts, i, classifier, hints)
|
|
213
|
+
render_segment_row(entry, ev, hints)
|
|
214
|
+
end
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
def query_rows(params, classifier)
|
|
218
|
+
params.keys.sort.map do |k|
|
|
219
|
+
v = params[k].to_s
|
|
220
|
+
ev = query_param_evidence(k, v, classifier)
|
|
221
|
+
render_query_row(k, v, ev)
|
|
222
|
+
end
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
def render_segment_row(entry, evidence, hints)
|
|
226
|
+
notes = collect_notes(evidence)
|
|
227
|
+
value = entry[:value]
|
|
228
|
+
type = entry[:type]
|
|
229
|
+
|
|
230
|
+
return { value: value, type: type, output: value, notes: notes } unless entry[:variable]
|
|
231
|
+
|
|
232
|
+
canon_policy = find_payload(evidence, :canonical_date) || find_payload(evidence, :canonical_currency)
|
|
233
|
+
if canon_policy
|
|
234
|
+
return { value: value, type: type, output: canon_policy[:after], notes: notes }
|
|
235
|
+
end
|
|
236
|
+
|
|
237
|
+
placeholder = hints && entry[:hint] ? entry[:hint].to_s : SegmentClassifier.display_type(type).to_s
|
|
238
|
+
{ value: value, type: type, output: "{#{placeholder}}", notes: notes }
|
|
239
|
+
end
|
|
240
|
+
|
|
241
|
+
def render_query_row(name, value, evidence)
|
|
242
|
+
notes = collect_notes(evidence)
|
|
243
|
+
cls = find_evidence(evidence, source: :recognizer)
|
|
244
|
+
effective = cls ? cls.payload[:type] : SegmentClassifier::DEFAULT.classify(value)
|
|
245
|
+
canon_policy = find_payload(evidence, :canonical_date) || find_payload(evidence, :canonical_currency)
|
|
246
|
+
|
|
247
|
+
output =
|
|
248
|
+
if canon_policy
|
|
249
|
+
canon_policy[:after]
|
|
250
|
+
elsif SegmentClassifier::DEFAULT.variable?(effective)
|
|
251
|
+
"{#{SegmentClassifier.display_type(effective)}}"
|
|
252
|
+
else
|
|
253
|
+
value
|
|
254
|
+
end
|
|
255
|
+
|
|
256
|
+
{ name: name, value: value, type: effective, output: output, notes: notes }
|
|
257
|
+
end
|
|
258
|
+
|
|
259
|
+
# ── Helpers ──────────────────────────────────────────────────────────
|
|
260
|
+
|
|
261
|
+
def coerce(input)
|
|
262
|
+
input.is_a?(Identifier) ? input : Parser.parse(input)
|
|
263
|
+
end
|
|
264
|
+
|
|
265
|
+
def urn_parts(iri)
|
|
266
|
+
return [] unless iri.nss
|
|
267
|
+
iri.nss.include?(":") ? iri.nss.split(":", 2) : [iri.nss]
|
|
268
|
+
end
|
|
269
|
+
|
|
270
|
+
def collect_notes(evidence)
|
|
271
|
+
evidence.flat_map(&:notes)
|
|
272
|
+
end
|
|
273
|
+
|
|
274
|
+
def find_evidence(evidence, source:)
|
|
275
|
+
evidence.find { |r| r.source == source }
|
|
276
|
+
end
|
|
277
|
+
|
|
278
|
+
def find_payload(evidence, rule)
|
|
279
|
+
r = evidence.find { |e| e.source == :policy && e.payload[:rule] == rule }
|
|
280
|
+
r&.payload
|
|
281
|
+
end
|
|
282
|
+
|
|
283
|
+
def would_be_hint(segments, idx, type, classifier)
|
|
284
|
+
return nil if idx.zero?
|
|
285
|
+
|
|
286
|
+
prev = segments[idx - 1]
|
|
287
|
+
return nil unless classifier.classify(prev) == :literal
|
|
288
|
+
|
|
289
|
+
base = Inflector.singularize(prev)
|
|
290
|
+
suffix = type == :uuid ? "_uuid" : "_id"
|
|
291
|
+
"#{base}#{suffix}"
|
|
292
|
+
end
|
|
293
|
+
end
|
|
294
|
+
end
|
data/lib/iriq/version.rb
CHANGED
data/lib/iriq.rb
CHANGED
|
@@ -3,16 +3,33 @@ require "iriq/errors"
|
|
|
3
3
|
require "iriq/inflector"
|
|
4
4
|
require "iriq/identifier"
|
|
5
5
|
require "iriq/parser"
|
|
6
|
+
require "iriq/specificity"
|
|
7
|
+
require "iriq/recognizer"
|
|
8
|
+
require "iriq/recognizers/uuid"
|
|
9
|
+
require "iriq/recognizers/date"
|
|
10
|
+
require "iriq/recognizers/integer"
|
|
6
11
|
require "iriq/segment_classifier"
|
|
7
12
|
require "iriq/segment_hints"
|
|
13
|
+
require "iriq/shape"
|
|
8
14
|
require "iriq/path_shape"
|
|
9
15
|
require "iriq/normalizer"
|
|
10
16
|
require "iriq/explanation"
|
|
17
|
+
require "iriq/evidence"
|
|
18
|
+
require "iriq/trace"
|
|
11
19
|
require "iriq/cluster"
|
|
12
20
|
require "iriq/clusterer"
|
|
13
21
|
require "iriq/position_stats"
|
|
22
|
+
require "set"
|
|
23
|
+
|
|
14
24
|
require "iriq/observation"
|
|
25
|
+
require "iriq/position"
|
|
26
|
+
require "iriq/event"
|
|
27
|
+
require "iriq/reducer"
|
|
28
|
+
require "iriq/registrable_domain"
|
|
15
29
|
require "iriq/storage"
|
|
30
|
+
require "iriq/recognizer_proposal"
|
|
31
|
+
require "iriq/synthesized_recognizer"
|
|
32
|
+
require "iriq/cross_host_shape"
|
|
16
33
|
require "iriq/corpus"
|
|
17
34
|
require "iriq/extractor"
|
|
18
35
|
require "iriq/cli"
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: iriq
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.33.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Daniel Pepper
|
|
@@ -86,12 +86,10 @@ extensions: []
|
|
|
86
86
|
extra_rdoc_files: []
|
|
87
87
|
files:
|
|
88
88
|
- CHANGELOG.md
|
|
89
|
-
- CLAUDE.md
|
|
90
|
-
- Gemfile
|
|
91
|
-
- Gemfile.lock
|
|
92
89
|
- LICENSE.txt
|
|
93
|
-
- Makefile
|
|
94
90
|
- README.md
|
|
91
|
+
- completions/_iriq
|
|
92
|
+
- completions/iriq.bash
|
|
95
93
|
- exe/iriq
|
|
96
94
|
- iriq.gemspec
|
|
97
95
|
- lib/iriq.rb
|
|
@@ -99,7 +97,10 @@ files:
|
|
|
99
97
|
- lib/iriq/cluster.rb
|
|
100
98
|
- lib/iriq/clusterer.rb
|
|
101
99
|
- lib/iriq/corpus.rb
|
|
100
|
+
- lib/iriq/cross_host_shape.rb
|
|
102
101
|
- lib/iriq/errors.rb
|
|
102
|
+
- lib/iriq/event.rb
|
|
103
|
+
- lib/iriq/evidence.rb
|
|
103
104
|
- lib/iriq/explanation.rb
|
|
104
105
|
- lib/iriq/extractor.rb
|
|
105
106
|
- lib/iriq/identifier.rb
|
|
@@ -108,13 +109,25 @@ files:
|
|
|
108
109
|
- lib/iriq/observation.rb
|
|
109
110
|
- lib/iriq/parser.rb
|
|
110
111
|
- lib/iriq/path_shape.rb
|
|
112
|
+
- lib/iriq/position.rb
|
|
111
113
|
- lib/iriq/position_stats.rb
|
|
114
|
+
- lib/iriq/recognizer.rb
|
|
115
|
+
- lib/iriq/recognizer_proposal.rb
|
|
116
|
+
- lib/iriq/recognizers/date.rb
|
|
117
|
+
- lib/iriq/recognizers/integer.rb
|
|
118
|
+
- lib/iriq/recognizers/uuid.rb
|
|
119
|
+
- lib/iriq/reducer.rb
|
|
120
|
+
- lib/iriq/registrable_domain.rb
|
|
112
121
|
- lib/iriq/segment_classifier.rb
|
|
113
122
|
- lib/iriq/segment_hints.rb
|
|
123
|
+
- lib/iriq/shape.rb
|
|
124
|
+
- lib/iriq/specificity.rb
|
|
114
125
|
- lib/iriq/storage.rb
|
|
115
126
|
- lib/iriq/storage/json.rb
|
|
116
127
|
- lib/iriq/storage/memory.rb
|
|
117
128
|
- lib/iriq/storage/sqlite.rb
|
|
129
|
+
- lib/iriq/synthesized_recognizer.rb
|
|
130
|
+
- lib/iriq/trace.rb
|
|
118
131
|
- lib/iriq/version.rb
|
|
119
132
|
homepage: https://github.com/dpep/iriq
|
|
120
133
|
licenses:
|
|
@@ -127,7 +140,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
127
140
|
requirements:
|
|
128
141
|
- - ">="
|
|
129
142
|
- !ruby/object:Gem::Version
|
|
130
|
-
version: '3.
|
|
143
|
+
version: '3.4'
|
|
131
144
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
132
145
|
requirements:
|
|
133
146
|
- - ">="
|
data/CLAUDE.md
DELETED
|
@@ -1,121 +0,0 @@
|
|
|
1
|
-
# Iriq development conventions
|
|
2
|
-
|
|
3
|
-
## Repo layout — Ruby and Go intermixed at the root
|
|
4
|
-
|
|
5
|
-
We chose to mix Ruby and Go at the repo root rather than nest the Go module
|
|
6
|
-
under `/go/`. The signal is "both implementations are peers, not one-is-primary."
|
|
7
|
-
|
|
8
|
-
```
|
|
9
|
-
iriq/
|
|
10
|
-
lib/ exe/ spec/ ← Ruby gem (library, CLI, specs)
|
|
11
|
-
iriq.gemspec
|
|
12
|
-
Gemfile
|
|
13
|
-
|
|
14
|
-
go.mod ← module github.com/dpep/iriq
|
|
15
|
-
*.go ← Go package `iriq` at the root
|
|
16
|
-
cmd/iriq/ ← Go CLI binary
|
|
17
|
-
bin/ ← built Go binary (gitignored)
|
|
18
|
-
|
|
19
|
-
script/ ← shared dev scripts (fixture gen, parity, benches)
|
|
20
|
-
spec/fixtures/ ← golden JSON shared by Ruby specs + Go tests
|
|
21
|
-
.github/workflows/ ← Ruby CI, Go CI, parity CI
|
|
22
|
-
```
|
|
23
|
-
|
|
24
|
-
Trade-offs of this layout:
|
|
25
|
-
|
|
26
|
-
- Clean import path: `github.com/dpep/iriq` (no `/go/` artifact in consumers' code).
|
|
27
|
-
- One version tag (`vX.Y.Z`) serves both runtimes — Ruby's gemspec and Go's
|
|
28
|
-
module use the same tag stream.
|
|
29
|
-
- Root `ls` is busier (~15 `.go` files next to Ruby ones), accepted in exchange.
|
|
30
|
-
- The gemspec explicitly excludes Go files so `gem build` doesn't ship them:
|
|
31
|
-
`git ls-files * ':!:spec' ':!:script' ':!:cmd' ':!:bin' ':!:*.go' ':!:go.mod' ':!:go.sum'`.
|
|
32
|
-
|
|
33
|
-
## Building
|
|
34
|
-
|
|
35
|
-
```sh
|
|
36
|
-
# Ruby gem
|
|
37
|
-
bundle install
|
|
38
|
-
bundle exec exe/iriq --help # runs the CLI from source
|
|
39
|
-
|
|
40
|
-
# Go binary — convenience targets in the Makefile
|
|
41
|
-
make build # → ./bin/iriq
|
|
42
|
-
make install # go install into $GOBIN
|
|
43
|
-
make uninstall # remove from $GOBIN
|
|
44
|
-
make clean # remove ./bin/
|
|
45
|
-
make test # go test ./...
|
|
46
|
-
|
|
47
|
-
# Both via Homebrew
|
|
48
|
-
brew install dpep/tools/iriq # uses the Ruby gem under the hood
|
|
49
|
-
```
|
|
50
|
-
|
|
51
|
-
## Keeping Ruby and Go in sync
|
|
52
|
-
|
|
53
|
-
The Ruby gem is the **reference implementation**. Go mirrors its public API
|
|
54
|
-
and behavior. Two layers of parity testing keep them aligned:
|
|
55
|
-
|
|
56
|
-
1. **Golden JSON fixtures** (`spec/fixtures/*.json`)
|
|
57
|
-
Generated by `script/generate_fixtures.rb` from the Ruby implementation
|
|
58
|
-
over a curated set of inputs. Go's `fixtures_test.go` loads each file
|
|
59
|
-
and asserts the same outputs from the Go side.
|
|
60
|
-
|
|
61
|
-
2. **CLI parity harness** (`script/cli_parity.sh`)
|
|
62
|
-
Runs the same input through `bundle exec exe/iriq` and the Go binary and
|
|
63
|
-
diffs stdout. Lives in CI as the `Ruby ↔ Go parity` job.
|
|
64
|
-
|
|
65
|
-
When changing behavior:
|
|
66
|
-
|
|
67
|
-
1. Update the Ruby code + specs first.
|
|
68
|
-
2. Regenerate fixtures: `bundle exec ruby script/generate_fixtures.rb`.
|
|
69
|
-
3. Port the change to Go.
|
|
70
|
-
4. `go test ./...` (uses the updated fixtures).
|
|
71
|
-
5. `script/cli_parity.sh` should pass.
|
|
72
|
-
6. Commit fixtures with the change — CI will fail if they're stale.
|
|
73
|
-
|
|
74
|
-
## Tests
|
|
75
|
-
|
|
76
|
-
```sh
|
|
77
|
-
bundle exec rspec # Ruby suite (305+ examples)
|
|
78
|
-
go test ./... # Go suite (native + fixture parity tests)
|
|
79
|
-
script/cli_parity.sh # CLI parity (13+ scenarios)
|
|
80
|
-
```
|
|
81
|
-
|
|
82
|
-
## Releases
|
|
83
|
-
|
|
84
|
-
- One version tag covers both runtimes — bump `lib/iriq/version.rb` (and
|
|
85
|
-
optionally a matching constant on the Go side if we add one), tag `vX.Y.Z`,
|
|
86
|
-
push.
|
|
87
|
-
- `gem push iriq-X.Y.Z.gem` to publish to RubyGems.
|
|
88
|
-
- Update `Formula/iriq.rb` in the homebrew-tools tap to the new version.
|
|
89
|
-
- Go consumers pick up the tag automatically via `go get @vX.Y.Z`.
|
|
90
|
-
|
|
91
|
-
## Corpus storage backends
|
|
92
|
-
|
|
93
|
-
The `Corpus` class delegates state to a `Storage` backend; three backends ship:
|
|
94
|
-
|
|
95
|
-
- **Memory** — default, in-process only.
|
|
96
|
-
- **JSON** — Memory wrapped with atomic load/save against a JSON file
|
|
97
|
-
(`.json` by default). Same shape both runtimes have always written.
|
|
98
|
-
- **SQLite** — incremental UPSERTs against a `.db` / `.sqlite` / `.sqlite3`
|
|
99
|
-
file with WAL journaling. Supports concurrent observers and avoids
|
|
100
|
-
loading the whole corpus into memory.
|
|
101
|
-
|
|
102
|
-
`Corpus.open(path)` (Ruby) / `iriq.OpenCorpus(path)` (Go) picks the backend
|
|
103
|
-
by file extension. `corpus.save(other_path)` exports as JSON regardless of
|
|
104
|
-
the live backend; `corpus.save(same_path)` is idempotent (no clobbering a
|
|
105
|
-
SQLite file with JSON, etc.).
|
|
106
|
-
|
|
107
|
-
The Ruby `sqlite3` gem is loaded lazily (only when a `.db` path is opened),
|
|
108
|
-
keeping the iriq install footprint minimal for users that stick with JSON.
|
|
109
|
-
On the Go side we use `modernc.org/sqlite` (pure Go — no cgo).
|
|
110
|
-
|
|
111
|
-
When adding a new backend, replicate the contract in both languages and
|
|
112
|
-
add a parity scenario in `script/cli_parity.sh`'s `corpus_pair` section.
|
|
113
|
-
|
|
114
|
-
## What lives where in scripts
|
|
115
|
-
|
|
116
|
-
- `script/benchmark.rb` — Ruby-only throughput benchmark.
|
|
117
|
-
- `script/memory.rb` — Ruby-only memory profile.
|
|
118
|
-
- `script/generate_fixtures.rb` — produces `spec/fixtures/*.json` for cross-runtime parity.
|
|
119
|
-
- `script/cli_parity.sh` — Ruby ↔ Go CLI diff.
|
|
120
|
-
- `script/bench_compare.sh` — Ruby vs Go CLI wall-time comparison.
|
|
121
|
-
- `script/bench_storage.sh` — JSON vs SQLite backend timing (single-process, incremental, concurrent).
|
data/Gemfile
DELETED
data/Gemfile.lock
DELETED
|
@@ -1,103 +0,0 @@
|
|
|
1
|
-
PATH
|
|
2
|
-
remote: .
|
|
3
|
-
specs:
|
|
4
|
-
iriq (0.2.0)
|
|
5
|
-
|
|
6
|
-
GEM
|
|
7
|
-
remote: https://rubygems.org/
|
|
8
|
-
specs:
|
|
9
|
-
date (3.5.1)
|
|
10
|
-
debug (1.11.1)
|
|
11
|
-
irb (~> 1.10)
|
|
12
|
-
reline (>= 0.3.8)
|
|
13
|
-
diff-lcs (1.6.2)
|
|
14
|
-
docile (1.4.1)
|
|
15
|
-
erb (6.0.4)
|
|
16
|
-
io-console (0.8.2)
|
|
17
|
-
irb (1.17.0)
|
|
18
|
-
pp (>= 0.6.0)
|
|
19
|
-
prism (>= 1.3.0)
|
|
20
|
-
rdoc (>= 4.0.0)
|
|
21
|
-
reline (>= 0.4.2)
|
|
22
|
-
mini_portile2 (2.8.9)
|
|
23
|
-
pp (0.6.3)
|
|
24
|
-
prettyprint
|
|
25
|
-
prettyprint (0.2.0)
|
|
26
|
-
prism (1.9.0)
|
|
27
|
-
psych (5.3.1)
|
|
28
|
-
date
|
|
29
|
-
stringio
|
|
30
|
-
rdoc (7.2.0)
|
|
31
|
-
erb
|
|
32
|
-
psych (>= 4.0.0)
|
|
33
|
-
tsort
|
|
34
|
-
reline (0.6.3)
|
|
35
|
-
io-console (~> 0.5)
|
|
36
|
-
rspec (3.13.2)
|
|
37
|
-
rspec-core (~> 3.13.0)
|
|
38
|
-
rspec-expectations (~> 3.13.0)
|
|
39
|
-
rspec-mocks (~> 3.13.0)
|
|
40
|
-
rspec-core (3.13.6)
|
|
41
|
-
rspec-support (~> 3.13.0)
|
|
42
|
-
rspec-debugging (0.0.4)
|
|
43
|
-
rspec-expectations (>= 3)
|
|
44
|
-
rspec-expectations (3.13.5)
|
|
45
|
-
diff-lcs (>= 1.2.0, < 2.0)
|
|
46
|
-
rspec-support (~> 3.13.0)
|
|
47
|
-
rspec-mocks (3.13.8)
|
|
48
|
-
diff-lcs (>= 1.2.0, < 2.0)
|
|
49
|
-
rspec-support (~> 3.13.0)
|
|
50
|
-
rspec-support (3.13.7)
|
|
51
|
-
simplecov (0.22.0)
|
|
52
|
-
docile (~> 1.1)
|
|
53
|
-
simplecov-html (~> 0.11)
|
|
54
|
-
simplecov_json_formatter (~> 0.1)
|
|
55
|
-
simplecov-html (0.13.2)
|
|
56
|
-
simplecov_json_formatter (0.1.4)
|
|
57
|
-
sqlite3 (2.9.4)
|
|
58
|
-
mini_portile2 (~> 2.8.0)
|
|
59
|
-
stringio (3.2.0)
|
|
60
|
-
tsort (0.2.0)
|
|
61
|
-
|
|
62
|
-
PLATFORMS
|
|
63
|
-
ruby
|
|
64
|
-
|
|
65
|
-
DEPENDENCIES
|
|
66
|
-
debug (>= 1)
|
|
67
|
-
iriq!
|
|
68
|
-
rspec (>= 3.10)
|
|
69
|
-
rspec-debugging
|
|
70
|
-
simplecov (>= 0.22)
|
|
71
|
-
sqlite3 (>= 1.6)
|
|
72
|
-
|
|
73
|
-
CHECKSUMS
|
|
74
|
-
date (3.5.1) sha256=750d06384d7b9c15d562c76291407d89e368dda4d4fff957eb94962d325a0dc0
|
|
75
|
-
debug (1.11.1) sha256=2e0b0ac6119f2207a6f8ac7d4a73ca8eb4e440f64da0a3136c30343146e952b6
|
|
76
|
-
diff-lcs (1.6.2) sha256=9ae0d2cba7d4df3075fe8cd8602a8604993efc0dfa934cff568969efb1909962
|
|
77
|
-
docile (1.4.1) sha256=96159be799bfa73cdb721b840e9802126e4e03dfc26863db73647204c727f21e
|
|
78
|
-
erb (6.0.4) sha256=38e3803694be357fe2bfe312487c74beaf9fb4e5beb3e22498952fe1645b95d9
|
|
79
|
-
io-console (0.8.2) sha256=d6e3ae7a7cc7574f4b8893b4fca2162e57a825b223a177b7afa236c5ef9814cc
|
|
80
|
-
irb (1.17.0) sha256=168c4ddb93d8a361a045c41d92b2952c7a118fa73f23fe14e55609eb7a863aae
|
|
81
|
-
iriq (0.2.0)
|
|
82
|
-
mini_portile2 (2.8.9) sha256=0cd7c7f824e010c072e33f68bc02d85a00aeb6fce05bb4819c03dfd3c140c289
|
|
83
|
-
pp (0.6.3) sha256=2951d514450b93ccfeb1df7d021cae0da16e0a7f95ee1e2273719669d0ab9df6
|
|
84
|
-
prettyprint (0.2.0) sha256=2bc9e15581a94742064a3cc8b0fb9d45aae3d03a1baa6ef80922627a0766f193
|
|
85
|
-
prism (1.9.0) sha256=7b530c6a9f92c24300014919c9dcbc055bf4cdf51ec30aed099b06cd6674ef85
|
|
86
|
-
psych (5.3.1) sha256=eb7a57cef10c9d70173ff74e739d843ac3b2c019a003de48447b2963d81b1974
|
|
87
|
-
rdoc (7.2.0) sha256=8650f76cd4009c3b54955eb5d7e3a075c60a57276766ebf36f9085e8c9f23192
|
|
88
|
-
reline (0.6.3) sha256=1198b04973565b36ec0f11542ab3f5cfeeec34823f4e54cebde90968092b1835
|
|
89
|
-
rspec (3.13.2) sha256=206284a08ad798e61f86d7ca3e376718d52c0bc944626b2349266f239f820587
|
|
90
|
-
rspec-core (3.13.6) sha256=a8823c6411667b60a8bca135364351dda34cd55e44ff94c4be4633b37d828b2d
|
|
91
|
-
rspec-debugging (0.0.4) sha256=7a8e2dc240c140f0ed27b452a5661a56474ee8cf7b84c5bcbefd827ad36f0a6f
|
|
92
|
-
rspec-expectations (3.13.5) sha256=33a4d3a1d95060aea4c94e9f237030a8f9eae5615e9bd85718fe3a09e4b58836
|
|
93
|
-
rspec-mocks (3.13.8) sha256=086ad3d3d17533f4237643de0b5c42f04b66348c28bf6b9c2d3f4a3b01af1d47
|
|
94
|
-
rspec-support (3.13.7) sha256=0640e5570872aafefd79867901deeeeb40b0c9875a36b983d85f54fb7381c47c
|
|
95
|
-
simplecov (0.22.0) sha256=fe2622c7834ff23b98066bb0a854284b2729a569ac659f82621fc22ef36213a5
|
|
96
|
-
simplecov-html (0.13.2) sha256=bd0b8e54e7c2d7685927e8d6286466359b6f16b18cb0df47b508e8d73c777246
|
|
97
|
-
simplecov_json_formatter (0.1.4) sha256=529418fbe8de1713ac2b2d612aa3daa56d316975d307244399fa4838c601b428
|
|
98
|
-
sqlite3 (2.9.4) sha256=6161c5b9c17886b289558e6c8082b28a22a814736d2433c9a67f4c6bfcde5c97
|
|
99
|
-
stringio (3.2.0) sha256=c37cb2e58b4ffbd33fe5cd948c05934af997b36e0b6ca6fdf43afa234cf222e1
|
|
100
|
-
tsort (0.2.0) sha256=9650a793f6859a43b6641671278f79cfead60ac714148aabe4e3f0060480089f
|
|
101
|
-
|
|
102
|
-
BUNDLED WITH
|
|
103
|
-
4.0.9
|