@riddledc/riddle-proof 0.5.20 → 0.5.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/runtime/lib/recon.py +50 -12
- package/runtime/lib/verify.py +47 -7
- package/runtime/tests/recon_verify_smoke.py +115 -0
package/package.json
CHANGED
package/runtime/lib/recon.py
CHANGED
|
@@ -11,7 +11,7 @@ with updated state inputs such as server_path or wait_for_selector.
|
|
|
11
11
|
"""
|
|
12
12
|
|
|
13
13
|
import json, os, re, sys, time
|
|
14
|
-
from urllib.parse import urlparse
|
|
14
|
+
from urllib.parse import parse_qsl, urlparse
|
|
15
15
|
|
|
16
16
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
17
17
|
from util import ( # noqa: E402
|
|
@@ -255,7 +255,7 @@ def route_candidates(route_hints, prod_url=''):
|
|
|
255
255
|
|
|
256
256
|
parsed_prod = urlparse((prod_url or '').strip())
|
|
257
257
|
if parsed_prod.path and parsed_prod.path.strip() and parsed_prod.path != '/':
|
|
258
|
-
add(parsed_prod.path, 'prod_url path')
|
|
258
|
+
add(parsed_prod.path + (('?' + parsed_prod.query) if parsed_prod.query else ''), 'prod_url path')
|
|
259
259
|
|
|
260
260
|
for hint in route_hints:
|
|
261
261
|
code_fragment = hint.split(':', 2)[-1]
|
|
@@ -424,17 +424,54 @@ def has_enriched_page_state(page_state):
|
|
|
424
424
|
|
|
425
425
|
|
|
426
426
|
def normalize_observed_path(value):
|
|
427
|
-
|
|
428
|
-
if not
|
|
427
|
+
raw = str(value or '').strip()
|
|
428
|
+
if not raw:
|
|
429
429
|
return ''
|
|
430
|
-
|
|
430
|
+
parsed = urlparse(raw.split('#', 1)[0])
|
|
431
|
+
path = parsed.path or ''
|
|
432
|
+
query = parsed.query or ''
|
|
431
433
|
if not path.startswith('/'):
|
|
432
|
-
|
|
433
|
-
path = parsed.path or path
|
|
434
|
+
path = '/' + path.lstrip('/')
|
|
434
435
|
parts = path.split('/')
|
|
435
436
|
if len(parts) >= 4 and parts[1] == 's':
|
|
436
437
|
path = '/' + '/'.join(parts[3:])
|
|
437
|
-
|
|
438
|
+
path = path.rstrip('/') or '/'
|
|
439
|
+
return path + (('?' + query) if query else '')
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
def observed_location_from_page_state(page_state):
|
|
443
|
+
if not isinstance(page_state, dict):
|
|
444
|
+
return ''
|
|
445
|
+
pathname = str(page_state.get('pathname') or '').strip()
|
|
446
|
+
search = str(page_state.get('search') or '').strip()
|
|
447
|
+
if search and not search.startswith('?'):
|
|
448
|
+
search = '?' + search
|
|
449
|
+
if pathname:
|
|
450
|
+
return pathname + search
|
|
451
|
+
return str(page_state.get('href') or '').strip()
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
def route_matches_expected(expected_path, observed_path):
|
|
455
|
+
expected = normalize_observed_path(expected_path)
|
|
456
|
+
observed = normalize_observed_path(observed_path)
|
|
457
|
+
if not expected or not observed:
|
|
458
|
+
return False
|
|
459
|
+
expected_parsed = urlparse(expected)
|
|
460
|
+
observed_parsed = urlparse(observed)
|
|
461
|
+
expected_pathname = expected_parsed.path.rstrip('/') or '/'
|
|
462
|
+
observed_pathname = observed_parsed.path.rstrip('/') or '/'
|
|
463
|
+
if observed_pathname != expected_pathname:
|
|
464
|
+
return False
|
|
465
|
+
expected_query = parse_qsl(expected_parsed.query, keep_blank_values=True)
|
|
466
|
+
if not expected_query:
|
|
467
|
+
return True
|
|
468
|
+
observed_query = parse_qsl(observed_parsed.query, keep_blank_values=True)
|
|
469
|
+
remaining = list(observed_query)
|
|
470
|
+
for pair in expected_query:
|
|
471
|
+
if pair not in remaining:
|
|
472
|
+
return False
|
|
473
|
+
remaining.remove(pair)
|
|
474
|
+
return True
|
|
438
475
|
|
|
439
476
|
|
|
440
477
|
def build_probe_capture_script(base_script='', screenshot_label=''):
|
|
@@ -471,6 +508,8 @@ def build_probe_capture_script(base_script='', screenshot_label=''):
|
|
|
471
508
|
' canvasCount: document.querySelectorAll("canvas").length,',
|
|
472
509
|
' largeVisibleElements,',
|
|
473
510
|
' pathname: window.location.pathname,',
|
|
511
|
+
' search: window.location.search,',
|
|
512
|
+
' href: window.location.href,',
|
|
474
513
|
' title: document.title,',
|
|
475
514
|
' };',
|
|
476
515
|
'});',
|
|
@@ -517,7 +556,7 @@ def evaluate_capture_quality(payload, expected_path):
|
|
|
517
556
|
|
|
518
557
|
page_state = extract_page_state(payload)
|
|
519
558
|
if isinstance(page_state, dict):
|
|
520
|
-
raw_observed_path = page_state
|
|
559
|
+
raw_observed_path = observed_location_from_page_state(page_state)
|
|
521
560
|
details.update({
|
|
522
561
|
'body_text_length': page_state.get('bodyTextLength', 0),
|
|
523
562
|
'interactive_elements': page_state.get('interactiveElements', 0),
|
|
@@ -567,8 +606,7 @@ def evaluate_capture_quality(payload, expected_path):
|
|
|
567
606
|
reasons.append('page has console/runtime errors')
|
|
568
607
|
|
|
569
608
|
observed_path = normalize_observed_path(details.get('observed_path'))
|
|
570
|
-
|
|
571
|
-
if expected_path and observed_path and observed_path != normalized_expected:
|
|
609
|
+
if expected_path and observed_path and not route_matches_expected(expected_path, observed_path):
|
|
572
610
|
raw_observed = details.get('observed_path_raw') or details.get('observed_path') or observed_path
|
|
573
611
|
reasons.append(f'wrong route: expected {expected_path}, got {raw_observed}')
|
|
574
612
|
|
|
@@ -703,7 +741,7 @@ def capture_prod_baseline(prod_url, plan, capture_script=''):
|
|
|
703
741
|
return {
|
|
704
742
|
'source': 'prod_url',
|
|
705
743
|
'mode': 'remote',
|
|
706
|
-
'path':
|
|
744
|
+
'path': normalize_observed_path(target_url) or (plan.get('target_path') or '/'),
|
|
707
745
|
'capture_url': target_url,
|
|
708
746
|
'url': extract_screenshot_url(shot, 'prod'),
|
|
709
747
|
'static_fallback_reason': '',
|
package/runtime/lib/verify.py
CHANGED
|
@@ -8,6 +8,7 @@ must assess before the wrapper routes back into author/implement/recon work or s
|
|
|
8
8
|
"""
|
|
9
9
|
|
|
10
10
|
import json, os, sys, time
|
|
11
|
+
from urllib.parse import parse_qsl, urlparse
|
|
11
12
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
12
13
|
from util import (
|
|
13
14
|
append_capture_diagnostic,
|
|
@@ -248,6 +249,8 @@ def build_probe_capture_script(base_script='', verification_mode='proof'):
|
|
|
248
249
|
' canvasCount: document.querySelectorAll("canvas").length,',
|
|
249
250
|
' largeVisibleElements,',
|
|
250
251
|
' pathname: window.location.pathname,',
|
|
252
|
+
' search: window.location.search,',
|
|
253
|
+
' href: window.location.href,',
|
|
251
254
|
' title: document.title,',
|
|
252
255
|
' };',
|
|
253
256
|
'});',
|
|
@@ -575,16 +578,54 @@ def has_enriched_page_state(page_state):
|
|
|
575
578
|
|
|
576
579
|
|
|
577
580
|
def normalize_observed_path(value):
|
|
578
|
-
|
|
579
|
-
if not
|
|
581
|
+
raw = str(value or '').strip()
|
|
582
|
+
if not raw:
|
|
580
583
|
return ''
|
|
581
|
-
|
|
584
|
+
parsed = urlparse(raw.split('#', 1)[0])
|
|
585
|
+
path = parsed.path or ''
|
|
586
|
+
query = parsed.query or ''
|
|
582
587
|
if not path.startswith('/'):
|
|
583
588
|
path = '/' + path.lstrip('/')
|
|
584
589
|
parts = path.split('/')
|
|
585
590
|
if len(parts) >= 4 and parts[1] == 's':
|
|
586
591
|
path = '/' + '/'.join(parts[3:])
|
|
587
|
-
|
|
592
|
+
path = path.rstrip('/') or '/'
|
|
593
|
+
return path + (('?' + query) if query else '')
|
|
594
|
+
|
|
595
|
+
|
|
596
|
+
def observed_location_from_page_state(page_state):
|
|
597
|
+
if not isinstance(page_state, dict):
|
|
598
|
+
return ''
|
|
599
|
+
pathname = str(page_state.get('pathname') or '').strip()
|
|
600
|
+
search = str(page_state.get('search') or '').strip()
|
|
601
|
+
if search and not search.startswith('?'):
|
|
602
|
+
search = '?' + search
|
|
603
|
+
if pathname:
|
|
604
|
+
return pathname + search
|
|
605
|
+
return str(page_state.get('href') or '').strip()
|
|
606
|
+
|
|
607
|
+
|
|
608
|
+
def route_matches_expected(expected_path, observed_path):
|
|
609
|
+
expected = normalize_observed_path(expected_path)
|
|
610
|
+
observed = normalize_observed_path(observed_path)
|
|
611
|
+
if not expected or not observed:
|
|
612
|
+
return False
|
|
613
|
+
expected_parsed = urlparse(expected)
|
|
614
|
+
observed_parsed = urlparse(observed)
|
|
615
|
+
expected_pathname = expected_parsed.path.rstrip('/') or '/'
|
|
616
|
+
observed_pathname = observed_parsed.path.rstrip('/') or '/'
|
|
617
|
+
if observed_pathname != expected_pathname:
|
|
618
|
+
return False
|
|
619
|
+
expected_query = parse_qsl(expected_parsed.query, keep_blank_values=True)
|
|
620
|
+
if not expected_query:
|
|
621
|
+
return True
|
|
622
|
+
observed_query = parse_qsl(observed_parsed.query, keep_blank_values=True)
|
|
623
|
+
remaining = list(observed_query)
|
|
624
|
+
for pair in expected_query:
|
|
625
|
+
if pair not in remaining:
|
|
626
|
+
return False
|
|
627
|
+
remaining.remove(pair)
|
|
628
|
+
return True
|
|
588
629
|
|
|
589
630
|
|
|
590
631
|
def collect_supporting_artifacts(payload):
|
|
@@ -767,7 +808,7 @@ def evaluate_capture_quality(payload, expected_path, verification_mode='proof'):
|
|
|
767
808
|
|
|
768
809
|
page_state = extract_page_state(payload)
|
|
769
810
|
if isinstance(page_state, dict):
|
|
770
|
-
raw_observed_path = page_state
|
|
811
|
+
raw_observed_path = observed_location_from_page_state(page_state)
|
|
771
812
|
details.update({
|
|
772
813
|
'body_text_length': page_state.get('bodyTextLength', 0),
|
|
773
814
|
'interactive_elements': page_state.get('interactiveElements', 0),
|
|
@@ -830,8 +871,7 @@ def evaluate_capture_quality(payload, expected_path, verification_mode='proof'):
|
|
|
830
871
|
reasons.append('page has console/runtime errors')
|
|
831
872
|
|
|
832
873
|
observed_path = normalize_observed_path(details.get('observed_path'))
|
|
833
|
-
|
|
834
|
-
if isinstance(page_state, dict) and expected_path and observed_path and observed_path != normalized_expected:
|
|
874
|
+
if isinstance(page_state, dict) and expected_path and observed_path and not route_matches_expected(expected_path, observed_path):
|
|
835
875
|
raw_observed = details.get('observed_path_raw') or details.get('observed_path') or observed_path
|
|
836
876
|
reasons.append(f'wrong route: expected {expected_path}, got {raw_observed}')
|
|
837
877
|
|
|
@@ -182,6 +182,27 @@ class FakeRiddle:
|
|
|
182
182
|
'outputs': [{'name': 'prod.png', 'url': 'https://cdn.example.com/prod.png'}],
|
|
183
183
|
'console': ['RIDDLE_PROOF_STATE:{"bodyTextLength":180,"interactiveElements":4,"pathname":"/pricing","title":"Prod"}'],
|
|
184
184
|
}
|
|
185
|
+
if 'prod.example.com/games/drum-sequencer' in script:
|
|
186
|
+
page_state = {
|
|
187
|
+
'bodyTextLength': 240,
|
|
188
|
+
'visibleTextSample': 'Neon Step Sequencer Monkberry Moon Delight Mix Board Play All',
|
|
189
|
+
'interactiveElements': 8,
|
|
190
|
+
'visibleInteractiveElements': 8,
|
|
191
|
+
'pathname': '/games/drum-sequencer',
|
|
192
|
+
'search': '?song=monkberry-moon-delight-tab&mix=profile',
|
|
193
|
+
'title': 'Neon Step Sequencer',
|
|
194
|
+
'buttons': ['Play All', 'Shuffle'],
|
|
195
|
+
'headings': ['Neon Step Sequencer'],
|
|
196
|
+
'links': [],
|
|
197
|
+
'canvasCount': 1,
|
|
198
|
+
'largeVisibleElements': [{'tag': 'canvas', 'text': ''}],
|
|
199
|
+
}
|
|
200
|
+
return {
|
|
201
|
+
'ok': True,
|
|
202
|
+
'screenshots': [{'url': 'https://cdn.example.com/sequencer-prod.png'}],
|
|
203
|
+
'outputs': [{'name': 'prod.png', 'url': 'https://cdn.example.com/sequencer-prod.png'}],
|
|
204
|
+
'console': ['RIDDLE_PROOF_STATE:' + json.dumps(page_state)],
|
|
205
|
+
}
|
|
185
206
|
if 'preview.example.com' in script and '/pricing' in script:
|
|
186
207
|
return {
|
|
187
208
|
'ok': True,
|
|
@@ -189,6 +210,27 @@ class FakeRiddle:
|
|
|
189
210
|
'outputs': [{'name': 'before.png', 'url': 'https://cdn.example.com/before.png'}],
|
|
190
211
|
'console': ['RIDDLE_PROOF_STATE:{"bodyTextLength":180,"interactiveElements":4,"pathname":"/pricing","title":"Before"}'],
|
|
191
212
|
}
|
|
213
|
+
if 'preview.example.com' in script and '/games/drum-sequencer' in script:
|
|
214
|
+
page_state = {
|
|
215
|
+
'bodyTextLength': 240,
|
|
216
|
+
'visibleTextSample': 'Neon Step Sequencer Monkberry Moon Delight Mix Board Play All',
|
|
217
|
+
'interactiveElements': 8,
|
|
218
|
+
'visibleInteractiveElements': 8,
|
|
219
|
+
'pathname': '/s/pv-before/games/drum-sequencer',
|
|
220
|
+
'search': '?song=monkberry-moon-delight-tab&mix=profile',
|
|
221
|
+
'title': 'Neon Step Sequencer',
|
|
222
|
+
'buttons': ['Play All', 'Shuffle'],
|
|
223
|
+
'headings': ['Neon Step Sequencer'],
|
|
224
|
+
'links': [],
|
|
225
|
+
'canvasCount': 1,
|
|
226
|
+
'largeVisibleElements': [{'tag': 'canvas', 'text': ''}],
|
|
227
|
+
}
|
|
228
|
+
return {
|
|
229
|
+
'ok': True,
|
|
230
|
+
'screenshots': [{'url': 'https://cdn.example.com/sequencer-before.png'}],
|
|
231
|
+
'outputs': [{'name': 'before.png', 'url': 'https://cdn.example.com/sequencer-before.png'}],
|
|
232
|
+
'console': state_console(page_state),
|
|
233
|
+
}
|
|
192
234
|
if 'preview.example.com' in script and '/games/tic-tac-toe' in script:
|
|
193
235
|
return {
|
|
194
236
|
'ok': True,
|
|
@@ -408,6 +450,35 @@ def run_verify_quality_ignores_proof_telemetry_console_text():
|
|
|
408
450
|
assert runtime_error_quality['details']['has_errors'] is True
|
|
409
451
|
assert 'console/runtime errors' in runtime_error_quality['reason']
|
|
410
452
|
|
|
453
|
+
query_payload = dict(telemetry_payload)
|
|
454
|
+
query_payload.update({
|
|
455
|
+
'visibleTextSample': 'Neon Step Sequencer Monkberry Moon Delight Mix Board Play All',
|
|
456
|
+
'pathname': '/s/pv-after/games/drum-sequencer',
|
|
457
|
+
'search': '?song=monkberry-moon-delight-tab&mix=profile&utm_source=test',
|
|
458
|
+
'title': 'Neon Step Sequencer',
|
|
459
|
+
'headings': ['Neon Step Sequencer'],
|
|
460
|
+
'buttons': ['Play All'],
|
|
461
|
+
'canvasCount': 1,
|
|
462
|
+
})
|
|
463
|
+
query_quality = namespace['evaluate_capture_quality']({
|
|
464
|
+
'ok': True,
|
|
465
|
+
'screenshots': [{'name': 'after-proof.png', 'url': 'https://cdn.example.com/after-proof.png'}],
|
|
466
|
+
'outputs': [{'name': 'after-proof.png', 'url': 'https://cdn.example.com/after-proof.png'}],
|
|
467
|
+
'console': ['RIDDLE_PROOF_STATE:' + json.dumps(query_payload)],
|
|
468
|
+
}, '/games/drum-sequencer?mix=profile&song=monkberry-moon-delight-tab', 'visual')
|
|
469
|
+
assert query_quality['valid'] is True, query_quality
|
|
470
|
+
assert query_quality['details']['observed_path'] == '/games/drum-sequencer?song=monkberry-moon-delight-tab&mix=profile&utm_source=test'
|
|
471
|
+
assert query_quality['details']['observed_path_raw'] == '/s/pv-after/games/drum-sequencer?song=monkberry-moon-delight-tab&mix=profile&utm_source=test'
|
|
472
|
+
|
|
473
|
+
missing_query_quality = namespace['evaluate_capture_quality']({
|
|
474
|
+
'ok': True,
|
|
475
|
+
'screenshots': [{'name': 'after-proof.png', 'url': 'https://cdn.example.com/after-proof.png'}],
|
|
476
|
+
'outputs': [{'name': 'after-proof.png', 'url': 'https://cdn.example.com/after-proof.png'}],
|
|
477
|
+
'console': ['RIDDLE_PROOF_STATE:' + json.dumps({**query_payload, 'search': '?song=monkberry-moon-delight-tab'})],
|
|
478
|
+
}, '/games/drum-sequencer?mix=profile&song=monkberry-moon-delight-tab', 'visual')
|
|
479
|
+
assert missing_query_quality['valid'] is False
|
|
480
|
+
assert 'wrong route' in missing_query_quality['reason']
|
|
481
|
+
|
|
411
482
|
strong_delta = namespace['extract_visual_delta']({
|
|
412
483
|
'ok': True,
|
|
413
484
|
'result': {
|
|
@@ -659,6 +730,49 @@ def run_recon_then_author_request():
|
|
|
659
730
|
shutil.rmtree(tempdir, ignore_errors=True)
|
|
660
731
|
|
|
661
732
|
|
|
733
|
+
def run_recon_preserves_query_route():
|
|
734
|
+
tempdir = Path(tempfile.mkdtemp(prefix='riddle-proof-query-route-'))
|
|
735
|
+
state_path = tempdir / 'state.json'
|
|
736
|
+
try:
|
|
737
|
+
query_route = '/games/drum-sequencer?song=monkberry-moon-delight-tab&mix=profile'
|
|
738
|
+
state = base_state(
|
|
739
|
+
tempdir,
|
|
740
|
+
reference='both',
|
|
741
|
+
prod_url='https://prod.example.com' + query_route,
|
|
742
|
+
)
|
|
743
|
+
state.update({
|
|
744
|
+
'server_path': query_route,
|
|
745
|
+
'change_request': 'Make a tiny harmless visible sequencer helper-copy change.',
|
|
746
|
+
'success_criteria': 'The Monkberry profile sequencer route is loaded and visible.',
|
|
747
|
+
})
|
|
748
|
+
write_state(state_path, state)
|
|
749
|
+
os.environ['RIDDLE_PROOF_STATE_FILE'] = str(state_path)
|
|
750
|
+
|
|
751
|
+
fake = FakeRiddle()
|
|
752
|
+
load_util_with_fake(fake)
|
|
753
|
+
load_module('recon_query_route', RECON_PATH)
|
|
754
|
+
after_recon = json.loads(state_path.read_text())
|
|
755
|
+
|
|
756
|
+
latest_attempt = after_recon['recon_results']['attempt_history'][-1]
|
|
757
|
+
before = latest_attempt['captured_baselines']['before']
|
|
758
|
+
prod = latest_attempt['captured_baselines']['prod']
|
|
759
|
+
assert before['path'] == query_route
|
|
760
|
+
assert prod['path'] == query_route
|
|
761
|
+
assert before['observation']['valid'] is True, before['observation']
|
|
762
|
+
assert prod['observation']['valid'] is True, prod['observation']
|
|
763
|
+
assert before['observation']['details']['observed_path'] == query_route
|
|
764
|
+
assert prod['observation']['details']['observed_path'] == query_route
|
|
765
|
+
assert 'wrong route' not in before['observation']['reason']
|
|
766
|
+
assert 'wrong route' not in prod['observation']['reason']
|
|
767
|
+
return {
|
|
768
|
+
'ok': True,
|
|
769
|
+
'before_observed_path': before['observation']['details']['observed_path'],
|
|
770
|
+
'prod_observed_path': prod['observation']['details']['observed_path'],
|
|
771
|
+
}
|
|
772
|
+
finally:
|
|
773
|
+
shutil.rmtree(tempdir, ignore_errors=True)
|
|
774
|
+
|
|
775
|
+
|
|
662
776
|
def run_recon_prefers_route_literals_over_import_paths():
|
|
663
777
|
tempdir = Path(tempfile.mkdtemp(prefix='riddle-proof-route-literals-'))
|
|
664
778
|
state_path = tempdir / 'state.json'
|
|
@@ -1369,6 +1483,7 @@ if __name__ == '__main__':
|
|
|
1369
1483
|
'implement_records_detection_when_changes_missing': run_implement_records_detection_when_changes_missing(),
|
|
1370
1484
|
'verify_quality_ignores_proof_telemetry_console_text': run_verify_quality_ignores_proof_telemetry_console_text(),
|
|
1371
1485
|
'recon_then_author_request': run_recon_then_author_request(),
|
|
1486
|
+
'recon_preserves_query_route': run_recon_preserves_query_route(),
|
|
1372
1487
|
'recon_route_literal_preference': run_recon_prefers_route_literals_over_import_paths(),
|
|
1373
1488
|
'recon_hint_root_preference': run_recon_prefers_hint_root_over_single_route_literal(),
|
|
1374
1489
|
'author_applies_supervisor_packet': run_author_applies_supervisor_packet(),
|