optbrew 0.1.3 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/.codex-plugin/plugin.json +2 -2
- package/AGENT_INSTALL.md +13 -11
- package/CHANGELOG.md +52 -0
- package/README.ko.md +40 -15
- package/README.md +40 -15
- package/THIRD_PARTY_NOTICES.md +88 -0
- package/docs/platforms.md +10 -10
- package/docs/release.md +37 -5
- package/lib/cli.mjs +5 -2
- package/lib/install.mjs +2 -1
- package/lib/records.mjs +2 -1
- package/lib/transaction.mjs +12 -5
- package/lib/update.mjs +3 -2
- package/package.json +6 -4
- package/plugin.json +2 -2
- package/skills/.claude-plugin/icon.png +0 -0
- package/skills/.claude-plugin/plugin.json +23 -0
- package/skills/README.md +27 -0
- package/skills/optbrew/SKILL.md +223 -8
- package/skills/optbrew/THIRD_PARTY_NOTICES.md +88 -0
- package/skills/optbrew/bundle-manifest.json +407 -47
- package/skills/optbrew/modules/naver-searchadvisor/scripts/audit.mjs +4 -4
- package/skills/optbrew/references/api.md +5 -3
- package/skills/optbrew/references/check-registry.json +16 -16
- package/skills/optbrew/references/chrome-paths.json +31 -0
- package/skills/optbrew/references/endpoints.json +10 -0
- package/skills/optbrew/references/harness-signals.json +5 -0
- package/skills/optbrew/references/intake.md +76 -5
- package/skills/optbrew/references/judging.md +233 -0
- package/skills/optbrew/references/limits.json +26 -0
- package/skills/optbrew/references/measurement.md +1 -1
- package/skills/optbrew/references/messages.ko.json +170 -0
- package/skills/optbrew/references/report.md +18 -6
- package/skills/optbrew/references/routing.md +2 -2
- package/skills/optbrew/references/runbooks/anthropic-api-key.md +34 -0
- package/skills/optbrew/references/runbooks/chrome-install.md +56 -0
- package/skills/optbrew/references/runbooks/cwv-site-collection.md +38 -0
- package/skills/optbrew/references/runbooks/google-api-key.md +51 -0
- package/skills/optbrew/references/runbooks/gsc-export.md +32 -0
- package/skills/optbrew/references/runbooks/index.json +18 -0
- package/skills/optbrew/references/runbooks/key-set-claude-cloud.md +32 -0
- package/skills/optbrew/references/runbooks/key-set-codex.md +33 -0
- package/skills/optbrew/references/runbooks/key-set-local.md +40 -0
- package/skills/optbrew/references/runbooks/naver-searchadvisor-export.md +32 -0
- package/skills/optbrew/references/runbooks/network-claude-cloud-custom.md +33 -0
- package/skills/optbrew/references/runbooks/network-codex-approval.md +26 -0
- package/skills/optbrew/references/runbooks/network-codex-config.md +36 -0
- package/skills/optbrew/references/runbooks/seranking.md +38 -0
- package/skills/optbrew/references/schemas/contract.schema.json +42 -15
- package/skills/optbrew/references/scoring.md +40 -5
- package/skills/optbrew/references/source-analysis.md +40 -0
- package/skills/optbrew/references/versions.json +9 -0
- package/skills/optbrew/scripts/adapters/anthropic.mjs +5 -5
- package/skills/optbrew/scripts/adapters/imports.mjs +138 -1
- package/skills/optbrew/scripts/audit.mjs +10 -112
- package/skills/optbrew/scripts/cli.mjs +5 -2
- package/skills/optbrew/scripts/core/bundle.mjs +30 -13
- package/skills/optbrew/scripts/core/capabilities.mjs +8 -4
- package/skills/optbrew/scripts/core/cdp.mjs +105 -0
- package/skills/optbrew/scripts/core/checks/canonical-consistency.mjs +56 -0
- package/skills/optbrew/scripts/core/checks/cwv-field.mjs +77 -0
- package/skills/optbrew/scripts/core/checks/discovery.mjs +43 -0
- package/skills/optbrew/scripts/core/checks/document-title.mjs +62 -0
- package/skills/optbrew/scripts/core/checks/entity-identity.mjs +31 -0
- package/skills/optbrew/scripts/core/checks/fact-consistency.mjs +31 -0
- package/skills/optbrew/scripts/core/checks/fetch-response.mjs +18 -0
- package/skills/optbrew/scripts/core/checks/index-policy.mjs +16 -0
- package/skills/optbrew/scripts/core/checks/index.mjs +18 -0
- package/skills/optbrew/scripts/core/checks/internal-navigation.mjs +67 -0
- package/skills/optbrew/scripts/core/checks/locale-links.mjs +55 -0
- package/skills/optbrew/scripts/core/checks/naver-markup.mjs +21 -0
- package/skills/optbrew/scripts/core/checks/observation.mjs +37 -0
- package/skills/optbrew/scripts/core/checks/prepare.mjs +103 -0
- package/skills/optbrew/scripts/core/checks/render-content.mjs +104 -0
- package/skills/optbrew/scripts/core/checks/search-crawl-policy.mjs +29 -0
- package/skills/optbrew/scripts/core/checks/structured-data-truth.mjs +46 -0
- package/skills/optbrew/scripts/core/chrome-temp.mjs +68 -0
- package/skills/optbrew/scripts/core/chrome.mjs +546 -0
- package/skills/optbrew/scripts/core/collector.mjs +45 -49
- package/skills/optbrew/scripts/core/compare.mjs +18 -1
- package/skills/optbrew/scripts/core/config.mjs +31 -0
- package/skills/optbrew/scripts/core/contracts.mjs +31 -2
- package/skills/optbrew/scripts/core/cwv.mjs +100 -0
- package/skills/optbrew/scripts/core/derive.mjs +10 -186
- package/skills/optbrew/scripts/core/evidence-store.mjs +112 -0
- package/skills/optbrew/scripts/core/facts.mjs +175 -17
- package/skills/optbrew/scripts/core/findings.mjs +19 -1
- package/skills/optbrew/scripts/core/html.mjs +41 -3
- package/skills/optbrew/scripts/core/index-targets.mjs +118 -0
- package/skills/optbrew/scripts/core/judgments.mjs +675 -0
- package/skills/optbrew/scripts/core/keys.mjs +86 -0
- package/skills/optbrew/scripts/core/link-graph.mjs +93 -0
- package/skills/optbrew/scripts/core/messages.mjs +23 -0
- package/skills/optbrew/scripts/core/metrics.mjs +4 -2
- package/skills/optbrew/scripts/core/migration.mjs +37 -0
- package/skills/optbrew/scripts/core/page-select.mjs +136 -0
- package/skills/optbrew/scripts/core/probe.mjs +83 -0
- package/skills/optbrew/scripts/core/protocol.mjs +38 -14
- package/skills/optbrew/scripts/core/render.mjs +160 -0
- package/skills/optbrew/scripts/core/report-compose.mjs +377 -0
- package/skills/optbrew/scripts/core/report-html.mjs +198 -0
- package/skills/optbrew/scripts/core/report-page.mjs +14 -0
- package/skills/optbrew/scripts/core/report.mjs +327 -122
- package/skills/optbrew/scripts/core/runbooks.mjs +63 -0
- package/skills/optbrew/scripts/core/scope.mjs +23 -5
- package/skills/optbrew/scripts/core/score.mjs +12 -4
- package/skills/optbrew/scripts/core/secrets.mjs +2 -0
- package/skills/optbrew/scripts/core/stages.mjs +694 -0
- package/skills/optbrew/scripts/core/transport.mjs +157 -0
- package/skills/optbrew/scripts/core/weights.mjs +2 -1
- package/skills/optbrew/scripts/key.mjs +90 -0
- package/skills/optbrew/scripts/preflight.mjs +176 -0
- package/skills/optbrew/scripts/run.mjs +81 -0
- package/skills/optbrew/source-lock.json +1 -1
- package/source-lock.json +1 -1
|
@@ -1,27 +1,32 @@
|
|
|
1
|
-
import { createHash } from 'node:crypto';
|
|
2
1
|
import { readFileSync, realpathSync } from 'node:fs';
|
|
3
2
|
import { access, mkdir, rename, rm, writeFile } from 'node:fs/promises';
|
|
4
3
|
import path from 'node:path';
|
|
5
4
|
import { ContractError, validateEntity } from './contracts.mjs';
|
|
5
|
+
import { endpoints, limits, requestLimit, versions } from './config.mjs';
|
|
6
6
|
import { score } from './score.mjs';
|
|
7
|
+
import { assertCurrentEvidence, MigrationRequired } from './migration.mjs';
|
|
7
8
|
import { compare } from './compare.mjs';
|
|
8
9
|
import { freezeProtocol } from './protocol.mjs';
|
|
9
10
|
import { measurePanel } from './metrics.mjs';
|
|
10
11
|
import { normalizeImport } from '../adapters/imports.mjs';
|
|
11
|
-
import { mergeFindings, selectRemediations } from './findings.mjs';
|
|
12
|
+
import { mergeFindings, pageNeutral, problemsOf, selectRemediations } from './findings.mjs';
|
|
12
13
|
import { auditCollected, indexingDirectives } from '../../modules/naver-searchadvisor/scripts/audit.mjs';
|
|
13
14
|
import { deriveObservations } from './derive.mjs';
|
|
15
|
+
import { isDerivedIndexDecision, notIndexTargets } from './index-targets.mjs';
|
|
16
|
+
import { readVerified } from './evidence-store.mjs';
|
|
14
17
|
import { links } from './collector.mjs';
|
|
18
|
+
import { checkRenderLedger } from './render.mjs';
|
|
15
19
|
import { assertUrlInScope } from './scope.mjs';
|
|
20
|
+
import { composeReport, pagesLabel, renderReport } from './report-compose.mjs';
|
|
21
|
+
import { messageOf } from './messages.mjs';
|
|
22
|
+
import { runbookEntry } from './runbooks.mjs';
|
|
23
|
+
import { sensitive } from './secrets.mjs';
|
|
24
|
+
import { ENTITY_UNIT, OBSERVATION_SOURCES, QUEUE_KIND, QUESTION_UNIT, acceptance, alsoQuestionIds, basisAllowed, candidateSpans, modelFailMessages, sentenceBasis, checkSpan, deriveQuestionCoverage, homeOf, identityBasisOf, isCodeJudged, isEmptyPage, isEntityFallback, isModelJudged, operatorNames, queuedSpan, siteChrome } from './judgments.mjs';
|
|
16
25
|
import { parseRobots, robotsVerdict } from '../../modules/naver-searchadvisor/scripts/robots.mjs';
|
|
17
26
|
|
|
18
27
|
const catalog = JSON.parse(readFileSync(new URL('../../references/remediations.json', import.meta.url))).items;
|
|
19
28
|
const same = (a, b) => JSON.stringify(a) === JSON.stringify(b);
|
|
20
29
|
const named = value => typeof value === 'string' && value.trim().length > 0;
|
|
21
|
-
const sensitive = /(?:sk-ant-api03-[A-Za-z0-9_-]{32,}|sk-proj-[A-Za-z0-9_-]{32,}|ghp_[A-Za-z0-9]{36,}|AKIA[A-Z0-9]{16}|\b\d{6}-[1-8]\d{6}\b)/;
|
|
22
|
-
const html = value => String(value ?? 'unknown').replace(/\\/g, '\\\\').replace(/&/g, '&').replace(/</g, '<')
|
|
23
|
-
.replace(/>/g, '>').replace(/\|/g, '\\|').replace(/\r?\n/g, '<br>');
|
|
24
|
-
const table = (columns, rows) => `| ${columns.join(' | ')} |\n| ${columns.map(() => '---').join(' | ')} |\n${rows.map(row => `| ${row.map(html).join(' | ')} |`).join('\n') || `| ${columns.map(() => '—').join(' | ')} |`}`;
|
|
25
30
|
const days = window => (Date.parse(`${window.end}T00:00:00Z`) - Date.parse(`${window.start}T00:00:00Z`)) / 86400000 + 1;
|
|
26
31
|
const dayBefore = date => new Date(Date.parse(`${date}T00:00:00Z`) - 86400000).toISOString().slice(0, 10);
|
|
27
32
|
function completeWindow(metric) {
|
|
@@ -36,28 +41,6 @@ function completeWindow(metric) {
|
|
|
36
41
|
return completeDate <= dayBefore(localCollectionDate) && metric.window.end <= completeDate;
|
|
37
42
|
}
|
|
38
43
|
|
|
39
|
-
/** The optbrew operating rule (design spec, scoring.md): at least 80% of the applicable weight must be measured. */
|
|
40
|
-
export const COVERAGE_THRESHOLD = 80;
|
|
41
|
-
|
|
42
|
-
/** One headline per profile. Below the coverage threshold the single number is withheld and only the measured subset is shown. */
|
|
43
|
-
export function scoreHeadline(result) {
|
|
44
|
-
const exact = result.A > 0 ? 100 * result.M / result.A : null;
|
|
45
|
-
const coverageOk = result.score !== null && exact !== null && exact >= COVERAGE_THRESHOLD - 1e-9;
|
|
46
|
-
// The design spec also asks for verdicts on the core access and indexing checks before a score counts as sufficient.
|
|
47
|
-
const coreOk = (result.unknown_core ?? []).length === 0;
|
|
48
|
-
const status = !coverageOk ? 'insufficient_coverage' : !coreOk ? 'insufficient_core' : 'sufficient';
|
|
49
|
-
return { profile: result.profile, status, headline_score: status === 'sufficient' ? result.score : null,
|
|
50
|
-
measured_subset_score: result.score, coverage: result.coverage, coverage_exact: exact, coverage_threshold: COVERAGE_THRESHOLD,
|
|
51
|
-
unresolved_core: (result.unknown_core ?? []).length, missing_bounds: result.missing_bounds, readiness_status: result.readiness_status };
|
|
52
|
-
}
|
|
53
|
-
const floorTenth = value => value === null ? 'unknown' : Math.floor(value * 10 + 1e-9) / 10;
|
|
54
|
-
const headlineCell = result => {
|
|
55
|
-
const headline = scoreHeadline(result);
|
|
56
|
-
return headline.status === 'sufficient' ? `${headline.headline_score} (coverage ${headline.coverage}%)` :
|
|
57
|
-
headline.status === 'insufficient_core' ? `insufficient core checks (${headline.unresolved_core} unresolved; measured subset ${headline.measured_subset_score ?? 'unknown'}; coverage ${headline.coverage ?? 'unknown'}%; bounds ${headline.missing_bounds?.join('–') ?? 'unknown'})` :
|
|
58
|
-
`insufficient coverage (measured subset ${headline.measured_subset_score ?? 'unknown'}; coverage ${headline.coverage ?? 'unknown'}%; bounds ${headline.missing_bounds?.join('–') ?? 'unknown'})`;
|
|
59
|
-
};
|
|
60
|
-
|
|
61
44
|
function metricRows(metrics) {
|
|
62
45
|
const groups = new Map();
|
|
63
46
|
for (const metric of metrics) {
|
|
@@ -97,18 +80,16 @@ function indexedEvidence(packet) {
|
|
|
97
80
|
if (!entry || typeof entry.ref !== 'string' || !entry.ref || !/^[a-f0-9]{64}$/.test(entry.sha256) ||
|
|
98
81
|
typeof entry.kind !== 'string' || !entry.kind || entry.run_id !== packet.run_context.run_id || index.has(entry.ref))
|
|
99
82
|
throw new ContractError('invalid or duplicate run evidence entry');
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
const bytes = readFileSync(actual);
|
|
83
|
+
let bytes;
|
|
84
|
+
try { bytes = readVerified(root, entry.ref, entry.sha256); }
|
|
85
|
+
catch (error) { throw new ContractError(error.message); }
|
|
104
86
|
if (sensitive.test(bytes.toString('utf8'))) throw new ContractError('private identifier or secret in report evidence');
|
|
105
|
-
|
|
106
|
-
throw new ContractError(`evidence hash mismatch: ${entry.ref}`);
|
|
107
|
-
index.set(entry.ref, { ...entry, text: bytes.toString('utf8') });
|
|
87
|
+
index.set(entry.ref, { ...entry, text: bytes.toString('utf8'), bytes });
|
|
108
88
|
}
|
|
109
|
-
|
|
89
|
+
/** `source` is the `source_ref` the entry must carry, or `{ anySource: true }` for evidence cited from another source. */
|
|
90
|
+
const requireRef = (ref, source) => {
|
|
110
91
|
const entry = index.get(ref);
|
|
111
|
-
if (!entry ||
|
|
92
|
+
if (!entry || source && !source.anySource && entry.source_ref !== source) throw new ContractError(`evidence/source link missing: ${ref}`);
|
|
112
93
|
return entry;
|
|
113
94
|
};
|
|
114
95
|
const context = requireRef(packet.run_context.evidence_ref);
|
|
@@ -168,8 +149,9 @@ function exportPath(value, host) {
|
|
|
168
149
|
}
|
|
169
150
|
if (!text.startsWith('/') || /^\/[/\\]/.test(text)) return null;
|
|
170
151
|
try {
|
|
171
|
-
const
|
|
172
|
-
|
|
152
|
+
const base = new URL(endpoints().relative_url_base);
|
|
153
|
+
const url = new URL(text, base);
|
|
154
|
+
return url.host === base.host ? url.pathname : null;
|
|
173
155
|
} catch { return null; }
|
|
174
156
|
}
|
|
175
157
|
const pathWithin = (value, prefix, host) => {
|
|
@@ -188,8 +170,8 @@ function boundFailure(artifact, audit, requireRef) {
|
|
|
188
170
|
const raw = requireRef(artifact.failure_ref, 'optbrew:collector');
|
|
189
171
|
if (raw.sha256 !== entry.content_hash) throw new ContractError('failure record hash differs from the audit index');
|
|
190
172
|
const record = JSON.parse(raw.text);
|
|
191
|
-
// At most
|
|
192
|
-
if (record.kind !== 'failure' || ![0, 1].includes(record.unanswered) || !Array.isArray(record.redirect_chain) || record.redirect_chain.length >
|
|
173
|
+
// At most `max_redirects` redirects are followed, and the answer after the last one (a refused redirect) is the last hop that can be recorded.
|
|
174
|
+
if (record.kind !== 'failure' || ![0, 1].includes(record.unanswered) || !Array.isArray(record.redirect_chain) || record.redirect_chain.length > limits().max_redirects + 1 ||
|
|
193
175
|
record.redirect_chain.some((hop, index) => typeof hop?.url !== 'string' || !(hop.status >= 300 && hop.status <= 399) ||
|
|
194
176
|
hop.url !== (index ? record.redirect_chain[index - 1].location : record.normalized_url)))
|
|
195
177
|
throw new ContractError('failure record is malformed');
|
|
@@ -234,13 +216,31 @@ function checkDecisions(input, requireRef, collectedFor) {
|
|
|
234
216
|
item.condition === decision.condition && item.reason === decision.reason))
|
|
235
217
|
throw new ContractError(`applicability decision record does not match ${scope} ${identity.value}`);
|
|
236
218
|
};
|
|
237
|
-
for (const type of input.run_context.protocol.page_types) for (const url of type.urls)
|
|
238
|
-
if (url.indexing_decision)
|
|
219
|
+
for (const type of input.run_context.protocol.page_types) for (const url of type.urls) {
|
|
220
|
+
if (!url.indexing_decision) continue;
|
|
221
|
+
// A page the audit found deliberately not indexed has no approval record: checkAuditPhase derives the decision again from the
|
|
222
|
+
// stored bytes, which exist only for an audited phase.
|
|
223
|
+
if (!isDerivedIndexDecision(url.indexing_decision)) decisionRecord(url.indexing_decision, 'url', { key: 'normalized_url', value: url.normalized });
|
|
224
|
+
else if (input.run_context.audited !== true) throw new ContractError(`indexing decision of ${url.normalized} is derived from an audit this phase does not have`);
|
|
225
|
+
}
|
|
239
226
|
for (const family of input.run_context.protocol.families)
|
|
240
227
|
if (family.applicability_decision) decisionRecord(family.applicability_decision, 'family', { key: 'family_id', value: family.family_id });
|
|
241
228
|
for (const observation of input.observations) {
|
|
242
|
-
|
|
243
|
-
|
|
229
|
+
// A model's verdict cites evidence the collector stored; modelJudgmentRows binds it to that page's raw evidence.
|
|
230
|
+
const anySource = observation.evaluator_type === 'host_model' || observation.source_ref === OBSERVATION_SOURCES.derived;
|
|
231
|
+
const evidence = requireRef(observation.evidence_ref, anySource ? { anySource: true } : observation.source_ref);
|
|
232
|
+
// A collector verdict's own N/A decision cites the page evidence it was derived from, and replay re-derives it (the audit
|
|
233
|
+
// phase compares the decision too). A model's N/A decision cites its verdict's page evidence the same way; its condition is one of
|
|
234
|
+
// the family's registry conditions (the schema check) and its spans are replayed against the judge queue (modelJudgmentRows).
|
|
235
|
+
// Only an outside decision needs an approved decision record.
|
|
236
|
+
if (observation.applicability_decision) {
|
|
237
|
+
const own = observation.evaluator_type === 'code' && observation.source_ref === OBSERVATION_SOURCES.code ||
|
|
238
|
+
observation.evaluator_type === 'host_model' && observation.source_ref === OBSERVATION_SOURCES.host_model;
|
|
239
|
+
if (own) {
|
|
240
|
+
if (observation.applicability_decision.evidence_ref !== observation.evidence_ref)
|
|
241
|
+
throw new ContractError(`N/A decision does not cite the verdict's evidence: ${observation.unit_key}`);
|
|
242
|
+
} else decisionRecord(observation.applicability_decision, 'unit', { key: 'unit_key', value: observation.unit_key });
|
|
243
|
+
}
|
|
244
244
|
if (observation.check_id === 'index-policy' && observation.state === 'fail') {
|
|
245
245
|
if (evidence.kind !== 'raw_html' && evidence.kind !== 'http_response')
|
|
246
246
|
throw new ContractError('index-policy failure requires raw HTML or response-header evidence');
|
|
@@ -336,10 +336,16 @@ function checkMetricEvidence(metric, packet, requireRef) {
|
|
|
336
336
|
observed = measurePanel(source.panel, source.responses, source.evaluations);
|
|
337
337
|
} else throw new ContractError('unsupported metric evidence kind');
|
|
338
338
|
if (!observed.some(item => same(item, metric))) throw new ContractError('metric differs from bound export or panel evidence');
|
|
339
|
-
return source.kind === 'native_export' ? 'owner export origin and provider completeness are not independently authenticated' :
|
|
340
|
-
'provider response origin and semantic evaluations are attributed to indexed receipts, not independently authenticated';
|
|
339
|
+
return source.kind === 'native_export' ? ['owner export origin and provider completeness are not independently authenticated', 'metric-origin.owner-export'] :
|
|
340
|
+
['provider response origin and semantic evaluations are attributed to indexed receipts, not independently authenticated', 'metric-origin.panel'];
|
|
341
341
|
}
|
|
342
342
|
|
|
343
|
+
/**
|
|
344
|
+
* A verdict that replay re-derived against its phase's audit snapshot: the collector's own pair, in a phase whose context
|
|
345
|
+
* records an audit (the report refuses a packet whose snapshots differ from that record). Only such a verdict can verify a change.
|
|
346
|
+
*/
|
|
347
|
+
const replayed = (obs, input) => input.run_context.audited === true && obs.evaluator_type === 'code' && obs.source_ref === OBSERVATION_SOURCES.code;
|
|
348
|
+
|
|
343
349
|
function statusFor(finding, changes, afterInput, target, requireRef) {
|
|
344
350
|
const linked = changes.filter(change => change.finding_ids.some(id => finding.finding_ids.includes(id)));
|
|
345
351
|
if (!linked.length) return finding.remediation_ids.length ? 'proposed_only' :
|
|
@@ -350,7 +356,7 @@ function statusFor(finding, changes, afterInput, target, requireRef) {
|
|
|
350
356
|
afterInput?.run_context.environment === 'production' && linked.every(change => afterInput.observations.some(obs => {
|
|
351
357
|
if (obs.check_id !== finding.check_id || obs.unit_key !== finding.unit_key ||
|
|
352
358
|
!['pass', 'partial'].includes(obs.state) || obs.observation_id !== change.verification.observation_id ||
|
|
353
|
-
obs.evidence_ref !== change.verification.observation_evidence_ref) return false;
|
|
359
|
+
obs.evidence_ref !== change.verification.observation_evidence_ref || !replayed(obs, afterInput)) return false;
|
|
354
360
|
const source = requireRef(obs.evidence_ref, obs.source_ref);
|
|
355
361
|
return source.phase === 'after' && source.environment === 'production' && same(source.target, target);
|
|
356
362
|
}));
|
|
@@ -363,7 +369,7 @@ function statusFor(finding, changes, afterInput, target, requireRef) {
|
|
|
363
369
|
linked.every(change => change.environment === 'local' && change.verification.observation_id &&
|
|
364
370
|
change.verification.observation_evidence_ref && afterInput.observations.some(obs => {
|
|
365
371
|
if (obs.check_id !== finding.check_id || obs.unit_key !== finding.unit_key || !['pass', 'partial'].includes(obs.state) ||
|
|
366
|
-
obs
|
|
372
|
+
!replayed(obs, afterInput) || obs.observation_id !== change.verification.observation_id ||
|
|
367
373
|
obs.evidence_ref !== change.verification.observation_evidence_ref) return false;
|
|
368
374
|
const source = requireRef(obs.evidence_ref, obs.source_ref);
|
|
369
375
|
return source.phase === 'after' && source.environment === 'local' && same(source.target, target);
|
|
@@ -381,7 +387,7 @@ function checkLedger(input, audit, collection, artifacts, requireRef) {
|
|
|
381
387
|
const entries = ledger.entries;
|
|
382
388
|
const fail = message => { throw new ContractError(`request ledger: ${message}`); };
|
|
383
389
|
if (!Array.isArray(entries)) fail('entries missing');
|
|
384
|
-
const limit =
|
|
390
|
+
const limit = requestLimit(input.run_context.scope);
|
|
385
391
|
if (ledger.limit !== limit) fail('request limit differs from the run scope');
|
|
386
392
|
const total = entries.reduce((sum, entry) => sum + entry.requests, 0);
|
|
387
393
|
// The limit is a hard ceiling: every request, redirect hops included, takes one slot before it is sent.
|
|
@@ -440,6 +446,50 @@ function checkLedger(input, audit, collection, artifacts, requireRef) {
|
|
|
440
446
|
entries.some(x => x.requests > 0 && !planned.has(x.url))) fail('a selected page is skipped for the request limit although a discovered page was fetched');
|
|
441
447
|
}
|
|
442
448
|
|
|
449
|
+
/**
|
|
450
|
+
* The render records of an audit index, read through the run's evidence (hash and secret checked): every record against the
|
|
451
|
+
* hash the index lists, its DOM against the hash the record holds, and the render ledger against each record's own requests.
|
|
452
|
+
* An index from before rendering has none of these sections. `renderTextOf` returns the stored DOM of a record.
|
|
453
|
+
*/
|
|
454
|
+
function storedRenders(audit, renderBudget, requireRef) {
|
|
455
|
+
const fail = message => { throw new ContractError(message); };
|
|
456
|
+
const dom = new Map();
|
|
457
|
+
const records = (audit.renders ?? []).map(entry => {
|
|
458
|
+
const stored = requireRef(entry.evidence_ref);
|
|
459
|
+
if (stored.sha256 !== entry.sha256) fail('render record hash differs from the audit index');
|
|
460
|
+
let record;
|
|
461
|
+
try { record = JSON.parse(stored.text); } catch { fail('render record is not JSON'); }
|
|
462
|
+
if (typeof record.url !== 'string' || !Array.isArray(record.requests)) fail('render record is malformed');
|
|
463
|
+
// The ref it was read from, as the audit's own reading gives it, so a verdict inherited from a rendered page cites the same record.
|
|
464
|
+
record.evidence_ref = entry.evidence_ref;
|
|
465
|
+
if (record.dom_ref) {
|
|
466
|
+
const rendered = requireRef(record.dom_ref);
|
|
467
|
+
if (rendered.sha256 !== record.dom_sha256) fail('render DOM hash differs from its record');
|
|
468
|
+
dom.set(record, rendered.text);
|
|
469
|
+
}
|
|
470
|
+
return record;
|
|
471
|
+
});
|
|
472
|
+
const ledger = audit.render_ledger ?? [];
|
|
473
|
+
for (const record of records) checkRenderLedger(ledger, record, renderBudget);
|
|
474
|
+
if (ledger.some(entry => !records.some(record => record.url === entry.page))) fail('render ledger: an entry names a page that was not rendered');
|
|
475
|
+
return { records, renderTextOf: record => dom.get(record) ?? '' };
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
/**
|
|
479
|
+
* The field CWV answers of an audit index, read through the run's evidence (hash and secret checked), each one with the ref
|
|
480
|
+
* it was read from. An index from before field data has no `cwv` section.
|
|
481
|
+
*/
|
|
482
|
+
function storedCwv(audit, requireRef) {
|
|
483
|
+
return (audit.cwv ?? []).map(entry => {
|
|
484
|
+
const stored = requireRef(entry.evidence_ref);
|
|
485
|
+
if (stored.sha256 !== entry.sha256) throw new ContractError('CWV record hash differs from the audit index');
|
|
486
|
+
let record;
|
|
487
|
+
try { record = JSON.parse(stored.text); } catch { throw new ContractError('CWV record is not JSON'); }
|
|
488
|
+
if (record.source !== entry.source || typeof record.origin !== 'string') throw new ContractError('CWV record is malformed');
|
|
489
|
+
return { ...record, evidence_ref: entry.evidence_ref };
|
|
490
|
+
});
|
|
491
|
+
}
|
|
492
|
+
|
|
443
493
|
function checkAuditPhase(input, phase, snapshot, requireRef) {
|
|
444
494
|
if (!input) return;
|
|
445
495
|
const observations = input.observations.filter(x => x.source_ref === 'optbrew:collector');
|
|
@@ -455,7 +505,7 @@ function checkAuditPhase(input, phase, snapshot, requireRef) {
|
|
|
455
505
|
if (!Array.isArray(collection.artifacts) || !Array.isArray(audit.raw) || !Array.isArray(savedObservations))
|
|
456
506
|
throw new ContractError('audit snapshot files incomplete');
|
|
457
507
|
if (!Array.isArray(audit.modules) || !audit.ledger_ref || !Array.isArray(audit.responses) || !Array.isArray(audit.failures))
|
|
458
|
-
throw new
|
|
508
|
+
throw new MigrationRequired('report packet', 'comes from an older optbrew release (it lacks the request ledger, module list or failure records)');
|
|
459
509
|
for (const name of audit.modules) if (!Array.isArray(moduleOutputs[name])) throw new ContractError(`planned module output missing: ${name}`);
|
|
460
510
|
const naverPlanned = audit.modules.includes('naver-searchadvisor');
|
|
461
511
|
const rawRefs = new Set();
|
|
@@ -470,9 +520,21 @@ function checkAuditPhase(input, phase, snapshot, requireRef) {
|
|
|
470
520
|
throw new ContractError('audit raw content hash mismatch');
|
|
471
521
|
rawRefs.add(raw.evidence_ref);
|
|
472
522
|
}
|
|
523
|
+
// Field CWV answers are collected evidence too; their hashes are checked where the verdicts are re-derived below.
|
|
524
|
+
for (const entry of audit.cwv ?? []) rawRefs.add(entry.evidence_ref);
|
|
525
|
+
// So are the render records: a verdict inherited from a rendered page cites that page's record (the replay compares the exact ref).
|
|
526
|
+
for (const entry of audit.renders ?? []) rawRefs.add(entry.evidence_ref);
|
|
473
527
|
// Final URL, redirect chain and header lines come from the hashed response records, not from collection.json alone.
|
|
474
528
|
const artifacts = collection.artifacts.map(x => boundArtifact(x, audit, requireRef));
|
|
475
529
|
checkLedger(input, audit, collection, artifacts, requireRef);
|
|
530
|
+
// A page marked as deliberately not indexed must be one the stored bytes show so (noindex, not in the sitemap the run read).
|
|
531
|
+
const marked = input.run_context.protocol.page_types.flatMap(type => type.urls).filter(url => isDerivedIndexDecision(url.indexing_decision));
|
|
532
|
+
if (marked.length) {
|
|
533
|
+
const found = new Map(notIndexTargets({ context: input.run_context, artifacts,
|
|
534
|
+
textOf: artifact => artifact.content_hash ? requireRef(artifact.evidence_ref).text : null }).map(target => [target.url, target.evidence_ref]));
|
|
535
|
+
for (const url of marked) if (url.public_indexing !== false || found.get(url.normalized) !== url.indexing_decision.evidence_ref)
|
|
536
|
+
throw new ContractError(`indexing decision is not derivable from the raw evidence: ${url.normalized}`);
|
|
537
|
+
}
|
|
476
538
|
for (const obs of observations) {
|
|
477
539
|
if (!savedObservations.some(x => same(x, obs)) || !rawRefs.has(obs.evidence_ref))
|
|
478
540
|
throw new ContractError('collector observation is not bound to audit index and raw evidence');
|
|
@@ -532,10 +594,18 @@ function checkAuditPhase(input, phase, snapshot, requireRef) {
|
|
|
532
594
|
throw new ContractError(`selected page missing from the collection and its skipped list: ${url}`);
|
|
533
595
|
// Re-derive every collector verdict from the hashed raw evidence. Each derived observation must be present in the
|
|
534
596
|
// packet and in the saved observation file, and no collector observation may lack a derivation.
|
|
597
|
+
const { records, renderTextOf } = storedRenders(audit, input.run_context.protocol.render?.max_render_requests, requireRef);
|
|
535
598
|
const derived = deriveObservations({ context: input.run_context, artifacts, collection,
|
|
536
|
-
textOf: artifact => artifact.content_hash ? requireRef(artifact.evidence_ref).text : null
|
|
537
|
-
|
|
599
|
+
textOf: artifact => artifact.content_hash ? requireRef(artifact.evidence_ref).text : null,
|
|
600
|
+
renders: records, renderTextOf, chrome: audit.chrome ?? null, cwv: storedCwv(audit, requireRef) });
|
|
601
|
+
const shape = x => [x.state, x.q, x.partial_basis, x.evidence_ref, x.reason, x.evaluator_type, x.source_ref, x.applicability_decision ?? null, x.measurement ?? null, x.problem_messages ?? null];
|
|
602
|
+
// The site's identity is the one code-judged unit a model may judge, and only where code gave no verdict: its placeholder is replaced by
|
|
603
|
+
// the model's verdict (modelJudgmentRows replays that one), and a model verdict next to a code verdict of the unit is refused.
|
|
604
|
+
const modelEntity = input.observations.some(x => x.unit_key === ENTITY_UNIT && x.evaluator_type === 'host_model');
|
|
605
|
+
if (modelEntity && derived.some(x => x.unit_key === ENTITY_UNIT && x.state !== 'unknown'))
|
|
606
|
+
throw new ContractError(`${ENTITY_UNIT} has a collector verdict; a model verdict cannot stand beside it`);
|
|
538
607
|
for (const expected of derived) {
|
|
608
|
+
if (modelEntity && expected.unit_key === ENTITY_UNIT) continue;
|
|
539
609
|
if (!observations.some(x => x.unit_key === expected.unit_key) || !savedObservations.some(x => x.unit_key === expected.unit_key))
|
|
540
610
|
throw new ContractError(`planned collector observation missing: ${expected.unit_key}`);
|
|
541
611
|
const have = observations.find(x => x.unit_key === expected.unit_key);
|
|
@@ -548,10 +618,129 @@ function checkAuditPhase(input, phase, snapshot, requireRef) {
|
|
|
548
618
|
throw new ContractError(`collector observation is not derivable from raw evidence: ${have.unit_key}`);
|
|
549
619
|
}
|
|
550
620
|
|
|
621
|
+
const MODEL_JUDGMENT_LABEL = '근거 확인됨, 판정은 모델';
|
|
622
|
+
|
|
623
|
+
/**
|
|
624
|
+
* A verdict of the derived source is the question coverage that code computes from the answer-clarity judgments and the fixed
|
|
625
|
+
* panel. Compute it again from the same input: a verdict that does not follow from them, or any other unit with that source, is refused.
|
|
626
|
+
*/
|
|
627
|
+
function replayedQuestionCoverage(obs, input, fail) {
|
|
628
|
+
const rest = input.observations.filter(x => x !== obs);
|
|
629
|
+
const expected = obs.unit_key === QUESTION_UNIT && obs.evaluator_type === 'code' ?
|
|
630
|
+
deriveQuestionCoverage({ observations: rest, panel: input.run_context.protocol.question_panel, evidence_ref: input.run_context.evidence_ref }).find(x => x.unit_key === QUESTION_UNIT) : null;
|
|
631
|
+
const shape = x => [x.family_id, x.state, x.q, x.partial_basis, x.evidence_ref, x.reason, x.evaluator_type, x.source_ref, x.measurement ?? null, x.problem_messages ?? null];
|
|
632
|
+
if (!expected || expected.source_ref !== OBSERVATION_SOURCES.derived || !same(shape(obs), shape(expected)))
|
|
633
|
+
fail(obs, 'it does not follow from the answer-clarity judgments and the fixed question list');
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
/**
|
|
637
|
+
* The model judgments of one score input, each replayed against the run's audit evidence. A `host_model` observation is
|
|
638
|
+
* refused for a code-judged family and unless it carries the model source. Its evidence must be the audit raw entry of
|
|
639
|
+
* the page its unit names. A verdict other than unknown cites ranges that are candidates of the queue stored in the audit
|
|
640
|
+
* index, each with that raw entry's hash and a byte range inside it on character boundaries. The label says the evidence was
|
|
641
|
+
* checked and the verdict is the model's.
|
|
642
|
+
*/
|
|
643
|
+
function modelJudgmentRows(phase, input, snapshot, requireRef, confirmedNames = []) {
|
|
644
|
+
const fail = (obs, message) => { throw new ContractError(`verdict ${obs.unit_key}: ${message}`); };
|
|
645
|
+
for (const obs of input?.observations ?? []) {
|
|
646
|
+
const type = obs.evaluator_type;
|
|
647
|
+
const entityFallback = isEntityFallback(obs.family_id, obs.unit_key) && type === 'host_model' && obs.source_ref === OBSERVATION_SOURCES.host_model;
|
|
648
|
+
// Next to an audit, a code-judged family is the collector's alone and is re-derived; without one, only a model is refused (an outside source stays unverified).
|
|
649
|
+
// The one exception is the site's identity judged by the model, which checkAuditPhase allows only where code gave no verdict.
|
|
650
|
+
if (isCodeJudged(obs.family_id) && !entityFallback && (type === 'host_model' || snapshot && (type !== 'code' || obs.source_ref !== OBSERVATION_SOURCES.code)))
|
|
651
|
+
fail(obs, `${obs.family_id} is judged by code; a ${type} observation with source_ref ${obs.source_ref} cannot stand for it`);
|
|
652
|
+
if (obs.source_ref === OBSERVATION_SOURCES.derived) replayedQuestionCoverage(obs, input, fail);
|
|
653
|
+
// A content family has exactly one allowed pair: the model's, bound to the judge queue.
|
|
654
|
+
if (isModelJudged(obs.family_id) && (type !== 'host_model' || obs.source_ref !== OBSERVATION_SOURCES.host_model))
|
|
655
|
+
fail(obs, `${obs.family_id} is judged by a model through the judge queue; evaluator_type ${type} with source_ref ${obs.source_ref} is not allowed`);
|
|
656
|
+
if (type === 'host_model' && obs.source_ref !== OBSERVATION_SOURCES.host_model) fail(obs, `source_ref must be ${OBSERVATION_SOURCES.host_model}`);
|
|
657
|
+
// The collector's and the model's source stand for replayed work; without the phase's audit there is nothing to replay against.
|
|
658
|
+
if (!snapshot && Object.values(OBSERVATION_SOURCES).includes(obs.source_ref))
|
|
659
|
+
fail(obs, `source_ref ${obs.source_ref} needs the ${phase} audit snapshot; without an audit only outside inputs are accepted, and they stay unverified`);
|
|
660
|
+
}
|
|
661
|
+
const judgments = (input?.observations ?? []).filter(obs => obs.evaluator_type === 'host_model');
|
|
662
|
+
if (!judgments.length) return [];
|
|
663
|
+
if (!snapshot) throw new ContractError(`${phase} model judgments require their audit snapshot`);
|
|
664
|
+
const audit = JSON.parse(requireRef(snapshot.audit_ref).text);
|
|
665
|
+
const queues = (audit.judge_queue ?? []).map(entry => {
|
|
666
|
+
const stored = requireRef(entry.evidence_ref);
|
|
667
|
+
if (stored.sha256 !== entry.sha256) throw new ContractError('judge queue hash differs from the audit index');
|
|
668
|
+
if (stored.kind !== QUEUE_KIND) throw new ContractError(`judge queue evidence has kind ${stored.kind}, not ${QUEUE_KIND}`);
|
|
669
|
+
return JSON.parse(stored.text);
|
|
670
|
+
});
|
|
671
|
+
// The site's identity is judged on the home page, chosen as the queue chose it: from the pages whose fetch verdict passed.
|
|
672
|
+
const audited = input.observations.filter(x => x.family_id === 'fetch-response' && x.state === 'pass')
|
|
673
|
+
.filter(x => (audit.raw ?? []).some(entry => entry.content_hash && entry.evidence_ref === x.evidence_ref && entry.normalized_url === unitUrl(x.unit_key)));
|
|
674
|
+
const home = homeOf(audited.map(x => unitUrl(x.unit_key)));
|
|
675
|
+
// What the gates of an absence fail need, derived here and not read from the queue: the lines most audited pages repeat, and (below) the names the owner confirmed in the intake answers, the code identity check and the host.
|
|
676
|
+
const chrome = siteChrome([...new Map(audited.map(x => [unitUrl(x.unit_key), requireRef(x.evidence_ref).bytes])).values()]);
|
|
677
|
+
return judgments.filter(obs => obs.state !== 'unknown').map(obs => {
|
|
678
|
+
if (obs.state === 'not_applicable' && !isModelJudged(obs.family_id)) fail(obs, `${obs.family_id} is not a family a model may declare not applicable`);
|
|
679
|
+
if (obs.family_id === 'answer-clarity') {
|
|
680
|
+
const named = obs.measurement?.question_id;
|
|
681
|
+
if (obs.state === 'not_applicable' ? named !== undefined : !(input.run_context.protocol.question_panel ?? []).some(question => question.id === named))
|
|
682
|
+
fail(obs, obs.state === 'not_applicable' ? 'a not applicable answer-clarity verdict names no question' : 'an answer-clarity verdict names a question of the fixed panel');
|
|
683
|
+
// The other questions it says the page answers: other, distinct questions of the fixed panel, on a pass or partial only.
|
|
684
|
+
const also = obs.measurement?.also_question_ids;
|
|
685
|
+
if (also !== undefined && (!['pass', 'partial'].includes(obs.state) || alsoQuestionIds({ question_id: named, also_question_ids: also }, new Set(input.run_context.protocol.question_panel.map(question => question.id))) === null))
|
|
686
|
+
fail(obs, 'its other questions are not other, distinct questions of the fixed panel on a pass or partial verdict');
|
|
687
|
+
}
|
|
688
|
+
if (isEntityFallback(obs.family_id, obs.unit_key) && obs.measurement?.identity_basis !== identityBasisOf(obs.state)) fail(obs, 'its identity basis is not the one its state records');
|
|
689
|
+
const raw = (audit.raw ?? []).find(x => x.evidence_ref === obs.evidence_ref && x.normalized_url === (isEntityFallback(obs.family_id, obs.unit_key) ? home : unitUrl(obs.unit_key)));
|
|
690
|
+
if (!raw) fail(obs, 'its evidence is not the audit raw entry of the unit page');
|
|
691
|
+
// Candidates are recomputed from the raw bytes, so a consistently forged queue cannot widen what a model may cite.
|
|
692
|
+
const candidates = candidateSpans(requireRef(obs.evidence_ref).bytes, raw.content_hash);
|
|
693
|
+
// A verdict that rests on something missing from the page needs a queue that offered enough of it and a page that does not show it; a fail of an absence family names its basis; the empty-page mark is true of the page.
|
|
694
|
+
const bytes = requireRef(obs.evidence_ref).bytes;
|
|
695
|
+
if (!basisAllowed({ family_id: obs.family_id, state: obs.state, basis: obs.measurement?.basis })) fail(obs, 'its basis is not the one this verdict must carry');
|
|
696
|
+
const cited = (obs.evidence_span ?? []).map(span => candidates.find(x => x.start === span.start && x.end === span.end)).filter(Boolean);
|
|
697
|
+
const unaccepted = acceptance({ family_id: obs.family_id, state: obs.state, condition: obs.applicability_decision?.condition, basis: obs.measurement?.basis }, bytes, cited, candidates,
|
|
698
|
+
{ chrome, ...operatorNames({ confirmed: confirmedNames, observations: input.observations, url: unitUrl(obs.unit_key) }) });
|
|
699
|
+
if (unaccepted) fail(obs, `its verdict is not accepted: ${unaccepted.reason}`);
|
|
700
|
+
// What the owner reads for a fail is the catalog sentence of its family and basis with the text it cites; any other verdict states no problem.
|
|
701
|
+
const failed = { family_id: obs.family_id, state: obs.state, basis: obs.measurement?.basis };
|
|
702
|
+
const question = obs.family_id === 'answer-clarity' ? input.run_context.protocol.question_panel.find(item => item.id === obs.measurement?.question_id)?.question : null;
|
|
703
|
+
if (!same(obs.problem_messages ?? null, obs.state === 'fail' ? modelFailMessages({ family_id: obs.family_id, basis: sentenceBasis(failed, bytes, chrome) }, cited, chrome, question) : null))
|
|
704
|
+
fail(obs, 'its problem message is not the one its family, basis and cited text give');
|
|
705
|
+
if (Boolean(obs.measurement?.empty_page) !== isEmptyPage(bytes)) fail(obs, 'its empty-page mark differs from the page');
|
|
706
|
+
for (const span of obs.evidence_span ?? []) {
|
|
707
|
+
if (span.evidence_sha256 !== raw.content_hash) fail(obs, 'a span names evidence other than the unit page');
|
|
708
|
+
try { checkSpan(requireRef(obs.evidence_ref).bytes, span); } catch (error) { fail(obs, error.message); }
|
|
709
|
+
const candidate = candidates.find(x => x.start === span.start && x.end === span.end);
|
|
710
|
+
if (!candidate) fail(obs, `span ${span.start}-${span.end} is not a text run of the page`);
|
|
711
|
+
if (!queues.some(queue => queuedSpan(queue, obs, span))) fail(obs, `span ${span.start}-${span.end} is not a candidate of the stored judge queue`);
|
|
712
|
+
// The text the model read is the text of that range, with inline tags left out: the queue cannot show it something else.
|
|
713
|
+
const offered = queues.flatMap(queue => queue.items.filter(x => x.unit_key === obs.unit_key && x.family_id === obs.family_id && x.evidence_ref === obs.evidence_ref))
|
|
714
|
+
.flatMap(x => x.spans).find(x => x.evidence_sha256 === span.evidence_sha256 && x.start === span.start && x.end === span.end);
|
|
715
|
+
if (offered.text !== candidate.text) fail(obs, `the queued text of span ${span.start}-${span.end} differs from the text of the page`);
|
|
716
|
+
}
|
|
717
|
+
return { phase, family_id: obs.family_id, unit_key: obs.unit_key, state: obs.state, evidence_ref: obs.evidence_ref, evidence_span: obs.evidence_span, label: MODEL_JUDGMENT_LABEL,
|
|
718
|
+
...(obs.measurement?.basis ? { basis: obs.measurement.basis, sentence_basis: sentenceBasis(failed, bytes, chrome) } : {}),
|
|
719
|
+
...(obs.measurement?.identity_basis ? { identity_basis: obs.measurement.identity_basis } : {}) };
|
|
720
|
+
});
|
|
721
|
+
}
|
|
722
|
+
|
|
723
|
+
/**
|
|
724
|
+
* The audited pages that show no text, for the phase the report describes: the pages the judge queue lists as empty, each checked
|
|
725
|
+
* against the raw bytes the audit stored, so a queue cannot name a page that has text.
|
|
726
|
+
*/
|
|
727
|
+
function emptyPagesOf(snapshot, requireRef) {
|
|
728
|
+
if (!snapshot) return [];
|
|
729
|
+
const audit = JSON.parse(requireRef(snapshot.audit_ref).text);
|
|
730
|
+
const listed = new Set((audit.judge_queue ?? []).flatMap(entry => JSON.parse(requireRef(entry.evidence_ref).text).empty_pages ?? []));
|
|
731
|
+
return [...listed].filter(url => {
|
|
732
|
+
const raw = (audit.raw ?? []).find(entry => entry.content_hash && entry.normalized_url === url);
|
|
733
|
+
return raw && isEmptyPage(requireRef(raw.evidence_ref).bytes);
|
|
734
|
+
}).sort();
|
|
735
|
+
}
|
|
736
|
+
|
|
551
737
|
/** Recompute scores and metrics from contained run evidence; render four design tables. */
|
|
552
738
|
export function buildReport(packet) {
|
|
739
|
+
assertCurrentEvidence(packet, 'report packet');
|
|
740
|
+
assertCurrentEvidence(packet?.after?.input, 'report packet (follow-up phase)');
|
|
553
741
|
validateEntity('ReportPacket', packet);
|
|
554
742
|
if (sensitive.test(JSON.stringify(packet))) throw new ContractError('private identifier or secret in report packet');
|
|
743
|
+
for (const id of packet.answered_runbooks ?? []) if (!runbookEntry(id)) throw new ContractError(`answered_runbooks names an unknown runbook: ${id}`);
|
|
555
744
|
if (packet.baseline.input.run_context.run_id !== packet.run_context.run_id ||
|
|
556
745
|
!same(packet.baseline.input.run_context, packet.run_context) || packet.costs.run_id !== packet.run_context.run_id)
|
|
557
746
|
throw new ContractError('report run context or cost ledger mismatch');
|
|
@@ -562,9 +751,12 @@ export function buildReport(packet) {
|
|
|
562
751
|
snapshots.some(x => !['baseline', 'after'].includes(x.phase) || !x.audit_ref || !x.context_ref))
|
|
563
752
|
throw new ContractError('audit snapshot phase or reference invalid');
|
|
564
753
|
const snapshotFor = phase => snapshots.find(x => x.phase === phase);
|
|
565
|
-
//
|
|
566
|
-
|
|
567
|
-
|
|
754
|
+
// Each phase's run context, hashed as run evidence, records whether the phase was audited. A packet cannot add or drop a
|
|
755
|
+
// snapshot to change how its phase is judged: the snapshot must exist exactly when the context says the phase was audited.
|
|
756
|
+
for (const [phase, input] of [['baseline', packet.baseline.input], ['after', packet.after?.input]]) {
|
|
757
|
+
if (input && (input.run_context.audited === true) !== Boolean(snapshotFor(phase)))
|
|
758
|
+
throw new ContractError(`${phase} audit snapshot differs from the audit recorded in its run context`);
|
|
759
|
+
}
|
|
568
760
|
const baselineScores = score(packet.baseline.input);
|
|
569
761
|
if (!same(packet.baseline.scores, baselineScores)) throw new ContractError('baseline score differs from raw observations');
|
|
570
762
|
const collectedIn = phase => collectedResponses(snapshotFor(phase) ? [snapshotFor(phase)] : [], requireRef);
|
|
@@ -572,7 +764,8 @@ export function buildReport(packet) {
|
|
|
572
764
|
let afterScores = null;
|
|
573
765
|
if (packet.after) {
|
|
574
766
|
if (packet.after.input.run_context.run_id !== packet.run_context.run_id) throw new ContractError('after run ID mismatch');
|
|
575
|
-
|
|
767
|
+
const afterContext = requireRef(packet.after.input.run_context.evidence_ref);
|
|
768
|
+
if (afterContext.kind !== 'context' || !same(JSON.parse(afterContext.text), packet.after.input.run_context))
|
|
576
769
|
throw new ContractError('after context evidence differs from input');
|
|
577
770
|
afterScores = score(packet.after.input);
|
|
578
771
|
if (!same(packet.after.scores, afterScores)) throw new ContractError('after score differs from raw observations');
|
|
@@ -580,11 +773,24 @@ export function buildReport(packet) {
|
|
|
580
773
|
}
|
|
581
774
|
const comparison = compare(packet.baseline.input, packet.after?.input ?? null);
|
|
582
775
|
if (!same(packet.comparison, comparison)) throw new ContractError('comparison differs from raw inputs');
|
|
583
|
-
const
|
|
776
|
+
const originNotes = [...new Map(packet.metrics.map(metric => checkMetricEvidence(metric, packet, requireRef))).entries()];
|
|
777
|
+
const metricOriginLimitations = originNotes.map(([text]) => text);
|
|
584
778
|
if (!packet.costs.evidence_ref || !same(JSON.parse(requireRef(packet.costs.evidence_ref).text), packet.costs))
|
|
585
779
|
throw new ContractError('cost ledger snapshot evidence differs from report');
|
|
780
|
+
// Model verdicts are replayed first, so a host_model observation never reaches the collector replay under a collector source.
|
|
781
|
+
const intakeEntry = packet.evidence_index.find(x => x.kind === 'intake');
|
|
782
|
+
const intake = intakeEntry ? JSON.parse(requireRef(intakeEntry.ref).text) : null;
|
|
783
|
+
const confirmedNames = (intake?.answers?.facts ?? []).filter(fact => fact.key === 'name').map(fact => fact.value);
|
|
784
|
+
const modelRows = [['baseline', packet.baseline.input], ['after', packet.after?.input]].flatMap(([phase, input]) => modelJudgmentRows(phase, input, snapshotFor(phase), requireRef, confirmedNames));
|
|
785
|
+
const emptyPages = emptyPagesOf(snapshotFor(packet.after ? 'after' : 'baseline'), requireRef);
|
|
586
786
|
checkAuditPhase(packet.baseline.input, 'baseline', snapshotFor('baseline'), requireRef);
|
|
587
787
|
checkAuditPhase(packet.after?.input, 'after', snapshotFor('after'), requireRef);
|
|
788
|
+
// An adopted start address is said in the run conditions, so it must match the intake record and the run's target.
|
|
789
|
+
const startRedirect = packet.start_redirect ?? null;
|
|
790
|
+
if (startRedirect || intake?.selection?.start_redirect) {
|
|
791
|
+
if (startRedirect?.to !== packet.run_context.target || !same(intake?.selection?.start_redirect ?? null, startRedirect))
|
|
792
|
+
throw new ContractError('start redirect differs from the intake record or the run target');
|
|
793
|
+
}
|
|
588
794
|
const merged = mergeFindings(packet.findings);
|
|
589
795
|
const observations = [...packet.baseline.input.observations, ...(packet.after?.input.observations ?? [])];
|
|
590
796
|
const collectedAny = collectedResponses(snapshots, requireRef);
|
|
@@ -595,8 +801,27 @@ export function buildReport(packet) {
|
|
|
595
801
|
if (finding.blocker && !observations.some(obs => obs.check_id === finding.check_id &&
|
|
596
802
|
obs.unit_key === finding.unit_key && obs.state === 'fail'))
|
|
597
803
|
throw new ContractError('blocker requires confirmed failed check observation');
|
|
804
|
+
// A finding that carries the model's source must rest on a model verdict of its unit; its evidence is the page the collector stored.
|
|
805
|
+
const modelFinding = finding.source_ref === OBSERVATION_SOURCES.host_model;
|
|
806
|
+
// A finding that groups several pages names each of them, and each must itself be a failed verdict of the same problem (the same text
|
|
807
|
+
// once its own address is left out, among the problems the verdict states) by the same kind of judge: a listed page cannot be one
|
|
808
|
+
// that was not measured.
|
|
809
|
+
for (const unit of finding.unit_keys ?? []) if (!observations.some(obs => obs.check_id === finding.check_id && obs.unit_key === unit && obs.state === 'fail' &&
|
|
810
|
+
problemsOf(obs).some(problem => pageNeutral(problem, unit) === pageNeutral(finding.message, finding.unit_key)) &&
|
|
811
|
+
(obs.evaluator_type === 'host_model') === modelFinding))
|
|
812
|
+
throw new ContractError(`finding names a page that is not a failed verdict of the same problem: ${unit}`);
|
|
813
|
+
// The Korean text rests on the same verdict as the English message: its key and values are the ones the verdict recorded for this problem.
|
|
814
|
+
if (finding.message_key !== undefined || finding.message_values !== undefined) {
|
|
815
|
+
const stated = { message_key: finding.message_key ?? null, message_values: finding.message_values ?? {} };
|
|
816
|
+
if (!observations.some(obs => obs.check_id === finding.check_id && obs.unit_key === finding.unit_key && obs.state === 'fail' &&
|
|
817
|
+
problemsOf(obs).some((problem, index) => pageNeutral(problem, obs.unit_key) === pageNeutral(finding.message, finding.unit_key) && same(obs.problem_messages?.[index] ?? null, stated))))
|
|
818
|
+
throw new ContractError(`finding message key differs from its verdict: ${finding.finding_id}`);
|
|
819
|
+
}
|
|
820
|
+
if (modelFinding && !observations.some(obs => obs.check_id === finding.check_id && obs.unit_key === finding.unit_key &&
|
|
821
|
+
obs.evaluator_type === 'host_model' && obs.state === 'fail'))
|
|
822
|
+
throw new ContractError(`model finding has no model verdict: ${finding.finding_id}`);
|
|
598
823
|
for (const source of finding.sources) {
|
|
599
|
-
const evidence = requireRef(source.evidence_ref, source.source_ref);
|
|
824
|
+
const evidence = requireRef(source.evidence_ref, [OBSERVATION_SOURCES.host_model, OBSERVATION_SOURCES.derived].includes(source.source_ref) ? { anySource: true } : source.source_ref);
|
|
600
825
|
// The same HTML plus response-header replay as the observation, bound to the finding's own unit.
|
|
601
826
|
if (finding.check_id === 'index-policy' && finding.blocker &&
|
|
602
827
|
!evidenceBlocksIndexing(evidence, packet.run_context.protocol.engines, collectedAny, unitUrl(finding.unit_key)))
|
|
@@ -616,7 +841,7 @@ export function buildReport(packet) {
|
|
|
616
841
|
const resolution = resolving.length ? { observation_id: resolving[0].observation_id,
|
|
617
842
|
evaluator_type: [...new Set(resolving.map(obs => obs.evaluator_type))].join(', '),
|
|
618
843
|
source_ref: [...new Set(resolving.map(obs => obs.source_ref))].join(', ') } : null;
|
|
619
|
-
return { ...finding, remediation: selected ?? null, status, resolution };
|
|
844
|
+
return { ...finding, remediation: selected ?? null, status, resolution, ...(modelFinding ? { label: MODEL_JUDGMENT_LABEL } : {}) };
|
|
620
845
|
});
|
|
621
846
|
for (const change of packet.changes) {
|
|
622
847
|
validateEntity('ChangeRecord', change);
|
|
@@ -673,91 +898,71 @@ export function buildReport(packet) {
|
|
|
673
898
|
problem: `Production application and observation pending for ${finding.unit_key}`,
|
|
674
899
|
reason: 'production_deployment_pending', required_input_or_authority: 'site owner deployment authority and production target',
|
|
675
900
|
next_action: 'Deploy the locally verified change, then fetch the production response',
|
|
676
|
-
reobserve_when: 'after authorized production deployment and a new raw response' });
|
|
677
|
-
} else remaining.push({ finding_id: finding.finding_id, problem: `${finding.message} ${finding
|
|
901
|
+
reobserve_when: 'after authorized production deployment and a new raw response', ...messageOf('follow-up.production-pending', { pages: pagesLabel(finding) }) });
|
|
902
|
+
} else remaining.push({ finding_id: finding.finding_id, problem: `${finding.message} ${pagesLabel(finding)}`,
|
|
678
903
|
reason: finding.status, required_input_or_authority: finding.remediation?.required_authority ?? 'source evidence and owner review',
|
|
679
904
|
next_action: finding.remediation?.mechanism ?? 'Confirm the issue from raw evidence',
|
|
680
|
-
reobserve_when: finding.remediation?.validation ?? 'after new evidence is collected' });
|
|
905
|
+
reobserve_when: finding.remediation?.validation ?? 'after new evidence is collected', ...messageOf('follow-up.finding') });
|
|
681
906
|
}
|
|
682
|
-
|
|
907
|
+
// Each cost note keeps its English text beside the catalog key and values of its Korean sentence.
|
|
908
|
+
const costNote = (text, message) => ({ text, ...message });
|
|
909
|
+
const costNotes = [
|
|
683
910
|
...packet.costs.reservations.filter(x => ['unknown', 'overrun'].includes(x.status)).map(x => {
|
|
684
911
|
const settlement = packet.costs.settlements.find(item => item.reservation_id === x.reservation_id);
|
|
685
|
-
const format = amounts => amounts?.map(item => `${item.amount} ${item.currency ?? item.unit}`).join(', ')
|
|
686
|
-
return `${x.provider}: ${x.status} (reserved ${format(x.upperBounds)}; actual ${format(settlement?.actual)})
|
|
912
|
+
const format = amounts => amounts?.map(item => `${item.amount} ${item.currency ?? item.unit}`).join(', ');
|
|
913
|
+
return costNote(`${x.provider}: ${x.status} (reserved ${format(x.upperBounds) ?? 'unknown'}; actual ${format(settlement?.actual) ?? 'unknown'})`,
|
|
914
|
+
messageOf('cost.reservation', { provider: x.provider, status: x.status, reserved: format(x.upperBounds), actual: format(settlement?.actual) }));
|
|
687
915
|
}),
|
|
688
|
-
...(packet.capabilities?.selected ?? []).filter(x => x.cost_enforcement === 'host_policy').map(x => `${x.provider}: host_policy
|
|
689
|
-
...(packet.capabilities?.limitations ?? []),
|
|
916
|
+
...(packet.capabilities?.selected ?? []).filter(x => x.cost_enforcement === 'host_policy').map(x => costNote(`${x.provider}: host_policy`, messageOf('cost.host-policy', { provider: x.provider }))),
|
|
917
|
+
...(packet.capabilities?.limitations ?? []).map((text, index) => costNote(text, packet.capabilities.limitation_messages?.[index] ?? messageOf(null))),
|
|
690
918
|
];
|
|
691
919
|
if (packet.costs.reservations.some(x => x.provider === 'anthropic') ||
|
|
692
920
|
(packet.capabilities?.selected ?? []).some(x => x.provider === 'anthropic'))
|
|
693
|
-
|
|
694
|
-
|
|
921
|
+
// The direct adapter ships with the release, so this label is the release version (the panel label in metrics.mjs is the evaluation version).
|
|
922
|
+
costNotes.push(costNote(`Anthropic direct v${versions().release}: text-only; claude-fable-5-1, claude-opus-5-5, claude-sonnet-5-5, claude-haiku-5-5; max output 128000; one token reserve >=1000000+max_tokens per attempt; tools/search and currency-only or mixed bounds unsupported; reservation is capacity, not actual billing`,
|
|
923
|
+
messageOf('cost.anthropic', { version: versions().release })));
|
|
924
|
+
const verified = { schema_version: versions().contract_schema, run_context: packet.run_context,
|
|
695
925
|
data_class: packet.data_class ?? 'unknown', baseline: { input: packet.baseline.input, scores: baselineScores },
|
|
696
926
|
after: packet.after ? { input: packet.after.input, scores: afterScores } : null, comparison,
|
|
697
927
|
metrics: packet.metrics, findings, changes: packet.changes, costs: packet.costs,
|
|
698
|
-
capabilities: packet.capabilities ?? null, cost_issues:
|
|
928
|
+
capabilities: packet.capabilities ?? null, cost_issues: costNotes.map(item => item.text),
|
|
929
|
+
cost_issue_messages: costNotes.map(({ message_key, message_values }) => ({ message_key, message_values })), metric_origin_limitations: metricOriginLimitations,
|
|
930
|
+
metric_origin_limitation_messages: originNotes.map(([, key]) => messageOf(key)), follow_ups: remaining,
|
|
699
931
|
evidence_root: packet.evidence_root, evidence_index: packet.evidence_index,
|
|
700
|
-
audit_ref: packet.audit_ref ?? null, audit_snapshots: snapshots, source_ref: packet.source_ref
|
|
701
|
-
|
|
702
|
-
const
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
`${before.missing_bounds?.join('–') ?? 'unknown'} → ${after?.missing_bounds?.join('–') ?? 'unknown'}`,
|
|
710
|
-
`${before.readiness_status} → ${after?.readiness_status ?? 'unknown'}`];
|
|
711
|
-
});
|
|
712
|
-
const headlines = baselineScores.map(scoreHeadline);
|
|
713
|
-
result.headline = headlines;
|
|
714
|
-
const headlineText = headlines.map(h => h.status === 'sufficient' ? `${h.profile}: ${h.headline_score} (coverage ${h.coverage}%)` :
|
|
715
|
-
h.status === 'insufficient_core' ? `${h.profile}: insufficient core checks (${h.unresolved_core} access or indexing check(s) have no verdict, although ${h.coverage}% is measured); measured subset ${h.measured_subset_score ?? 'unknown'}, bounds ${h.missing_bounds?.join('–') ?? 'unknown'}` :
|
|
716
|
-
`${h.profile}: insufficient coverage (${floorTenth(h.coverage_exact)}% of the applicable weight is measured; at least ${h.coverage_threshold}% is needed); measured subset ${h.measured_subset_score ?? 'unknown'}, bounds ${h.missing_bounds?.join('–') ?? 'unknown'}`);
|
|
717
|
-
const remediationRows = findings.map(f => [f.priority, `${f.message} ${f.unit_key}`, f.sources.map(x => `${x.source_ref} ${x.evidence_ref}`).join('; '),
|
|
718
|
-
f.remediation ? `${f.remediation.mechanism} (${f.remediation.source_url}, checked ${f.remediation.checked_date})` : 'unknown',
|
|
719
|
-
packet.changes.filter(x => x.finding_ids.some(id => f.finding_ids.includes(id))).map(x => x.path).join('; ') || f.remediation?.location_hint || 'owner implementation location required',
|
|
720
|
-
f.remediation ? `${f.remediation.cost}; ${f.remediation.effort}` : 'unknown', f.remediation?.validation ?? 'unknown',
|
|
721
|
-
f.resolution ? `${f.status} (resolved by ${f.resolution.evaluator_type} observation from ${f.resolution.source_ref})` : f.status]);
|
|
722
|
-
const outcomeRows = metricRows(packet.metrics);
|
|
723
|
-
const remainingRows = remaining.map(x => [x.problem, x.reason, x.required_input_or_authority, x.next_action, x.reobserve_when]);
|
|
724
|
-
const markdown = `# optbrew report\n\n${packet.data_class === 'synthetic' ? '**Synthetic example — no real site or provider outcome.**\n\n' : ''}` +
|
|
725
|
-
`Target: ${html(packet.run_context.target)} \nEnvironment: ${html(packet.run_context.environment)} \nGoals: ${html(packet.run_context.goals.join(', '))} \n` +
|
|
726
|
-
`Comparison: ${html(comparison.status)} (${html(comparison.reasons.join(', ') || 'none')}) \n` +
|
|
727
|
-
`Headline: \n${headlineText.map(line => `- ${html(line)} \n`).join('')}` +
|
|
728
|
-
`Baseline blockers: ${html(baselineScores[0]?.blockers.length ?? 0)} (${html(baselineScores[0]?.blockers.map(x => `${x.family_id} ${x.unit_key}`).join('; ') || 'none')}) → ` +
|
|
729
|
-
`After blockers: ${afterScores ? html(afterScores[0]?.blockers.length ?? 0) : 'unknown'} (${afterScores ? html(afterScores[0]?.blockers.map(x => `${x.family_id} ${x.unit_key}`).join('; ') || 'none') : 'after not observed'}); ` +
|
|
730
|
-
`after core unknown: ${afterScores ? html(afterScores[0]?.unknown_core.length ?? 0) : 'unknown'}; actual outcomes: ${packet.metrics.length ? 'see measured and missing states below' : 'not_connected / unknown'} \n` +
|
|
731
|
-
`Cost and capability limitations: ${html(costIssues.join('; ') || 'none recorded')} \n` +
|
|
732
|
-
`Metric source limitations: ${html(metricOriginLimitations.join('; ') || 'no measured provider input')}\n\n` +
|
|
733
|
-
`## Score\n\n${table(['Area', 'Before', 'After', 'Difference', 'Coverage before → after', 'Missing bounds', 'Status'], scoreRows)}\n\n` +
|
|
734
|
-
`## Remediation\n\n${table(['Priority', 'Problem / URL', 'Evidence', 'Selected remedy', 'Change path or owner location', 'Cost / effort', 'Verification', 'Status'], remediationRows)}\n\n` +
|
|
735
|
-
`## Actual outcomes\n\n${table(['Engine / property', 'Metric', 'Window / sample', 'Before raw', 'After raw', 'Difference', 'Completion', 'Limitations'], outcomeRows)}\n\n` +
|
|
736
|
-
`## Remaining work\n\n${table(['Problem', 'Reason', 'Needed input / authority', 'Next action', 'Reobserve when'], remainingRows)}\n\n` +
|
|
737
|
-
`Evidence root: ${html(packet.evidence_root)}; run ID: ${html(packet.run_context.run_id)}. Read report.json to replay raw score inputs.\n`;
|
|
738
|
-
result.tables = { score: scoreRows, remediation: remediationRows, actual_outcomes: outcomeRows, remaining_work: remainingRows };
|
|
739
|
-
return { json: result, markdown };
|
|
932
|
+
audit_ref: packet.audit_ref ?? null, audit_snapshots: snapshots, model_judgments: modelRows, empty_pages: emptyPages, source_ref: packet.source_ref,
|
|
933
|
+
outcome_rows: metricRows(packet.metrics), lab_cwv: packet.lab_cwv ?? null, start_redirect: startRedirect, answered_runbooks: packet.answered_runbooks ?? [] };
|
|
934
|
+
const composed = composeReport(verified);
|
|
935
|
+
const { markdown, tables } = renderReport(composed, verified);
|
|
936
|
+
// The composed report leads; the checked data it was built from follows. `findings` is the sorted fixes again, for readers of earlier reports.
|
|
937
|
+
const checked = { ...verified };
|
|
938
|
+
for (const key of ['schema_version', 'data_class', 'outcome_rows', 'lab_cwv', 'answered_runbooks']) delete checked[key];
|
|
939
|
+
return { json: { schema_version: verified.schema_version, data_class: verified.data_class, composed, ...checked,
|
|
940
|
+
findings: composed.fixes, tables }, markdown };
|
|
740
941
|
}
|
|
741
942
|
|
|
742
|
-
/**
|
|
743
|
-
|
|
943
|
+
/**
|
|
944
|
+
* Write the outputs after all validation; temporary files stay in the destination. `renderHtml`, when given, turns the composed
|
|
945
|
+
* report into the page written as `report.html` beside the other two. All files appear or none does.
|
|
946
|
+
*/
|
|
947
|
+
export async function writeReport(packet, outputDir, { renderHtml = null } = {}) {
|
|
744
948
|
const built = buildReport(packet);
|
|
949
|
+
const html = renderHtml ? renderHtml(built.json.composed) : null;
|
|
745
950
|
await mkdir(outputDir, { recursive: true });
|
|
746
951
|
const jsonPath = path.resolve(outputDir, 'report.json'), markdownPath = path.resolve(outputDir, 'report.md');
|
|
747
|
-
|
|
952
|
+
const htmlPath = html === null ? null : path.resolve(outputDir, 'report.html');
|
|
953
|
+
const outputs = [[jsonPath, `${JSON.stringify(built.json, null, 2)}\n`], [markdownPath, built.markdown], ...(html === null ? [] : [[htmlPath, html]])];
|
|
954
|
+
for (const [file] of outputs) {
|
|
748
955
|
try { await access(file); throw new ContractError(`report output already exists: ${file}`); }
|
|
749
956
|
catch (error) { if (error.code !== 'ENOENT') throw error; }
|
|
750
957
|
}
|
|
751
|
-
const temporary = [
|
|
752
|
-
|
|
958
|
+
const temporary = outputs.map(([file]) => `${file}.${process.pid}.tmp`);
|
|
959
|
+
const written = [];
|
|
753
960
|
try {
|
|
754
|
-
await writeFile(temporary[
|
|
755
|
-
|
|
756
|
-
await rename(temporary[0], jsonPath); jsonWritten = true;
|
|
757
|
-
await rename(temporary[1], markdownPath);
|
|
961
|
+
for (const [index, [, text]] of outputs.entries()) await writeFile(temporary[index], text, { flag: 'wx' });
|
|
962
|
+
for (const [index, [file]] of outputs.entries()) { await rename(temporary[index], file); written.push(file); }
|
|
758
963
|
} catch (error) {
|
|
759
|
-
|
|
964
|
+
await Promise.all(written.map(file => rm(file, { force: true })));
|
|
760
965
|
throw error;
|
|
761
966
|
} finally { await Promise.all(temporary.map(file => rm(file, { force: true }))); }
|
|
762
|
-
return { jsonPath, markdownPath };
|
|
967
|
+
return { jsonPath, markdownPath, ...(htmlPath ? { htmlPath } : {}) };
|
|
763
968
|
}
|