@moda-ai/cli 1.12.3 → 1.12.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/harness-report.js +60 -3
- package/dist/harness.d.ts +6 -0
- package/dist/harness.js +16 -10
- package/package.json +1 -1
- package/skills/integration/index.json +2 -2
package/dist/harness-report.js
CHANGED
|
@@ -622,7 +622,8 @@ function graphEvidenceById(graph) {
|
|
|
622
622
|
const evidence = item.evidence
|
|
623
623
|
.filter((entry) => (isRecord(entry) &&
|
|
624
624
|
typeof entry.path === 'string' &&
|
|
625
|
-
typeof entry.reason === 'string'))
|
|
625
|
+
typeof entry.reason === 'string'))
|
|
626
|
+
.map((entry) => normalizeEvidenceEntry(entry));
|
|
626
627
|
if (evidence.length > 0)
|
|
627
628
|
byId.set(item.id, evidence);
|
|
628
629
|
}
|
|
@@ -637,10 +638,15 @@ function dedupeEvidence(evidence) {
|
|
|
637
638
|
if (seen.has(key))
|
|
638
639
|
continue;
|
|
639
640
|
seen.add(key);
|
|
640
|
-
deduped.push(
|
|
641
|
+
deduped.push(normalizeEvidenceEntry(item));
|
|
641
642
|
}
|
|
642
643
|
return deduped;
|
|
643
644
|
}
|
|
645
|
+
/** Keep the `{ path, reason }` shape, preserving a valid structured `line`. */
|
|
646
|
+
function normalizeEvidenceEntry(item) {
|
|
647
|
+
const line = evidenceLine(item);
|
|
648
|
+
return line ? { path: item.path, reason: item.reason, line } : { path: item.path, reason: item.reason };
|
|
649
|
+
}
|
|
644
650
|
function normalizeHarnessClaim(claim, graph) {
|
|
645
651
|
if (!isRecord(claim))
|
|
646
652
|
return claim;
|
|
@@ -792,11 +798,52 @@ function validateGraphCollection(graph, errors, field, shape) {
|
|
|
792
798
|
for (const [evidenceIndex, evidence] of item.evidence.entries()) {
|
|
793
799
|
if (!isRecord(evidence) || typeof evidence.path !== 'string' || typeof evidence.reason !== 'string') {
|
|
794
800
|
errors.push(`Graph ${label} evidence ${evidenceIndex + 1} must include path and reason.`);
|
|
801
|
+
continue;
|
|
802
|
+
}
|
|
803
|
+
if (evidence.line !== undefined && (typeof evidence.line !== 'number' || !Number.isFinite(evidence.line) || evidence.line <= 0)) {
|
|
804
|
+
errors.push(`Graph ${label} evidence ${evidenceIndex + 1} line must be a positive number.`);
|
|
795
805
|
}
|
|
796
806
|
}
|
|
797
807
|
}
|
|
808
|
+
if (field === 'artifacts' && requiresArtifactEvidenceLine(item.type) && !artifactHasRetrievableLocation(item)) {
|
|
809
|
+
errors.push(`Graph ${label} is a ${String(item.type)} artifact and must include a numeric evidence line (or a dedicated whole-file source) so its exact text can be retrieved.`);
|
|
810
|
+
}
|
|
798
811
|
}
|
|
799
812
|
}
|
|
813
|
+
/** prompt/tool artifacts must pin an exact source location for downstream retrieval. */
|
|
814
|
+
function requiresArtifactEvidenceLine(type) {
|
|
815
|
+
return type === 'prompt' || type === 'tool';
|
|
816
|
+
}
|
|
817
|
+
/**
|
|
818
|
+
* A prompt/tool artifact's text is retrievable when either (a) at least one
|
|
819
|
+
* evidence entry carries a resolvable line, or (b) the artifact is a whole
|
|
820
|
+
* dedicated file — its evidence points at the same file named in `sourcePath`,
|
|
821
|
+
* so the entire file IS the artifact and no line is needed.
|
|
822
|
+
*/
|
|
823
|
+
function artifactHasRetrievableLocation(item) {
|
|
824
|
+
if (hasEvidenceLine(item.evidence))
|
|
825
|
+
return true;
|
|
826
|
+
return isWholeFileArtifact(item);
|
|
827
|
+
}
|
|
828
|
+
function isWholeFileArtifact(item) {
|
|
829
|
+
if (typeof item.sourcePath !== 'string' || item.sourcePath.length === 0)
|
|
830
|
+
return false;
|
|
831
|
+
if (!Array.isArray(item.evidence))
|
|
832
|
+
return false;
|
|
833
|
+
return item.evidence.some((entry) => isRecord(entry) && entry.path === item.sourcePath);
|
|
834
|
+
}
|
|
835
|
+
function hasEvidenceLine(evidence) {
|
|
836
|
+
if (!Array.isArray(evidence))
|
|
837
|
+
return false;
|
|
838
|
+
return evidence.some((entry) => (isRecord(entry) &&
|
|
839
|
+
typeof entry.path === 'string' &&
|
|
840
|
+
typeof entry.reason === 'string' &&
|
|
841
|
+
evidenceLine({
|
|
842
|
+
path: entry.path,
|
|
843
|
+
reason: entry.reason,
|
|
844
|
+
line: typeof entry.line === 'number' ? entry.line : undefined,
|
|
845
|
+
}) !== undefined));
|
|
846
|
+
}
|
|
800
847
|
function normalizeGraphItemFieldAliases(collection, item) {
|
|
801
848
|
if (!isRecord(item))
|
|
802
849
|
return;
|
|
@@ -1155,7 +1202,7 @@ function upsertCitation(citations, citationIndex, evidence, rootDir) {
|
|
|
1155
1202
|
return undefined;
|
|
1156
1203
|
if (evidence.path === '.')
|
|
1157
1204
|
return undefined;
|
|
1158
|
-
const line =
|
|
1205
|
+
const line = evidenceLine(evidence);
|
|
1159
1206
|
const key = `${evidence.path}:${line ?? ''}:${evidence.reason}`;
|
|
1160
1207
|
const existing = citationIndex.get(key);
|
|
1161
1208
|
if (existing)
|
|
@@ -1389,6 +1436,16 @@ function lineFromReason(reason) {
|
|
|
1389
1436
|
const value = Number(match[1]);
|
|
1390
1437
|
return Number.isFinite(value) && value > 0 ? value : undefined;
|
|
1391
1438
|
}
|
|
1439
|
+
/**
|
|
1440
|
+
* Resolve the 1-based source line for an evidence entry, preferring the
|
|
1441
|
+
* structured `line` field and falling back to a "line N" mention in the reason.
|
|
1442
|
+
*/
|
|
1443
|
+
function evidenceLine(evidence) {
|
|
1444
|
+
if (typeof evidence.line === 'number' && Number.isFinite(evidence.line) && evidence.line > 0) {
|
|
1445
|
+
return Math.trunc(evidence.line);
|
|
1446
|
+
}
|
|
1447
|
+
return lineFromReason(evidence.reason);
|
|
1448
|
+
}
|
|
1392
1449
|
function inferenceFromConfidence(confidence) {
|
|
1393
1450
|
if (confidence >= 0.85)
|
|
1394
1451
|
return 'direct';
|
package/dist/harness.d.ts
CHANGED
|
@@ -4,6 +4,12 @@ declare const SCHEMA_VERSION = "harness.v0.1";
|
|
|
4
4
|
export interface HarnessEvidence {
|
|
5
5
|
path: string;
|
|
6
6
|
reason: string;
|
|
7
|
+
/**
|
|
8
|
+
* 1-based line where the cited item is defined. Optional in general, but
|
|
9
|
+
* required for `prompt` and `tool` artifacts so downstream prompt/tool
|
|
10
|
+
* optimization can retrieve the exact source text of every enumerated item.
|
|
11
|
+
*/
|
|
12
|
+
line?: number;
|
|
7
13
|
}
|
|
8
14
|
export interface HarnessDetection {
|
|
9
15
|
confidence: number;
|
package/dist/harness.js
CHANGED
|
@@ -1343,11 +1343,12 @@ function renderExternalHarnessAnalystPrompt(options) {
|
|
|
1343
1343
|
'',
|
|
1344
1344
|
'Your task:',
|
|
1345
1345
|
renderExternalAnalystProtocol(),
|
|
1346
|
-
'1. Identify the
|
|
1347
|
-
'2.
|
|
1348
|
-
|
|
1346
|
+
'1. Identify the runtime agent harnesses, agent families, channel adapters, identity mappings, model providers, frameworks, deployments, and open questions.',
|
|
1347
|
+
'2. Exhaustively enumerate EVERY prompt, tool, guardrail, and eval as its own graph artifact — the complete inventory, not a sample. This report feeds prompt optimization and tool optimization, so a missing tool or prompt is a real defect. Runtime agents may be modeled representatively on framework/example repos, but prompts and tools must be complete.',
|
|
1348
|
+
'3. Cite every narrative claim with local file paths, exact line numbers, and excerpts. File citations without line numbers are invalid for agent-authored reports. Enumerated artifacts only need evidence:[{path, reason}], so completeness of the artifact inventory is not limited by the citation cap.',
|
|
1349
|
+
`4. After completing investigation, print the report JSON between ${REPORT_JSON_BEGIN} and ${REPORT_JSON_END}. Moda CLI will validate it and write the local artifacts.`,
|
|
1349
1350
|
'Your final response must contain the sentinel-wrapped JSON report. Do not end with commentary about being read-only, do not summarize without JSON, and do not omit the sentinel markers.',
|
|
1350
|
-
'If the repository is
|
|
1351
|
+
'If the repository is very large, keep enumerating prompts/tools/guardrails/evals as artifacts (uncapped) and only summarize runtime AGENTS representatively — do not drop tools or prompts to save space.',
|
|
1351
1352
|
'',
|
|
1352
1353
|
'Final output format:',
|
|
1353
1354
|
REPORT_JSON_BEGIN,
|
|
@@ -1372,7 +1373,8 @@ function renderExternalHarnessAnalystPrompt(options) {
|
|
|
1372
1373
|
'Use exact field names and numeric confidence values from 0 to 1. Do not write string confidence values like "high". Do not use confidence >= 0.90 unless the claim is direct evidence from the cited line(s).',
|
|
1373
1374
|
'Each graph item with evidence must use evidence: [{ path, reason }], not citationIds/citations.',
|
|
1374
1375
|
'Runtime agents must include id, name, kind:"runtime_agent", root, entrypointIds, channelAdapterIds, artifactIds, identityNamespaceIds, deploymentIds, frameworkIds, modelProviderIds, confidence, evidence.',
|
|
1375
|
-
'Artifacts must include id, type, name, scope, usedByAgentIds, confidence, evidence, and optional sourcePath/contentHash/ownedByAgentId.',
|
|
1376
|
+
'Artifacts must include id, type, name, scope, usedByAgentIds, confidence, evidence, and optional sourcePath/contentHash/ownedByAgentId. Emit one artifact PER tool, PER prompt (including each inline system prompt), PER guardrail, and PER eval — enumerate them all; artifact types are entrypoint, prompt, tool, skill, eval, retrieval_index, memory, guardrail, model_config, unknown.',
|
|
1377
|
+
'For every `prompt` and `tool` artifact, at least one evidence entry MUST pin an exact source location so downstream optimization can retrieve the item text: carry a numeric `line` (the definition line, or the first line of the range) — evidence: [{ path, line, reason }]. The only exception is an artifact whose entire dedicated file is the text (e.g. a standalone prompt file); set its sourcePath and cite that same file and no line is required. When a single file holds multiple inline prompts or tool definitions, give each its own distinct line — a bare { path, reason } is a defect there.',
|
|
1376
1378
|
'Agent families must include id, name, runtimeAgentIds, sharedArtifactIds, confidence, evidence.',
|
|
1377
1379
|
'Frameworks/providers must include packageNames and runtimeAgentIds arrays; providers also include envVars and modelHints arrays.',
|
|
1378
1380
|
'Unknowns in graph must include id, question, candidateIds, evidence. Report-level unknowns must include id, question, citationIds.',
|
|
@@ -1393,11 +1395,15 @@ function renderExternalAnalystProtocol() {
|
|
|
1393
1395
|
return [
|
|
1394
1396
|
'Focused investigation protocol:',
|
|
1395
1397
|
'- First build a quick repo map with read-only commands such as pwd, git ls-files, rg, and targeted manifest/doc reads.',
|
|
1396
|
-
'-
|
|
1397
|
-
'-
|
|
1398
|
-
'-
|
|
1399
|
-
'-
|
|
1400
|
-
'-
|
|
1398
|
+
'- EXHAUSTIVELY enumerate every prompt, tool, guardrail, eval, memory/retrieval store, and model config in the repository — one graph artifact each. This inventory feeds downstream prompt optimization and tool optimization, so completeness matters more than brevity: do NOT stop at representative examples and do NOT sample.',
|
|
1399
|
+
'- Tools: emit a `tool` artifact for EVERY tool definition — each `@function_tool` / `function_tool(...)`-decorated function, every hosted/first-party tool class (e.g. WebSearchTool, FileSearchTool, ComputerTool, CodeInterpreterTool, HostedMCPTool, ImageGenerationTool, LocalShellTool), and every `tool({...})` / `createTool(...)` / `new *Tool(...)` factory. Name each artifact after the tool/function and cite its definition line.',
|
|
1400
|
+
'- Prompts: emit a `prompt` artifact for EVERY distinct prompt — dedicated prompt files AND inline system prompts (`instructions=`, `system=`, `SYSTEM_PROMPT`, prompt-template strings), one per agent/definition, each with its exact file+line location so the prompt text can be retrieved and optimized later.',
|
|
1401
|
+
'- Also emit artifacts for guardrails (type "guardrail"), evals/tests-of-agents (type "eval"), memory/session stores (type "memory"), retrieval indexes (type "retrieval_index"), and model configuration (type "model_config").',
|
|
1402
|
+
'- Only runtime AGENTS may be modeled representatively: if the repo is a framework/SDK/examples collection, you need not treat every example as a distinct production agent — but you MUST still enumerate every distinct tool, prompt, guardrail, and eval as its own artifact.',
|
|
1403
|
+
'- Each enumerated artifact needs evidence: [{ path, reason }] (file path + short reason); a full line+excerpt citation is required only for the narrative `claims`, NOT for every artifact — so listing hundreds of tools/prompts is cheap and expected.',
|
|
1404
|
+
'- EXCEPTION — `prompt` and `tool` artifacts MUST pin an exact location: put a numeric `line` on at least one evidence entry (evidence: [{ path, line, reason }]) at the definition line. That structured location is what lets downstream optimization retrieve the exact prompt/tool text; when one file has several inline prompts or tools, each artifact gets its own line. The only case that needs no line is a whole dedicated file whose entire content is the artifact (set sourcePath to that file and cite it). Still no excerpt required — just the line.',
|
|
1405
|
+
'- For every runtime agent, read and cite at least one concrete entrypoint or orchestration line, and link its prompts/tools via artifactIds.',
|
|
1406
|
+
'- Keep going until every prompt, tool, guardrail, and eval in scope is represented as an artifact. Use unknowns only for genuinely ambiguous items — never to skip enumeration. Respect the hard caps (<=250 claims, <=400 citations): artifacts are uncapped, so when there are many items keep enumerating them as artifacts and reserve claims/citations for the most important ones.',
|
|
1401
1407
|
].join('\n');
|
|
1402
1408
|
}
|
|
1403
1409
|
function renderAnalystPreflightSection(rootDir) {
|
package/package.json
CHANGED