@tiangong-ai/cli 0.0.50 → 0.0.55

package/dist/research/workspace/runtime.js

@@ -2,10 +2,12 @@ import { randomUUID } from "node:crypto";
 import { cp, lstat, readFile, rm } from "node:fs/promises";
 import { basename, dirname, isAbsolute, join, relative, resolve } from "node:path";
 import { CliError } from "../../errors.js";
+import { dataPublicSchemas } from "../../data/schemas.js";
 import { loadCapabilityDeclarations, stageLockedCapabilities, verifyCapabilities, } from "./capabilities.js";
 import { startCapabilityBroker } from "./broker.js";
 import { commitAcquisitionAssessments, freezeEvidenceSnapshot, loadCurrentEvidenceSnapshot, loadInferenceReadyEvidenceSnapshot, loadImmutableEvidenceSnapshotChain, materializeAcquisitionAudit, parseMaterializedAcquisitionAudit, } from "./acquisition.js";
 import { stageEvidenceArtifacts } from "./artifacts.js";
+import { projectResearchDataCapabilities, } from "./data-evidence-adapter.js";
 import { commitDiscoveryDecisions, materializeDiscoveryEvidence } from "./discovery.js";
 import { inspectDiscoveryProgress } from "./discovery-status.js";
 import { downloadBindingRecordSchema } from "./downloads.js";
@@ -392,12 +394,24 @@ export async function prepareNativeResearchStage(input) {
             const stageContextContent = await stageContextForPackage(capsule.projectRoot, project, workPackage, config);
             const declarations = await loadCapabilityDeclarations(input.root);
             const hasBrokeredEvidence = declarations.capabilities.some((capability) => capability.permissions.includes("brokered-network"));
+            const dataCapabilities = projectResearchDataCapabilities();
+            const hasDataEvidence = dataCapabilities.capabilities.length > 0;
             const basePrompt = packagePrompt(project, workPackage, capsule.inputManifest, capsule.stagedSkills, capsule.capabilityDocumentation, null, capsule.contextBundle, capsule.contextBundleContent, stageContextContent, discovery, await listEvidenceCandidates(input.root, project.id));
             const prompt = [
                 "Perform this producer stage in the current interactive host session. Do not launch codex exec, claude -p, or any other nested reasoning agent.",
                 capsule.publicationPolicyDocumentation,
-                input.stage === "discover" && hasBrokeredEvidence
-                    ? "Use native Web/Browser broadly for discovery when useful, but record every native search/navigation with recordActivity and register its candidates. Before admitting any native lead, formalize the same URL or DOI through fetchEvidence so it receives an immutable broker receipt. Assess candidates in bounded batches with recordAssessment as they arrive; the final output is only a small coverage closeout. Native results without broker/input provenance are discovery leads, never evidence."
+                input.stage === "discover" && (hasBrokeredEvidence || hasDataEvidence)
+                    ? [
+                        hasBrokeredEvidence
+                            ? "Use native Web/Browser broadly for discovery when useful, but record every native search/navigation with recordActivity and register its candidates. Before admitting any native lead, formalize the same URL or DOI through fetchEvidence so it receives an immutable broker receipt."
+                            : null,
+                        hasDataEvidence
+                            ? "Use the dynamically projected structured data capabilities when their source coverage and operation semantics fit the question. Inspect a selected capability with the supplied describe command, then call runDataCapability with the exact published DataRunRequest. The Research adapter invokes the same TypeScript runtime in-process and returns an immutable data receipt plus candidate ID; do not invoke standalone data run for project evidence."
+                            : null,
+                        "Assess candidates in bounded batches with recordAssessment as they arrive; the final output is only a small coverage closeout. Native results without broker/data/input provenance are discovery leads, never evidence.",
+                    ]
+                        .filter((value) => value !== null)
+                        .join(" ")
                     : input.stage === "acquire"
                         ? "Acquire the provisionally admitted sources with the installed external acquisition/document Skills or an explicitly selected user-authorized browser. Capture the exact browser/adapter Download object, save it to the planned unique staging path, and call bindDownload before registerArtifact. Record browser/download/file-inspection activity with recordActivity. Failed or cancelled downloads cannot create bindings or artifacts. Never scan a download directory or infer success from file existence."
                         : "Do not acquire additional evidence in this stage.",
@@ -418,7 +432,7 @@ export async function prepareNativeResearchStage(input) {
                 preparedAt,
                 bindingSha256,
                 prompt,
-                outputSchema: schemaForStage(input.stage, null, input.stage === "discover" && !hasBrokeredEvidence
+                outputSchema: schemaForStage(input.stage, null, input.stage === "discover" && !hasBrokeredEvidence && !hasDataEvidence
                     ? { inputOnlyProvenanceIds: capsule.inputManifest.map((record) => record.id) }
                     : {}),
                 publicationPolicy: capsule.publicationPolicy,
@@ -499,6 +513,27 @@ export async function prepareNativeResearchStage(input) {
                             requestSchema: nativeEvidenceRequestSchema(Boolean(project.scientificDesign)),
                         }
                         : null,
+                    runDataCapability: input.stage === "discover" && hasDataEvidence
+                        ? {
+                            argv: [
+                                "tiangong-ai",
+                                "research",
+                                "project",
+                                "evidence",
+                                "data",
+                                "run",
+                                project.id,
+                                "--request",
+                                "<absolute-data-run-request.json>",
+                                "--workspace",
+                                input.root,
+                                "--json",
+                            ],
+                            describeArgv: ["tiangong-ai", "data", "describe", "<capability-id>", "--json"],
+                            requestSchema: structuredClone(dataPublicSchemas.runRequest),
+                            catalog: dataCapabilities,
+                        }
+                        : null,
                     registerArtifact: input.stage === "acquire"
                         ? {
                             argv: [
@@ -651,10 +686,10 @@ export async function prepareNativeResearchStage(input) {
                 },
                 rules: [
                     "Current native host performs producer reasoning; the CLI does not spawn a producer.",
-                    "Native Web/Browser activity is visible in the evidence ledger but becomes admissible only after broker/input formalization and strict assessment.",
+                    "Native Web/Browser activity is visible in the evidence ledger but becomes admissible only after broker/data/input formalization and strict assessment.",
                     "An interactive challenge pauses immediately. A material evidence-exhausted handoff is valid only after inspectAccess proves every required plan-bound agent route with exact terminal event hashes.",
                     "When the next material step requires user authorization or an external response, request a durable handoff and stop; do not keep searching low-yield substitutes.",
-                    "Only broker receipts or registered immutable inputs may support discover output.",
+                    "Only broker receipts, data-runtime receipts, or registered immutable inputs may support discover output.",
                     "Only exact, structurally validated, content-addressed artifacts may support full-text acquisition claims.",
                     "Do not place credentials, cookies, authorization data, or sensitive URL parameters in request/output files.",
                     "A file's existence is not success; submit performs schema, provenance, budget, hash, and atomic-commit checks.",
@@ -1588,9 +1623,11 @@ async function createCapsule(root, project, workPackage, runId, config) {
     const evidenceArtifacts = frozenSnapshot
         ? await stageEvidenceArtifacts(root, project.id, capsuleProject, new Set(frozenSnapshot.artifacts.map((artifact) => artifact.artifactId)))
         : [];
+    const inputSha256s = new Set(project.inputs.map((input) => input.sha256));
+    const contextualEvidenceArtifacts = evidenceArtifacts.filter((artifact) => !inputSha256s.has(artifact.sha256));
     if ((workPackage.stage === "analyze" || workPackage.stage === "synthesize") &&
-        evidenceArtifacts.length) {
-        await writeProducerArtifactContext(capsuleProject, evidenceArtifacts, Math.floor((config.budget.maxInputContextTokens * RESEARCH_ESTIMATED_BYTES_PER_TOKEN) / 2));
+        contextualEvidenceArtifacts.length) {
+        await writeProducerArtifactContext(capsuleProject, contextualEvidenceArtifacts, Math.floor((config.budget.maxInputContextTokens * RESEARCH_ESTIMATED_BYTES_PER_TOKEN) / 2));
     }
     await writeJsonAtomic(join(capsuleProject, "inputs", "evidence-receipts.json"), evidenceReceipts.map(reviewSafeReceipt));
     await writeJsonAtomic(join(capsuleProject, "project.json"), {
@@ -1614,7 +1651,7 @@ async function createCapsule(root, project, workPackage, runId, config) {
     const stagedSkills = await stageLockedCapabilities(root, join(capsuleProject, "skills"));
     const capabilityDocumentation = await buildCapabilityDocumentation(capsuleProject, stagedSkills, config.budget.maxInputContextTokens * RESEARCH_ESTIMATED_BYTES_PER_TOKEN);
     const reviewEvidenceContext = workPackage.stage === "review"
-        ? await writeReviewEvidenceContext(root, project.id, capsuleProject, contextBundleContent, evidenceReceipts, evidenceArtifacts, config.budget.maxInputContextTokens * RESEARCH_ESTIMATED_BYTES_PER_TOKEN)
+        ? await writeReviewEvidenceContext(root, project.id, capsuleProject, contextBundleContent, evidenceReceipts, contextualEvidenceArtifacts, config.budget.maxInputContextTokens * RESEARCH_ESTIMATED_BYTES_PER_TOKEN)
         : null;
     const reviewPacket = reviewEvidenceContext
         ? await writeReviewPacket(root, capsuleProject, project, inputManifest, evidenceReceipts, evidenceArtifacts, reviewEvidenceContext.persistent)
@@ -1697,7 +1734,11 @@ async function writeReviewPacket(root, capsuleProject, project, inputManifest, e
     ];
     const evidenceFiles = new Map();
     for (const receipt of evidenceReceipts) {
-        for (const locator of [receipt.locator, receipt.contextLocator]) {
+        for (const locator of [
+            receipt.locator,
+            receipt.contextLocator,
+            ...(receipt.data?.artifacts.map((artifact) => artifact.locator) ?? []),
+        ]) {
             if (!evidenceFiles.has(locator)) {
                 evidenceFiles.set(locator, await fileRecord(resolveContained(capsuleProject, locator), locator));
             }
@@ -1789,7 +1830,7 @@ async function writeReviewEvidenceContext(root, projectId, capsuleProject, input
             : "[No admitted evidence source cites this receipt; its raw object and bounded context remain hash-bound in the review packet.]";
         views.push({
             prefix: [
-                `--- BROKER RECEIPT ${receipt.attemptId} ---`,
+                `--- ${receipt.evidenceKind === "data" ? "DATA" : "BROKER"} RECEIPT ${receipt.attemptId} ---`,
                 `metadata: ${JSON.stringify(metadata)}`,
                 "--- BEGIN BOUNDED REVIEW EXCERPT ---",
                 "",
@@ -1934,7 +1975,8 @@ async function loadBrokerReviewReferences(capsuleProject) {
     for (const source of evidence.sources) {
         if (!isObject(source) || !isObject(source.provenance))
             continue;
-        if (source.provenance.kind !== "broker" || typeof source.provenance.id !== "string")
+        if (!["broker", "data"].includes(String(source.provenance.kind)) ||
+            typeof source.provenance.id !== "string")
             continue;
         if (typeof source.id !== "string" || typeof source.title !== "string")
             continue;
@@ -2200,6 +2242,7 @@ async function reviewEnvironmentPacket(root, projectId) {
 function reviewSafeReceipt(receipt) {
     return {
         schemaVersion: receipt.schemaVersion,
+        evidenceKind: receipt.evidenceKind ?? "broker",
         attemptId: receipt.attemptId,
         capabilityId: receipt.capabilityId,
         status: receipt.status,
@@ -2220,6 +2263,7 @@ function reviewSafeReceipt(receipt) {
         retrievedAt: receipt.retrievedAt,
         servedAt: receipt.servedAt,
         cacheHit: receipt.cacheHit,
+        data: receipt.data ?? null,
     };
 }
 function agentRequest(input) {
@@ -2352,7 +2396,13 @@ async function validateEvidenceSources(root, project, sources) {
         join("inputs", input.id, basename(input.path)).replaceAll("\\", "/"),
     ]));
     const receipts = await loadProjectEvidenceReceipts(root, project.id);
-    const brokerLocators = new Map(receipts.map((receipt) => [receipt.attemptId, receipt.locator]));
+    const receiptLocators = new Map(receipts.map((receipt) => [
+        receipt.attemptId,
+        {
+            kind: receipt.evidenceKind === "data" ? "data" : "broker",
+            locator: receipt.locator,
+        },
+    ]));
     const sourceIds = new Set();
     for (const source of sources) {
         if (!isObject(source) || !isObject(source.provenance)) {
@@ -2366,13 +2416,15 @@ async function validateEvidenceSources(root, project, sources) {
         }
         const expectedLocator = source.provenance.kind === "input"
             ? inputLocators.get(String(source.provenance.id))
-            : source.provenance.kind === "broker"
-                ? brokerLocators.get(String(source.provenance.id))
+            : source.provenance.kind === "broker" || source.provenance.kind === "data"
+                ? receiptLocators.get(String(source.provenance.id))?.kind === source.provenance.kind
+                    ? receiptLocators.get(String(source.provenance.id))?.locator
+                    : undefined
                 : undefined;
         if (!expectedLocator || expectedLocator !== source.locator) {
             throw new StructuredOutputError(`discover output contains invalid provenance for evidence source ${String(id)}.`, {
                 validation: [
-                    "provenance.id must be an exact immutable input ID or broker receipt attemptId",
+                    "provenance.id must be an exact immutable input ID or evidence receipt attemptId",
                     "locator must exactly match the locator bound to that provenance record",
                 ],
                 actual: {
@@ -2385,10 +2437,10 @@ async function validateEvidenceSources(root, project, sources) {
                     id: inputId,
                     locator,
                 })),
-                allowedBrokerReceipts: [...brokerLocators].map(([attemptId, locator]) => ({
-                    kind: "broker",
+                allowedEvidenceReceipts: [...receiptLocators].map(([attemptId, receipt]) => ({
+                    kind: receipt.kind,
                     id: attemptId,
-                    locator,
+                    locator: receipt.locator,
                 })),
             });
         }
@@ -2691,12 +2743,18 @@ async function assertDiscoveryCoverage(root, project, evidencePath) {
     // establish breadth, dates, source types and dimensions, but cannot claim
     // that a search-result receipt is already acquired full text.
     const gaps = computed.mechanicalGaps.filter((gap) => !/full-text source\(s\)/.test(gap));
-    const requiredCapabilities = requiredDiscoveryCapabilityIds(await loadCapabilityDeclarations(root));
+    const requiredCapabilities = [
+        ...new Set([
+            ...requiredDiscoveryCapabilityIds(await loadCapabilityDeclarations(root)),
+            ...(project.evidenceRequirements.requiredCapabilityIds ?? []),
+        ]),
+    ];
     const exercisedCapabilities = new Set((await loadProjectEvidenceReceipts(root, project.id)).map((receipt) => receipt.capabilityId));
     const journalEvents = await readJournal(workspacePaths(root).journal);
     const attemptedCapabilities = new Map();
     for (const event of journalEvents) {
-        if (event.scope !== project.id || event.type !== "capability.fetch.attempted")
+        if (event.scope !== project.id ||
+            (event.type !== "capability.fetch.attempted" && event.type !== "data.capability.requested"))
             continue;
         const capabilityId = event.payload.capabilityId;
         if (typeof capabilityId !== "string")
@@ -2709,7 +2767,8 @@ async function assertDiscoveryCoverage(root, project, evidencePath) {
         attemptedCapabilities.set(capabilityId, current);
     }
     for (const event of journalEvents) {
-        if (event.scope !== project.id || event.type !== "capability.fetch.failed")
+        if (event.scope !== project.id ||
+            (event.type !== "capability.fetch.failed" && event.type !== "data.capability.failed"))
             continue;
         const capabilityId = event.payload.capabilityId;
         const failureKind = event.payload.failureKind;
@@ -3028,7 +3087,7 @@ async function stageContextForPackage(capsuleProject, project, workPackage, conf
 }
 function packagePrompt(project, workPackage, inputs, stagedSkills, capabilityDocumentation, reviewPacketSha256, contextBundle, contextBundleContent, stageContextContent, discovery, evidenceCandidates) {
     const stageInstructions = {
-        discover: "Assess candidates incrementally through the packet's recordAssessment command; do not accumulate a source-sized final response. The control plane has already assigned every immutable input and broker result a candidateId and retains its title, URL, DOI, dates, receipt, locator, JSON Pointer, hashes, and retrieval metadata. Reference candidateId; never repeat or invent those deterministic fields. Give each admitted candidate a concise sourceId plus source type, relevance, quality, applicability, coverage dimensions, and limitations. Record meaningful explicit rejections; omitted candidates remain unassessed for later gap filling. Native Web or Browser discoveries are supplemental candidates only and cannot be admitted until an immutable broker receipt is attached to the same canonical URL or DOI. After broad search, strict assessment, and focused gap filling, return only the small closeout object with one judgment for every reviewed dimension plus limitations and remaining gaps. The CLI mechanically joins the latest recorded assessments to provenance, derives counts/date range/coverage, and rejects unknown or unformalized candidates.",
+        discover: "Assess candidates incrementally through the packet's recordAssessment command; do not accumulate a source-sized final response. The control plane has already assigned every immutable input, broker result, and structured data result a candidateId and retains its title, URL, DOI, dates, receipt, locator, JSON Pointer, hashes, and retrieval metadata. Reference candidateId; never repeat or invent those deterministic fields. Give each admitted candidate a concise sourceId plus source type, relevance, quality, applicability, coverage dimensions, and limitations. Record meaningful explicit rejections; omitted candidates remain unassessed for later gap filling. Native Web or Browser discoveries are supplemental candidates only and cannot be admitted until an immutable broker receipt is attached to the same canonical URL or DOI. After broad search, strict assessment, and focused gap filling, return only the small closeout object with one judgment for every reviewed dimension plus limitations and remaining gaps. The CLI mechanically joins the latest recorded assessments to provenance, derives counts/date range/coverage, and rejects unknown or unformalized candidates.",
         acquire: "Audit every provisionally admitted source exactly once. For each source, bind its ledger candidateId, list only artifactIds returned by the exact artifact registration command, and choose accepted, limited, or rejected with a concise rationale and explicit limitations. A broker receipt is an immutable discovery record but is not full text. Use an empty artifactIds array only when intentionally retaining a source as metadata/abstract evidence or when the source is an already registered local input. Put unresolved blocking acquisition or coverage deficiencies in gaps; put honest non-blocking scope constraints in limitations. Do not invent file paths, hashes, URLs, artifact IDs, or successful downloads.",
         analyze: "Use only the complete embedded inference snapshot below and return analysis schema v2. Bind the exact inference snapshot hash and one reproducible analysisRun. Every finding must cite admitted source IDs and exact evidenceAtomIds; a top-journal finding must also bind design claimIds. State uncertainty and applicability. Never promote a source-level citation when no exact atom supports the statement.",
         synthesize: "Use only the complete embedded inference snapshot, analysis, and Claim–Evidence Graph below. Return the schema-defined object whose reportMarkdown separates supported conclusions, uncertainty, limitations, and next actions. Every material statement must remain within a graph-bound finding. Use real Markdown line breaks encoded exactly once for JSON; never place literal /n or double-escaped \\n markers in reportMarkdown.",
@@ -3050,7 +3109,7 @@ function packagePrompt(project, workPackage, inputs, stagedSkills, capabilityDoc
                 ? "Use the installed external acquisition/document Skills in the current native host, but treat the CLI artifact registry and acquisition schema as the only authority for durable evidence."
                 : "Capability files are provenance-bound but are not available as execution tools in this stage.",
         workPackage.stage === "discover"
-            ? `Follow the reviewed discovery plan: ${JSON.stringify(discovery)}. The plan's max broker-view count is a hard working ceiling, not a target to exhaust. Execute required first-pass channels before supplemental channels, prefer broad high-yield queries, assess registered candidates between batches, and use the next gap-fill batch only for explicit uncovered dimensions, source types, date ranges, full text, limitations, or counterevidence. Stop fetching as soon as the declared coverage minimums are supportable. Native Web/Browser may broaden lead discovery, but every such action must be recorded through recordActivity, every useful result must be registered as a candidate, and the same URL/DOI must then be formalized through the broker before admission. The broker or an immutable registered input is the sole admissible evidence path. Do not execute a staged Skill's curl/CLI examples or read provider environment variables. Invoke fetch_candidate_source with the manifest capability ID and obey its exact declared HTTP method. GET capabilities use only declared query parameters; POST capabilities require request_body containing only the documented non-secret request JSON. Never place API keys, tokens, authorization data, cookies, or other credential-like fields in request_body. The broker injects the sole declared logical credential, disables caching for credentialed requests, and never persists the POST body (only its hash). The tool result includes exact bounded context, registered candidate IDs, its receipt, whether a network call was avoided, and remaining view budget. Exercise every manifest capability with requiredForDiscovery=true, or the mechanical coverage gate will stop downstream work.`
+            ? `Follow the reviewed discovery plan: ${JSON.stringify(discovery)}. The plan's max evidence-call count is a hard working ceiling, not a target to exhaust. Execute required first-pass channels before supplemental channels, prefer broad high-yield queries, assess registered candidates between batches, and use the next gap-fill batch only for explicit uncovered dimensions, source types, date ranges, full text, limitations, or counterevidence. Stop fetching as soon as the declared coverage minimums are supportable. Native Web/Browser may broaden lead discovery, but every such action must be recorded through recordActivity, every useful result must be registered as a candidate, and the same URL/DOI must then be formalized through the broker before admission. A broker receipt, structured data-runtime receipt, or immutable registered input is an admissible evidence path. Do not execute a staged Skill's curl/CLI examples or read provider environment variables. For generic broker capabilities, invoke fetch_candidate_source with the manifest capability ID and obey its exact declared HTTP method. For structured data capabilities, use the packet's dynamic catalog and describe command, then invoke runDataCapability; never call standalone data run for project evidence. Never place API keys, tokens, authorization data, cookies, or other credential-like fields in request files. The Research control plane injects declared logical credentials and persists only safe hash-bound results. Exercise every manifest capability with requiredForDiscovery=true and every project-required data capability ID, or the mechanical coverage gate will stop downstream work.`
             : "Use only the complete embedded stage context; no tools or additional source reads are allowed.",
         stageInstructions[workPackage.stage],
         "Do not write stage output files directly. Your final response must be only the JSON object required by the supplied output schema; the CLI will validate and atomically materialize it.",