@harness-engineering/intelligence 0.2.6 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +263 -1
- package/dist/index.d.ts +263 -1
- package/dist/index.js +419 -27
- package/dist/index.mjs +410 -25
- package/package.json +6 -3
package/dist/index.d.mts
CHANGED
|
@@ -283,6 +283,84 @@ interface ManualInput {
|
|
|
283
283
|
*/
|
|
284
284
|
declare function manualToRawWorkItem(input: ManualInput): RawWorkItem;
|
|
285
285
|
|
|
286
|
+
/**
|
|
287
|
+
* Canary adapter — a total, gracefully-degrading boundary around the deterministic
|
|
288
|
+
* `canary` test CLI (`canary-test-cli`, declared as an optionalDependency).
|
|
289
|
+
*
|
|
290
|
+
* All `canary` / `canary-test-cli` references are confined to this module
|
|
291
|
+
* (enforced by a boundary test). The adapter never throws on a missing or
|
|
292
|
+
* misbehaving CLI: every method resolves a degraded/empty result instead.
|
|
293
|
+
*/
|
|
294
|
+
/** Why probe() degraded. */
|
|
295
|
+
type CanaryDegradeReason = 'not-installed' | 'binary-missing' | 'exec-failed' | 'bad-output';
|
|
296
|
+
interface CanaryProbe {
|
|
297
|
+
status: 'available' | 'degraded';
|
|
298
|
+
version?: string;
|
|
299
|
+
reason?: CanaryDegradeReason;
|
|
300
|
+
}
|
|
301
|
+
declare const frameworkRecommendationSchema: z.ZodObject<{
|
|
302
|
+
status: z.ZodString;
|
|
303
|
+
test_type: z.ZodString;
|
|
304
|
+
framework: z.ZodString;
|
|
305
|
+
file_extension: z.ZodString;
|
|
306
|
+
reasoning: z.ZodArray<z.ZodString, "many">;
|
|
307
|
+
alternatives: z.ZodArray<z.ZodString, "many">;
|
|
308
|
+
}, "strip", z.ZodTypeAny, {
|
|
309
|
+
status: string;
|
|
310
|
+
test_type: string;
|
|
311
|
+
framework: string;
|
|
312
|
+
file_extension: string;
|
|
313
|
+
reasoning: string[];
|
|
314
|
+
alternatives: string[];
|
|
315
|
+
}, {
|
|
316
|
+
status: string;
|
|
317
|
+
test_type: string;
|
|
318
|
+
framework: string;
|
|
319
|
+
file_extension: string;
|
|
320
|
+
reasoning: string[];
|
|
321
|
+
alternatives: string[];
|
|
322
|
+
}>;
|
|
323
|
+
type FrameworkRecommendation = z.infer<typeof frameworkRecommendationSchema>;
|
|
324
|
+
declare const canaryFindingSchema: z.ZodObject<{
|
|
325
|
+
file: z.ZodString;
|
|
326
|
+
line: z.ZodNumber;
|
|
327
|
+
rule: z.ZodString;
|
|
328
|
+
severity: z.ZodString;
|
|
329
|
+
message: z.ZodString;
|
|
330
|
+
suggestion: z.ZodString;
|
|
331
|
+
}, "strip", z.ZodTypeAny, {
|
|
332
|
+
message: string;
|
|
333
|
+
file: string;
|
|
334
|
+
line: number;
|
|
335
|
+
rule: string;
|
|
336
|
+
severity: string;
|
|
337
|
+
suggestion: string;
|
|
338
|
+
}, {
|
|
339
|
+
message: string;
|
|
340
|
+
file: string;
|
|
341
|
+
line: number;
|
|
342
|
+
rule: string;
|
|
343
|
+
severity: string;
|
|
344
|
+
suggestion: string;
|
|
345
|
+
}>;
|
|
346
|
+
type CanaryFinding = z.infer<typeof canaryFindingSchema>;
|
|
347
|
+
interface CanaryAdapter {
|
|
348
|
+
probe(): Promise<CanaryProbe>;
|
|
349
|
+
recommendFramework(prompt: string): Promise<FrameworkRecommendation>;
|
|
350
|
+
reviewTest(path: string, framework?: string): Promise<CanaryFinding[]>;
|
|
351
|
+
}
|
|
352
|
+
/**
|
|
353
|
+
* The raw exec seam: runs a `canary` subcommand and resolves its stdout, or
|
|
354
|
+
* rejects with the spawn/exit error (carrying `code` and `stderr`). This is the
|
|
355
|
+
* single injection point — the default talks to the real CLI; tests pass a fake.
|
|
356
|
+
* Injecting here (rather than at a higher level) keeps the degrade-classification
|
|
357
|
+
* in `execCanary` fully under test.
|
|
358
|
+
*/
|
|
359
|
+
type CanaryExec = (cmd: string, args: string[]) => Promise<{
|
|
360
|
+
stdout: string;
|
|
361
|
+
}>;
|
|
362
|
+
declare function createCanaryAdapter(exec?: CanaryExec): CanaryAdapter;
|
|
363
|
+
|
|
286
364
|
interface AnalysisRequest {
|
|
287
365
|
prompt: string;
|
|
288
366
|
systemPrompt?: string;
|
|
@@ -545,6 +623,14 @@ interface ExecutionOutcome {
|
|
|
545
623
|
agentPersona?: string;
|
|
546
624
|
/** Task type categorization (e.g., 'feature', 'bugfix', 'refactor', 'docs'). */
|
|
547
625
|
taskType?: TaskType;
|
|
626
|
+
/**
|
|
627
|
+
* Optional caller-supplied metadata merged into the node's metadata.
|
|
628
|
+
* Used by judgment sources (e.g. outcome-eval) to record verdict-specific
|
|
629
|
+
* signal -- verdict, confidence, judgedAgainst, source -- without bloating
|
|
630
|
+
* the core ExecutionOutcome contract. Reserved core keys (id/result/etc.)
|
|
631
|
+
* always win and are not overridable.
|
|
632
|
+
*/
|
|
633
|
+
metadata?: Record<string, unknown>;
|
|
548
634
|
}
|
|
549
635
|
|
|
550
636
|
interface OutcomeIngestResult {
|
|
@@ -566,6 +652,182 @@ declare class ExecutionOutcomeConnector {
|
|
|
566
652
|
ingest(outcome: ExecutionOutcome): OutcomeIngestResult;
|
|
567
653
|
}
|
|
568
654
|
|
|
655
|
+
/**
|
|
656
|
+
* outcome-eval contract types.
|
|
657
|
+
*
|
|
658
|
+
* `authority` is DERIVED in TypeScript from (verdict, confidence) via
|
|
659
|
+
* `deriveAuthority` in `./authority.js`. It is NEVER read from the LLM
|
|
660
|
+
* response — see `verdictSchema` in `./prompts.js`, which omits it.
|
|
661
|
+
*/
|
|
662
|
+
type Verdict = 'SATISFIED' | 'NOT_SATISFIED' | 'INCONCLUSIVE';
|
|
663
|
+
type Confidence = 'low' | 'medium' | 'high';
|
|
664
|
+
type JudgedAgainst = 'success-criteria' | 'user-visible-behavior' | 'overview';
|
|
665
|
+
/** Ship authority DERIVED in TS from (verdict, confidence); never from the LLM. */
|
|
666
|
+
type Authority = 'blocking' | 'advisory';
|
|
667
|
+
interface OutcomeEvalInput {
|
|
668
|
+
/** Absolute or repo-relative path to the spec markdown. */
|
|
669
|
+
specPath: string;
|
|
670
|
+
/** Unified diff of the change under judgment. */
|
|
671
|
+
diff: string;
|
|
672
|
+
/** Captured test-runner output. */
|
|
673
|
+
testOutput: string;
|
|
674
|
+
/** Pre-resolved judgment section; otherwise the section-resolver runs. */
|
|
675
|
+
specSection?: string;
|
|
676
|
+
}
|
|
677
|
+
interface OutcomeVerdict {
|
|
678
|
+
verdict: Verdict;
|
|
679
|
+
confidence: Confidence;
|
|
680
|
+
/** Cites specific met / unmet criteria. */
|
|
681
|
+
rationale: string;
|
|
682
|
+
judgedAgainst: JudgedAgainst;
|
|
683
|
+
/** Empty when SATISFIED. */
|
|
684
|
+
unmetCriteria: string[];
|
|
685
|
+
/** DERIVED in TS from (verdict, confidence); never from the LLM. */
|
|
686
|
+
authority: Authority;
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
/**
|
|
690
|
+
* Pure mapping from (verdict, confidence) to ship authority.
|
|
691
|
+
*
|
|
692
|
+
* Blocking iff a NOT_SATISFIED verdict is held with high confidence; every
|
|
693
|
+
* other combination — including all INCONCLUSIVE and SATISFIED cases — is
|
|
694
|
+
* advisory. Missing inputs never punish the change.
|
|
695
|
+
*
|
|
696
|
+
* This function is the false-positive-critical seam. Authority is computed
|
|
697
|
+
* here in TypeScript and is NEVER trusted from the LLM response.
|
|
698
|
+
*/
|
|
699
|
+
declare function deriveAuthority(verdict: Verdict, confidence: Confidence): Authority;
|
|
700
|
+
|
|
701
|
+
/**
|
|
702
|
+
* Zod schema for the LLM verdict response.
|
|
703
|
+
*
|
|
704
|
+
* `authority` is intentionally ABSENT: it is derived in TypeScript by
|
|
705
|
+
* `deriveAuthority` and must never be supplied by the model. The schema is
|
|
706
|
+
* `.strict()` so an injected `authority` (or any other extra key) is rejected
|
|
707
|
+
* at the parse boundary rather than silently passing through.
|
|
708
|
+
*
|
|
709
|
+
*/
|
|
710
|
+
declare const verdictSchema: z.ZodObject<{
|
|
711
|
+
verdict: z.ZodEnum<["SATISFIED", "NOT_SATISFIED", "INCONCLUSIVE"]>;
|
|
712
|
+
confidence: z.ZodEnum<["low", "medium", "high"]>;
|
|
713
|
+
rationale: z.ZodString;
|
|
714
|
+
unmetCriteria: z.ZodArray<z.ZodString, "many">;
|
|
715
|
+
}, "strict", z.ZodTypeAny, {
|
|
716
|
+
verdict: "SATISFIED" | "NOT_SATISFIED" | "INCONCLUSIVE";
|
|
717
|
+
confidence: "low" | "medium" | "high";
|
|
718
|
+
rationale: string;
|
|
719
|
+
unmetCriteria: string[];
|
|
720
|
+
}, {
|
|
721
|
+
verdict: "SATISFIED" | "NOT_SATISFIED" | "INCONCLUSIVE";
|
|
722
|
+
confidence: "low" | "medium" | "high";
|
|
723
|
+
rationale: string;
|
|
724
|
+
unmetCriteria: string[];
|
|
725
|
+
}>;
|
|
726
|
+
type LlmVerdict = z.infer<typeof verdictSchema>;
|
|
727
|
+
/**
|
|
728
|
+
* System prompt for outcome-eval. Conservative-confidence posture copied from
|
|
729
|
+
* security-craft (SKILL.md): the model defaults to `medium` confidence; `high`
|
|
730
|
+
* requires naming a specific met or unmet criterion; the bias is toward
|
|
731
|
+
* advisory, not blocking. `authority` is derived in TypeScript and must never
|
|
732
|
+
* be supplied by the model — the schema is `.strict()` and rejects it.
|
|
733
|
+
*/
|
|
734
|
+
declare const OUTCOME_EVAL_SYSTEM_PROMPT = "You are a post-execution outcome judge. Given a spec acceptance section, a unified diff, and test output, decide whether the change SATISFIED, NOT_SATISFIED, or is INCONCLUSIVE against that section.\n\nConfidence calibration (be conservative \u2014 false alarms are costly):\n- Default to \"medium\" confidence.\n- Use \"high\" ONLY when you can name a SPECIFIC criterion from the section that the diff and test output clearly met or clearly failed to meet, and quote or paraphrase it in the rationale.\n- Use \"low\" when the diff or test output is ambiguous, partial, or insufficient to judge.\n- When the change only PARTIALLY meets the criteria, do not exceed \"medium\" confidence.\n- Bias toward advisory caution: if unsure between two confidence levels, choose the lower one.\n\nRules:\n- The rationale MUST cite specific met or unmet criteria from the section.\n- \"unmetCriteria\" lists the section criteria the change failed to meet; it is empty when the verdict is SATISFIED.\n- Do NOT emit an \"authority\" field. Authority is computed downstream in TypeScript from (verdict, confidence) and must never come from you.\n\nReturn your judgment using the structured_output tool.";
|
|
735
|
+
/**
|
|
736
|
+
* Build the user prompt from the resolved spec section body, the change diff,
|
|
737
|
+
* and the captured test output. Mirrors the labeled-section structure of
|
|
738
|
+
* sel/pesl prompts.
|
|
739
|
+
*
|
|
740
|
+
* The diff and test output are clamped to PROMPT_FIELD_MAX_CHARS and wrapped in
|
|
741
|
+
* a 4-backtick fence so a triple-backtick sequence inside the diff cannot close
|
|
742
|
+
* the fence early.
|
|
743
|
+
*/
|
|
744
|
+
declare function buildUserPrompt(section: string, diff: string, testOutput: string): string;
|
|
745
|
+
|
|
746
|
+
/**
|
|
747
|
+
* Result of resolving the judgment section from a spec's markdown.
|
|
748
|
+
* `body` is the matched section's content (heading excluded, blank-trimmed).
|
|
749
|
+
*/
|
|
750
|
+
interface ResolvedSection {
|
|
751
|
+
judgedAgainst: JudgedAgainst;
|
|
752
|
+
body: string;
|
|
753
|
+
}
|
|
754
|
+
/**
|
|
755
|
+
* Resolve the judgment input from a spec's markdown via the fallback chain
|
|
756
|
+
* Success Criteria -> User-Visible Behavior -> Overview, returning the matched
|
|
757
|
+
* section body plus which heading matched.
|
|
758
|
+
*
|
|
759
|
+
* Returns `null` when no judgable section exists. The caller (a later phase)
|
|
760
|
+
* maps that null to an INCONCLUSIVE verdict — this resolver never throws and
|
|
761
|
+
* never decides verdict authority.
|
|
762
|
+
*
|
|
763
|
+
* Self-contained string parsing: imports only the JudgedAgainst type, honoring
|
|
764
|
+
* the intelligence layer rule (no `core` dependency).
|
|
765
|
+
*/
|
|
766
|
+
declare function resolveSection(markdown: string): ResolvedSection | null;
|
|
767
|
+
|
|
768
|
+
interface OutcomeEvaluatorOptions {
|
|
769
|
+
/** Override model for the outcome-eval LLM call. */
|
|
770
|
+
model?: string;
|
|
771
|
+
}
|
|
772
|
+
/**
|
|
773
|
+
* Post-execution spec-satisfaction judge. Mirrors PeslSimulator's
|
|
774
|
+
* (provider, store, options) constructor shape. The store is held for the
|
|
775
|
+
* Phase 4 execution_outcome graph write; see `persistOutcome`.
|
|
776
|
+
*/
|
|
777
|
+
declare class OutcomeEvaluator {
|
|
778
|
+
private readonly provider;
|
|
779
|
+
private readonly store;
|
|
780
|
+
private readonly options;
|
|
781
|
+
constructor(provider: AnalysisProvider, store: GraphStore, options?: OutcomeEvaluatorOptions);
|
|
782
|
+
evaluate(input: OutcomeEvalInput): Promise<OutcomeVerdict>;
|
|
783
|
+
/**
|
|
784
|
+
* Run the provider call and strict re-parse. ANY failure here — provider
|
|
785
|
+
* rejection (rate limit/network), or a strict-parse rejection of a malformed
|
|
786
|
+
* or authority-injected payload — degrades safely to INCONCLUSIVE/low/
|
|
787
|
+
* advisory rather than throwing. This reconciles Criterion 3 (never block on
|
|
788
|
+
* noise) with Criterion 4: an injected `authority` key is discarded by the
|
|
789
|
+
* .strict() re-parse, and the degraded verdict's authority is DERIVED from
|
|
790
|
+
* INCONCLUSIVE/low = advisory — so the LLM gains nothing by injecting it.
|
|
791
|
+
*/
|
|
792
|
+
private judge;
|
|
793
|
+
/**
|
|
794
|
+
* Build the safe-degradation verdict. INCONCLUSIVE/low yields advisory
|
|
795
|
+
* authority via deriveAuthority — never blocking. The rationale names only a
|
|
796
|
+
* coarse reason category, never a stack trace or secret.
|
|
797
|
+
*/
|
|
798
|
+
private degradedVerdict;
|
|
799
|
+
/** Persist (Phase 4 seam) then return the verdict. */
|
|
800
|
+
private finish;
|
|
801
|
+
private resolveJudgmentSection;
|
|
802
|
+
private buildVerdict;
|
|
803
|
+
/**
|
|
804
|
+
* Map an OutcomeVerdict + OutcomeEvalInput to the connector's ExecutionOutcome.
|
|
805
|
+
* - result: SATISFIED -> 'success'; otherwise 'failure'. INCONCLUSIVE is
|
|
806
|
+
* 'failure' for type-validity but omits agentPersona/affected systems so
|
|
807
|
+
* the effectiveness scorer ignores it (plan D2).
|
|
808
|
+
* - linkedSpecId: input.specPath (metadata only; no spec edge — plan D1).
|
|
809
|
+
* - affectedSystemNodeIds: [] in v1 (not available from OutcomeEvalInput — D4).
|
|
810
|
+
* - id: one node per EVALUATION. GraphStore.addNode upserts by id, so the id
|
|
811
|
+
* carries a collision-free randomUUID() — two evaluate() calls in the same
|
|
812
|
+
* millisecond can never overwrite each other (data-loss fix). specPath is
|
|
813
|
+
* included for human readability only.
|
|
814
|
+
* - taskType: OMITTED. The outcome-eval judge has no task categorization, and
|
|
815
|
+
* asserting a false 'feature' would mislead specialization analytics (SUG-2).
|
|
816
|
+
* - metadata: verdict-specific signal carried through the connector's
|
|
817
|
+
* additive pass-through (verdict/confidence/judgedAgainst/source) so the
|
|
818
|
+
* true 3-valued verdict is durable on the node (Truth 3).
|
|
819
|
+
*/
|
|
820
|
+
private toExecutionOutcome;
|
|
821
|
+
/**
|
|
822
|
+
* Phase 4: writes exactly one execution_outcome node via
|
|
823
|
+
* ExecutionOutcomeConnector. Degrade-safe (plan D3): a graph-write failure is
|
|
824
|
+
* swallowed-and-logged, never thrown — the verdict is already computed before
|
|
825
|
+
* this runs, so swallowing keeps evaluate() total. No secrets/stack frames in
|
|
826
|
+
* the log message.
|
|
827
|
+
*/
|
|
828
|
+
private persistOutcome;
|
|
829
|
+
}
|
|
830
|
+
|
|
569
831
|
/**
|
|
570
832
|
* Compute historical complexity from past execution outcomes in the graph.
|
|
571
833
|
*
|
|
@@ -906,4 +1168,4 @@ declare function saveProfiles(projectRoot: string, store: ProfileStore): void;
|
|
|
906
1168
|
*/
|
|
907
1169
|
declare function refreshProfiles(projectRoot: string, graphStore: GraphStore, opts?: Omit<SpecializationOptions, 'persona'>): ProfileStore;
|
|
908
1170
|
|
|
909
|
-
export { type AffectedSystem, type AnalysisProvider, type AnalysisRequest, type AnalysisResponse, AnthropicAnalysisProvider, type BlastRadius, type BlindSpot, ClaudeCliAnalysisProvider, type ComplexityScore, type EnrichedSpec, type ExecutionOutcome, ExecutionOutcomeConnector, type ExpertiseLevel, type GitHubIssue, GraphValidator, IntelligencePipeline, type JiraIssue, type LinearIssue, type ManualInput, OpenAICompatibleAnalysisProvider, type OutcomeIngestResult, type PersonaEffectivenessScore, type PersonaRecommendation, PeslSimulator, type PreprocessResult, type ProfileStore, type RawWorkItem, type SimulationResult, type SpecializationEntry, type SpecializationProfile, type SpecializationScore, type TaskType, type TemporalConfig, type WeightedRecommendation, buildSpecializationProfile, computeExpertiseLevel, computeHistoricalComplexity, computePersonaEffectiveness, computeSemanticComplexity, computeSpecialization, computeStructuralComplexity, decayWeight, detectBlindSpots, enrich, githubToRawWorkItem, jiraToRawWorkItem, linearToRawWorkItem, loadProfiles, manualToRawWorkItem, recommendPersona, refreshProfiles, runGraphOnlyChecks, runLlmSimulation, saveProfiles, score as scoreCML, scoreToConcernSignals, temporalSuccessRate, toRawWorkItem, weightedRecommendPersona };
|
|
1171
|
+
export { type AffectedSystem, type AnalysisProvider, type AnalysisRequest, type AnalysisResponse, AnthropicAnalysisProvider, type Authority, type BlastRadius, type BlindSpot, type CanaryAdapter, type CanaryDegradeReason, type CanaryExec, type CanaryFinding, type CanaryProbe, ClaudeCliAnalysisProvider, type ComplexityScore, type Confidence, type EnrichedSpec, type ExecutionOutcome, ExecutionOutcomeConnector, type ExpertiseLevel, type FrameworkRecommendation, type GitHubIssue, GraphValidator, IntelligencePipeline, type JiraIssue, type JudgedAgainst, type LinearIssue, type LlmVerdict, type ManualInput, OUTCOME_EVAL_SYSTEM_PROMPT, OpenAICompatibleAnalysisProvider, type OutcomeEvalInput, OutcomeEvaluator, type OutcomeEvaluatorOptions, type OutcomeIngestResult, type OutcomeVerdict, type PersonaEffectivenessScore, type PersonaRecommendation, PeslSimulator, type PreprocessResult, type ProfileStore, type RawWorkItem, type ResolvedSection, type SimulationResult, type SpecializationEntry, type SpecializationProfile, type SpecializationScore, type TaskType, type TemporalConfig, type Verdict, type WeightedRecommendation, buildSpecializationProfile, buildUserPrompt, computeExpertiseLevel, computeHistoricalComplexity, computePersonaEffectiveness, computeSemanticComplexity, computeSpecialization, computeStructuralComplexity, createCanaryAdapter, decayWeight, deriveAuthority, detectBlindSpots, enrich, githubToRawWorkItem, jiraToRawWorkItem, linearToRawWorkItem, loadProfiles, manualToRawWorkItem, recommendPersona, refreshProfiles, resolveSection, runGraphOnlyChecks, runLlmSimulation, saveProfiles, score as scoreCML, scoreToConcernSignals, temporalSuccessRate, toRawWorkItem, verdictSchema, weightedRecommendPersona };
|
package/dist/index.d.ts
CHANGED
|
@@ -283,6 +283,84 @@ interface ManualInput {
|
|
|
283
283
|
*/
|
|
284
284
|
declare function manualToRawWorkItem(input: ManualInput): RawWorkItem;
|
|
285
285
|
|
|
286
|
+
/**
|
|
287
|
+
* Canary adapter — a total, gracefully-degrading boundary around the deterministic
|
|
288
|
+
* `canary` test CLI (`canary-test-cli`, declared as an optionalDependency).
|
|
289
|
+
*
|
|
290
|
+
* All `canary` / `canary-test-cli` references are confined to this module
|
|
291
|
+
* (enforced by a boundary test). The adapter never throws on a missing or
|
|
292
|
+
* misbehaving CLI: every method resolves a degraded/empty result instead.
|
|
293
|
+
*/
|
|
294
|
+
/** Why probe() degraded. */
|
|
295
|
+
type CanaryDegradeReason = 'not-installed' | 'binary-missing' | 'exec-failed' | 'bad-output';
|
|
296
|
+
interface CanaryProbe {
|
|
297
|
+
status: 'available' | 'degraded';
|
|
298
|
+
version?: string;
|
|
299
|
+
reason?: CanaryDegradeReason;
|
|
300
|
+
}
|
|
301
|
+
declare const frameworkRecommendationSchema: z.ZodObject<{
|
|
302
|
+
status: z.ZodString;
|
|
303
|
+
test_type: z.ZodString;
|
|
304
|
+
framework: z.ZodString;
|
|
305
|
+
file_extension: z.ZodString;
|
|
306
|
+
reasoning: z.ZodArray<z.ZodString, "many">;
|
|
307
|
+
alternatives: z.ZodArray<z.ZodString, "many">;
|
|
308
|
+
}, "strip", z.ZodTypeAny, {
|
|
309
|
+
status: string;
|
|
310
|
+
test_type: string;
|
|
311
|
+
framework: string;
|
|
312
|
+
file_extension: string;
|
|
313
|
+
reasoning: string[];
|
|
314
|
+
alternatives: string[];
|
|
315
|
+
}, {
|
|
316
|
+
status: string;
|
|
317
|
+
test_type: string;
|
|
318
|
+
framework: string;
|
|
319
|
+
file_extension: string;
|
|
320
|
+
reasoning: string[];
|
|
321
|
+
alternatives: string[];
|
|
322
|
+
}>;
|
|
323
|
+
type FrameworkRecommendation = z.infer<typeof frameworkRecommendationSchema>;
|
|
324
|
+
declare const canaryFindingSchema: z.ZodObject<{
|
|
325
|
+
file: z.ZodString;
|
|
326
|
+
line: z.ZodNumber;
|
|
327
|
+
rule: z.ZodString;
|
|
328
|
+
severity: z.ZodString;
|
|
329
|
+
message: z.ZodString;
|
|
330
|
+
suggestion: z.ZodString;
|
|
331
|
+
}, "strip", z.ZodTypeAny, {
|
|
332
|
+
message: string;
|
|
333
|
+
file: string;
|
|
334
|
+
line: number;
|
|
335
|
+
rule: string;
|
|
336
|
+
severity: string;
|
|
337
|
+
suggestion: string;
|
|
338
|
+
}, {
|
|
339
|
+
message: string;
|
|
340
|
+
file: string;
|
|
341
|
+
line: number;
|
|
342
|
+
rule: string;
|
|
343
|
+
severity: string;
|
|
344
|
+
suggestion: string;
|
|
345
|
+
}>;
|
|
346
|
+
type CanaryFinding = z.infer<typeof canaryFindingSchema>;
|
|
347
|
+
interface CanaryAdapter {
|
|
348
|
+
probe(): Promise<CanaryProbe>;
|
|
349
|
+
recommendFramework(prompt: string): Promise<FrameworkRecommendation>;
|
|
350
|
+
reviewTest(path: string, framework?: string): Promise<CanaryFinding[]>;
|
|
351
|
+
}
|
|
352
|
+
/**
|
|
353
|
+
* The raw exec seam: runs a `canary` subcommand and resolves its stdout, or
|
|
354
|
+
* rejects with the spawn/exit error (carrying `code` and `stderr`). This is the
|
|
355
|
+
* single injection point — the default talks to the real CLI; tests pass a fake.
|
|
356
|
+
* Injecting here (rather than at a higher level) keeps the degrade-classification
|
|
357
|
+
* in `execCanary` fully under test.
|
|
358
|
+
*/
|
|
359
|
+
type CanaryExec = (cmd: string, args: string[]) => Promise<{
|
|
360
|
+
stdout: string;
|
|
361
|
+
}>;
|
|
362
|
+
declare function createCanaryAdapter(exec?: CanaryExec): CanaryAdapter;
|
|
363
|
+
|
|
286
364
|
interface AnalysisRequest {
|
|
287
365
|
prompt: string;
|
|
288
366
|
systemPrompt?: string;
|
|
@@ -545,6 +623,14 @@ interface ExecutionOutcome {
|
|
|
545
623
|
agentPersona?: string;
|
|
546
624
|
/** Task type categorization (e.g., 'feature', 'bugfix', 'refactor', 'docs'). */
|
|
547
625
|
taskType?: TaskType;
|
|
626
|
+
/**
|
|
627
|
+
* Optional caller-supplied metadata merged into the node's metadata.
|
|
628
|
+
* Used by judgment sources (e.g. outcome-eval) to record verdict-specific
|
|
629
|
+
* signal -- verdict, confidence, judgedAgainst, source -- without bloating
|
|
630
|
+
* the core ExecutionOutcome contract. Reserved core keys (id/result/etc.)
|
|
631
|
+
* always win and are not overridable.
|
|
632
|
+
*/
|
|
633
|
+
metadata?: Record<string, unknown>;
|
|
548
634
|
}
|
|
549
635
|
|
|
550
636
|
interface OutcomeIngestResult {
|
|
@@ -566,6 +652,182 @@ declare class ExecutionOutcomeConnector {
|
|
|
566
652
|
ingest(outcome: ExecutionOutcome): OutcomeIngestResult;
|
|
567
653
|
}
|
|
568
654
|
|
|
655
|
+
/**
|
|
656
|
+
* outcome-eval contract types.
|
|
657
|
+
*
|
|
658
|
+
* `authority` is DERIVED in TypeScript from (verdict, confidence) via
|
|
659
|
+
* `deriveAuthority` in `./authority.js`. It is NEVER read from the LLM
|
|
660
|
+
* response — see `verdictSchema` in `./prompts.js`, which omits it.
|
|
661
|
+
*/
|
|
662
|
+
type Verdict = 'SATISFIED' | 'NOT_SATISFIED' | 'INCONCLUSIVE';
|
|
663
|
+
type Confidence = 'low' | 'medium' | 'high';
|
|
664
|
+
type JudgedAgainst = 'success-criteria' | 'user-visible-behavior' | 'overview';
|
|
665
|
+
/** Ship authority DERIVED in TS from (verdict, confidence); never from the LLM. */
|
|
666
|
+
type Authority = 'blocking' | 'advisory';
|
|
667
|
+
interface OutcomeEvalInput {
|
|
668
|
+
/** Absolute or repo-relative path to the spec markdown. */
|
|
669
|
+
specPath: string;
|
|
670
|
+
/** Unified diff of the change under judgment. */
|
|
671
|
+
diff: string;
|
|
672
|
+
/** Captured test-runner output. */
|
|
673
|
+
testOutput: string;
|
|
674
|
+
/** Pre-resolved judgment section; otherwise the section-resolver runs. */
|
|
675
|
+
specSection?: string;
|
|
676
|
+
}
|
|
677
|
+
interface OutcomeVerdict {
|
|
678
|
+
verdict: Verdict;
|
|
679
|
+
confidence: Confidence;
|
|
680
|
+
/** Cites specific met / unmet criteria. */
|
|
681
|
+
rationale: string;
|
|
682
|
+
judgedAgainst: JudgedAgainst;
|
|
683
|
+
/** Empty when SATISFIED. */
|
|
684
|
+
unmetCriteria: string[];
|
|
685
|
+
/** DERIVED in TS from (verdict, confidence); never from the LLM. */
|
|
686
|
+
authority: Authority;
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
/**
|
|
690
|
+
* Pure mapping from (verdict, confidence) to ship authority.
|
|
691
|
+
*
|
|
692
|
+
* Blocking iff a NOT_SATISFIED verdict is held with high confidence; every
|
|
693
|
+
* other combination — including all INCONCLUSIVE and SATISFIED cases — is
|
|
694
|
+
* advisory. Missing inputs never punish the change.
|
|
695
|
+
*
|
|
696
|
+
* This function is the false-positive-critical seam. Authority is computed
|
|
697
|
+
* here in TypeScript and is NEVER trusted from the LLM response.
|
|
698
|
+
*/
|
|
699
|
+
declare function deriveAuthority(verdict: Verdict, confidence: Confidence): Authority;
|
|
700
|
+
|
|
701
|
+
/**
|
|
702
|
+
* Zod schema for the LLM verdict response.
|
|
703
|
+
*
|
|
704
|
+
* `authority` is intentionally ABSENT: it is derived in TypeScript by
|
|
705
|
+
* `deriveAuthority` and must never be supplied by the model. The schema is
|
|
706
|
+
* `.strict()` so an injected `authority` (or any other extra key) is rejected
|
|
707
|
+
* at the parse boundary rather than silently passing through.
|
|
708
|
+
*
|
|
709
|
+
*/
|
|
710
|
+
declare const verdictSchema: z.ZodObject<{
|
|
711
|
+
verdict: z.ZodEnum<["SATISFIED", "NOT_SATISFIED", "INCONCLUSIVE"]>;
|
|
712
|
+
confidence: z.ZodEnum<["low", "medium", "high"]>;
|
|
713
|
+
rationale: z.ZodString;
|
|
714
|
+
unmetCriteria: z.ZodArray<z.ZodString, "many">;
|
|
715
|
+
}, "strict", z.ZodTypeAny, {
|
|
716
|
+
verdict: "SATISFIED" | "NOT_SATISFIED" | "INCONCLUSIVE";
|
|
717
|
+
confidence: "low" | "medium" | "high";
|
|
718
|
+
rationale: string;
|
|
719
|
+
unmetCriteria: string[];
|
|
720
|
+
}, {
|
|
721
|
+
verdict: "SATISFIED" | "NOT_SATISFIED" | "INCONCLUSIVE";
|
|
722
|
+
confidence: "low" | "medium" | "high";
|
|
723
|
+
rationale: string;
|
|
724
|
+
unmetCriteria: string[];
|
|
725
|
+
}>;
|
|
726
|
+
type LlmVerdict = z.infer<typeof verdictSchema>;
|
|
727
|
+
/**
|
|
728
|
+
* System prompt for outcome-eval. Conservative-confidence posture copied from
|
|
729
|
+
* security-craft (SKILL.md): the model defaults to `medium` confidence; `high`
|
|
730
|
+
* requires naming a specific met or unmet criterion; the bias is toward
|
|
731
|
+
* advisory, not blocking. `authority` is derived in TypeScript and must never
|
|
732
|
+
* be supplied by the model — the schema is `.strict()` and rejects it.
|
|
733
|
+
*/
|
|
734
|
+
declare const OUTCOME_EVAL_SYSTEM_PROMPT = "You are a post-execution outcome judge. Given a spec acceptance section, a unified diff, and test output, decide whether the change SATISFIED, NOT_SATISFIED, or is INCONCLUSIVE against that section.\n\nConfidence calibration (be conservative \u2014 false alarms are costly):\n- Default to \"medium\" confidence.\n- Use \"high\" ONLY when you can name a SPECIFIC criterion from the section that the diff and test output clearly met or clearly failed to meet, and quote or paraphrase it in the rationale.\n- Use \"low\" when the diff or test output is ambiguous, partial, or insufficient to judge.\n- When the change only PARTIALLY meets the criteria, do not exceed \"medium\" confidence.\n- Bias toward advisory caution: if unsure between two confidence levels, choose the lower one.\n\nRules:\n- The rationale MUST cite specific met or unmet criteria from the section.\n- \"unmetCriteria\" lists the section criteria the change failed to meet; it is empty when the verdict is SATISFIED.\n- Do NOT emit an \"authority\" field. Authority is computed downstream in TypeScript from (verdict, confidence) and must never come from you.\n\nReturn your judgment using the structured_output tool.";
|
|
735
|
+
/**
|
|
736
|
+
* Build the user prompt from the resolved spec section body, the change diff,
|
|
737
|
+
* and the captured test output. Mirrors the labeled-section structure of
|
|
738
|
+
* sel/pesl prompts.
|
|
739
|
+
*
|
|
740
|
+
* The diff and test output are clamped to PROMPT_FIELD_MAX_CHARS and wrapped in
|
|
741
|
+
* a 4-backtick fence so a triple-backtick sequence inside the diff cannot close
|
|
742
|
+
* the fence early.
|
|
743
|
+
*/
|
|
744
|
+
declare function buildUserPrompt(section: string, diff: string, testOutput: string): string;
|
|
745
|
+
|
|
746
|
+
/**
|
|
747
|
+
* Result of resolving the judgment section from a spec's markdown.
|
|
748
|
+
* `body` is the matched section's content (heading excluded, blank-trimmed).
|
|
749
|
+
*/
|
|
750
|
+
interface ResolvedSection {
|
|
751
|
+
judgedAgainst: JudgedAgainst;
|
|
752
|
+
body: string;
|
|
753
|
+
}
|
|
754
|
+
/**
|
|
755
|
+
* Resolve the judgment input from a spec's markdown via the fallback chain
|
|
756
|
+
* Success Criteria -> User-Visible Behavior -> Overview, returning the matched
|
|
757
|
+
* section body plus which heading matched.
|
|
758
|
+
*
|
|
759
|
+
* Returns `null` when no judgable section exists. The caller (a later phase)
|
|
760
|
+
* maps that null to an INCONCLUSIVE verdict — this resolver never throws and
|
|
761
|
+
* never decides verdict authority.
|
|
762
|
+
*
|
|
763
|
+
* Self-contained string parsing: imports only the JudgedAgainst type, honoring
|
|
764
|
+
* the intelligence layer rule (no `core` dependency).
|
|
765
|
+
*/
|
|
766
|
+
declare function resolveSection(markdown: string): ResolvedSection | null;
|
|
767
|
+
|
|
768
|
+
interface OutcomeEvaluatorOptions {
|
|
769
|
+
/** Override model for the outcome-eval LLM call. */
|
|
770
|
+
model?: string;
|
|
771
|
+
}
|
|
772
|
+
/**
|
|
773
|
+
* Post-execution spec-satisfaction judge. Mirrors PeslSimulator's
|
|
774
|
+
* (provider, store, options) constructor shape. The store is held for the
|
|
775
|
+
* Phase 4 execution_outcome graph write; see `persistOutcome`.
|
|
776
|
+
*/
|
|
777
|
+
declare class OutcomeEvaluator {
|
|
778
|
+
private readonly provider;
|
|
779
|
+
private readonly store;
|
|
780
|
+
private readonly options;
|
|
781
|
+
constructor(provider: AnalysisProvider, store: GraphStore, options?: OutcomeEvaluatorOptions);
|
|
782
|
+
evaluate(input: OutcomeEvalInput): Promise<OutcomeVerdict>;
|
|
783
|
+
/**
|
|
784
|
+
* Run the provider call and strict re-parse. ANY failure here — provider
|
|
785
|
+
* rejection (rate limit/network), or a strict-parse rejection of a malformed
|
|
786
|
+
* or authority-injected payload — degrades safely to INCONCLUSIVE/low/
|
|
787
|
+
* advisory rather than throwing. This reconciles Criterion 3 (never block on
|
|
788
|
+
* noise) with Criterion 4: an injected `authority` key is discarded by the
|
|
789
|
+
* .strict() re-parse, and the degraded verdict's authority is DERIVED from
|
|
790
|
+
* INCONCLUSIVE/low = advisory — so the LLM gains nothing by injecting it.
|
|
791
|
+
*/
|
|
792
|
+
private judge;
|
|
793
|
+
/**
|
|
794
|
+
* Build the safe-degradation verdict. INCONCLUSIVE/low yields advisory
|
|
795
|
+
* authority via deriveAuthority — never blocking. The rationale names only a
|
|
796
|
+
* coarse reason category, never a stack trace or secret.
|
|
797
|
+
*/
|
|
798
|
+
private degradedVerdict;
|
|
799
|
+
/** Persist (Phase 4 seam) then return the verdict. */
|
|
800
|
+
private finish;
|
|
801
|
+
private resolveJudgmentSection;
|
|
802
|
+
private buildVerdict;
|
|
803
|
+
/**
|
|
804
|
+
* Map an OutcomeVerdict + OutcomeEvalInput to the connector's ExecutionOutcome.
|
|
805
|
+
* - result: SATISFIED -> 'success'; otherwise 'failure'. INCONCLUSIVE is
|
|
806
|
+
* 'failure' for type-validity but omits agentPersona/affected systems so
|
|
807
|
+
* the effectiveness scorer ignores it (plan D2).
|
|
808
|
+
* - linkedSpecId: input.specPath (metadata only; no spec edge — plan D1).
|
|
809
|
+
* - affectedSystemNodeIds: [] in v1 (not available from OutcomeEvalInput — D4).
|
|
810
|
+
* - id: one node per EVALUATION. GraphStore.addNode upserts by id, so the id
|
|
811
|
+
* carries a collision-free randomUUID() — two evaluate() calls in the same
|
|
812
|
+
* millisecond can never overwrite each other (data-loss fix). specPath is
|
|
813
|
+
* included for human readability only.
|
|
814
|
+
* - taskType: OMITTED. The outcome-eval judge has no task categorization, and
|
|
815
|
+
* asserting a false 'feature' would mislead specialization analytics (SUG-2).
|
|
816
|
+
* - metadata: verdict-specific signal carried through the connector's
|
|
817
|
+
* additive pass-through (verdict/confidence/judgedAgainst/source) so the
|
|
818
|
+
* true 3-valued verdict is durable on the node (Truth 3).
|
|
819
|
+
*/
|
|
820
|
+
private toExecutionOutcome;
|
|
821
|
+
/**
|
|
822
|
+
* Phase 4: writes exactly one execution_outcome node via
|
|
823
|
+
* ExecutionOutcomeConnector. Degrade-safe (plan D3): a graph-write failure is
|
|
824
|
+
* swallowed-and-logged, never thrown — the verdict is already computed before
|
|
825
|
+
* this runs, so swallowing keeps evaluate() total. No secrets/stack frames in
|
|
826
|
+
* the log message.
|
|
827
|
+
*/
|
|
828
|
+
private persistOutcome;
|
|
829
|
+
}
|
|
830
|
+
|
|
569
831
|
/**
|
|
570
832
|
* Compute historical complexity from past execution outcomes in the graph.
|
|
571
833
|
*
|
|
@@ -906,4 +1168,4 @@ declare function saveProfiles(projectRoot: string, store: ProfileStore): void;
|
|
|
906
1168
|
*/
|
|
907
1169
|
declare function refreshProfiles(projectRoot: string, graphStore: GraphStore, opts?: Omit<SpecializationOptions, 'persona'>): ProfileStore;
|
|
908
1170
|
|
|
909
|
-
export { type AffectedSystem, type AnalysisProvider, type AnalysisRequest, type AnalysisResponse, AnthropicAnalysisProvider, type BlastRadius, type BlindSpot, ClaudeCliAnalysisProvider, type ComplexityScore, type EnrichedSpec, type ExecutionOutcome, ExecutionOutcomeConnector, type ExpertiseLevel, type GitHubIssue, GraphValidator, IntelligencePipeline, type JiraIssue, type LinearIssue, type ManualInput, OpenAICompatibleAnalysisProvider, type OutcomeIngestResult, type PersonaEffectivenessScore, type PersonaRecommendation, PeslSimulator, type PreprocessResult, type ProfileStore, type RawWorkItem, type SimulationResult, type SpecializationEntry, type SpecializationProfile, type SpecializationScore, type TaskType, type TemporalConfig, type WeightedRecommendation, buildSpecializationProfile, computeExpertiseLevel, computeHistoricalComplexity, computePersonaEffectiveness, computeSemanticComplexity, computeSpecialization, computeStructuralComplexity, decayWeight, detectBlindSpots, enrich, githubToRawWorkItem, jiraToRawWorkItem, linearToRawWorkItem, loadProfiles, manualToRawWorkItem, recommendPersona, refreshProfiles, runGraphOnlyChecks, runLlmSimulation, saveProfiles, score as scoreCML, scoreToConcernSignals, temporalSuccessRate, toRawWorkItem, weightedRecommendPersona };
|
|
1171
|
+
export { type AffectedSystem, type AnalysisProvider, type AnalysisRequest, type AnalysisResponse, AnthropicAnalysisProvider, type Authority, type BlastRadius, type BlindSpot, type CanaryAdapter, type CanaryDegradeReason, type CanaryExec, type CanaryFinding, type CanaryProbe, ClaudeCliAnalysisProvider, type ComplexityScore, type Confidence, type EnrichedSpec, type ExecutionOutcome, ExecutionOutcomeConnector, type ExpertiseLevel, type FrameworkRecommendation, type GitHubIssue, GraphValidator, IntelligencePipeline, type JiraIssue, type JudgedAgainst, type LinearIssue, type LlmVerdict, type ManualInput, OUTCOME_EVAL_SYSTEM_PROMPT, OpenAICompatibleAnalysisProvider, type OutcomeEvalInput, OutcomeEvaluator, type OutcomeEvaluatorOptions, type OutcomeIngestResult, type OutcomeVerdict, type PersonaEffectivenessScore, type PersonaRecommendation, PeslSimulator, type PreprocessResult, type ProfileStore, type RawWorkItem, type ResolvedSection, type SimulationResult, type SpecializationEntry, type SpecializationProfile, type SpecializationScore, type TaskType, type TemporalConfig, type Verdict, type WeightedRecommendation, buildSpecializationProfile, buildUserPrompt, computeExpertiseLevel, computeHistoricalComplexity, computePersonaEffectiveness, computeSemanticComplexity, computeSpecialization, computeStructuralComplexity, createCanaryAdapter, decayWeight, deriveAuthority, detectBlindSpots, enrich, githubToRawWorkItem, jiraToRawWorkItem, linearToRawWorkItem, loadProfiles, manualToRawWorkItem, recommendPersona, refreshProfiles, resolveSection, runGraphOnlyChecks, runLlmSimulation, saveProfiles, score as scoreCML, scoreToConcernSignals, temporalSuccessRate, toRawWorkItem, verdictSchema, weightedRecommendPersona };
|