@harness-engineering/intelligence 0.3.1 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +167 -2
- package/dist/index.d.ts +167 -2
- package/dist/index.js +188 -34
- package/dist/index.mjs +182 -34
- package/package.json +3 -3
package/dist/index.d.mts
CHANGED
|
@@ -741,7 +741,7 @@ declare const OUTCOME_EVAL_SYSTEM_PROMPT = "You are a post-execution outcome jud
|
|
|
741
741
|
* a 4-backtick fence so a triple-backtick sequence inside the diff cannot close
|
|
742
742
|
* the fence early.
|
|
743
743
|
*/
|
|
744
|
-
declare function buildUserPrompt(section: string, diff: string, testOutput: string): string;
|
|
744
|
+
declare function buildUserPrompt$1(section: string, diff: string, testOutput: string): string;
|
|
745
745
|
|
|
746
746
|
/**
|
|
747
747
|
* Result of resolving the judgment section from a spec's markdown.
|
|
@@ -828,6 +828,171 @@ declare class OutcomeEvaluator {
|
|
|
828
828
|
private persistOutcome;
|
|
829
829
|
}
|
|
830
830
|
|
|
831
|
+
/**
|
|
832
|
+
* acceptance-eval contract types — the upstream twin of outcome-eval.
|
|
833
|
+
*
|
|
834
|
+
* `authority` is DERIVED in TypeScript from (measurability, confidence) via
|
|
835
|
+
* `deriveAcceptanceAuthority` in `./authority.js`. It is NEVER read from the
|
|
836
|
+
* LLM response — see `acceptanceVerdictSchema` in `./prompts.js`, which omits
|
|
837
|
+
* it. `Confidence`, `JudgedAgainst`, and `Authority` are REUSED from the
|
|
838
|
+
* outcome-eval module — not forked — consistent with the imported section-resolver.
|
|
839
|
+
*/
|
|
840
|
+
|
|
841
|
+
/** (c) the measurability gate dimension. */
|
|
842
|
+
type Measurability = 'MEASURABLE' | 'NOT_MEASURABLE' | 'INCONCLUSIVE';
|
|
843
|
+
/** A single advisory observation about one criterion or behavior. */
|
|
844
|
+
interface Finding {
|
|
845
|
+
/** The specific criterion or user-visible behavior this finding references. */
|
|
846
|
+
target: string;
|
|
847
|
+
/** The advisory observation (e.g. 'not observable', 'no covering test'). */
|
|
848
|
+
message: string;
|
|
849
|
+
}
|
|
850
|
+
interface AcceptanceEvalInput {
|
|
851
|
+
/** Absolute or repo-relative path to the spec markdown. */
|
|
852
|
+
specPath: string;
|
|
853
|
+
/** Pre-resolved judgment section; otherwise the section-resolver runs. */
|
|
854
|
+
specSection?: string;
|
|
855
|
+
/**
|
|
856
|
+
* Located test snippets for coverage responsibility (b). Optional: absence
|
|
857
|
+
* degrades (b) coverageFindings to advisory-empty and never affects the
|
|
858
|
+
* (c) measurability gate.
|
|
859
|
+
*/
|
|
860
|
+
testContent?: string;
|
|
861
|
+
}
|
|
862
|
+
interface AcceptanceVerdict {
|
|
863
|
+
measurability: Measurability;
|
|
864
|
+
confidence: Confidence;
|
|
865
|
+
/** DERIVED in TS from (measurability, confidence); never from the LLM. */
|
|
866
|
+
authority: Authority;
|
|
867
|
+
/** Which spec section resolved. */
|
|
868
|
+
judgedAgainst: JudgedAgainst;
|
|
869
|
+
/** (a) advisory — observability / testability / completeness critique. */
|
|
870
|
+
criteriaFindings: Finding[];
|
|
871
|
+
/** (b) advisory — user-visible behaviors with no covering test. */
|
|
872
|
+
coverageFindings: Finding[];
|
|
873
|
+
rationale: string;
|
|
874
|
+
}
|
|
875
|
+
|
|
876
|
+
/**
|
|
877
|
+
* Pure mapping from (measurability, confidence) to gate authority.
|
|
878
|
+
*
|
|
879
|
+
* Blocking iff a spec is judged NOT_MEASURABLE with high confidence — i.e. it
|
|
880
|
+
* objectively lacks measurable success criteria; every other combination,
|
|
881
|
+
* including all INCONCLUSIVE and MEASURABLE cases, is advisory. Missing or
|
|
882
|
+
* uncertain inputs never punish the spec.
|
|
883
|
+
*
|
|
884
|
+
* This function is the false-positive-critical seam. Authority is computed
|
|
885
|
+
* here in TypeScript and is NEVER trusted from the LLM response.
|
|
886
|
+
*/
|
|
887
|
+
declare function deriveAcceptanceAuthority(measurability: Measurability, confidence: Confidence): Authority;
|
|
888
|
+
|
|
889
|
+
/** A single advisory finding. `.strict()` rejects unexpected keys. */
|
|
890
|
+
declare const findingSchema: z.ZodObject<{
|
|
891
|
+
target: z.ZodString;
|
|
892
|
+
message: z.ZodString;
|
|
893
|
+
}, "strict", z.ZodTypeAny, {
|
|
894
|
+
message: string;
|
|
895
|
+
target: string;
|
|
896
|
+
}, {
|
|
897
|
+
message: string;
|
|
898
|
+
target: string;
|
|
899
|
+
}>;
|
|
900
|
+
/**
|
|
901
|
+
* Zod schema for the LLM verdict response.
|
|
902
|
+
*
|
|
903
|
+
* `authority` is intentionally ABSENT: it is derived in TypeScript by
|
|
904
|
+
* `deriveAcceptanceAuthority` and must never be supplied by the model. The
|
|
905
|
+
* schema is `.strict()`, so an injected `authority` (or any other extra key)
|
|
906
|
+
* is rejected at the parse boundary rather than silently passing through.
|
|
907
|
+
*/
|
|
908
|
+
declare const acceptanceVerdictSchema: z.ZodObject<{
|
|
909
|
+
measurability: z.ZodEnum<["MEASURABLE", "NOT_MEASURABLE", "INCONCLUSIVE"]>;
|
|
910
|
+
confidence: z.ZodEnum<["low", "medium", "high"]>;
|
|
911
|
+
rationale: z.ZodString;
|
|
912
|
+
criteriaFindings: z.ZodArray<z.ZodObject<{
|
|
913
|
+
target: z.ZodString;
|
|
914
|
+
message: z.ZodString;
|
|
915
|
+
}, "strict", z.ZodTypeAny, {
|
|
916
|
+
message: string;
|
|
917
|
+
target: string;
|
|
918
|
+
}, {
|
|
919
|
+
message: string;
|
|
920
|
+
target: string;
|
|
921
|
+
}>, "many">;
|
|
922
|
+
coverageFindings: z.ZodArray<z.ZodObject<{
|
|
923
|
+
target: z.ZodString;
|
|
924
|
+
message: z.ZodString;
|
|
925
|
+
}, "strict", z.ZodTypeAny, {
|
|
926
|
+
message: string;
|
|
927
|
+
target: string;
|
|
928
|
+
}, {
|
|
929
|
+
message: string;
|
|
930
|
+
target: string;
|
|
931
|
+
}>, "many">;
|
|
932
|
+
}, "strict", z.ZodTypeAny, {
|
|
933
|
+
confidence: "low" | "medium" | "high";
|
|
934
|
+
rationale: string;
|
|
935
|
+
measurability: "INCONCLUSIVE" | "MEASURABLE" | "NOT_MEASURABLE";
|
|
936
|
+
criteriaFindings: {
|
|
937
|
+
message: string;
|
|
938
|
+
target: string;
|
|
939
|
+
}[];
|
|
940
|
+
coverageFindings: {
|
|
941
|
+
message: string;
|
|
942
|
+
target: string;
|
|
943
|
+
}[];
|
|
944
|
+
}, {
|
|
945
|
+
confidence: "low" | "medium" | "high";
|
|
946
|
+
rationale: string;
|
|
947
|
+
measurability: "INCONCLUSIVE" | "MEASURABLE" | "NOT_MEASURABLE";
|
|
948
|
+
criteriaFindings: {
|
|
949
|
+
message: string;
|
|
950
|
+
target: string;
|
|
951
|
+
}[];
|
|
952
|
+
coverageFindings: {
|
|
953
|
+
message: string;
|
|
954
|
+
target: string;
|
|
955
|
+
}[];
|
|
956
|
+
}>;
|
|
957
|
+
type LlmAcceptanceVerdict = z.infer<typeof acceptanceVerdictSchema>;
|
|
958
|
+
/**
|
|
959
|
+
* System prompt for acceptance-eval. Conservative-confidence posture mirrors
|
|
960
|
+
* outcome-eval: default to medium; high requires naming a specific criterion;
|
|
961
|
+
* bias toward advisory. `authority` is derived in TypeScript and must never be
|
|
962
|
+
* supplied by the model — the schema is `.strict()` and rejects it.
|
|
963
|
+
*/
|
|
964
|
+
declare const ACCEPTANCE_EVAL_SYSTEM_PROMPT = "You are a PRE-execution acceptance-criteria judge. Given a spec acceptance section (and optionally located test snippets), assess three things:\n(a) criteria quality \u2014 are the success criteria observable, testable, and complete? (advisory findings)\n(b) coverage \u2014 do any user-visible behaviors lack a covering test? (advisory findings)\n(c) measurability \u2014 does the spec state MEASURABLE, NOT_MEASURABLE, or is it INCONCLUSIVE on whether any measurable success criteria exist at all?\n\nConfidence calibration (be conservative \u2014 false alarms are costly):\n- Default to \"medium\" confidence.\n- Use \"high\" ONLY when you can name a SPECIFIC criterion (or its absence) and quote or paraphrase it in the rationale.\n- Use \"low\" when the section is ambiguous, partial, or insufficient to judge.\n- Bias toward advisory caution: if unsure between two confidence levels, choose the lower one.\n\nRules:\n- \"measurability\" is NOT_MEASURABLE only when the section states no observable, testable success criterion at all.\n- \"criteriaFindings\" holds advisory (a) observations; \"coverageFindings\" holds advisory (b) observations; both may be empty.\n- Do NOT emit an \"authority\" field. Authority is computed downstream in TypeScript from (measurability, confidence) and must never come from you.\n\nReturn your judgment using the structured_output tool.";
|
|
965
|
+
/**
|
|
966
|
+
* Build the user prompt from the resolved spec section body and optional
|
|
967
|
+
* located test snippets. Test content is clamped to PROMPT_FIELD_MAX_CHARS and
|
|
968
|
+
* wrapped in a 4-backtick fence so an inner ``` cannot close the fence early.
|
|
969
|
+
*/
|
|
970
|
+
declare function buildUserPrompt(section: string, testContent?: string): string;
|
|
971
|
+
|
|
972
|
+
interface AcceptanceEvaluatorOptions {
|
|
973
|
+
/** Override model for the acceptance-eval LLM call. */
|
|
974
|
+
model?: string;
|
|
975
|
+
}
|
|
976
|
+
/**
|
|
977
|
+
* Pre-execution acceptance-criteria judge — the upstream twin of
|
|
978
|
+
* OutcomeEvaluator. Built on the cli AnalysisProvider. The LLM returns only
|
|
979
|
+
* measurability/confidence/criteriaFindings/coverageFindings/rationale;
|
|
980
|
+
* `authority` is derived in TypeScript and never read from the model.
|
|
981
|
+
*
|
|
982
|
+
* Unlike OutcomeEvaluator it holds no GraphStore: there is no acceptance
|
|
983
|
+
* outcome node type and Phase 1 does not persist (see plan D-P1-3).
|
|
984
|
+
*/
|
|
985
|
+
declare class AcceptanceEvaluator {
|
|
986
|
+
private readonly provider;
|
|
987
|
+
private readonly options;
|
|
988
|
+
constructor(provider: AnalysisProvider, options?: AcceptanceEvaluatorOptions);
|
|
989
|
+
evaluate(input: AcceptanceEvalInput): Promise<AcceptanceVerdict>;
|
|
990
|
+
private judge;
|
|
991
|
+
private degradedVerdict;
|
|
992
|
+
private resolveJudgmentSection;
|
|
993
|
+
private buildVerdict;
|
|
994
|
+
}
|
|
995
|
+
|
|
831
996
|
/**
|
|
832
997
|
* Compute historical complexity from past execution outcomes in the graph.
|
|
833
998
|
*
|
|
@@ -1168,4 +1333,4 @@ declare function saveProfiles(projectRoot: string, store: ProfileStore): void;
|
|
|
1168
1333
|
*/
|
|
1169
1334
|
declare function refreshProfiles(projectRoot: string, graphStore: GraphStore, opts?: Omit<SpecializationOptions, 'persona'>): ProfileStore;
|
|
1170
1335
|
|
|
1171
|
-
export { type AffectedSystem, type AnalysisProvider, type AnalysisRequest, type AnalysisResponse, AnthropicAnalysisProvider, type Authority, type BlastRadius, type BlindSpot, type CanaryAdapter, type CanaryDegradeReason, type CanaryExec, type CanaryFinding, type CanaryProbe, ClaudeCliAnalysisProvider, type ComplexityScore, type Confidence, type EnrichedSpec, type ExecutionOutcome, ExecutionOutcomeConnector, type ExpertiseLevel, type FrameworkRecommendation, type GitHubIssue, GraphValidator, IntelligencePipeline, type JiraIssue, type JudgedAgainst, type LinearIssue, type LlmVerdict, type ManualInput, OUTCOME_EVAL_SYSTEM_PROMPT, OpenAICompatibleAnalysisProvider, type OutcomeEvalInput, OutcomeEvaluator, type OutcomeEvaluatorOptions, type OutcomeIngestResult, type OutcomeVerdict, type PersonaEffectivenessScore, type PersonaRecommendation, PeslSimulator, type PreprocessResult, type ProfileStore, type RawWorkItem, type ResolvedSection, type SimulationResult, type SpecializationEntry, type SpecializationProfile, type SpecializationScore, type TaskType, type TemporalConfig, type Verdict, type WeightedRecommendation, buildSpecializationProfile, buildUserPrompt, computeExpertiseLevel, computeHistoricalComplexity, computePersonaEffectiveness, computeSemanticComplexity, computeSpecialization, computeStructuralComplexity, createCanaryAdapter, decayWeight, deriveAuthority, detectBlindSpots, enrich, githubToRawWorkItem, jiraToRawWorkItem, linearToRawWorkItem, loadProfiles, manualToRawWorkItem, recommendPersona, refreshProfiles, resolveSection, runGraphOnlyChecks, runLlmSimulation, saveProfiles, score as scoreCML, scoreToConcernSignals, temporalSuccessRate, toRawWorkItem, verdictSchema, weightedRecommendPersona };
|
|
1336
|
+
export { ACCEPTANCE_EVAL_SYSTEM_PROMPT, type AcceptanceEvalInput, AcceptanceEvaluator, type AcceptanceEvaluatorOptions, type AcceptanceVerdict, type AffectedSystem, type AnalysisProvider, type AnalysisRequest, type AnalysisResponse, AnthropicAnalysisProvider, type Authority, type BlastRadius, type BlindSpot, type CanaryAdapter, type CanaryDegradeReason, type CanaryExec, type CanaryFinding, type CanaryProbe, ClaudeCliAnalysisProvider, type ComplexityScore, type Confidence, type EnrichedSpec, type ExecutionOutcome, ExecutionOutcomeConnector, type ExpertiseLevel, type Finding, type FrameworkRecommendation, type GitHubIssue, GraphValidator, IntelligencePipeline, type JiraIssue, type JudgedAgainst, type LinearIssue, type LlmAcceptanceVerdict, type LlmVerdict, type ManualInput, type Measurability, OUTCOME_EVAL_SYSTEM_PROMPT, OpenAICompatibleAnalysisProvider, type OutcomeEvalInput, OutcomeEvaluator, type OutcomeEvaluatorOptions, type OutcomeIngestResult, type OutcomeVerdict, type PersonaEffectivenessScore, type PersonaRecommendation, PeslSimulator, type PreprocessResult, type ProfileStore, type RawWorkItem, type ResolvedSection, type SimulationResult, type SpecializationEntry, type SpecializationProfile, type SpecializationScore, type TaskType, type TemporalConfig, type Verdict, type WeightedRecommendation, acceptanceVerdictSchema, buildUserPrompt as buildAcceptanceUserPrompt, buildSpecializationProfile, buildUserPrompt$1 as buildUserPrompt, computeExpertiseLevel, computeHistoricalComplexity, computePersonaEffectiveness, computeSemanticComplexity, computeSpecialization, computeStructuralComplexity, createCanaryAdapter, decayWeight, deriveAcceptanceAuthority, deriveAuthority, detectBlindSpots, enrich, findingSchema, githubToRawWorkItem, jiraToRawWorkItem, linearToRawWorkItem, loadProfiles, manualToRawWorkItem, recommendPersona, refreshProfiles, resolveSection, runGraphOnlyChecks, runLlmSimulation, saveProfiles, score as scoreCML, scoreToConcernSignals, temporalSuccessRate, toRawWorkItem, verdictSchema, weightedRecommendPersona };
|
package/dist/index.d.ts
CHANGED
|
@@ -741,7 +741,7 @@ declare const OUTCOME_EVAL_SYSTEM_PROMPT = "You are a post-execution outcome jud
|
|
|
741
741
|
* a 4-backtick fence so a triple-backtick sequence inside the diff cannot close
|
|
742
742
|
* the fence early.
|
|
743
743
|
*/
|
|
744
|
-
declare function buildUserPrompt(section: string, diff: string, testOutput: string): string;
|
|
744
|
+
declare function buildUserPrompt$1(section: string, diff: string, testOutput: string): string;
|
|
745
745
|
|
|
746
746
|
/**
|
|
747
747
|
* Result of resolving the judgment section from a spec's markdown.
|
|
@@ -828,6 +828,171 @@ declare class OutcomeEvaluator {
|
|
|
828
828
|
private persistOutcome;
|
|
829
829
|
}
|
|
830
830
|
|
|
831
|
+
/**
|
|
832
|
+
* acceptance-eval contract types — the upstream twin of outcome-eval.
|
|
833
|
+
*
|
|
834
|
+
* `authority` is DERIVED in TypeScript from (measurability, confidence) via
|
|
835
|
+
* `deriveAcceptanceAuthority` in `./authority.js`. It is NEVER read from the
|
|
836
|
+
* LLM response — see `acceptanceVerdictSchema` in `./prompts.js`, which omits
|
|
837
|
+
* it. `Confidence`, `JudgedAgainst`, and `Authority` are REUSED from the
|
|
838
|
+
* outcome-eval module — not forked — consistent with the imported section-resolver.
|
|
839
|
+
*/
|
|
840
|
+
|
|
841
|
+
/** (c) the measurability gate dimension. */
|
|
842
|
+
type Measurability = 'MEASURABLE' | 'NOT_MEASURABLE' | 'INCONCLUSIVE';
|
|
843
|
+
/** A single advisory observation about one criterion or behavior. */
|
|
844
|
+
interface Finding {
|
|
845
|
+
/** The specific criterion or user-visible behavior this finding references. */
|
|
846
|
+
target: string;
|
|
847
|
+
/** The advisory observation (e.g. 'not observable', 'no covering test'). */
|
|
848
|
+
message: string;
|
|
849
|
+
}
|
|
850
|
+
interface AcceptanceEvalInput {
|
|
851
|
+
/** Absolute or repo-relative path to the spec markdown. */
|
|
852
|
+
specPath: string;
|
|
853
|
+
/** Pre-resolved judgment section; otherwise the section-resolver runs. */
|
|
854
|
+
specSection?: string;
|
|
855
|
+
/**
|
|
856
|
+
* Located test snippets for coverage responsibility (b). Optional: absence
|
|
857
|
+
* degrades (b) coverageFindings to advisory-empty and never affects the
|
|
858
|
+
* (c) measurability gate.
|
|
859
|
+
*/
|
|
860
|
+
testContent?: string;
|
|
861
|
+
}
|
|
862
|
+
interface AcceptanceVerdict {
|
|
863
|
+
measurability: Measurability;
|
|
864
|
+
confidence: Confidence;
|
|
865
|
+
/** DERIVED in TS from (measurability, confidence); never from the LLM. */
|
|
866
|
+
authority: Authority;
|
|
867
|
+
/** Which spec section resolved. */
|
|
868
|
+
judgedAgainst: JudgedAgainst;
|
|
869
|
+
/** (a) advisory — observability / testability / completeness critique. */
|
|
870
|
+
criteriaFindings: Finding[];
|
|
871
|
+
/** (b) advisory — user-visible behaviors with no covering test. */
|
|
872
|
+
coverageFindings: Finding[];
|
|
873
|
+
rationale: string;
|
|
874
|
+
}
|
|
875
|
+
|
|
876
|
+
/**
|
|
877
|
+
* Pure mapping from (measurability, confidence) to gate authority.
|
|
878
|
+
*
|
|
879
|
+
* Blocking iff a spec is judged NOT_MEASURABLE with high confidence — i.e. it
|
|
880
|
+
* objectively lacks measurable success criteria; every other combination,
|
|
881
|
+
* including all INCONCLUSIVE and MEASURABLE cases, is advisory. Missing or
|
|
882
|
+
* uncertain inputs never punish the spec.
|
|
883
|
+
*
|
|
884
|
+
* This function is the false-positive-critical seam. Authority is computed
|
|
885
|
+
* here in TypeScript and is NEVER trusted from the LLM response.
|
|
886
|
+
*/
|
|
887
|
+
declare function deriveAcceptanceAuthority(measurability: Measurability, confidence: Confidence): Authority;
|
|
888
|
+
|
|
889
|
+
/** A single advisory finding. `.strict()` rejects unexpected keys. */
|
|
890
|
+
declare const findingSchema: z.ZodObject<{
|
|
891
|
+
target: z.ZodString;
|
|
892
|
+
message: z.ZodString;
|
|
893
|
+
}, "strict", z.ZodTypeAny, {
|
|
894
|
+
message: string;
|
|
895
|
+
target: string;
|
|
896
|
+
}, {
|
|
897
|
+
message: string;
|
|
898
|
+
target: string;
|
|
899
|
+
}>;
|
|
900
|
+
/**
|
|
901
|
+
* Zod schema for the LLM verdict response.
|
|
902
|
+
*
|
|
903
|
+
* `authority` is intentionally ABSENT: it is derived in TypeScript by
|
|
904
|
+
* `deriveAcceptanceAuthority` and must never be supplied by the model. The
|
|
905
|
+
* schema is `.strict()`, so an injected `authority` (or any other extra key)
|
|
906
|
+
* is rejected at the parse boundary rather than silently passing through.
|
|
907
|
+
*/
|
|
908
|
+
declare const acceptanceVerdictSchema: z.ZodObject<{
|
|
909
|
+
measurability: z.ZodEnum<["MEASURABLE", "NOT_MEASURABLE", "INCONCLUSIVE"]>;
|
|
910
|
+
confidence: z.ZodEnum<["low", "medium", "high"]>;
|
|
911
|
+
rationale: z.ZodString;
|
|
912
|
+
criteriaFindings: z.ZodArray<z.ZodObject<{
|
|
913
|
+
target: z.ZodString;
|
|
914
|
+
message: z.ZodString;
|
|
915
|
+
}, "strict", z.ZodTypeAny, {
|
|
916
|
+
message: string;
|
|
917
|
+
target: string;
|
|
918
|
+
}, {
|
|
919
|
+
message: string;
|
|
920
|
+
target: string;
|
|
921
|
+
}>, "many">;
|
|
922
|
+
coverageFindings: z.ZodArray<z.ZodObject<{
|
|
923
|
+
target: z.ZodString;
|
|
924
|
+
message: z.ZodString;
|
|
925
|
+
}, "strict", z.ZodTypeAny, {
|
|
926
|
+
message: string;
|
|
927
|
+
target: string;
|
|
928
|
+
}, {
|
|
929
|
+
message: string;
|
|
930
|
+
target: string;
|
|
931
|
+
}>, "many">;
|
|
932
|
+
}, "strict", z.ZodTypeAny, {
|
|
933
|
+
confidence: "low" | "medium" | "high";
|
|
934
|
+
rationale: string;
|
|
935
|
+
measurability: "INCONCLUSIVE" | "MEASURABLE" | "NOT_MEASURABLE";
|
|
936
|
+
criteriaFindings: {
|
|
937
|
+
message: string;
|
|
938
|
+
target: string;
|
|
939
|
+
}[];
|
|
940
|
+
coverageFindings: {
|
|
941
|
+
message: string;
|
|
942
|
+
target: string;
|
|
943
|
+
}[];
|
|
944
|
+
}, {
|
|
945
|
+
confidence: "low" | "medium" | "high";
|
|
946
|
+
rationale: string;
|
|
947
|
+
measurability: "INCONCLUSIVE" | "MEASURABLE" | "NOT_MEASURABLE";
|
|
948
|
+
criteriaFindings: {
|
|
949
|
+
message: string;
|
|
950
|
+
target: string;
|
|
951
|
+
}[];
|
|
952
|
+
coverageFindings: {
|
|
953
|
+
message: string;
|
|
954
|
+
target: string;
|
|
955
|
+
}[];
|
|
956
|
+
}>;
|
|
957
|
+
type LlmAcceptanceVerdict = z.infer<typeof acceptanceVerdictSchema>;
|
|
958
|
+
/**
|
|
959
|
+
* System prompt for acceptance-eval. Conservative-confidence posture mirrors
|
|
960
|
+
* outcome-eval: default to medium; high requires naming a specific criterion;
|
|
961
|
+
* bias toward advisory. `authority` is derived in TypeScript and must never be
|
|
962
|
+
* supplied by the model — the schema is `.strict()` and rejects it.
|
|
963
|
+
*/
|
|
964
|
+
declare const ACCEPTANCE_EVAL_SYSTEM_PROMPT = "You are a PRE-execution acceptance-criteria judge. Given a spec acceptance section (and optionally located test snippets), assess three things:\n(a) criteria quality \u2014 are the success criteria observable, testable, and complete? (advisory findings)\n(b) coverage \u2014 do any user-visible behaviors lack a covering test? (advisory findings)\n(c) measurability \u2014 does the spec state MEASURABLE, NOT_MEASURABLE, or is it INCONCLUSIVE on whether any measurable success criteria exist at all?\n\nConfidence calibration (be conservative \u2014 false alarms are costly):\n- Default to \"medium\" confidence.\n- Use \"high\" ONLY when you can name a SPECIFIC criterion (or its absence) and quote or paraphrase it in the rationale.\n- Use \"low\" when the section is ambiguous, partial, or insufficient to judge.\n- Bias toward advisory caution: if unsure between two confidence levels, choose the lower one.\n\nRules:\n- \"measurability\" is NOT_MEASURABLE only when the section states no observable, testable success criterion at all.\n- \"criteriaFindings\" holds advisory (a) observations; \"coverageFindings\" holds advisory (b) observations; both may be empty.\n- Do NOT emit an \"authority\" field. Authority is computed downstream in TypeScript from (measurability, confidence) and must never come from you.\n\nReturn your judgment using the structured_output tool.";
|
|
965
|
+
/**
|
|
966
|
+
* Build the user prompt from the resolved spec section body and optional
|
|
967
|
+
* located test snippets. Test content is clamped to PROMPT_FIELD_MAX_CHARS and
|
|
968
|
+
* wrapped in a 4-backtick fence so an inner ``` cannot close the fence early.
|
|
969
|
+
*/
|
|
970
|
+
declare function buildUserPrompt(section: string, testContent?: string): string;
|
|
971
|
+
|
|
972
|
+
interface AcceptanceEvaluatorOptions {
|
|
973
|
+
/** Override model for the acceptance-eval LLM call. */
|
|
974
|
+
model?: string;
|
|
975
|
+
}
|
|
976
|
+
/**
|
|
977
|
+
* Pre-execution acceptance-criteria judge — the upstream twin of
|
|
978
|
+
* OutcomeEvaluator. Built on the cli AnalysisProvider. The LLM returns only
|
|
979
|
+
* measurability/confidence/criteriaFindings/coverageFindings/rationale;
|
|
980
|
+
* `authority` is derived in TypeScript and never read from the model.
|
|
981
|
+
*
|
|
982
|
+
* Unlike OutcomeEvaluator it holds no GraphStore: there is no acceptance
|
|
983
|
+
* outcome node type and Phase 1 does not persist (see plan D-P1-3).
|
|
984
|
+
*/
|
|
985
|
+
declare class AcceptanceEvaluator {
|
|
986
|
+
private readonly provider;
|
|
987
|
+
private readonly options;
|
|
988
|
+
constructor(provider: AnalysisProvider, options?: AcceptanceEvaluatorOptions);
|
|
989
|
+
evaluate(input: AcceptanceEvalInput): Promise<AcceptanceVerdict>;
|
|
990
|
+
private judge;
|
|
991
|
+
private degradedVerdict;
|
|
992
|
+
private resolveJudgmentSection;
|
|
993
|
+
private buildVerdict;
|
|
994
|
+
}
|
|
995
|
+
|
|
831
996
|
/**
|
|
832
997
|
* Compute historical complexity from past execution outcomes in the graph.
|
|
833
998
|
*
|
|
@@ -1168,4 +1333,4 @@ declare function saveProfiles(projectRoot: string, store: ProfileStore): void;
|
|
|
1168
1333
|
*/
|
|
1169
1334
|
declare function refreshProfiles(projectRoot: string, graphStore: GraphStore, opts?: Omit<SpecializationOptions, 'persona'>): ProfileStore;
|
|
1170
1335
|
|
|
1171
|
-
export { type AffectedSystem, type AnalysisProvider, type AnalysisRequest, type AnalysisResponse, AnthropicAnalysisProvider, type Authority, type BlastRadius, type BlindSpot, type CanaryAdapter, type CanaryDegradeReason, type CanaryExec, type CanaryFinding, type CanaryProbe, ClaudeCliAnalysisProvider, type ComplexityScore, type Confidence, type EnrichedSpec, type ExecutionOutcome, ExecutionOutcomeConnector, type ExpertiseLevel, type FrameworkRecommendation, type GitHubIssue, GraphValidator, IntelligencePipeline, type JiraIssue, type JudgedAgainst, type LinearIssue, type LlmVerdict, type ManualInput, OUTCOME_EVAL_SYSTEM_PROMPT, OpenAICompatibleAnalysisProvider, type OutcomeEvalInput, OutcomeEvaluator, type OutcomeEvaluatorOptions, type OutcomeIngestResult, type OutcomeVerdict, type PersonaEffectivenessScore, type PersonaRecommendation, PeslSimulator, type PreprocessResult, type ProfileStore, type RawWorkItem, type ResolvedSection, type SimulationResult, type SpecializationEntry, type SpecializationProfile, type SpecializationScore, type TaskType, type TemporalConfig, type Verdict, type WeightedRecommendation, buildSpecializationProfile, buildUserPrompt, computeExpertiseLevel, computeHistoricalComplexity, computePersonaEffectiveness, computeSemanticComplexity, computeSpecialization, computeStructuralComplexity, createCanaryAdapter, decayWeight, deriveAuthority, detectBlindSpots, enrich, githubToRawWorkItem, jiraToRawWorkItem, linearToRawWorkItem, loadProfiles, manualToRawWorkItem, recommendPersona, refreshProfiles, resolveSection, runGraphOnlyChecks, runLlmSimulation, saveProfiles, score as scoreCML, scoreToConcernSignals, temporalSuccessRate, toRawWorkItem, verdictSchema, weightedRecommendPersona };
|
|
1336
|
+
export { ACCEPTANCE_EVAL_SYSTEM_PROMPT, type AcceptanceEvalInput, AcceptanceEvaluator, type AcceptanceEvaluatorOptions, type AcceptanceVerdict, type AffectedSystem, type AnalysisProvider, type AnalysisRequest, type AnalysisResponse, AnthropicAnalysisProvider, type Authority, type BlastRadius, type BlindSpot, type CanaryAdapter, type CanaryDegradeReason, type CanaryExec, type CanaryFinding, type CanaryProbe, ClaudeCliAnalysisProvider, type ComplexityScore, type Confidence, type EnrichedSpec, type ExecutionOutcome, ExecutionOutcomeConnector, type ExpertiseLevel, type Finding, type FrameworkRecommendation, type GitHubIssue, GraphValidator, IntelligencePipeline, type JiraIssue, type JudgedAgainst, type LinearIssue, type LlmAcceptanceVerdict, type LlmVerdict, type ManualInput, type Measurability, OUTCOME_EVAL_SYSTEM_PROMPT, OpenAICompatibleAnalysisProvider, type OutcomeEvalInput, OutcomeEvaluator, type OutcomeEvaluatorOptions, type OutcomeIngestResult, type OutcomeVerdict, type PersonaEffectivenessScore, type PersonaRecommendation, PeslSimulator, type PreprocessResult, type ProfileStore, type RawWorkItem, type ResolvedSection, type SimulationResult, type SpecializationEntry, type SpecializationProfile, type SpecializationScore, type TaskType, type TemporalConfig, type Verdict, type WeightedRecommendation, acceptanceVerdictSchema, buildUserPrompt as buildAcceptanceUserPrompt, buildSpecializationProfile, buildUserPrompt$1 as buildUserPrompt, computeExpertiseLevel, computeHistoricalComplexity, computePersonaEffectiveness, computeSemanticComplexity, computeSpecialization, computeStructuralComplexity, createCanaryAdapter, decayWeight, deriveAcceptanceAuthority, deriveAuthority, detectBlindSpots, enrich, findingSchema, githubToRawWorkItem, jiraToRawWorkItem, linearToRawWorkItem, loadProfiles, manualToRawWorkItem, recommendPersona, refreshProfiles, resolveSection, runGraphOnlyChecks, runLlmSimulation, saveProfiles, score as scoreCML, scoreToConcernSignals, temporalSuccessRate, toRawWorkItem, verdictSchema, weightedRecommendPersona };
|
package/dist/index.js
CHANGED
|
@@ -30,6 +30,8 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
|
|
|
30
30
|
// src/index.ts
|
|
31
31
|
var index_exports = {};
|
|
32
32
|
__export(index_exports, {
|
|
33
|
+
ACCEPTANCE_EVAL_SYSTEM_PROMPT: () => ACCEPTANCE_EVAL_SYSTEM_PROMPT,
|
|
34
|
+
AcceptanceEvaluator: () => AcceptanceEvaluator,
|
|
33
35
|
AnthropicAnalysisProvider: () => AnthropicAnalysisProvider,
|
|
34
36
|
ClaudeCliAnalysisProvider: () => ClaudeCliAnalysisProvider,
|
|
35
37
|
ExecutionOutcomeConnector: () => ExecutionOutcomeConnector,
|
|
@@ -39,6 +41,8 @@ __export(index_exports, {
|
|
|
39
41
|
OpenAICompatibleAnalysisProvider: () => OpenAICompatibleAnalysisProvider,
|
|
40
42
|
OutcomeEvaluator: () => OutcomeEvaluator,
|
|
41
43
|
PeslSimulator: () => PeslSimulator,
|
|
44
|
+
acceptanceVerdictSchema: () => acceptanceVerdictSchema,
|
|
45
|
+
buildAcceptanceUserPrompt: () => buildUserPrompt3,
|
|
42
46
|
buildSpecializationProfile: () => buildSpecializationProfile,
|
|
43
47
|
buildUserPrompt: () => buildUserPrompt2,
|
|
44
48
|
computeExpertiseLevel: () => computeExpertiseLevel,
|
|
@@ -49,9 +53,11 @@ __export(index_exports, {
|
|
|
49
53
|
computeStructuralComplexity: () => computeStructuralComplexity,
|
|
50
54
|
createCanaryAdapter: () => createCanaryAdapter,
|
|
51
55
|
decayWeight: () => decayWeight,
|
|
56
|
+
deriveAcceptanceAuthority: () => deriveAcceptanceAuthority,
|
|
52
57
|
deriveAuthority: () => deriveAuthority,
|
|
53
58
|
detectBlindSpots: () => detectBlindSpots,
|
|
54
59
|
enrich: () => enrich,
|
|
60
|
+
findingSchema: () => findingSchema,
|
|
55
61
|
githubToRawWorkItem: () => githubToRawWorkItem,
|
|
56
62
|
jiraToRawWorkItem: () => jiraToRawWorkItem,
|
|
57
63
|
linearToRawWorkItem: () => linearToRawWorkItem,
|
|
@@ -239,43 +245,46 @@ function degradedRecommendation() {
|
|
|
239
245
|
alternatives: []
|
|
240
246
|
};
|
|
241
247
|
}
|
|
242
|
-
function
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
return { ok: false, reason: "binary-missing" };
|
|
253
|
-
}
|
|
254
|
-
return { ok: false, reason: "exec-failed" };
|
|
248
|
+
async function execCanary(exec, subArgs) {
|
|
249
|
+
const [cmd, args] = canaryInvocation(subArgs);
|
|
250
|
+
try {
|
|
251
|
+
const { stdout } = await exec(cmd, args);
|
|
252
|
+
return { ok: true, stdout };
|
|
253
|
+
} catch (err) {
|
|
254
|
+
const e = err;
|
|
255
|
+
if (e.code === "ENOENT") return { ok: false, reason: "not-installed" };
|
|
256
|
+
if (e.code === 1 && /canary binary not found/i.test(e.stderr ?? "")) {
|
|
257
|
+
return { ok: false, reason: "binary-missing" };
|
|
255
258
|
}
|
|
259
|
+
return { ok: false, reason: "exec-failed" };
|
|
256
260
|
}
|
|
261
|
+
}
|
|
262
|
+
async function probeCanary(exec) {
|
|
263
|
+
const res = await execCanary(exec, ["version"]);
|
|
264
|
+
if (!res.ok) return { status: "degraded", reason: res.reason };
|
|
265
|
+
if (res.stdout.trim() === "") return { status: "degraded", reason: "bad-output" };
|
|
266
|
+
const version = parseVersion(res.stdout);
|
|
267
|
+
return version ? { status: "available", version } : { status: "available" };
|
|
268
|
+
}
|
|
269
|
+
async function recommendFrameworkCanary(exec, prompt) {
|
|
270
|
+
const res = await execCanary(exec, ["recommend", prompt, "--json"]);
|
|
271
|
+
if (!res.ok) return degradedRecommendation();
|
|
272
|
+
const parsed = frameworkRecommendationSchema.safeParse(safeJson(res.stdout));
|
|
273
|
+
return parsed.success ? parsed.data : degradedRecommendation();
|
|
274
|
+
}
|
|
275
|
+
async function reviewTestCanary(exec, path2, framework) {
|
|
276
|
+
const args = ["review-test", path2, "--json"];
|
|
277
|
+
if (framework) args.push("--framework", framework);
|
|
278
|
+
const res = await execCanary(exec, args);
|
|
279
|
+
if (!res.ok) return [];
|
|
280
|
+
const parsed = canaryFindingsSchema.safeParse(safeJson(res.stdout));
|
|
281
|
+
return parsed.success ? parsed.data : [];
|
|
282
|
+
}
|
|
283
|
+
function createCanaryAdapter(exec = defaultExec) {
|
|
257
284
|
let cachedProbe;
|
|
258
|
-
const probe = () => cachedProbe ??= (
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
if (res.stdout.trim() === "") return { status: "degraded", reason: "bad-output" };
|
|
262
|
-
const version = parseVersion(res.stdout);
|
|
263
|
-
return version ? { status: "available", version } : { status: "available" };
|
|
264
|
-
})();
|
|
265
|
-
const recommendFramework = async (prompt) => {
|
|
266
|
-
const res = await execCanary(["recommend", prompt, "--json"]);
|
|
267
|
-
if (!res.ok) return degradedRecommendation();
|
|
268
|
-
const parsed = frameworkRecommendationSchema.safeParse(safeJson(res.stdout));
|
|
269
|
-
return parsed.success ? parsed.data : degradedRecommendation();
|
|
270
|
-
};
|
|
271
|
-
const reviewTest = async (path2, framework) => {
|
|
272
|
-
const args = ["review-test", path2, "--json"];
|
|
273
|
-
if (framework) args.push("--framework", framework);
|
|
274
|
-
const res = await execCanary(args);
|
|
275
|
-
if (!res.ok) return [];
|
|
276
|
-
const parsed = canaryFindingsSchema.safeParse(safeJson(res.stdout));
|
|
277
|
-
return parsed.success ? parsed.data : [];
|
|
278
|
-
};
|
|
285
|
+
const probe = () => cachedProbe ??= probeCanary(exec);
|
|
286
|
+
const recommendFramework = (prompt) => recommendFrameworkCanary(exec, prompt);
|
|
287
|
+
const reviewTest = (path2, framework) => reviewTestCanary(exec, path2, framework);
|
|
279
288
|
return { probe, recommendFramework, reviewTest };
|
|
280
289
|
}
|
|
281
290
|
|
|
@@ -1497,6 +1506,145 @@ var OutcomeEvaluator = class {
|
|
|
1497
1506
|
}
|
|
1498
1507
|
};
|
|
1499
1508
|
|
|
1509
|
+
// src/acceptance-eval/authority.ts
|
|
1510
|
+
function deriveAcceptanceAuthority(measurability, confidence) {
|
|
1511
|
+
return measurability === "NOT_MEASURABLE" && confidence === "high" ? "blocking" : "advisory";
|
|
1512
|
+
}
|
|
1513
|
+
|
|
1514
|
+
// src/acceptance-eval/prompts.ts
|
|
1515
|
+
var import_zod5 = require("zod");
|
|
1516
|
+
var findingSchema = import_zod5.z.object({
|
|
1517
|
+
target: import_zod5.z.string().describe("The criterion or user-visible behavior referenced"),
|
|
1518
|
+
message: import_zod5.z.string().describe("The advisory observation")
|
|
1519
|
+
}).strict();
|
|
1520
|
+
var acceptanceVerdictSchema = import_zod5.z.object({
|
|
1521
|
+
measurability: import_zod5.z.enum(["MEASURABLE", "NOT_MEASURABLE", "INCONCLUSIVE"]).describe("Whether the spec section states measurable, testable success criteria"),
|
|
1522
|
+
confidence: import_zod5.z.enum(["low", "medium", "high"]).describe("Confidence in the verdict; high requires a named criterion"),
|
|
1523
|
+
rationale: import_zod5.z.string().describe("Cites specific criteria/behaviors"),
|
|
1524
|
+
criteriaFindings: import_zod5.z.array(findingSchema).describe("(a) advisory observability/testability/completeness critique"),
|
|
1525
|
+
coverageFindings: import_zod5.z.array(findingSchema).describe("(b) advisory user-visible behaviors with no covering test")
|
|
1526
|
+
}).strict();
|
|
1527
|
+
var ACCEPTANCE_EVAL_SYSTEM_PROMPT = `You are a PRE-execution acceptance-criteria judge. Given a spec acceptance section (and optionally located test snippets), assess three things:
|
|
1528
|
+
(a) criteria quality \u2014 are the success criteria observable, testable, and complete? (advisory findings)
|
|
1529
|
+
(b) coverage \u2014 do any user-visible behaviors lack a covering test? (advisory findings)
|
|
1530
|
+
(c) measurability \u2014 does the spec state MEASURABLE, NOT_MEASURABLE, or is it INCONCLUSIVE on whether any measurable success criteria exist at all?
|
|
1531
|
+
|
|
1532
|
+
Confidence calibration (be conservative \u2014 false alarms are costly):
|
|
1533
|
+
- Default to "medium" confidence.
|
|
1534
|
+
- Use "high" ONLY when you can name a SPECIFIC criterion (or its absence) and quote or paraphrase it in the rationale.
|
|
1535
|
+
- Use "low" when the section is ambiguous, partial, or insufficient to judge.
|
|
1536
|
+
- Bias toward advisory caution: if unsure between two confidence levels, choose the lower one.
|
|
1537
|
+
|
|
1538
|
+
Rules:
|
|
1539
|
+
- "measurability" is NOT_MEASURABLE only when the section states no observable, testable success criterion at all.
|
|
1540
|
+
- "criteriaFindings" holds advisory (a) observations; "coverageFindings" holds advisory (b) observations; both may be empty.
|
|
1541
|
+
- Do NOT emit an "authority" field. Authority is computed downstream in TypeScript from (measurability, confidence) and must never come from you.
|
|
1542
|
+
|
|
1543
|
+
Return your judgment using the structured_output tool.`;
|
|
1544
|
+
var PROMPT_FIELD_MAX_CHARS2 = 12e3;
|
|
1545
|
+
var FENCE2 = "````";
|
|
1546
|
+
function clampField2(text) {
|
|
1547
|
+
const trimmed = text.trim();
|
|
1548
|
+
if (trimmed.length <= PROMPT_FIELD_MAX_CHARS2) return trimmed;
|
|
1549
|
+
const dropped = trimmed.length - PROMPT_FIELD_MAX_CHARS2;
|
|
1550
|
+
return `${trimmed.slice(0, PROMPT_FIELD_MAX_CHARS2)}
|
|
1551
|
+
\u2026 [truncated ${dropped} chars]`;
|
|
1552
|
+
}
|
|
1553
|
+
function buildUserPrompt3(section, testContent) {
|
|
1554
|
+
const tests = (testContent ?? "").trim();
|
|
1555
|
+
return [
|
|
1556
|
+
"## Spec Acceptance Criteria (judge against this section)",
|
|
1557
|
+
section.trim() || "(empty \u2014 treat as inconclusive)",
|
|
1558
|
+
"",
|
|
1559
|
+
"## Located Test Snippets (coverage evidence \u2014 may be absent)",
|
|
1560
|
+
`${FENCE2}`,
|
|
1561
|
+
tests ? clampField2(tests) : "(no test content provided)",
|
|
1562
|
+
FENCE2,
|
|
1563
|
+
"",
|
|
1564
|
+
"## Instructions",
|
|
1565
|
+
"Judge measurability (c), and emit advisory criteria (a) and coverage (b) findings. Calibrate confidence conservatively per your system instructions. Cite specific criteria in the rationale."
|
|
1566
|
+
].join("\n");
|
|
1567
|
+
}
|
|
1568
|
+
|
|
1569
|
+
// src/acceptance-eval/evaluator.ts
|
|
1570
|
+
var import_promises2 = require("fs/promises");
|
|
1571
|
+
var AcceptanceEvaluator = class {
|
|
1572
|
+
provider;
|
|
1573
|
+
options;
|
|
1574
|
+
constructor(provider, options = {}) {
|
|
1575
|
+
this.provider = provider;
|
|
1576
|
+
this.options = options;
|
|
1577
|
+
}
|
|
1578
|
+
async evaluate(input) {
|
|
1579
|
+
let resolved;
|
|
1580
|
+
try {
|
|
1581
|
+
resolved = await this.resolveJudgmentSection(input);
|
|
1582
|
+
} catch {
|
|
1583
|
+
return this.degradedVerdict("overview");
|
|
1584
|
+
}
|
|
1585
|
+
if (resolved === null) {
|
|
1586
|
+
return this.buildVerdict(
|
|
1587
|
+
"INCONCLUSIVE",
|
|
1588
|
+
"low",
|
|
1589
|
+
"No judgable spec section found.",
|
|
1590
|
+
"overview",
|
|
1591
|
+
[],
|
|
1592
|
+
[]
|
|
1593
|
+
);
|
|
1594
|
+
}
|
|
1595
|
+
return this.judge(resolved, input);
|
|
1596
|
+
}
|
|
1597
|
+
async judge(resolved, input) {
|
|
1598
|
+
try {
|
|
1599
|
+
const response = await this.provider.analyze({
|
|
1600
|
+
prompt: buildUserPrompt3(resolved.body, input.testContent),
|
|
1601
|
+
systemPrompt: ACCEPTANCE_EVAL_SYSTEM_PROMPT,
|
|
1602
|
+
responseSchema: acceptanceVerdictSchema,
|
|
1603
|
+
...this.options.model !== void 0 && { model: this.options.model }
|
|
1604
|
+
});
|
|
1605
|
+
const llm = acceptanceVerdictSchema.parse(response.result);
|
|
1606
|
+
return this.buildVerdict(
|
|
1607
|
+
llm.measurability,
|
|
1608
|
+
llm.confidence,
|
|
1609
|
+
llm.rationale,
|
|
1610
|
+
resolved.judgedAgainst,
|
|
1611
|
+
llm.criteriaFindings,
|
|
1612
|
+
llm.coverageFindings
|
|
1613
|
+
);
|
|
1614
|
+
} catch {
|
|
1615
|
+
return this.degradedVerdict(resolved.judgedAgainst);
|
|
1616
|
+
}
|
|
1617
|
+
}
|
|
1618
|
+
degradedVerdict(judgedAgainst) {
|
|
1619
|
+
return this.buildVerdict(
|
|
1620
|
+
"INCONCLUSIVE",
|
|
1621
|
+
"low",
|
|
1622
|
+
"Evaluation could not be completed; defaulting to an inconclusive, advisory verdict.",
|
|
1623
|
+
judgedAgainst,
|
|
1624
|
+
[],
|
|
1625
|
+
[]
|
|
1626
|
+
);
|
|
1627
|
+
}
|
|
1628
|
+
async resolveJudgmentSection(input) {
|
|
1629
|
+
if (input.specSection !== void 0) {
|
|
1630
|
+
return input.specSection.trim() === "" ? null : { judgedAgainst: "success-criteria", body: input.specSection };
|
|
1631
|
+
}
|
|
1632
|
+
const markdown = await (0, import_promises2.readFile)(input.specPath, "utf8");
|
|
1633
|
+
return resolveSection(markdown);
|
|
1634
|
+
}
|
|
1635
|
+
buildVerdict(measurability, confidence, rationale, judgedAgainst, criteriaFindings, coverageFindings) {
|
|
1636
|
+
return {
|
|
1637
|
+
measurability,
|
|
1638
|
+
confidence,
|
|
1639
|
+
rationale,
|
|
1640
|
+
judgedAgainst,
|
|
1641
|
+
criteriaFindings,
|
|
1642
|
+
coverageFindings,
|
|
1643
|
+
authority: deriveAcceptanceAuthority(measurability, confidence)
|
|
1644
|
+
};
|
|
1645
|
+
}
|
|
1646
|
+
};
|
|
1647
|
+
|
|
1500
1648
|
// src/pipeline.ts
|
|
1501
1649
|
var IntelligencePipeline = class {
|
|
1502
1650
|
provider;
|
|
@@ -1969,6 +2117,8 @@ function refreshProfiles(projectRoot, graphStore, opts) {
|
|
|
1969
2117
|
}
|
|
1970
2118
|
// Annotate the CommonJS export names for ESM import in node:
|
|
1971
2119
|
0 && (module.exports = {
|
|
2120
|
+
ACCEPTANCE_EVAL_SYSTEM_PROMPT,
|
|
2121
|
+
AcceptanceEvaluator,
|
|
1972
2122
|
AnthropicAnalysisProvider,
|
|
1973
2123
|
ClaudeCliAnalysisProvider,
|
|
1974
2124
|
ExecutionOutcomeConnector,
|
|
@@ -1978,6 +2128,8 @@ function refreshProfiles(projectRoot, graphStore, opts) {
|
|
|
1978
2128
|
OpenAICompatibleAnalysisProvider,
|
|
1979
2129
|
OutcomeEvaluator,
|
|
1980
2130
|
PeslSimulator,
|
|
2131
|
+
acceptanceVerdictSchema,
|
|
2132
|
+
buildAcceptanceUserPrompt,
|
|
1981
2133
|
buildSpecializationProfile,
|
|
1982
2134
|
buildUserPrompt,
|
|
1983
2135
|
computeExpertiseLevel,
|
|
@@ -1988,9 +2140,11 @@ function refreshProfiles(projectRoot, graphStore, opts) {
|
|
|
1988
2140
|
computeStructuralComplexity,
|
|
1989
2141
|
createCanaryAdapter,
|
|
1990
2142
|
decayWeight,
|
|
2143
|
+
deriveAcceptanceAuthority,
|
|
1991
2144
|
deriveAuthority,
|
|
1992
2145
|
detectBlindSpots,
|
|
1993
2146
|
enrich,
|
|
2147
|
+
findingSchema,
|
|
1994
2148
|
githubToRawWorkItem,
|
|
1995
2149
|
jiraToRawWorkItem,
|
|
1996
2150
|
linearToRawWorkItem,
|
package/dist/index.mjs
CHANGED
|
@@ -165,43 +165,46 @@ function degradedRecommendation() {
|
|
|
165
165
|
alternatives: []
|
|
166
166
|
};
|
|
167
167
|
}
|
|
168
|
-
function
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
return { ok: false, reason: "binary-missing" };
|
|
179
|
-
}
|
|
180
|
-
return { ok: false, reason: "exec-failed" };
|
|
168
|
+
async function execCanary(exec, subArgs) {
|
|
169
|
+
const [cmd, args] = canaryInvocation(subArgs);
|
|
170
|
+
try {
|
|
171
|
+
const { stdout } = await exec(cmd, args);
|
|
172
|
+
return { ok: true, stdout };
|
|
173
|
+
} catch (err) {
|
|
174
|
+
const e = err;
|
|
175
|
+
if (e.code === "ENOENT") return { ok: false, reason: "not-installed" };
|
|
176
|
+
if (e.code === 1 && /canary binary not found/i.test(e.stderr ?? "")) {
|
|
177
|
+
return { ok: false, reason: "binary-missing" };
|
|
181
178
|
}
|
|
179
|
+
return { ok: false, reason: "exec-failed" };
|
|
182
180
|
}
|
|
181
|
+
}
|
|
182
|
+
async function probeCanary(exec) {
|
|
183
|
+
const res = await execCanary(exec, ["version"]);
|
|
184
|
+
if (!res.ok) return { status: "degraded", reason: res.reason };
|
|
185
|
+
if (res.stdout.trim() === "") return { status: "degraded", reason: "bad-output" };
|
|
186
|
+
const version = parseVersion(res.stdout);
|
|
187
|
+
return version ? { status: "available", version } : { status: "available" };
|
|
188
|
+
}
|
|
189
|
+
async function recommendFrameworkCanary(exec, prompt) {
|
|
190
|
+
const res = await execCanary(exec, ["recommend", prompt, "--json"]);
|
|
191
|
+
if (!res.ok) return degradedRecommendation();
|
|
192
|
+
const parsed = frameworkRecommendationSchema.safeParse(safeJson(res.stdout));
|
|
193
|
+
return parsed.success ? parsed.data : degradedRecommendation();
|
|
194
|
+
}
|
|
195
|
+
async function reviewTestCanary(exec, path2, framework) {
|
|
196
|
+
const args = ["review-test", path2, "--json"];
|
|
197
|
+
if (framework) args.push("--framework", framework);
|
|
198
|
+
const res = await execCanary(exec, args);
|
|
199
|
+
if (!res.ok) return [];
|
|
200
|
+
const parsed = canaryFindingsSchema.safeParse(safeJson(res.stdout));
|
|
201
|
+
return parsed.success ? parsed.data : [];
|
|
202
|
+
}
|
|
203
|
+
function createCanaryAdapter(exec = defaultExec) {
|
|
183
204
|
let cachedProbe;
|
|
184
|
-
const probe = () => cachedProbe ??= (
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
if (res.stdout.trim() === "") return { status: "degraded", reason: "bad-output" };
|
|
188
|
-
const version = parseVersion(res.stdout);
|
|
189
|
-
return version ? { status: "available", version } : { status: "available" };
|
|
190
|
-
})();
|
|
191
|
-
const recommendFramework = async (prompt) => {
|
|
192
|
-
const res = await execCanary(["recommend", prompt, "--json"]);
|
|
193
|
-
if (!res.ok) return degradedRecommendation();
|
|
194
|
-
const parsed = frameworkRecommendationSchema.safeParse(safeJson(res.stdout));
|
|
195
|
-
return parsed.success ? parsed.data : degradedRecommendation();
|
|
196
|
-
};
|
|
197
|
-
const reviewTest = async (path2, framework) => {
|
|
198
|
-
const args = ["review-test", path2, "--json"];
|
|
199
|
-
if (framework) args.push("--framework", framework);
|
|
200
|
-
const res = await execCanary(args);
|
|
201
|
-
if (!res.ok) return [];
|
|
202
|
-
const parsed = canaryFindingsSchema.safeParse(safeJson(res.stdout));
|
|
203
|
-
return parsed.success ? parsed.data : [];
|
|
204
|
-
};
|
|
205
|
+
const probe = () => cachedProbe ??= probeCanary(exec);
|
|
206
|
+
const recommendFramework = (prompt) => recommendFrameworkCanary(exec, prompt);
|
|
207
|
+
const reviewTest = (path2, framework) => reviewTestCanary(exec, path2, framework);
|
|
205
208
|
return { probe, recommendFramework, reviewTest };
|
|
206
209
|
}
|
|
207
210
|
|
|
@@ -1423,6 +1426,145 @@ var OutcomeEvaluator = class {
|
|
|
1423
1426
|
}
|
|
1424
1427
|
};
|
|
1425
1428
|
|
|
1429
|
+
// src/acceptance-eval/authority.ts
|
|
1430
|
+
function deriveAcceptanceAuthority(measurability, confidence) {
|
|
1431
|
+
return measurability === "NOT_MEASURABLE" && confidence === "high" ? "blocking" : "advisory";
|
|
1432
|
+
}
|
|
1433
|
+
|
|
1434
|
+
// src/acceptance-eval/prompts.ts
|
|
1435
|
+
import { z as z5 } from "zod";
|
|
1436
|
+
var findingSchema = z5.object({
|
|
1437
|
+
target: z5.string().describe("The criterion or user-visible behavior referenced"),
|
|
1438
|
+
message: z5.string().describe("The advisory observation")
|
|
1439
|
+
}).strict();
|
|
1440
|
+
var acceptanceVerdictSchema = z5.object({
|
|
1441
|
+
measurability: z5.enum(["MEASURABLE", "NOT_MEASURABLE", "INCONCLUSIVE"]).describe("Whether the spec section states measurable, testable success criteria"),
|
|
1442
|
+
confidence: z5.enum(["low", "medium", "high"]).describe("Confidence in the verdict; high requires a named criterion"),
|
|
1443
|
+
rationale: z5.string().describe("Cites specific criteria/behaviors"),
|
|
1444
|
+
criteriaFindings: z5.array(findingSchema).describe("(a) advisory observability/testability/completeness critique"),
|
|
1445
|
+
coverageFindings: z5.array(findingSchema).describe("(b) advisory user-visible behaviors with no covering test")
|
|
1446
|
+
}).strict();
|
|
1447
|
+
var ACCEPTANCE_EVAL_SYSTEM_PROMPT = `You are a PRE-execution acceptance-criteria judge. Given a spec acceptance section (and optionally located test snippets), assess three things:
|
|
1448
|
+
(a) criteria quality \u2014 are the success criteria observable, testable, and complete? (advisory findings)
|
|
1449
|
+
(b) coverage \u2014 do any user-visible behaviors lack a covering test? (advisory findings)
|
|
1450
|
+
(c) measurability \u2014 does the spec state MEASURABLE, NOT_MEASURABLE, or is it INCONCLUSIVE on whether any measurable success criteria exist at all?
|
|
1451
|
+
|
|
1452
|
+
Confidence calibration (be conservative \u2014 false alarms are costly):
|
|
1453
|
+
- Default to "medium" confidence.
|
|
1454
|
+
- Use "high" ONLY when you can name a SPECIFIC criterion (or its absence) and quote or paraphrase it in the rationale.
|
|
1455
|
+
- Use "low" when the section is ambiguous, partial, or insufficient to judge.
|
|
1456
|
+
- Bias toward advisory caution: if unsure between two confidence levels, choose the lower one.
|
|
1457
|
+
|
|
1458
|
+
Rules:
|
|
1459
|
+
- "measurability" is NOT_MEASURABLE only when the section states no observable, testable success criterion at all.
|
|
1460
|
+
- "criteriaFindings" holds advisory (a) observations; "coverageFindings" holds advisory (b) observations; both may be empty.
|
|
1461
|
+
- Do NOT emit an "authority" field. Authority is computed downstream in TypeScript from (measurability, confidence) and must never come from you.
|
|
1462
|
+
|
|
1463
|
+
Return your judgment using the structured_output tool.`;
|
|
1464
|
+
var PROMPT_FIELD_MAX_CHARS2 = 12e3;
|
|
1465
|
+
var FENCE2 = "````";
|
|
1466
|
+
function clampField2(text) {
|
|
1467
|
+
const trimmed = text.trim();
|
|
1468
|
+
if (trimmed.length <= PROMPT_FIELD_MAX_CHARS2) return trimmed;
|
|
1469
|
+
const dropped = trimmed.length - PROMPT_FIELD_MAX_CHARS2;
|
|
1470
|
+
return `${trimmed.slice(0, PROMPT_FIELD_MAX_CHARS2)}
|
|
1471
|
+
\u2026 [truncated ${dropped} chars]`;
|
|
1472
|
+
}
|
|
1473
|
+
function buildUserPrompt3(section, testContent) {
|
|
1474
|
+
const tests = (testContent ?? "").trim();
|
|
1475
|
+
return [
|
|
1476
|
+
"## Spec Acceptance Criteria (judge against this section)",
|
|
1477
|
+
section.trim() || "(empty \u2014 treat as inconclusive)",
|
|
1478
|
+
"",
|
|
1479
|
+
"## Located Test Snippets (coverage evidence \u2014 may be absent)",
|
|
1480
|
+
`${FENCE2}`,
|
|
1481
|
+
tests ? clampField2(tests) : "(no test content provided)",
|
|
1482
|
+
FENCE2,
|
|
1483
|
+
"",
|
|
1484
|
+
"## Instructions",
|
|
1485
|
+
"Judge measurability (c), and emit advisory criteria (a) and coverage (b) findings. Calibrate confidence conservatively per your system instructions. Cite specific criteria in the rationale."
|
|
1486
|
+
].join("\n");
|
|
1487
|
+
}
|
|
1488
|
+
|
|
1489
|
+
// src/acceptance-eval/evaluator.ts
|
|
1490
|
+
import { readFile as readFile2 } from "fs/promises";
|
|
1491
|
+
var AcceptanceEvaluator = class {
|
|
1492
|
+
provider;
|
|
1493
|
+
options;
|
|
1494
|
+
constructor(provider, options = {}) {
|
|
1495
|
+
this.provider = provider;
|
|
1496
|
+
this.options = options;
|
|
1497
|
+
}
|
|
1498
|
+
async evaluate(input) {
|
|
1499
|
+
let resolved;
|
|
1500
|
+
try {
|
|
1501
|
+
resolved = await this.resolveJudgmentSection(input);
|
|
1502
|
+
} catch {
|
|
1503
|
+
return this.degradedVerdict("overview");
|
|
1504
|
+
}
|
|
1505
|
+
if (resolved === null) {
|
|
1506
|
+
return this.buildVerdict(
|
|
1507
|
+
"INCONCLUSIVE",
|
|
1508
|
+
"low",
|
|
1509
|
+
"No judgable spec section found.",
|
|
1510
|
+
"overview",
|
|
1511
|
+
[],
|
|
1512
|
+
[]
|
|
1513
|
+
);
|
|
1514
|
+
}
|
|
1515
|
+
return this.judge(resolved, input);
|
|
1516
|
+
}
|
|
1517
|
+
async judge(resolved, input) {
|
|
1518
|
+
try {
|
|
1519
|
+
const response = await this.provider.analyze({
|
|
1520
|
+
prompt: buildUserPrompt3(resolved.body, input.testContent),
|
|
1521
|
+
systemPrompt: ACCEPTANCE_EVAL_SYSTEM_PROMPT,
|
|
1522
|
+
responseSchema: acceptanceVerdictSchema,
|
|
1523
|
+
...this.options.model !== void 0 && { model: this.options.model }
|
|
1524
|
+
});
|
|
1525
|
+
const llm = acceptanceVerdictSchema.parse(response.result);
|
|
1526
|
+
return this.buildVerdict(
|
|
1527
|
+
llm.measurability,
|
|
1528
|
+
llm.confidence,
|
|
1529
|
+
llm.rationale,
|
|
1530
|
+
resolved.judgedAgainst,
|
|
1531
|
+
llm.criteriaFindings,
|
|
1532
|
+
llm.coverageFindings
|
|
1533
|
+
);
|
|
1534
|
+
} catch {
|
|
1535
|
+
return this.degradedVerdict(resolved.judgedAgainst);
|
|
1536
|
+
}
|
|
1537
|
+
}
|
|
1538
|
+
degradedVerdict(judgedAgainst) {
|
|
1539
|
+
return this.buildVerdict(
|
|
1540
|
+
"INCONCLUSIVE",
|
|
1541
|
+
"low",
|
|
1542
|
+
"Evaluation could not be completed; defaulting to an inconclusive, advisory verdict.",
|
|
1543
|
+
judgedAgainst,
|
|
1544
|
+
[],
|
|
1545
|
+
[]
|
|
1546
|
+
);
|
|
1547
|
+
}
|
|
1548
|
+
async resolveJudgmentSection(input) {
|
|
1549
|
+
if (input.specSection !== void 0) {
|
|
1550
|
+
return input.specSection.trim() === "" ? null : { judgedAgainst: "success-criteria", body: input.specSection };
|
|
1551
|
+
}
|
|
1552
|
+
const markdown = await readFile2(input.specPath, "utf8");
|
|
1553
|
+
return resolveSection(markdown);
|
|
1554
|
+
}
|
|
1555
|
+
buildVerdict(measurability, confidence, rationale, judgedAgainst, criteriaFindings, coverageFindings) {
|
|
1556
|
+
return {
|
|
1557
|
+
measurability,
|
|
1558
|
+
confidence,
|
|
1559
|
+
rationale,
|
|
1560
|
+
judgedAgainst,
|
|
1561
|
+
criteriaFindings,
|
|
1562
|
+
coverageFindings,
|
|
1563
|
+
authority: deriveAcceptanceAuthority(measurability, confidence)
|
|
1564
|
+
};
|
|
1565
|
+
}
|
|
1566
|
+
};
|
|
1567
|
+
|
|
1426
1568
|
// src/pipeline.ts
|
|
1427
1569
|
var IntelligencePipeline = class {
|
|
1428
1570
|
provider;
|
|
@@ -1894,6 +2036,8 @@ function refreshProfiles(projectRoot, graphStore, opts) {
|
|
|
1894
2036
|
return store;
|
|
1895
2037
|
}
|
|
1896
2038
|
export {
|
|
2039
|
+
ACCEPTANCE_EVAL_SYSTEM_PROMPT,
|
|
2040
|
+
AcceptanceEvaluator,
|
|
1897
2041
|
AnthropicAnalysisProvider,
|
|
1898
2042
|
ClaudeCliAnalysisProvider,
|
|
1899
2043
|
ExecutionOutcomeConnector,
|
|
@@ -1903,6 +2047,8 @@ export {
|
|
|
1903
2047
|
OpenAICompatibleAnalysisProvider,
|
|
1904
2048
|
OutcomeEvaluator,
|
|
1905
2049
|
PeslSimulator,
|
|
2050
|
+
acceptanceVerdictSchema,
|
|
2051
|
+
buildUserPrompt3 as buildAcceptanceUserPrompt,
|
|
1906
2052
|
buildSpecializationProfile,
|
|
1907
2053
|
buildUserPrompt2 as buildUserPrompt,
|
|
1908
2054
|
computeExpertiseLevel,
|
|
@@ -1913,9 +2059,11 @@ export {
|
|
|
1913
2059
|
computeStructuralComplexity,
|
|
1914
2060
|
createCanaryAdapter,
|
|
1915
2061
|
decayWeight,
|
|
2062
|
+
deriveAcceptanceAuthority,
|
|
1916
2063
|
deriveAuthority,
|
|
1917
2064
|
detectBlindSpots,
|
|
1918
2065
|
enrich,
|
|
2066
|
+
findingSchema,
|
|
1919
2067
|
githubToRawWorkItem,
|
|
1920
2068
|
jiraToRawWorkItem,
|
|
1921
2069
|
linearToRawWorkItem,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@harness-engineering/intelligence",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.1",
|
|
4
4
|
"description": "Intelligence pipeline for spec enrichment, complexity modeling, and pre-execution simulation",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"module": "./dist/index.mjs",
|
|
@@ -40,8 +40,8 @@
|
|
|
40
40
|
"@anthropic-ai/sdk": "^0.95.1",
|
|
41
41
|
"openai": "^6.0.0",
|
|
42
42
|
"zod": "^3.25.76",
|
|
43
|
-
"@harness-engineering/graph": "0.11.
|
|
44
|
-
"@harness-engineering/types": "0.
|
|
43
|
+
"@harness-engineering/graph": "0.11.3",
|
|
44
|
+
"@harness-engineering/types": "0.17.0"
|
|
45
45
|
},
|
|
46
46
|
"optionalDependencies": {
|
|
47
47
|
"canary-test-cli": "^5.4.0"
|