impact-compass 0.2.0

package/src/domain/scoring.test.ts

@@ -0,0 +1,184 @@
+import { describe, expect, it } from "vitest";
+import {
+  calculateCompassScore,
+  calculateConfidence,
+  calculateActivityScore,
+  calculateCompetitionFitScore,
+  calculateDemandScore,
+  calculateMomentumScore,
+  calculatePainScore,
+  calculateRecencyWeight,
+  calculateScoreRange,
+  rankIdeasByEvidence,
+  summarizeIntegrity,
+} from "./scoring";
+
+const balancedPillars = {
+  demand: 70,
+  pain: 80,
+  momentum: 60,
+  competitionFit: 65,
+  activity: 50,
+  channelFit: 75,
+  evidenceQuality: 60,
+};
+
+describe("scoring methodology", () => {
+  it("calculates the Compass Score as a weighted pillar average", () => {
+    const result = calculateCompassScore(balancedPillars);
+
+    expect(result.score).toBe(67);
+    expect(result.weights).toEqual({
+      demand: 20,
+      pain: 20,
+      momentum: 15,
+      competitionFit: 15,
+      activity: 10,
+      channelFit: 10,
+      evidenceQuality: 10,
+    });
+  });
+
+  it("keeps uncertainty separate from the Compass Score", () => {
+    const highUncertainty = calculateCompassScore(balancedPillars, {
+      uncertainty: 22,
+    });
+    const lowUncertainty = calculateCompassScore(balancedPillars, {
+      uncertainty: 6,
+    });
+
+    expect(highUncertainty.score).toBe(lowUncertainty.score);
+    expect(highUncertainty.uncertainty).toBe(22);
+    expect(lowUncertainty.uncertainty).toBe(6);
+  });
+
+  it("maps uncertainty bands to confidence labels", () => {
+    expect(calculateConfidence(7)).toBe("High");
+    expect(calculateConfidence(14)).toBe("Medium");
+    expect(calculateConfidence(24)).toBe("Low");
+    expect(calculateConfidence(25)).toBe("Very Low");
+  });
+
+  it("clamps score ranges to 0-100", () => {
+    expect(calculateScoreRange({ score: 96, uncertainty: 9 })).toEqual({
+      lower: 87,
+      upper: 100,
+    });
+    expect(calculateScoreRange({ score: 4, uncertainty: 10 })).toEqual({
+      lower: 0,
+      upper: 14,
+    });
+  });
+
+  it("ranks ideas by lower confidence bound, not raw score", () => {
+    const ranked = rankIdeasByEvidence([
+      { id: "spiky", name: "Spiky idea", score: 74, uncertainty: 18 },
+      { id: "steady", name: "Steady idea", score: 69, uncertainty: 5 },
+    ]);
+
+    expect(ranked.map((idea) => idea.id)).toEqual(["steady", "spiky"]);
+    expect(ranked[0].rankBasis).toBe(64);
+    expect(ranked[1].rankBasis).toBe(56);
+  });
+
+  it("summarizes integrity caps when evidence quality is weak", () => {
+    const integrity = summarizeIntegrity({
+      evidenceQuality: 38,
+      sourceDiversity: 4,
+      relevancePrecision: 72,
+      relevantEvidenceCount: 50,
+      dominantSourceShare: 0.42,
+      queryLocked: true,
+    });
+
+    expect(integrity.finalScoreAvailable).toBe(true);
+    expect(integrity.confidenceCap).toBe("Low");
+    expect(integrity.warnings).toContain(
+      "Evidence Quality below 40 caps confidence at Low.",
+    );
+  });
+
+  it("withholds final score when query bundle is not locked", () => {
+    const integrity = summarizeIntegrity({
+      evidenceQuality: 80,
+      sourceDiversity: 5,
+      relevancePrecision: 90,
+      relevantEvidenceCount: 100,
+      dominantSourceShare: 0.4,
+      queryLocked: false,
+    });
+
+    expect(integrity.finalScoreAvailable).toBe(false);
+    expect(integrity.warnings).toContain(
+      "Query bundle is not locked; only preview scoring is allowed.",
+    );
+  });
+
+  it("uses deterministic recency weight boundaries", () => {
+    expect(calculateRecencyWeight(0)).toBe(1);
+    expect(calculateRecencyWeight(30)).toBe(1);
+    expect(calculateRecencyWeight(31)).toBe(0.75);
+    expect(calculateRecencyWeight(90)).toBe(0.75);
+    expect(calculateRecencyWeight(91)).toBe(0.45);
+    expect(calculateRecencyWeight(365)).toBe(0.45);
+    expect(calculateRecencyWeight(366)).toBe(0.2);
+  });
+
+  it("calculates Demand from weighted normalized signals", () => {
+    expect(
+      calculateDemandScore({
+        volumePercentile: 80,
+        uniqueAuthorPercentile: 60,
+        questionPercentile: 70,
+        engagementPercentile: 50,
+      }),
+    ).toBe(70);
+  });
+
+  it("calculates Pain from weighted normalized signals", () => {
+    expect(
+      calculatePainScore({
+        painDensity: 90,
+        workaroundDensity: 80,
+        alternativeDensity: 50,
+        discussionDepthPercentile: 40,
+      }),
+    ).toBe(70);
+  });
+
+  it("clamps Momentum after spike penalties", () => {
+    expect(
+      calculateMomentumScore({
+        shortGrowth: 100,
+        mediumGrowth: 100,
+        sustainedGrowth: 100,
+        spikePenalty: 500,
+      }),
+    ).toBe(40);
+    expect(
+      calculateMomentumScore({
+        shortGrowth: 0,
+        mediumGrowth: 0,
+        sustainedGrowth: 0,
+        spikePenalty: 500,
+      }),
+    ).toBe(0);
+  });
+
+  it("scores Competition Fit highest near moderate supply", () => {
+    expect(calculateCompetitionFitScore(60)).toBe(100);
+    expect(calculateCompetitionFitScore(0)).toBe(6);
+    expect(calculateCompetitionFitScore(100)).toBe(28);
+  });
+
+  it("excludes non-relevant Activity metrics instead of treating them as zero", () => {
+    expect(
+      calculateActivityScore({
+        repoActivity: 80,
+        packageActivity: null,
+        launchRecency: 60,
+        discussionFreshness: 70,
+      }),
+    ).toBe(72);
+  });
+});

package/src/domain/scoring.ts

@@ -0,0 +1,322 @@
+export type ConfidenceLabel = "Very Low" | "Low" | "Medium" | "High";
+
+export type PillarScores = {
+  demand: number;
+  pain: number;
+  momentum: number;
+  competitionFit: number;
+  activity: number;
+  channelFit: number;
+  evidenceQuality: number;
+};
+
+export type ScoreWeights = Record<keyof PillarScores, number>;
+
+export type ScoreSummary = {
+  score: number;
+  uncertainty: number;
+  confidence: ConfidenceLabel;
+  weights: ScoreWeights;
+};
+
+export type RankedIdea = {
+  id: string;
+  name: string;
+  score: number;
+  uncertainty: number;
+};
+
+export type RankedIdeaWithBasis = RankedIdea & {
+  rankBasis: number;
+  lower: number;
+  upper: number;
+};
+
+export type IntegrityInput = {
+  evidenceQuality: number;
+  sourceDiversity: number;
+  relevancePrecision: number;
+  relevantEvidenceCount: number;
+  dominantSourceShare: number;
+  queryLocked: boolean;
+};
+
+export type IntegritySummary = {
+  finalScoreAvailable: boolean;
+  confidenceCap?: ConfidenceLabel;
+  warnings: string[];
+};
+
+export type DemandSignals = {
+  volumePercentile: number;
+  uniqueAuthorPercentile: number;
+  questionPercentile: number;
+  engagementPercentile: number;
+};
+
+export type PainSignals = {
+  painDensity: number;
+  workaroundDensity: number;
+  alternativeDensity: number;
+  discussionDepthPercentile: number;
+};
+
+export type MomentumSignals = {
+  shortGrowth: number;
+  mediumGrowth: number;
+  sustainedGrowth: number;
+  spikePenalty: number;
+};
+
+export type ActivitySignals = {
+  repoActivity: number | null;
+  packageActivity: number | null;
+  launchRecency: number | null;
+  discussionFreshness: number | null;
+};
+
+export const defaultWeights: ScoreWeights = {
+  demand: 20,
+  pain: 20,
+  momentum: 15,
+  competitionFit: 15,
+  activity: 10,
+  channelFit: 10,
+  evidenceQuality: 10,
+};
+
+const minimumEvidenceCount = 10;
+
+function clampScore(value: number) {
+  return Math.min(100, Math.max(0, Math.round(value)));
+}
+
+function weightedAverage<T extends string>(
+  signals: Record<T, number | null>,
+  weights: Record<T, number>,
+) {
+  let weightedTotal = 0;
+  let totalWeight = 0;
+
+  for (const key of Object.keys(weights) as T[]) {
+    const value = signals[key];
+
+    if (value === null) {
+      continue;
+    }
+
+    weightedTotal += value * weights[key];
+    totalWeight += weights[key];
+  }
+
+  return totalWeight === 0 ? 0 : weightedTotal / totalWeight;
+}
+
+function lowestConfidenceCap(
+  current: ConfidenceLabel | undefined,
+  next: ConfidenceLabel,
+) {
+  const order: ConfidenceLabel[] = ["Very Low", "Low", "Medium", "High"];
+
+  if (!current) {
+    return next;
+  }
+
+  return order.indexOf(next) < order.indexOf(current) ? next : current;
+}
+
+export function calculateConfidence(uncertainty: number): ConfidenceLabel {
+  if (uncertainty <= 7) {
+    return "High";
+  }
+
+  if (uncertainty <= 14) {
+    return "Medium";
+  }
+
+  if (uncertainty <= 24) {
+    return "Low";
+  }
+
+  return "Very Low";
+}
+
+export function calculateScoreRange(params: { score: number; uncertainty: number }) {
+  return {
+    lower: clampScore(params.score - params.uncertainty),
+    upper: clampScore(params.score + params.uncertainty),
+  };
+}
+
+export function calculateRecencyWeight(ageInDays: number) {
+  if (ageInDays <= 30) {
+    return 1;
+  }
+
+  if (ageInDays <= 90) {
+    return 0.75;
+  }
+
+  if (ageInDays <= 365) {
+    return 0.45;
+  }
+
+  return 0.2;
+}
+
+export function calculateDemandScore(signals: DemandSignals) {
+  return clampScore(
+    weightedAverage(
+      {
+        volumePercentile: signals.volumePercentile,
+        uniqueAuthorPercentile: signals.uniqueAuthorPercentile,
+        questionPercentile: signals.questionPercentile,
+        engagementPercentile: signals.engagementPercentile,
+      },
+      {
+        volumePercentile: 0.45,
+        uniqueAuthorPercentile: 0.25,
+        questionPercentile: 0.2,
+        engagementPercentile: 0.1,
+      },
+    ),
+  );
+}
+
+export function calculatePainScore(signals: PainSignals) {
+  return clampScore(
+    weightedAverage(
+      {
+        painDensity: signals.painDensity,
+        workaroundDensity: signals.workaroundDensity,
+        alternativeDensity: signals.alternativeDensity,
+        discussionDepthPercentile: signals.discussionDepthPercentile,
+      },
+      {
+        painDensity: 0.35,
+        workaroundDensity: 0.25,
+        alternativeDensity: 0.2,
+        discussionDepthPercentile: 0.2,
+      },
+    ),
+  );
+}
+
+export function calculateMomentumScore(signals: MomentumSignals) {
+  return clampScore(
+    0.4 * signals.shortGrowth +
+      0.3 * signals.mediumGrowth +
+      0.2 * signals.sustainedGrowth -
+      0.1 * signals.spikePenalty,
+  );
+}
+
+export function calculateCompetitionFitScore(supplyPercentile: number) {
+  const variance = 2 * 25 ** 2;
+  const rawScore = 100 * Math.exp(-((supplyPercentile - 60) ** 2) / variance);
+
+  return clampScore(rawScore);
+}
+
+export function calculateActivityScore(signals: ActivitySignals) {
+  return clampScore(
+    weightedAverage(
+      {
+        repoActivity: signals.repoActivity,
+        packageActivity: signals.packageActivity,
+        launchRecency: signals.launchRecency,
+        discussionFreshness: signals.discussionFreshness,
+      },
+      {
+        repoActivity: 0.35,
+        packageActivity: 0.25,
+        launchRecency: 0.2,
+        discussionFreshness: 0.2,
+      },
+    ),
+  );
+}
+
+export function calculateCompassScore(
+  pillars: PillarScores,
+  options: { uncertainty?: number; weights?: ScoreWeights } = {},
+): ScoreSummary {
+  const weights = options.weights ?? defaultWeights;
+  const totalWeight = Object.values(weights).reduce((sum, weight) => sum + weight, 0);
+  let weightedScore =
+    Object.entries(weights).reduce((sum, [pillar, weight]) => {
+      return sum + pillars[pillar as keyof PillarScores] * weight;
+    }, 0) / totalWeight;
+  const uncertainty = options.uncertainty ?? 10;
+
+  // Red Ocean Saturation Penalty
+  // High demand but very low Competition Fit (meaning massive competition volume)
+  let saturationPenalty = 0;
+  if (pillars.demand > 70 && pillars.competitionFit < 30) {
+    // Dock up to 15 extra points for extreme saturation
+    saturationPenalty = (30 - pillars.competitionFit) * 0.5;
+  }
+  
+  weightedScore -= saturationPenalty;
+
+  return {
+    score: clampScore(weightedScore),
+    uncertainty,
+    confidence: calculateConfidence(uncertainty),
+    weights,
+  };
+}
+
+export function rankIdeasByEvidence(ideas: RankedIdea[]): RankedIdeaWithBasis[] {
+  return ideas
+    .map((idea) => {
+      const range = calculateScoreRange(idea);
+
+      return {
+        ...idea,
+        ...range,
+        rankBasis: range.lower,
+      };
+    })
+    .sort((a, b) => b.rankBasis - a.rankBasis || b.score - a.score);
+}
+
+export function summarizeIntegrity(input: IntegrityInput): IntegritySummary {
+  const warnings: string[] = [];
+  let confidenceCap: ConfidenceLabel | undefined;
+
+  if (!input.queryLocked) {
+    warnings.push("Query bundle is not locked; only preview scoring is allowed.");
+  }
+
+  if (input.evidenceQuality < 40) {
+    warnings.push("Evidence Quality below 40 caps confidence at Low.");
+    confidenceCap = lowestConfidenceCap(confidenceCap, "Low");
+  }
+
+  if (input.sourceDiversity < 2) {
+    warnings.push("Source diversity below 2 caps confidence at Low.");
+    confidenceCap = lowestConfidenceCap(confidenceCap, "Low");
+  }
+
+  if (input.relevancePrecision < 50) {
+    warnings.push("Relevance precision below 50 caps confidence at Low.");
+    confidenceCap = lowestConfidenceCap(confidenceCap, "Low");
+  }
+
+  if (input.relevantEvidenceCount < minimumEvidenceCount) {
+    warnings.push("Relevant evidence count is below the minimum confidence threshold.");
+    confidenceCap = lowestConfidenceCap(confidenceCap, "Low");
+  }
+
+  if (input.dominantSourceShare > 0.7) {
+    warnings.push("One source contributes over 70% of evidence; confidence caps at Medium.");
+    confidenceCap = lowestConfidenceCap(confidenceCap, "Medium");
+  }
+
+  return {
+    finalScoreAvailable: input.queryLocked,
+    confidenceCap,
+    warnings,
+  };
+}

package/src/services/comparison.test.ts

@@ -0,0 +1,89 @@
+import { describe, expect, it } from "vitest";
+import { createLockedQueryBundle } from "../domain/queryBundle";
+import { buildCompassReport } from "./reportBuilder";
+import { buildComparisonRows, buildSnapshotComparisonRows } from "./comparison";
+import { createReportSnapshot } from "./reportStorage";
+import { therapyEvidenceSeed, therapyPillarScores } from "./therapySeed";
+
+function report(name: string, scoreShape: typeof therapyPillarScores, uncertainty: number) {
+  return buildCompassReport({
+    idea: {
+      name,
+      problem: `${name} problem`,
+      targetUser: "Indie builders",
+      lens: "Productivity / Prosumer SaaS",
+    },
+    queryBundle: createLockedQueryBundle({
+      problemKeywords: name,
+      solutionKeywords: `${name} tool`,
+      audienceKeywords: "indie builders",
+      competitorKeywords: "",
+      exclusions: "jobs",
+    }),
+    evidence: therapyEvidenceSeed,
+    pillarScores: scoreShape,
+    uncertainty,
+  });
+}
+
+describe("comparison service", () => {
+  it("ranks reports by lower confidence bound", () => {
+    const rows = buildComparisonRows([
+      report(
+        "Spiky high score",
+        { ...therapyPillarScores, demand: 95, pain: 95, evidenceQuality: 70 },
+        25,
+      ),
+      report(
+        "Steady evidence",
+        { ...therapyPillarScores, demand: 72, pain: 74, evidenceQuality: 80 },
+        6,
+      ),
+    ]);
+
+    expect(rows.map((row) => row.ideaName)).toEqual([
+      "Steady evidence",
+      "Spiky high score",
+    ]);
+    expect(rows[0].rankBasis).toBeGreaterThan(rows[1].rankBasis);
+  });
+
+  it("includes comparison diagnostics for each idea", () => {
+    const rows = buildComparisonRows([
+      report("Therapy notes", therapyPillarScores, 11),
+    ]);
+
+    expect(rows[0]).toMatchObject({
+      ideaName: "Therapy notes",
+      strongestPillar: "Pain",
+      weakestPillar: "Activity",
+      bestChannel: "Reddit",
+    });
+  });
+
+  it("ranks saved snapshots without seeded fake alternatives", () => {
+    const rows = buildSnapshotComparisonRows([
+      createReportSnapshot(
+        report(
+          "Spiky high score",
+          { ...therapyPillarScores, demand: 95, pain: 95, evidenceQuality: 70 },
+          25,
+        ),
+        "2026-05-24T04:00:00.000Z",
+      ),
+      createReportSnapshot(
+        report(
+          "Steady evidence",
+          { ...therapyPillarScores, demand: 72, pain: 74, evidenceQuality: 80 },
+          6,
+        ),
+        "2026-05-24T05:00:00.000Z",
+      ),
+    ]);
+
+    expect(rows.map((row) => row.ideaName)).toEqual([
+      "Steady evidence",
+      "Spiky high score",
+    ]);
+  });
+});

package/src/services/comparison.ts

@@ -0,0 +1,84 @@
+import { rankIdeasByEvidence } from "../domain/scoring";
+import { findBestEvidenceSource, summarizeEvidenceGap } from "./reportInsights";
+import type { ReportSnapshot } from "./reportStorage";
+import type { CompassReportModel } from "./reportTypes";
+
+export type ComparisonRow = {
+  ideaName: string;
+  score: number;
+  confidence: string;
+  range: string;
+  rankBasis: number;
+  strongestPillar: string;
+  weakestPillar: string;
+  bestChannel: string;
+  evidenceGap: string;
+};
+
+export function buildComparisonRows(
+  reports: CompassReportModel[],
+): ComparisonRow[] {
+  const ranked = rankIdeasByEvidence(
+    reports.map((report) => ({
+      id: report.idea.name,
+      name: report.idea.name,
+      score: report.summary.score,
+      uncertainty: report.summary.uncertainty,
+    })),
+  );
+  const reportByName = new Map(reports.map((report) => [report.idea.name, report]));
+
+  return ranked.map((rankedIdea) => {
+    const report = reportByName.get(rankedIdea.name);
+
+    if (!report) {
+      throw new Error(`Missing report for ${rankedIdea.name}`);
+    }
+
+    return {
+      ideaName: report.idea.name,
+      score: report.summary.score,
+      confidence: report.summary.confidence,
+      range: `${report.summary.range.lower}-${report.summary.range.upper}`,
+      rankBasis: rankedIdea.rankBasis,
+      strongestPillar: report.strongestPillar.label,
+      weakestPillar: report.weakestPillar.label,
+      bestChannel: findBestEvidenceSource(report),
+      evidenceGap: summarizeEvidenceGap(report),
+    };
+  });
+}
+
+export function buildSnapshotComparisonRows(
+  snapshots: ReportSnapshot[],
+): ComparisonRow[] {
+  const ranked = rankIdeasByEvidence(
+    snapshots.map((snapshot) => ({
+      id: snapshot.id,
+      name: snapshot.ideaName,
+      score: snapshot.score,
+      uncertainty: snapshot.uncertainty,
+    })),
+  );
+  const snapshotById = new Map(snapshots.map((snapshot) => [snapshot.id, snapshot]));
+
+  return ranked.map((rankedIdea) => {
+    const snapshot = snapshotById.get(rankedIdea.id);
+
+    if (!snapshot) {
+      throw new Error(`Missing snapshot for ${rankedIdea.id}`);
+    }
+
+    return {
+      ideaName: snapshot.ideaName,
+      score: snapshot.score,
+      confidence: snapshot.confidence,
+      range: snapshot.range,
+      rankBasis: rankedIdea.rankBasis,
+      strongestPillar: snapshot.strongestPillar,
+      weakestPillar: snapshot.weakestPillar,
+      bestChannel: snapshot.bestChannel,
+      evidenceGap: snapshot.evidenceGap,
+    };
+  });
+}

package/src/services/demoReport.test.ts

@@ -0,0 +1,37 @@
+import { describe, expect, it } from "vitest";
+import { createDemoReport } from "./demoReport";
+
+describe("demo report fixture", () => {
+  it("builds a locked public-evidence report from deterministic scoring", () => {
+    const report = createDemoReport();
+
+    expect(report.idea.name).toBe("Privacy-safe session note drafts");
+    expect(report.queryBundle.locked).toBe(true);
+    expect(report.methodologyVersion).toBe("0.1");
+    expect(report.summary.score).toBe(66);
+    expect(report.summary.uncertainty).toBe(11);
+    expect(report.summary.confidence).toBe("Medium");
+    expect(report.summary.range).toEqual({ lower: 55, upper: 77 });
+    expect(report.integrity.finalScoreAvailable).toBe(true);
+  });
+
+  it("keeps evidence ledger entries auditable", () => {
+    const report = createDemoReport();
+
+    expect(report.evidence).toHaveLength(12);
+    expect(report.evidence.some((item) => item.included)).toBe(true);
+    expect(report.evidence.some((item) => !item.included)).toBe(true);
+    expect(report.evidence[0]).toMatchObject({
+      source: "Reddit",
+      metricContribution: "Pain",
+      query: "therapist paperwork",
+    });
+  });
+
+  it("identifies strongest and weakest pillars", () => {
+    const report = createDemoReport();
+
+    expect(report.strongestPillar.label).toBe("Pain");
+    expect(report.weakestPillar.label).toBe("Activity");
+  });
+});