@sourcepress/ai 0.1.0

package/src/__tests__/improve-prompt.test.ts

@@ -0,0 +1,42 @@
+import { describe, expect, it, vi } from "vitest";
+import { BudgetTracker } from "../budget.js";
+import { improvePrompt } from "../functions/improve-prompt.js";
+import type { ResolvedProvider } from "../provider.js";
+
+vi.mock("ai", () => ({ generateObject: vi.fn() }));
+import { generateObject } from "ai";
+
+const mockProvider: ResolvedProvider = {
+	provider: "anthropic",
+	model: "claude-sonnet-4-5-20250514",
+};
+
+describe("improvePrompt", () => {
+	it("improves a generation prompt based on judge feedback", async () => {
+		vi.mocked(generateObject).mockResolvedValueOnce({
+			object: {
+				improved_prompt:
+					"Write a case study with specific metrics, client quotes, and a clear before/after comparison.",
+				changes_summary: "Added requirements for metrics and quotes based on judge feedback.",
+			},
+			usage: { promptTokens: 1500, completionTokens: 300 },
+			// biome-ignore lint/suspicious/noExplicitAny: partial mock of generateObject return type
+		} as any);
+
+		const budget = new BudgetTracker({ daily_limit_usd: 5.0 });
+		const result = await improvePrompt(
+			{
+				current_prompt: "Write a case study.",
+				judge_reasoning: "Draft is generic, lacks specific metrics and client quotes.",
+				judge_score: 45,
+				content_type: "case-study",
+			},
+			mockProvider,
+			budget,
+		);
+
+		expect(result.improved_prompt).toContain("metrics");
+		expect(result.changes_summary).toBeTruthy();
+		expect(result.usage.function_name).toBe("improvePrompt");
+	});
+});

package/src/__tests__/intent-impact.test.ts

@@ -0,0 +1,62 @@
+import { describe, expect, it, vi } from "vitest";
+import { BudgetTracker } from "../budget.js";
+import { analyzeIntentImpact } from "../functions/intent-impact.js";
+import type { ResolvedProvider } from "../provider.js";
+
+vi.mock("ai", () => ({ generateObject: vi.fn() }));
+import { generateObject } from "ai";
+const mockProvider: ResolvedProvider = {
+	provider: "anthropic",
+	model: "claude-sonnet-4-5-20250514",
+};
+
+describe("analyzeIntentImpact", () => {
+	it("identifies pages affected by intent change", async () => {
+		vi.mocked(generateObject).mockResolvedValueOnce({
+			object: {
+				affected_pages: [
+					{ path: "content/cases/acme.mdx", impact: "high", reason: "Uses forbidden word" },
+					{ path: "content/posts/tech-update.mdx", impact: "low", reason: "Minor tone adjustment" },
+				],
+				summary: "2 pages affected",
+			},
+			usage: { promptTokens: 3000, completionTokens: 200 },
+			// biome-ignore lint/suspicious/noExplicitAny: partial mock of generateObject return type
+		} as any);
+		const budget = new BudgetTracker({ daily_limit_usd: 5.0 });
+		const result = await analyzeIntentImpact(
+			{
+				changed_intent: 'Forbidden words: "revolutionerande", "synergier"',
+				previous_intent: 'Forbidden words: "synergier"',
+				content_pages: [
+					{ path: "content/cases/acme.mdx", text: "En revolutionerande migration till Next.js." },
+					{ path: "content/posts/tech-update.mdx", text: "Vi uppdaterade vår tech-stack." },
+				],
+			},
+			mockProvider,
+			budget,
+		);
+		expect(result.affected_pages).toHaveLength(2);
+		expect(result.affected_pages[0].impact).toBe("high");
+		expect(result.usage.function_name).toBe("analyzeIntentImpact");
+	});
+
+	it("returns empty when no pages affected", async () => {
+		vi.mocked(generateObject).mockResolvedValueOnce({
+			object: { affected_pages: [], summary: "No pages affected." },
+			usage: { promptTokens: 2000, completionTokens: 80 },
+			// biome-ignore lint/suspicious/noExplicitAny: partial mock of generateObject return type
+		} as any);
+		const budget = new BudgetTracker({ daily_limit_usd: 5.0 });
+		const result = await analyzeIntentImpact(
+			{
+				changed_intent: "Use we-form.",
+				previous_intent: "Use we-form.",
+				content_pages: [{ path: "content/about.mdx", text: "Vi hjälper företag att växa." }],
+			},
+			mockProvider,
+			budget,
+		);
+		expect(result.affected_pages).toHaveLength(0);
+	});
+});

package/src/__tests__/judge.test.ts

@@ -0,0 +1,78 @@
+import { describe, expect, it, vi } from "vitest";
+import { BudgetTracker } from "../budget.js";
+import { judge } from "../functions/judge.js";
+import type { ResolvedProvider } from "../provider.js";
+
+vi.mock("ai", () => ({ generateObject: vi.fn() }));
+import { generateObject } from "ai";
+const mockProvider: ResolvedProvider = {
+	provider: "anthropic",
+	model: "claude-sonnet-4-5-20250514",
+};
+
+describe("judge", () => {
+	it("judges a draft against gold standard", async () => {
+		vi.mocked(generateObject).mockResolvedValueOnce({
+			object: {
+				score: 82,
+				reasoning: "Draft matches gold standard structure. Tone is correct. Missing one metric.",
+			},
+			usage: { promptTokens: 2000, completionTokens: 200 },
+			// biome-ignore lint/suspicious/noExplicitAny: partial mock of generateObject return type
+		} as any);
+		const budget = new BudgetTracker({ daily_limit_usd: 5.0 });
+		const result = await judge(
+			{
+				draft: "# Acme Case Study\n\nWe helped Acme migrate.",
+				gold_standard:
+					"# Perfect Case Study\n\nWe migrated 12 services in 8 weeks, reducing load time by 40%.",
+				judge_prompt: "Score 0-100.",
+			},
+			mockProvider,
+			budget,
+		);
+		expect(result.score).toBe(82);
+		expect(result.reasoning).toBeTruthy();
+		expect(result.usage.function_name).toBe("judge");
+	});
+
+	it("gives low score for poor match", async () => {
+		vi.mocked(generateObject).mockResolvedValueOnce({
+			object: { score: 35, reasoning: "Draft is generic, lacks specifics, wrong tone." },
+			usage: { promptTokens: 1500, completionTokens: 100 },
+			// biome-ignore lint/suspicious/noExplicitAny: partial mock of generateObject return type
+		} as any);
+		const budget = new BudgetTracker({ daily_limit_usd: 5.0 });
+		const result = await judge(
+			{
+				draft: "We are a great company.",
+				gold_standard: "# Perfect Case Study\n\nDetailed metrics.",
+				judge_prompt: "Score 0-100.",
+			},
+			mockProvider,
+			budget,
+		);
+		expect(result.score).toBeLessThan(50);
+	});
+
+	it("includes intent in judgment when provided", async () => {
+		vi.mocked(generateObject).mockResolvedValueOnce({
+			object: { score: 70, reasoning: "Matches intent tone but not all structural requirements." },
+			usage: { promptTokens: 2500, completionTokens: 180 },
+			// biome-ignore lint/suspicious/noExplicitAny: partial mock of generateObject return type
+		} as any);
+		const budget = new BudgetTracker({ daily_limit_usd: 5.0 });
+		const result = await judge(
+			{
+				draft: "Case study draft.",
+				gold_standard: "Perfect case study.",
+				judge_prompt: "Score 0-100.",
+				intent: 'Always use "we" form. No buzzwords.',
+			},
+			mockProvider,
+			budget,
+		);
+		expect(result.score).toBe(70);
+		expect(result.usage.function_name).toBe("judge");
+	});
+});

package/src/__tests__/score.test.ts

@@ -0,0 +1,61 @@
+import { describe, expect, it, vi } from "vitest";
+import { BudgetTracker } from "../budget.js";
+import { score } from "../functions/score.js";
+import type { ResolvedProvider } from "../provider.js";
+
+vi.mock("ai", () => ({ generateObject: vi.fn() }));
+import { generateObject } from "ai";
+const mockProvider: ResolvedProvider = {
+	provider: "anthropic",
+	model: "claude-sonnet-4-5-20250514",
+};
+
+describe("score", () => {
+	it("scores content against intent", async () => {
+		vi.mocked(generateObject).mockResolvedValueOnce({
+			object: {
+				score: 74,
+				issues: ["Generic heading", "Missing specific results"],
+				strengths: ["Correct tone of voice", "Good structure"],
+			},
+			usage: { promptTokens: 1000, completionTokens: 150 },
+			// biome-ignore lint/suspicious/noExplicitAny: partial mock of generateObject return type
+		} as any);
+		const budget = new BudgetTracker({ daily_limit_usd: 5.0 });
+		const result = await score(
+			{
+				content: "# Acme Corp Case Study\n\nWe helped Acme Corp modernize their tech stack.",
+				intent: "Use concrete results with numbers.",
+			},
+			mockProvider,
+			budget,
+		);
+		expect(result.score).toBe(74);
+		expect(result.issues).toContain("Generic heading");
+		expect(result.strengths.length).toBeGreaterThan(0);
+		expect(result.usage.function_name).toBe("score");
+	});
+
+	it("returns high score for well-matched content", async () => {
+		vi.mocked(generateObject).mockResolvedValueOnce({
+			object: {
+				score: 92,
+				issues: [],
+				strengths: ["Specific metrics", "Perfect tone", "Clear CTA"],
+			},
+			usage: { promptTokens: 800, completionTokens: 100 },
+			// biome-ignore lint/suspicious/noExplicitAny: partial mock of generateObject return type
+		} as any);
+		const budget = new BudgetTracker({ daily_limit_usd: 5.0 });
+		const result = await score(
+			{
+				content: "# Acme Corp: 40% Faster with Next.js\n\nWe migrated 12 services in 8 weeks.",
+				intent: "Use concrete results with numbers.",
+			},
+			mockProvider,
+			budget,
+		);
+		expect(result.score).toBeGreaterThanOrEqual(90);
+		expect(result.issues).toHaveLength(0);
+	});
+});

package/src/__tests__/staleness.test.ts

@@ -0,0 +1,77 @@
+import { describe, expect, it, vi } from "vitest";
+import { BudgetTracker } from "../budget.js";
+import { detectStaleness } from "../functions/staleness.js";
+import type { ResolvedProvider } from "../provider.js";
+
+vi.mock("ai", () => ({ generateObject: vi.fn() }));
+import { generateObject } from "ai";
+const mockProvider: ResolvedProvider = {
+	provider: "anthropic",
+	model: "claude-sonnet-4-5-20250514",
+};
+
+describe("detectStaleness", () => {
+	it("detects stale content when sources have changed", async () => {
+		vi.mocked(generateObject).mockResolvedValueOnce({
+			object: {
+				stale: true,
+				reason: "Source file has new information about project completion.",
+				stale_sources: ["knowledge/clients/acme.md"],
+				confidence: 0.92,
+			},
+			usage: { promptTokens: 3000, completionTokens: 150 },
+			// biome-ignore lint/suspicious/noExplicitAny: partial mock of generateObject return type
+		} as any);
+		const budget = new BudgetTracker({ daily_limit_usd: 5.0 });
+		const result = await detectStaleness(
+			{
+				content: "Acme Corp is currently migrating to Next.js.",
+				content_generated_at: "2026-03-01T10:00:00Z",
+				source_texts: [
+					{
+						path: "knowledge/clients/acme.md",
+						text: "Acme Corp completed their Next.js migration on 2026-03-15.",
+						updated_at: "2026-03-16T10:00:00Z",
+					},
+				],
+			},
+			mockProvider,
+			budget,
+		);
+		expect(result.stale).toBe(true);
+		expect(result.stale_sources).toContain("knowledge/clients/acme.md");
+		expect(result.confidence).toBeGreaterThan(0.8);
+		expect(result.usage.function_name).toBe("detectStaleness");
+	});
+
+	it("reports fresh content when no meaningful changes", async () => {
+		vi.mocked(generateObject).mockResolvedValueOnce({
+			object: {
+				stale: false,
+				reason: "Source changes are minor formatting edits.",
+				stale_sources: [],
+				confidence: 0.88,
+			},
+			usage: { promptTokens: 2500, completionTokens: 100 },
+			// biome-ignore lint/suspicious/noExplicitAny: partial mock of generateObject return type
+		} as any);
+		const budget = new BudgetTracker({ daily_limit_usd: 5.0 });
+		const result = await detectStaleness(
+			{
+				content: "Acme Corp case study content.",
+				content_generated_at: "2026-04-01T10:00:00Z",
+				source_texts: [
+					{
+						path: "knowledge/clients/acme.md",
+						text: "Acme Corp case study notes (reformatted).",
+						updated_at: "2026-04-02T10:00:00Z",
+					},
+				],
+			},
+			mockProvider,
+			budget,
+		);
+		expect(result.stale).toBe(false);
+		expect(result.stale_sources).toHaveLength(0);
+	});
+});

package/src/budget.ts

@@ -0,0 +1,47 @@
+import type { BudgetConfig, BudgetStatus, TokenUsage } from "./types.js";
+
+export class BudgetTracker {
+	private config: BudgetConfig;
+	private usage: TokenUsage[] = [];
+	private dailySpent = 0;
+
+	constructor(config: BudgetConfig) {
+		this.config = config;
+	}
+
+	record(usage: TokenUsage): void {
+		this.usage.push(usage);
+		this.dailySpent += usage.estimated_cost_usd;
+	}
+
+	getStatus(): BudgetStatus {
+		const remaining = Math.max(0, this.config.daily_limit_usd - this.dailySpent);
+		const warnAt = this.config.warn_at_usd ?? this.config.daily_limit_usd * 0.8;
+		const now = new Date();
+		const resetAt = new Date(now);
+		resetAt.setUTCDate(resetAt.getUTCDate() + 1);
+		resetAt.setUTCHours(0, 0, 0, 0);
+		return {
+			spent_today_usd: Math.round(this.dailySpent * 1000) / 1000,
+			limit_usd: this.config.daily_limit_usd,
+			warn_at_usd: warnAt,
+			remaining_usd: Math.round(remaining * 1000) / 1000,
+			is_over_limit: this.dailySpent >= this.config.daily_limit_usd,
+			is_warned: this.dailySpent >= warnAt,
+			reset_at: resetAt.toISOString(),
+		};
+	}
+
+	canSpend(estimated_cost_usd: number): boolean {
+		return this.dailySpent + estimated_cost_usd <= this.config.daily_limit_usd;
+	}
+
+	getHistory(): TokenUsage[] {
+		return [...this.usage];
+	}
+
+	resetDaily(): void {
+		this.usage = [];
+		this.dailySpent = 0;
+	}
+}

package/src/eval/runner.ts

@@ -0,0 +1,163 @@
+import type { BudgetTracker } from "../budget.js";
+import { generate } from "../functions/generate.js";
+import { improvePrompt } from "../functions/improve-prompt.js";
+import { judge } from "../functions/judge.js";
+import type { ResolvedProvider } from "../provider.js";
+import type { EvalResult, EvalRunConfig, EvalRunResult, TokenUsage } from "../types.js";
+
+function mergeUsage(a: TokenUsage, b: TokenUsage): TokenUsage {
+	return {
+		input_tokens: a.input_tokens + b.input_tokens,
+		output_tokens: a.output_tokens + b.output_tokens,
+		estimated_cost_usd: a.estimated_cost_usd + b.estimated_cost_usd,
+		function_name: "eval-run",
+		timestamp: new Date().toISOString(),
+	};
+}
+
+function emptyUsage(): TokenUsage {
+	return {
+		input_tokens: 0,
+		output_tokens: 0,
+		estimated_cost_usd: 0,
+		function_name: "eval-run",
+		timestamp: new Date().toISOString(),
+	};
+}
+
+export class EvalRunner {
+	private provider: ResolvedProvider;
+	private budget: BudgetTracker;
+
+	constructor(provider: ResolvedProvider, budget: BudgetTracker) {
+		this.provider = provider;
+		this.budget = budget;
+	}
+
+	/**
+	 * Run the generate-judge-decide-improve loop.
+	 *
+	 * 1. GENERATE — AI creates content with current prompt
+	 * 2. JUDGE   — Locked judge-prompt scores 0-100
+	 * 3. DECIDE  — Score >= threshold? Keep. Otherwise discard.
+	 * 4. IMPROVE — If discard: agent adjusts generation prompt
+	 * 5. LOG     — Result recorded
+	 * 6. REPEAT  — Until threshold met or max iterations
+	 */
+	async run(config: EvalRunConfig): Promise<EvalRunResult> {
+		const iterations: EvalResult[] = [];
+		let currentPrompt = config.generation_prompt;
+		let totalUsage = emptyUsage();
+		let finalContent: string | undefined;
+		let finalScore = 0;
+		let finalStatus: "keep" | "discard" = "discard";
+		let promptImproved = false;
+
+		for (let i = 0; i < config.max_iterations; i++) {
+			// 1. GENERATE
+			const generated = await generate(
+				{
+					content_type: config.content_type,
+					generation_prompt: currentPrompt,
+					knowledge_context: config.knowledge_context,
+					intent: config.intent,
+					collection_schema: config.collection_schema,
+				},
+				this.provider,
+				this.budget,
+			);
+			totalUsage = mergeUsage(totalUsage, generated.usage);
+
+			// Compose the draft as frontmatter + body for judging
+			const draft = `---\n${Object.entries(generated.frontmatter)
+				.map(([k, v]) => `${k}: ${JSON.stringify(v)}`)
+				.join("\n")}\n---\n\n${generated.content}`;
+
+			// 2. JUDGE
+			const judged = await judge(
+				{
+					draft,
+					gold_standard: config.gold_standard,
+					judge_prompt: config.judge_prompt,
+					intent: config.intent,
+				},
+				this.provider,
+				this.budget,
+			);
+			totalUsage = mergeUsage(totalUsage, judged.usage);
+
+			// 3. DECIDE
+			const status: EvalResult["status"] = judged.score >= config.threshold ? "keep" : "discard";
+
+			const evalResult: EvalResult = {
+				id: `eval-${config.content_type}-${Date.now()}-${i}`,
+				content_type: config.content_type,
+				prompt_version: `v${i + 1}`,
+				score: judged.score,
+				status,
+				reasoning: judged.reasoning,
+				iteration: i + 1,
+				generated_content: draft,
+				timestamp: new Date().toISOString(),
+				total_usage: { ...totalUsage },
+			};
+
+			iterations.push(evalResult);
+			finalScore = judged.score;
+			finalContent = draft;
+
+			if (status === "keep") {
+				finalStatus = "keep";
+				break;
+			}
+
+			// 4. IMPROVE — adjust generation prompt (not on last iteration)
+			if (i < config.max_iterations - 1) {
+				const improved = await improvePrompt(
+					{
+						current_prompt: currentPrompt,
+						judge_reasoning: judged.reasoning,
+						judge_score: judged.score,
+						content_type: config.content_type,
+					},
+					this.provider,
+					this.budget,
+				);
+				totalUsage = mergeUsage(totalUsage, improved.usage);
+				currentPrompt = improved.improved_prompt;
+				promptImproved = true;
+
+				// Mark previous iteration as improved
+				evalResult.status = "improved";
+			}
+		}
+
+		return {
+			iterations,
+			final_score: finalScore,
+			final_status: finalStatus,
+			final_content: finalStatus === "keep" ? finalContent : undefined,
+			final_prompt: currentPrompt,
+			prompt_improved: promptImproved,
+			total_usage: totalUsage,
+		};
+	}
+
+	/**
+	 * Single judge pass — score existing content without the improve loop.
+	 * Used for quality scoring of already-published content.
+	 */
+	async judgeOnly(input: {
+		draft: string;
+		gold_standard: string;
+		judge_prompt: string;
+		intent?: string;
+	}): Promise<{ score: number; reasoning: string; usage: TokenUsage }> {
+		const result = await judge(input, this.provider, this.budget);
+		return {
+			score: result.score,
+			reasoning: result.reasoning,
+			usage: result.usage,
+		};
+	}
+}

package/src/functions/classify.ts

@@ -0,0 +1,54 @@
+import { generateObject } from "ai";
+import { z } from "zod";
+import type { BudgetTracker } from "../budget.js";
+import type { ResolvedProvider } from "../provider.js";
+import type { ClassifyInput, ClassifyResult } from "../types.js";
+import { createModel } from "./model-factory.js";
+import { recordUsage } from "./usage.js";
+
+const classifySchema = z.object({
+	quality: z.enum(["structured", "draft", "thoughts"]),
+	quality_score: z.number().min(1).max(10),
+	type: z.string(),
+	reasoning: z.string(),
+});
+
+export async function classify(
+	input: ClassifyInput,
+	provider: ResolvedProvider,
+	budget: BudgetTracker,
+): Promise<ClassifyResult> {
+	const typeConstraint = input.available_types
+		? `Choose the type from these options: ${input.available_types.join(", ")}`
+		: "Infer the most appropriate type (e.g. project-notes, meeting-notes, transcript, brainstorm, research, documentation)";
+
+	const { object, usage } = await generateObject({
+		model: await createModel(provider),
+		schema: classifySchema,
+		prompt: `You are a knowledge classifier for a content engine. Analyze the following text and classify it.
+
+Quality levels:
+- "structured": Well-organized, has clear sections, headings, or data points. Score 7-10.
+- "draft": Has useful information but needs editing, incomplete. Score 4-6.
+- "thoughts": Raw brainstorm, bullet points, rough ideas. Score 1-3.
+
+${typeConstraint}
+
+Text to classify:
+---
+${input.text}
+---
+
+Respond with quality, quality_score (1-10), type, and reasoning.`,
+	});
+
+	const tokenUsage = recordUsage(budget, provider, usage, "classify");
+
+	return {
+		quality: object.quality,
+		quality_score: object.quality_score,
+		type: object.type,
+		reasoning: object.reasoning,
+		usage: tokenUsage,
+	};
+}

package/src/functions/extract.ts

@@ -0,0 +1,72 @@
+import { generateObject } from "ai";
+import { z } from "zod";
+import type { BudgetTracker } from "../budget.js";
+import type { ResolvedProvider } from "../provider.js";
+import type { ExtractInput, ExtractResult } from "../types.js";
+import { createModel } from "./model-factory.js";
+import { recordUsage } from "./usage.js";
+
+const extractSchema = z.object({
+	entities: z.array(
+		z.object({
+			type: z.string(),
+			name: z.string(),
+			aliases: z.array(z.string()),
+			confidence: z.number().min(0).max(1),
+		}),
+	),
+	relations: z.array(
+		z.object({
+			from_entity: z.string(),
+			to_entity: z.string(),
+			relation_type: z.string(),
+			confidence: z.number().min(0).max(1),
+			evidence: z.string(),
+		}),
+	),
+});
+
+export async function extract(
+	input: ExtractInput,
+	provider: ResolvedProvider,
+	budget: BudgetTracker,
+): Promise<ExtractResult> {
+	const existingContext = input.existing_entities?.length
+		? `\nExisting entities in the knowledge graph (you may create relations to these):\n${input.existing_entities.map((e) => `- ${e.name} (${e.type})`).join("\n")}`
+		: "";
+
+	const { object, usage } = await generateObject({
+		model: await createModel(provider),
+		schema: extractSchema,
+		maxTokens: 16384,
+		prompt: `You are an entity and relation extractor for a knowledge graph. Extract the most important named entities and their relationships from the following text.
+
+Focus on concrete, named entities — not generic concepts. Limit to the 30 most significant entities.
+
+Entity types to look for: client, technology, service, person, organization, concept, project, location, event.
+
+For each entity provide:
+- type: the entity category
+- name: the canonical name
+- aliases: alternative names or abbreviations
+- confidence: 0.0-1.0 how confident you are this is a real entity
+
+For each relation provide:
+- from_entity / to_entity: entity names (must match extracted entities or existing ones)
+- relation_type: e.g. "uses", "built_on", "employs", "located_in", "needs", "provides"
+- confidence: 0.0-1.0
+- evidence: the text snippet supporting this relation
+${existingContext}
+
+Source file: ${input.file_path}
+
+Text to analyze:
+---
+${input.text}
+---`,
+	});
+
+	const tokenUsage = recordUsage(budget, provider, usage, "extract");
+
+	return { entities: object.entities, relations: object.relations, usage: tokenUsage };
+}