@asterxsk/kiln 0.3.0 → 0.4.2

package/agent.bak.20260905-190020/extensions/pi-web-access/source-check.ts

@@ -1,282 +0,0 @@
-// Structured source checking and machine-readable research artifacts.
-import { createHash } from "node:crypto";
-import { generateId, getResult, storeResult } from "./storage.ts";
-import type { SearchResult } from "./search-types.ts";
-import type { ExtractedContent } from "./extract.ts";
-
-export type SourceQuality = "official_docs" | "vendor_docs" | "repo_issue" | "blog" | "forum" | "news" | "unknown";
-export type ClaimStatus = "supported" | "contradicted" | "unclear" | "missing-evidence";
-export type RecencyFilter = "day" | "week" | "month" | "year";
-
-export interface ResearchSource {
-	rank: number;
-	url: string;
-	title: string;
-	snippet?: string;
-	fetch_timestamp?: number;
-	content_hash?: string;
-	quality: SourceQuality;
-	fetched?: boolean;
-	fetch_error?: string;
-}
-
-export interface ResearchPassage {
-	passage_id: string;
-	source_url: string;
-	source_rank: number;
-	text: string;
-	extraction_span?: { start: number; end: number };
-	content_hash?: string;
-}
-
-export interface ClaimAssessment {
-	claim: string;
-	status: ClaimStatus;
-	supporting_passages: string[];
-	contradicting_passages: string[];
-	rationale: string;
-	confidence: number;
-}
-
-export interface ResearchArtifact {
-	id: string;
-	type: "research";
-	timestamp: number;
-	query: string;
-	sources: ResearchSource[];
-	passages: ResearchPassage[];
-	claims?: ClaimAssessment[];
-	provider?: string;
-	summary?: string;
-	content_hash?: string;
-	filters?: { recency?: RecencyFilter; domain_include?: string[]; domain_exclude?: string[] };
-	errors?: Array<{ query: string; error: string }>;
-}
-
-export interface ResearchSearchRequest {
-	query: string;
-	numResults: number;
-	recencyFilter?: RecencyFilter;
-	domainFilter?: string[];
-}
-export interface ResearchSearchResult { url: string; title: string; snippet: string; rank: number }
-export interface ResearchSearchResponse { provider: string; results: ResearchSearchResult[]; summary?: string }
-export interface ResearchProvider { name: string; search(req: ResearchSearchRequest): Promise<ResearchSearchResponse> }
-
-const OFFICIAL_DOCS_HOSTS = /^(developers\.|docs\.|learn\.|reference\.)|\.github\.io$/i;
-const OFFICIAL_DOCS_PATHS = /\/(docs?|reference)(\/|\b)/i;
-const VENDOR_DOCS_PATHS = /\/(documentation|docs?)\//i;
-const REPO_ISSUE_PATHS = /\/(issues|pull|pulls)\//i;
-const BLOG_HOSTS = /(medium\.com|substack\.com|dev\.to|hashnode\.)/i;
-const BLOG_PATHS = /\/blogs?\//i;
-const FORUM_HOSTS = /(stackoverflow\.com|serverfault\.com|superuser\.com|discourse\.|community\.)/i;
-const FORUM_PATHS = /\/(forum|forums|threads)\//i;
-const NEWS_HOSTS = /(reuters\.com|bloomberg\.com|techcrunch\.com|theverge\.com|arstechnica\.com|wired\.com|cnet\.com|zdnet\.com)/i;
-const NEWS_PATHS = /\/news(\/|$)/i;
-
-export function classifySource(url: string): SourceQuality {
-	let host = "";
-	let path = "";
-	try {
-		const parsed = new URL(url);
-		host = parsed.hostname;
-		path = parsed.pathname;
-	} catch {
-		return "unknown";
-	}
-	if (REPO_ISSUE_PATHS.test(path)) return "repo_issue";
-	if (OFFICIAL_DOCS_HOSTS.test(host) || OFFICIAL_DOCS_PATHS.test(path)) return "official_docs";
-	if (VENDOR_DOCS_PATHS.test(path)) return "vendor_docs";
-	if (NEWS_HOSTS.test(host) || NEWS_PATHS.test(path)) return "news";
-	if (FORUM_HOSTS.test(host) || FORUM_PATHS.test(path)) return "forum";
-	if (BLOG_HOSTS.test(host) || BLOG_PATHS.test(path)) return "blog";
-	return "unknown";
-}
-
-export function hashContent(text: string): string {
-	return `sha256:${createHash("sha256").update(text, "utf8").digest("hex")}`;
-}
-
-interface Span { text: string; start: number; end: number }
-
-function tokenize(value: string): string[] {
-	return [...new Set(value.toLowerCase().split(/[^a-z0-9]+/).filter((term) => term.length > 3))];
-}
-
-function extractRelevantSpans(content: string, hint: string): Span[] {
-	const sentences: Span[] = [];
-	const sentencePattern = /[^.!?]+(?:[.!?]+(?=\s|$)|$)/g;
-	for (const match of content.matchAll(sentencePattern)) {
-		const raw = match[0];
-		const text = raw.trim();
-		if (text.length > 0 && text.length <= 400) {
-			const start = (match.index ?? 0) + raw.indexOf(text);
-			sentences.push({ text, start, end: start + text.length });
-		}
-	}
-	const terms = tokenize(hint);
-	if (terms.length === 0) return [];
-	return sentences
-		.map((sentence, index) => ({ sentence, index, score: terms.filter((term) => sentence.text.toLowerCase().includes(term)).length }))
-		.filter((item) => item.score > 0)
-		.sort((a, b) => b.score - a.score || a.index - b.index)
-		.slice(0, 3)
-		.map(({ sentence }) => sentence);
-}
-
-function passageId(sourceRank: number, index: number): string {
-	return `p-${sourceRank}-${index}`;
-}
-
-export function buildPassages(sources: ResearchSource[], fetched: ExtractedContent[] = [], hint = ""): ResearchPassage[] {
-	const passages: ResearchPassage[] = [];
-	const fetchedByUrl = new Map(fetched.map((item) => [item.url, item]));
-	for (const source of sources) {
-		if (source.snippet) {
-			passages.push({
-				passage_id: passageId(source.rank, 0),
-				source_url: source.url,
-				source_rank: source.rank,
-				text: source.snippet,
-				content_hash: hashContent(source.snippet),
-			});
-		}
-		const page = fetchedByUrl.get(source.url);
-		if (page && !page.error && page.content) {
-			const passageHint = source.snippet?.trim() || hint;
-			for (const [index, span] of extractRelevantSpans(page.content, passageHint).entries()) {
-				passages.push({
-					passage_id: passageId(source.rank, index + 1),
-					source_url: source.url,
-					source_rank: source.rank,
-					text: span.text,
-					extraction_span: { start: span.start, end: span.end },
-					content_hash: hashContent(span.text),
-				});
-			}
-		}
-	}
-	return passages;
-}
-
-const CONTRADICTION_MARKERS = ["not true", "false", "incorrect", "debunked", "retracted", "no longer", "never", "denied", "contrary", "misleading"];
-const SUPPORT_MARKERS = ["yes", "true", "correct", "confirmed", "according to", "shows that", "demonstrates", "reported", "verified", "established"];
-
-function containsPhrase(value: string, phrase: string): boolean {
-	const escaped = phrase.trim().toLowerCase().split(/\s+/).map((term) => term.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("\\s+");
-	return new RegExp(`(?:^|[^a-z0-9])${escaped}(?=$|[^a-z0-9])`, "i").test(value);
-}
-
-function markerIsNegated(value: string, marker: string): boolean {
-	const escaped = marker.trim().toLowerCase().split(/\s+/).map((term) => term.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("\\s+");
-	const markerPattern = new RegExp(`(?:^|[^a-z0-9])${escaped}(?=$|[^a-z0-9])`, "i");
-	const match = markerPattern.exec(value);
-	if (!match || match.index === undefined) return false;
-	const matchedMarker = match[0].replace(/^[^a-z0-9]+/i, "");
-	const beforeMarker = value.slice(0, match.index + match[0].length - matchedMarker.length);
-	return /(?:^|[^a-z0-9])(?:not|no|never|without)\s+$/i.test(beforeMarker);
-}
-
-function hasPolarityMarker(value: string, markers: string[], allowNegated = false): boolean {
-	return markers.some((marker) => containsPhrase(value, marker) && (allowNegated || !markerIsNegated(value, marker)));
-}
-
-export function assessClaim(claim: string, passages: ResearchPassage[]): ClaimAssessment {
-	const terms = tokenize(claim);
-	if (terms.length === 0 || passages.length === 0) {
-		return { claim, status: "missing-evidence", supporting_passages: [], contradicting_passages: [], rationale: "No passages available that discuss the claim's terms.", confidence: 0.2 };
-	}
-	const supporting: string[] = [];
-	const contradicting: string[] = [];
-	for (const passage of passages) {
-		const lower = passage.text.toLowerCase();
-		const overlap = terms.filter((term) => containsPhrase(lower, term)).length;
-		if (overlap < Math.max(2, Math.ceil(terms.length / 4))) continue;
-		const contra = hasPolarityMarker(lower, CONTRADICTION_MARKERS);
-		const support = hasPolarityMarker(lower, SUPPORT_MARKERS);
-		if (contra && !support) contradicting.push(passage.passage_id);
-		else if (support && !contra) supporting.push(passage.passage_id);
-	}
-	if (contradicting.length > 0 && supporting.length === 0) {
-		return { claim, status: "contradicted", supporting_passages: [], contradicting_passages: contradicting, rationale: `${contradicting.length} passage(s) contradict the claim; none support it.`, confidence: Math.min(0.85, 0.5 + contradicting.length * 0.1) };
-	}
-	if (supporting.length > 0 && contradicting.length === 0) {
-		return { claim, status: "supported", supporting_passages: supporting, contradicting_passages: [], rationale: `${supporting.length} passage(s) support the claim; none contradict it.`, confidence: Math.min(0.85, 0.5 + supporting.length * 0.1) };
-	}
-	if (supporting.length > 0 || contradicting.length > 0) {
-		return { claim, status: "unclear", supporting_passages: supporting, contradicting_passages: contradicting, rationale: `${supporting.length} supporting and ${contradicting.length} contradicting passage(s); evidence is mixed.`, confidence: 0.4 };
-	}
-	return { claim, status: "unclear", supporting_passages: [], contradicting_passages: [], rationale: "Passages mention the claim's terms but contain no clear support or contradiction markers.", confidence: 0.3 };
-}
-
-interface RankedSearchResult extends SearchResult {
-	rank?: number;
-}
-
-export interface BuildArtifactInput {
-	query: string;
-	provider?: string;
-	summary?: string;
-	results: RankedSearchResult[];
-	fetched?: ExtractedContent[];
-	recency?: RecencyFilter;
-	domainFilter?: string[];
-}
-
-export function buildResearchArtifact(input: BuildArtifactInput): ResearchArtifact {
-	const filters = input.domainFilter ?? [];
-	const fetchedByUrl = new Map((input.fetched ?? []).map((page) => [page.url, page]));
-	const sources: ResearchSource[] = [];
-	const seen = new Set<string>();
-	for (const [index, result] of input.results.entries()) {
-		if (seen.has(result.url)) continue;
-		seen.add(result.url);
-		const page = fetchedByUrl.get(result.url);
-		const fetched = Boolean(page && !page.error);
-		sources.push({
-			rank: result.rank ?? index + 1,
-			url: result.url,
-			title: result.title,
-			snippet: result.snippet,
-			quality: classifySource(result.url),
-			fetched,
-			...(page ? { fetch_timestamp: Date.now() } : {}),
-			...(page && !page.error ? { content_hash: hashContent(page.content) } : {}),
-			...(page?.error ? { fetch_error: page.error } : {}),
-		});
-	}
-	const passages = buildPassages(sources, input.fetched, input.query);
-	const domainInclude = filters.filter((domain) => !domain.startsWith("-"));
-	const domainExclude = filters.filter((domain) => domain.startsWith("-")).map((domain) => domain.slice(1));
-	return {
-		id: generateId(),
-		type: "research",
-		timestamp: Date.now(),
-		query: input.query,
-		sources,
-		passages,
-		...(input.provider !== undefined ? { provider: input.provider } : {}),
-		...(input.summary !== undefined ? { summary: input.summary } : {}),
-		...(passages.length > 0 ? { content_hash: hashContent(passages.map((passage) => passage.text).join("\n")) } : {}),
-		filters: {
-			...(input.recency !== undefined ? { recency: input.recency } : {}),
-			domain_include: domainInclude,
-			domain_exclude: domainExclude,
-		},
-	};
-}
-
-export function withClaimAssessment(artifact: ResearchArtifact, claims: string[]): ResearchArtifact {
-	return { ...artifact, claims: claims.map((claim) => assessClaim(claim, artifact.passages)) };
-}
-
-export function storeResearchArtifact(artifact: ResearchArtifact): void {
-	if (!artifact.id) throw new Error("Research artifact id must not be empty");
-	storeResult(artifact.id, { id: artifact.id, type: "research", timestamp: artifact.timestamp, artifact });
-}
-
-export function getResearchArtifact(id: string): ResearchArtifact | null {
-	const data = getResult(id);
-	if (!data || data.type !== "research" || !data.artifact || typeof data.artifact !== "object") return null;
-	return data.artifact as ResearchArtifact;
-}