@ever-works/valyu-plugin 1.0.0

package/src/valyu.plugin.ts

@@ -0,0 +1,365 @@
+import type {
+	IPlugin,
+	ISearchPlugin,
+	IContentExtractorPlugin,
+	PluginContext,
+	PluginCategory,
+	PluginManifest,
+	PluginHealthCheck,
+	JsonSchema,
+	PluginSettings,
+	SearchOptions,
+	SearchResponse,
+	SearchResult,
+	RateLimitInfo,
+	ContentExtractionOptions,
+	ContentExtractionResult,
+	ConnectionValidationResult
+} from '@ever-works/plugin';
+
+import { Valyu, type SearchOptions as ValyuSearchOptions } from 'valyu-js';
+import { isSafeWebhookUrl } from '@ever-works/plugin/helpers/ssrf-guard';
+
+export class ValyuSearchPlugin implements IPlugin, ISearchPlugin, IContentExtractorPlugin {
+	readonly id = 'valyu';
+	readonly name = 'Valyu';
+	readonly version = '1.0.0';
+	readonly category: PluginCategory = 'search';
+	readonly capabilities: readonly string[] = ['search', 'content-extractor'];
+	readonly providerName = 'Valyu';
+
+	readonly settingsSchema: JsonSchema = {
+		type: 'object',
+		properties: {
+			apiKey: {
+				type: 'string',
+				title: 'API Key',
+				description: 'Your Valyu API key. Get one at https://valyu.ai',
+				'x-secret': true,
+				'x-envVar': 'PLUGIN_VALYU_API_KEY',
+				'x-scope': 'user'
+			},
+			responseLength: {
+				type: 'string',
+				title: 'Response Length',
+				description:
+					'Content volume per result: short (~25k chars), medium (~50k), large (~100k), or max (unlimited)',
+				enum: ['short', 'medium', 'large', 'max'],
+				default: 'medium'
+			}
+		},
+		required: ['apiKey']
+	};
+
+	readonly configurationMode: 'admin-only' | 'user-required' | 'hybrid' = 'hybrid';
+
+	private context?: PluginContext;
+
+	async search(options: SearchOptions): Promise<SearchResponse> {
+		const client = this.getClient(options.settings);
+		const startTime = Date.now();
+
+		const maxNumResults = options.limit || 20;
+		const responseLength = (options.settings?.responseLength as ValyuSearchOptions['responseLength']) || 'medium';
+
+		try {
+			const searchOptions: ValyuSearchOptions = {
+				searchType: 'all',
+				maxNumResults,
+				responseLength
+			};
+
+			if (options.includeDomains && options.includeDomains.length > 0) {
+				searchOptions.includedSources = options.includeDomains as string[];
+			} else if (options.excludeDomains && options.excludeDomains.length > 0) {
+				searchOptions.excludeSources = options.excludeDomains as string[];
+			}
+
+			if (options.region) {
+				searchOptions.countryCode = options.region.toUpperCase() as ValyuSearchOptions['countryCode'];
+			}
+
+			if (options.timeRange && options.timeRange !== 'all') {
+				const now = new Date();
+				const start = new Date(now);
+				if (options.timeRange === 'day') start.setDate(now.getDate() - 1);
+				else if (options.timeRange === 'week') start.setDate(now.getDate() - 7);
+				else if (options.timeRange === 'month') start.setMonth(now.getMonth() - 1);
+				else if (options.timeRange === 'year') start.setFullYear(now.getFullYear() - 1);
+				searchOptions.startDate = start.toISOString().split('T')[0];
+				searchOptions.endDate = now.toISOString().split('T')[0];
+			}
+
+			const response = await client.search(options.query, searchOptions);
+
+			const results: SearchResult[] = (response.results || []).map((r, index) => ({
+				title: r.title,
+				url: r.url,
+				snippet: typeof r.content === 'string' ? r.content : String(r.content),
+				position: index + 1,
+				publishedDate: r.publication_date,
+				metadata: {
+					relevanceScore: r.relevance_score,
+					source: r.source,
+					description: r.description
+				}
+			}));
+
+			return {
+				results,
+				query: options.query,
+				totalResults: results.length,
+				hasMore: false,
+				duration: Date.now() - startTime
+			};
+		} catch (error) {
+			this.context?.logger.error(
+				`Valyu search failed: ${error instanceof Error ? error.message : String(error)}`
+			);
+			throw error;
+		}
+	}
+
+	async isAvailable(): Promise<boolean> {
+		if (!this.context) return false;
+		const settings = await this.context.getSettings();
+		return Boolean(settings?.apiKey);
+	}
+
+	async validateConnection(settings: Record<string, unknown>): Promise<ConnectionValidationResult> {
+		const apiKey = settings.apiKey as string | undefined;
+		if (!apiKey) {
+			return { success: false, message: 'Valyu API key is not configured.' };
+		}
+
+		try {
+			const client = this.getClient(settings);
+			await client.search('test', { maxNumResults: 1, searchType: 'all' });
+			return { success: true, message: 'Valyu connection verified.' };
+		} catch (error) {
+			return {
+				success: false,
+				message: `Valyu connection failed: ${error instanceof Error ? error.message : String(error)}`
+			};
+		}
+	}
+
+	async getRateLimitInfo(): Promise<RateLimitInfo> {
+		return {
+			remaining: -1,
+			limit: -1,
+			period: 'month'
+		};
+	}
+
+	async extract(options: ContentExtractionOptions): Promise<ContentExtractionResult> {
+		const client = this.getClient(options.settings);
+		const startTime = Date.now();
+
+		// Security (SSRF): refuse to hand private/loopback/link-local/cloud-metadata
+		// URLs to the Valyu SDK, which would fetch them server-side and return the
+		// rendered content. `canExtract()` is advisory and may be bypassed by a
+		// direct call, so enforce the guard here. The Valyu SDK performs the fetch
+		// internally, so redirect targets can't be re-checked — this is the lexical
+		// first line of defense for the initial URL.
+		if (!isSafeWebhookUrl(options.url)) {
+			return {
+				success: false,
+				url: options.url,
+				error: `URL host is not safe to fetch (SSRF guard blocked: ${options.url})`,
+				duration: Date.now() - startTime
+			};
+		}
+
+		try {
+			const response = await client.contents([options.url]);
+
+			if (!response.results || response.results.length === 0) {
+				return {
+					success: false,
+					url: options.url,
+					error: 'No content extracted',
+					duration: Date.now() - startTime
+				};
+			}
+
+			const result = response.results[0];
+			const content = typeof result.content === 'string' ? result.content : String(result.content || '');
+
+			return {
+				success: true,
+				url: options.url,
+				finalUrl: result.url !== options.url ? result.url : undefined,
+				title: result.title,
+				content,
+				markdown: content,
+				duration: Date.now() - startTime,
+				wordCount: content ? content.split(/\s+/).length : 0
+			};
+		} catch (error) {
+			return {
+				success: false,
+				url: options.url,
+				error: error instanceof Error ? error.message : String(error),
+				duration: Date.now() - startTime
+			};
+		}
+	}
+
+	async extractBatch(
+		urls: readonly string[],
+		options?: Partial<ContentExtractionOptions>
+	): Promise<readonly ContentExtractionResult[]> {
+		const client = this.getClient(options?.settings);
+		const startTime = Date.now();
+
+		// Security (SSRF): drop private/loopback/link-local/cloud-metadata URLs
+		// before they reach the Valyu SDK (which would fetch them server-side),
+		// emitting a failed result for each. This also stops a single batch call
+		// from being used to probe many internal hosts/ports at once. Legitimate
+		// public URLs are unaffected.
+		const safeUrls: string[] = [];
+		const blockedResults: ContentExtractionResult[] = [];
+		for (const url of urls) {
+			if (isSafeWebhookUrl(url)) {
+				safeUrls.push(url);
+			} else {
+				blockedResults.push({
+					success: false,
+					url,
+					error: `URL host is not safe to fetch (SSRF guard blocked: ${url})`,
+					duration: Date.now() - startTime
+				});
+			}
+		}
+
+		try {
+			const batchSize = 10;
+			const allResults: ContentExtractionResult[] = [];
+
+			for (let i = 0; i < safeUrls.length; i += batchSize) {
+				const batch = safeUrls.slice(i, i + batchSize);
+				const response = await client.contents(batch as string[]);
+
+				const results = (response.results || []).map((result, index) => {
+					const content = typeof result.content === 'string' ? result.content : String(result.content || '');
+					const requestedUrl = batch[index] || result.url;
+					return {
+						success: true,
+						url: requestedUrl,
+						finalUrl: result.url !== requestedUrl ? result.url : undefined,
+						title: result.title,
+						content,
+						markdown: content,
+						duration: Date.now() - startTime,
+						wordCount: content ? content.split(/\s+/).length : 0
+					};
+				});
+
+				allResults.push(...results);
+			}
+
+			return [...blockedResults, ...allResults];
+		} catch (error) {
+			// Only the safe URLs were sent to the SDK, so surface the fetch error
+			// for those; blocked URLs already carry their SSRF-guard reason.
+			const failed = safeUrls.map((url) => ({
+				success: false,
+				url,
+				error: error instanceof Error ? error.message : String(error),
+				duration: Date.now() - startTime
+			}));
+			return [...blockedResults, ...failed];
+		}
+	}
+
+	async canExtract(url: string): Promise<boolean> {
+		// Security (SSRF): `isSafeWebhookUrl` enforces http(s) AND rejects
+		// private/loopback/link-local/cloud-metadata hosts, so callers that gate
+		// on canExtract() won't even surface an internal URL as extractable.
+		// (extract()/extractBatch() re-check independently in case this is bypassed.)
+		return isSafeWebhookUrl(url);
+	}
+
+	getSupportedFormats(): readonly ('text' | 'html' | 'markdown')[] {
+		return ['text', 'markdown'];
+	}
+
+	private getClient(settings?: PluginSettings): Valyu {
+		const apiKey = settings?.apiKey as string;
+
+		if (!apiKey) {
+			throw new Error(
+				'Valyu API key not configured. ' +
+					'Set it in plugin settings or via PLUGIN_VALYU_API_KEY environment variable.'
+			);
+		}
+
+		return new Valyu(apiKey);
+	}
+
+	async onLoad(context: PluginContext): Promise<void> {
+		this.context = context;
+		context.logger.log('Valyu Plugin loaded');
+	}
+
+	async onUnload(): Promise<void> {
+		this.context = undefined;
+	}
+
+	async healthCheck(): Promise<PluginHealthCheck> {
+		return {
+			status: 'healthy',
+			message: 'Valyu plugin is ready (API key required for operations)',
+			checkedAt: Date.now()
+		};
+	}
+
+	getManifest(): PluginManifest {
+		return {
+			id: this.id,
+			name: this.name,
+			version: this.version,
+			description: 'AI-native search and content extraction across web and proprietary data sources',
+			category: this.category,
+			capabilities: [...this.capabilities],
+			author: { name: 'Ever Works Team' },
+			license: 'AGPL-3.0',
+			builtIn: true,
+			systemPlugin: false,
+			autoEnable: false,
+			homepage: 'https://valyu.ai',
+			icon: {
+				type: 'svg',
+				value: `<svg xmlns="http://www.w3.org/2000/svg" width="100" height="100" viewBox="0 0 100 100" preserveAspectRatio="xMidYMid meet"><g fill="#1e1d1d" fill-opacity="1"><g transform="translate(-3, 88)"><path d="M 48.3125 1.734375 L 59.78125 1.734375 L 91.5625 -71.453125 L 91.5625 -77.5 L 16.328125 -77.5 L 16.328125 -71.453125 Z M 64.421875 -32.640625 L 54.15625 -7.140625 L 53.609375 -7.140625 L 43.234375 -32.640625 L 27.78125 -68.96875 L 79.984375 -68.96875 Z M 64.421875 -32.640625 "/></g></g></svg>`,
+				darkValue: `<svg xmlns="http://www.w3.org/2000/svg" width="100" height="100" viewBox="0 0 100 100" preserveAspectRatio="xMidYMid meet"><g fill="#ebebeb" fill-opacity="1"><g transform="translate(-3, 88)"><path d="M 48.3125 1.734375 L 59.78125 1.734375 L 91.5625 -71.453125 L 91.5625 -77.5 L 16.328125 -77.5 L 16.328125 -71.453125 Z M 64.421875 -32.640625 L 54.15625 -7.140625 L 53.609375 -7.140625 L 43.234375 -32.640625 L 27.78125 -68.96875 L 79.984375 -68.96875 Z M 64.421875 -32.640625 "/></g></g></svg>`
+			},
+			readme: [
+				'## What does Valyu do?',
+				'',
+				'Valyu is an AI-native search and content extraction service that searches across web and proprietary data sources. It returns results optimized for AI applications and RAG pipelines, providing the source material that Ever Works uses to generate accurate work items.',
+				'',
+				'## Why use it?',
+				'',
+				'- **Multi-source search** — search across web, proprietary datasets (arXiv, PubMed, financial data), and news',
+				'- **AI-optimized results** — returns content formatted for AI processing with relevance scoring',
+				'- **Content extraction** — pulls clean text and markdown from web pages',
+				'- **Domain filtering** — include or exclude specific websites from search results',
+				'- **Date filtering** — restrict results to specific time periods',
+				'',
+				'## How it works in Ever Works',
+				'',
+				'During work generation, the search facade uses Valyu to find information about each item, discover relevant source URLs, and extract content from web pages. This powers automatic descriptions, source URL resolution, and content enrichment across the generation pipeline.',
+				'',
+				'## Getting started',
+				'',
+				'1. Create an account at [valyu.ai](https://valyu.ai)',
+				'2. Copy your API key from the Valyu dashboard',
+				'3. Enter the key in the **API Key** field below',
+				'4. Valyu will be used during work generation when selected as the search provider'
+			].join('\n')
+		};
+	}
+}
+
+export default ValyuSearchPlugin;

package/tsconfig.json

@@ -0,0 +1,22 @@
+{
+	"compilerOptions": {
+		"target": "ES2022",
+		"types": ["node"],
+		"module": "NodeNext",
+		"moduleResolution": "NodeNext",
+		"declaration": true,
+		"declarationMap": true,
+		"sourceMap": true,
+		"outDir": "./dist",
+		"rootDir": "./src",
+		"strict": true,
+		"esModuleInterop": true,
+		"skipLibCheck": true,
+		"forceConsistentCasingInFileNames": true,
+		"resolveJsonModule": true,
+		"isolatedModules": true,
+		"noEmit": true
+	},
+	"include": ["src/**/*"],
+	"exclude": ["node_modules", "dist", "**/*.test.ts", "**/*.spec.ts"]
+}

package/tsup.config.ts

@@ -0,0 +1,12 @@
+import { defineConfig } from 'tsup';
+
+export default defineConfig({
+	entry: ['src/index.ts'],
+	noExternal: ['@ever-works/plugin'],
+	format: ['cjs', 'esm'],
+	dts: true,
+	clean: true,
+	sourcemap: false,
+	splitting: false,
+	treeshake: true
+});