expandai 0.0.4 → 0.0.6

package/src/Mcp/Toolkit.ts

@@ -1,21 +0,0 @@
-import { McpServer, Toolkit } from '@effect/ai'
-import { Effect, Layer } from 'effect'
-import { ApiClient } from '../ApiClient.js'
-import { Fetch, FetchSearchTool, FetchTool } from './tools/Fetch.js'
-
-const toolkit = Toolkit.make(FetchTool, FetchSearchTool)
-
-const ToolkitLayer = toolkit
-	.toLayer(
-		Effect.gen(function* () {
-			const fetchService = yield* Fetch
-
-			return toolkit.of({
-				fetch: fetchService.fetch,
-				fetch_search: fetchService.fetchSearch,
-			})
-		}),
-	)
-	.pipe(Layer.provide(Fetch.Default), Layer.provide(ApiClient.Default))
-
-export const ExpandToolKit = McpServer.toolkit(toolkit).pipe(Layer.provide(ToolkitLayer))

package/src/Mcp/tools/Citations.ts

@@ -1,219 +0,0 @@
-// MCP-facing result wrappers that add a ready-made `citationUrl` to every search snippet, so the
-// agent can hand the user an auditable playground link without assembling URLs itself. Wraps the
-// SDK's `json`-recovering decode types (`@expandai/sdk/FetchJson`) and reuses generated leaf
-// schemas; `Generated.ts` is left untouched.
-
-import type { FetchObjectModeResultWithJson, FetchSnapshotSearchResultWithJson } from '@expandai/sdk/FetchJson'
-import * as Generated from '@expandai/sdk/Generated'
-import { citationUrl, playgroundBase, resolvePlaygroundHost } from '@expandai/sdk/Playground'
-import { Schema as S } from 'effect'
-
-export const FetchInvocationPath = S.Literal('cli-recovery', 'mcp-fetch-search')
-export type FetchInvocationPath = typeof FetchInvocationPath.Type
-
-export class CitedSnippet extends S.Class<CitedSnippet>('CitedSnippet')({
-	type: S.optionalWith(S.Literal('text'), { nullable: true, default: () => 'text' as const }),
-	source: S.Literal('markdown', 'appendix', 'statejson'),
-	text: S.String,
-	json: S.optionalWith(S.Unknown, { nullable: true }),
-	score: S.Number.pipe(S.greaterThanOrEqualTo(0), S.lessThanOrEqualTo(1)),
-	location: S.optionalWith(Generated.FetchSearchSnippetLocation, { nullable: true }),
-	citationUrl: S.String.annotations({
-		description: 'Public playground link auditing this exact snippet. Return it to the user verbatim.',
-	}),
-}) {}
-
-/**
- * Carried in the tool result itself, not only in `SKILL.md`. A harness may not load
- * skills at all, and the hook's snapshot note never reaches this path — so without
- * this the model receives perfect `citationUrl`s and no instruction to use them, and
- * answers with no citation. Observed on OpenCode before this was added.
- */
-export const CITATION_INSTRUCTIONS =
-	'Cite every claim you take from this result. Append the ready-made `citationBlock` from this ' +
-	'result to the end of your answer. Delete every entry whose marker you did not actually place ' +
-	'in your text, and renumber what remains from [1] — an entry with no matching marker is not a ' +
-	'citation, it is clutter. Do not ' +
-	'retype the URLs, do not substitute the page you fetched, and do not build a link yourself — ' +
-	"the citation points at expand.ai's stored capture, not at the original site. Never invent an " +
-	'evidence id and never reuse one from different text. A link without `?id=` is not a citation.'
-
-const CitationInstructionsField = S.optionalWith(S.String, { default: () => CITATION_INSTRUCTIONS }).annotations({
-	description: 'How to cite this result back to the user. Follow it exactly.',
-})
-
-const CitationBlockField = S.optionalWith(S.String, { default: () => '' }).annotations({
-	description:
-		'Ready-to-append citation block, one entry per citable snippet. Copy it to the end of your ' +
-		'answer, keeping only the entries whose markers you placed in your text and renumbering ' +
-		'from [1]. You may also correct the page title.',
-})
-
-/**
- * Prebuilds the citation markdown instead of asking the model to assemble it.
- * Selecting `citationUrl` out of a JSON result and pasting it correctly is the step
- * weaker models get wrong — observed repeatedly on OpenCode, which substituted the
- * fetched page URL for the capture link. Copying one block has no such failure mode.
- */
-const MAX_CITATION_MARKERS = 3
-const EVIDENCE_MARKER = /\{(\d{1,6})\}/gu
-
-/**
- * Citation URLs for a plain fetch with no search. The markdown still stamps evidence
- * ids, so a citation is always possible — without this a `fetch` called without a
- * query produced an empty block and the model answered uncited.
- */
-function markdownCitationUrls(markdown: string, snapshotId: string, host: string): string[] {
-	const urls: string[] = []
-	for (const match of markdown.matchAll(EVIDENCE_MARKER)) {
-		const id = Number(match[1])
-		if (!Number.isFinite(id)) continue
-		const url = citationUrl(snapshotId, { evidenceId: id }, host)
-		if (!urls.includes(url)) urls.push(url)
-		if (urls.length >= MAX_CITATION_MARKERS) break
-	}
-	return urls
-}
-
-/**
- * `statejson` snippets are extracted page state — arrays of routes, config blobs — not
- * prose anyone can quote. Offering one as a citable source produced a `Sources` entry
- * pointing at a list of URLs, so only quotable text earns a citation.
- */
-function isQuotable(snippet: CitedSnippet): boolean {
-	return snippet.source !== 'statejson'
-}
-
-function citationBlock(sourceLabel: string, snippets: ReadonlyArray<CitedSnippet>, fallback: string[] = []): string {
-	const urls: string[] = []
-	for (const snippet of snippets) {
-		if (!isQuotable(snippet)) continue
-		if (snippet.citationUrl.length > 0 && !urls.includes(snippet.citationUrl)) urls.push(snippet.citationUrl)
-	}
-	if (urls.length === 0) urls.push(...fallback)
-	if (urls.length === 0) return ''
-	// One linked source line per marker.
-	// Grouping all the sources first and all the links after loses every link when a
-	// model copies only the opening lines — observed on OpenCode.
-	const lines = ['**Sources**']
-	urls.slice(0, MAX_CITATION_MARKERS).forEach((url, index) => {
-		if (index > 0) lines.push('')
-		lines.push(`[${index + 1}] [${sourceLabel}](${url})`)
-	})
-	return lines.join('\n')
-}
-
-export class CitedSearchResult extends S.Class<CitedSearchResult>('CitedSearchResult')({
-	query: S.String,
-	snippets: S.Array(CitedSnippet),
-	meta: S.optionalWith(Generated.FetchSearchMeta, { nullable: true }),
-}) {}
-
-export class CitedObjectModeResult extends S.Class<CitedObjectModeResult>('CitedObjectModeResult')({
-	invocationPath: S.Literal('mcp-fetch-search'),
-	citationInstructions: CitationInstructionsField,
-	citationBlock: CitationBlockField,
-	meta: Generated.FetchObjectModeMeta,
-	markdown: S.String,
-	json: S.Array(Generated.FetchObjectModeEvidence),
-	data: S.optionalWith(
-		S.Struct({
-			assets: S.optionalWith(S.Array(Generated.FetchAssetReference), { nullable: true }),
-			search: S.optionalWith(CitedSearchResult, { nullable: true }),
-		}),
-		{ nullable: true },
-	),
-}) {}
-
-export class CitedSnapshotSearchResult extends S.Class<CitedSnapshotSearchResult>('CitedSnapshotSearchResult')({
-	invocationPath: FetchInvocationPath,
-	citationInstructions: CitationInstructionsField,
-	citationBlock: CitationBlockField,
-	snapshotId: S.String.pipe(S.minLength(1)),
-	response: Generated.ResponseInfo,
-	search: CitedSearchResult,
-	durationMs: S.Number.pipe(S.greaterThanOrEqualTo(0)),
-	playground: S.String.annotations({ description: 'Public playground link for the whole snapshot.' }),
-}) {}
-
-type SearchResultWithJson = NonNullable<NonNullable<FetchObjectModeResultWithJson['data']>['search']>
-type SnippetWithJson = SearchResultWithJson['snippets'][number]
-
-function toCitedSnippet(snapshotId: string, host: string, snippet: SnippetWithJson): CitedSnippet {
-	return new CitedSnippet({
-		type: snippet.type,
-		source: snippet.source,
-		text: snippet.text,
-		json: snippet.json,
-		score: snippet.score,
-		location: snippet.location,
-		citationUrl: citationUrl(snapshotId, snippet.location, host),
-	})
-}
-
-function toCitedSearch(snapshotId: string, host: string, search: SearchResultWithJson): CitedSearchResult {
-	return new CitedSearchResult({
-		query: search.query,
-		snippets: search.snippets.map((snippet) => toCitedSnippet(snapshotId, host, snippet)),
-		meta: search.meta,
-	})
-}
-
-/** Add `citationUrl` to each snippet of an object-mode fetch result. */
-export function enrichObjectModeResult(
-	result: FetchObjectModeResultWithJson,
-	fallbackHost: string,
-): CitedObjectModeResult {
-	// `meta.playground` is server-resolved (staging-safe); fall back to the configured host only if
-	// it is somehow not a parseable URL.
-	const host = resolvePlaygroundHost(result.meta.playground, fallbackHost)
-	const data = result.data
-	const citedSearch = data?.search == null ? undefined : toCitedSearch(result.meta.snapshotId, host, data.search)
-	const block = citationBlock(
-		result.meta.url,
-		citedSearch?.snippets ?? [],
-		markdownCitationUrls(result.markdown, result.meta.snapshotId, host),
-	)
-	return new CitedObjectModeResult({
-		invocationPath: 'mcp-fetch-search',
-		citationInstructions: CITATION_INSTRUCTIONS,
-		citationBlock: block,
-		meta: result.meta,
-		// The block is appended to the content, not just offered as a sibling field.
-		// A model summarising `markdown` carries an inline footer through; a separate
-		// JSON field it has to notice and copy gets dropped — repeatedly, on OpenCode.
-		markdown: block.length === 0 ? result.markdown : `${result.markdown.trimEnd()}\n\n---\n\n${block}\n`,
-		json: result.json,
-		data:
-			data == null
-				? data
-				: {
-						assets: data.assets,
-						search: citedSearch,
-					},
-	})
-}
-
-/** Add `citationUrl` to each snippet of a snapshot-search result. */
-export function enrichSnapshotSearchResult(
-	result: FetchSnapshotSearchResultWithJson,
-	fallbackHost: string,
-	invocationPath: FetchInvocationPath = 'mcp-fetch-search',
-): CitedSnapshotSearchResult {
-	// `playground` is server-resolved (staging-safe), exactly like `meta.playground`
-	// above; fall back to the configured host only if it is somehow not a parseable
-	// URL. The playground is a separate deployment from the API, so the server is
-	// the only component that can say where its snapshots are publicly served.
-	const host = resolvePlaygroundHost(result.playground, fallbackHost)
-	const citedSearch = toCitedSearch(result.snapshotId, host, result.search)
-	return new CitedSnapshotSearchResult({
-		invocationPath,
-		citationInstructions: CITATION_INSTRUCTIONS,
-		citationBlock: citationBlock(result.response.url, citedSearch.snippets),
-		snapshotId: result.snapshotId,
-		response: result.response,
-		search: citedSearch,
-		durationMs: result.durationMs,
-		playground: playgroundBase(result.snapshotId, host),
-	})
-}

package/src/Mcp/tools/Fetch.ts

@@ -1,203 +0,0 @@
-import { Tool } from '@effect/ai'
-import { Effect, Schema, String } from 'effect'
-import { ApiClient } from '../../ApiClient.js'
-import { ConfigManager } from '../../Config.js'
-import { Telemetry } from '../../Telemetry.js'
-import {
-	CitedObjectModeResult,
-	CitedSnapshotSearchResult,
-	enrichObjectModeResult,
-	enrichSnapshotSearchResult,
-} from './Citations.js'
-
-export class FetchSearchParams extends Schema.Class<FetchSearchParams>('FetchSearchParams')({
-	query: Schema.NonEmptyTrimmedString.annotations({
-		description: 'Search query. When provided, fetch returns search results only.',
-	}),
-	maxResults: Schema.optionalWith(Schema.Int.pipe(Schema.between(1, 50)), { default: () => 5 }).annotations({
-		description: 'Maximum snippets to return.',
-		default: 5,
-	}),
-	minScore: Schema.optionalWith(Schema.Number.pipe(Schema.between(0, 1)), { default: () => 0 }).annotations({
-		description: 'Minimum snippet relevance score from 0 to 1.',
-		default: 0,
-	}),
-}) {}
-
-export class FetchParams extends Schema.Class<FetchParams>('FetchParams')({
-	url: Schema.String.annotations({
-		description: 'The URL to fetch content from',
-	}),
-	search: Schema.optional(FetchSearchParams).annotations({
-		description: 'Semantic search configuration. When provided, the API returns snippets under data.search.snippets.',
-	}),
-	includeMeta: Schema.optionalWith(Schema.Boolean, { default: () => false }).annotations({
-		description: 'Include page meta tags (title, description, Open Graph)',
-		default: false,
-	}),
-	includeAppendix: Schema.optionalWith(Schema.Boolean, { default: () => false }).annotations({
-		description: 'Include extracted links and sidebar content',
-		default: false,
-	}),
-	includeJson: Schema.optionalWith(Schema.Boolean, { default: () => true }).annotations({
-		description: 'Include JSON extracted from network responses and DOM',
-		default: true,
-	}),
-}) {}
-
-export class FetchSnapshotSearchParams extends Schema.Class<FetchSnapshotSearchParams>('FetchSnapshotSearchParams')({
-	snapshotId: Schema.NonEmptyString.annotations({
-		description: 'Snapshot ID returned by a previous fetch response.',
-	}),
-	search: FetchSearchParams.annotations({
-		description: 'Semantic search configuration. Returns snippets from stored fetch artifacts.',
-	}),
-	includeMarkdown: Schema.optionalWith(Schema.Boolean, { default: () => true }).annotations({
-		description: 'Search the stored markdown corpus.',
-		default: true,
-	}),
-	includeAppendix: Schema.optionalWith(Schema.Boolean, { default: () => false }).annotations({
-		description: 'Search extracted appendix content.',
-		default: false,
-	}),
-	includeJson: Schema.optionalWith(Schema.Boolean, { default: () => true }).annotations({
-		description: 'Search stored State JSON.',
-		default: true,
-	}),
-}) {}
-
-export const FetchTool = Tool.make('fetch', {
-	description: String.stripMargin(`
-		|Fetch and extract content from any URL.
-		|
-		|You MUST cite what you take from the result. The result carries a ready-made
-		|'citationBlock' — append it verbatim to the end of your answer. Do not retype it, do not
-		|substitute the URL you fetched, and do not build a link yourself: the citation points at
-		|expand.ai's stored capture, not at the original site. Drop only the markers you did not use.
-		|A 'Verify it' link without '?id=' is not a citation. Never invent or reuse an evidence id.
-		|
-		|Returns object-mode output with markdown, front-matter metadata, and optional extracted JSON.
-		|When 'search' is set, the response also carries ranked snippets under data.search.snippets.
-		|For State JSON sources each snippet also carries the extracted 'json' value.`),
-	parameters: FetchParams.fields,
-	success: CitedObjectModeResult,
-	failure: Schema.Defect,
-	failureMode: 'return',
-})
-	.annotate(Tool.Readonly, true)
-	.annotate(Tool.Destructive, false)
-
-export const FetchSearchTool = Tool.make('fetch_search', {
-	description: String.stripMargin(`
-		|Search a previous fetch by snapshotId without starting a new capture.
-		|
-		|You MUST cite what you take from the result. The result carries a ready-made
-		|'citationBlock' — append it verbatim to the end of your answer. Do not retype it, do not
-		|substitute the URL you fetched, and do not build a link yourself: the citation points at
-		|expand.ai's stored capture, not at the original site. Drop only the markers you did not use.
-		|A 'Verify it' link without '?id=' is not a citation. Never invent or reuse an evidence id.
-		|
-		|Returns ranked snippets from stored markdown, appendix, or State JSON artifacts. For State JSON
-		|sources each snippet also carries the extracted 'json' value. The result also includes a
-		|whole-snapshot 'playground' link.`),
-	parameters: FetchSnapshotSearchParams.fields,
-	success: CitedSnapshotSearchResult,
-	failure: Schema.Defect,
-	failureMode: 'return',
-})
-	.annotate(Tool.Readonly, true)
-	.annotate(Tool.Destructive, false)
-
-export interface FetchService {
-	readonly fetch: (params: FetchParams) => Effect.Effect<CitedObjectModeResult, unknown>
-	readonly fetchSearch: (params: FetchSnapshotSearchParams) => Effect.Effect<CitedSnapshotSearchResult, unknown>
-}
-
-export class Fetch extends Effect.Service<Fetch>()('@expandai/cli/Mcp/Fetch', {
-	dependencies: [ApiClient.Default, ConfigManager.Default, Telemetry.Default],
-	scoped: Effect.gen(function* () {
-		const client = yield* ApiClient
-		const configManager = yield* ConfigManager
-		const telemetry = yield* Telemetry
-		const playgroundHost = yield* configManager.getPlaygroundHost()
-
-		const fetch: FetchService['fetch'] = Effect.fn('Fetch.fetch')(function* ({
-			url,
-			search,
-			includeMeta,
-			includeAppendix,
-			includeJson,
-		}: FetchParams) {
-			yield* Effect.logDebug('MCP fetch started').pipe(
-				Effect.annotateLogs({
-					operation: 'fetch',
-					hasSearch: search !== undefined,
-					includeMeta,
-					includeAppendix,
-					includeJson,
-				}),
-			)
-			yield* telemetry.capture('mcp_tool_called', { tool: 'fetch', has_search: search !== undefined })
-
-			const includeParam =
-				includeAppendix && includeJson
-					? 'appendix,statejson'
-					: includeAppendix
-						? 'appendix'
-						: includeJson
-							? undefined
-							: ''
-			const result = yield* client.fetchFetchJson({
-				params: { include: includeParam },
-				payload: {
-					url,
-					include: {
-						markdown: true,
-						meta: includeMeta,
-						json: includeJson,
-						appendix: includeAppendix,
-					},
-					search,
-				},
-			})
-
-			yield* Effect.logDebug('MCP fetch completed').pipe(
-				Effect.annotateLogs({
-					operation: 'fetch',
-					hasSearch: search !== undefined,
-					searchResultCount: result.data?.search?.snippets.length ?? 0,
-					jsonEvidenceCount: result.json.length,
-				}),
-			)
-
-			return enrichObjectModeResult(result, playgroundHost)
-		})
-
-		const fetchSearch: FetchService['fetchSearch'] = Effect.fn('Fetch.fetchSearch')(function* ({
-			snapshotId,
-			search,
-			includeMarkdown,
-			includeAppendix,
-			includeJson,
-		}: FetchSnapshotSearchParams) {
-			yield* Effect.logDebug('MCP snapshot search started').pipe(
-				Effect.annotateLogs({ operation: 'fetch_search', includeMarkdown, includeAppendix, includeJson }),
-			)
-			yield* telemetry.capture('mcp_tool_called', { tool: 'fetch_search', has_search: true })
-
-			const result = yield* client.fetchFetchSearch({
-				snapshotId,
-				search,
-				include: {
-					markdown: includeMarkdown,
-					json: includeJson,
-					appendix: includeAppendix,
-				},
-			})
-
-			return enrichSnapshotSearchResult(result, playgroundHost, 'mcp-fetch-search')
-		})
-
-		return { fetch, fetchSearch } satisfies FetchService
-	}),
-}) {}

package/src/SecureFile.ts

@@ -1,70 +0,0 @@
-import { FileSystem, Error as PlatformError } from '@effect/platform'
-import { Effect, Exit, Option } from 'effect'
-
-export interface CreatedFileIdentity {
-	readonly dev: number
-	readonly ino?: number
-	readonly mode: number
-	readonly type: FileSystem.File.Type
-}
-
-export interface ExclusiveFileCreation {
-	readonly identity: CreatedFileIdentity
-	readonly writeExit: Exit.Exit<void, PlatformError.PlatformError>
-	readonly pathMatchesHandle: boolean
-}
-
-/** Creates a file exclusively and binds its identity to the opened handle before writing bytes. */
-export function createExclusiveFile(
-	fileSystem: FileSystem.FileSystem,
-	targetPath: string,
-	contents: string,
-	mode: number,
-): Effect.Effect<ExclusiveFileCreation, PlatformError.PlatformError> {
-	return Effect.scoped(
-		Effect.gen(function* () {
-			const file = yield* fileSystem.open(targetPath, { flag: 'wx', mode })
-			const info = yield* file.stat
-			const identity: CreatedFileIdentity = {
-				dev: info.dev,
-				ino: Option.getOrUndefined(info.ino),
-				mode: info.mode,
-				type: info.type,
-			}
-			const writeExit = yield* Effect.exit(
-				file.writeAll(new TextEncoder().encode(contents)).pipe(Effect.zipRight(file.sync)),
-			)
-			if (Exit.isFailure(writeExit)) return { identity, writeExit, pathMatchesHandle: false } as const
-
-			// Validate while the creating handle is still open. In particular, Windows
-			// may reuse a deleted file's numeric identifier immediately after its final
-			// handle closes, making a later pathname-only comparison ambiguous.
-			const pathInfo = yield* fileSystem.stat(targetPath)
-			const pathMatchesHandle = matchesCreatedFile(identity, {
-				exists: true,
-				dev: pathInfo.dev,
-				ino: Option.getOrUndefined(pathInfo.ino),
-				type: pathInfo.type,
-			})
-			return { identity, writeExit, pathMatchesHandle } as const
-		}),
-	)
-}
-
-/** Compares a pathname snapshot with identity captured from the creating file handle. */
-export function matchesCreatedFile(
-	identity: CreatedFileIdentity,
-	pathIdentity: {
-		readonly exists: boolean
-		readonly dev?: number
-		readonly ino?: number
-		readonly type?: FileSystem.File.Type
-	},
-): boolean {
-	return (
-		pathIdentity.exists &&
-		pathIdentity.dev === identity.dev &&
-		pathIdentity.ino === identity.ino &&
-		pathIdentity.type === identity.type
-	)
-}

package/src/Snippets.ts

@@ -1,39 +0,0 @@
-import type { FetchObjectModeResultWithJson } from '@expandai/sdk/FetchJson'
-import { citationUrl } from '@expandai/sdk/Playground'
-import { Schema } from 'effect'
-
-const encodeJson = Schema.encodeSync(Schema.parseJson(Schema.Unknown, { space: 2 }))
-
-type SearchResultWithJson = NonNullable<NonNullable<FetchObjectModeResultWithJson['data']>['search']>
-type RenderableSnippet = SearchResultWithJson['snippets'][number]
-
-/**
- * Attaches the public citation link to each snippet. JSON consumers get the same
- * auditable URL the text renderer prints in its header — without it, a caller has
- * the snapshot and the evidence id but no host to build a link against.
- */
-export function withCitationUrls<A extends RenderableSnippet>(
-	snapshotId: string,
-	host: string,
-	snippets: ReadonlyArray<A>,
-): Array<A & { readonly citationUrl: string }> {
-	return snippets.map((snippet) => ({
-		...snippet,
-		citationUrl: citationUrl(snapshotId, snippet.location, host),
-	}))
-}
-
-/**
- * Render search snippets as auditable text blocks: a header carrying the source, score, and
- * citation link, the snippet text, and — for State JSON snippets — the extracted JSON value.
- */
-export function renderSnippets(snapshotId: string, host: string, snippets: ReadonlyArray<RenderableSnippet>): string {
-	return snippets
-		.map((snippet) => {
-			const url = citationUrl(snapshotId, snippet.location, host)
-			const header = `[${snippet.source} · score ${snippet.score.toFixed(2)}] ${url}`
-			const json = snippet.json == null ? '' : `\n\n${encodeJson(snippet.json)}`
-			return `${header}\n\n${snippet.text}${json}`
-		})
-		.join('\n\n---\n\n')
-}