expandai 0.0.2 → 0.0.4

package/src/Mcp/tools/Fetch.ts

@@ -2,6 +2,7 @@ import { Tool } from '@effect/ai'
 import { Effect, Schema, String } from 'effect'
 import { ApiClient } from '../../ApiClient.js'
 import { ConfigManager } from '../../Config.js'
+import { Telemetry } from '../../Telemetry.js'
 import {
 	CitedObjectModeResult,
 	CitedSnapshotSearchResult,
@@ -69,11 +70,15 @@ export const FetchTool = Tool.make('fetch', {
 	description: String.stripMargin(`
 		|Fetch and extract content from any URL.
 		|
+		|You MUST cite what you take from the result. The result carries a ready-made
+		|'citationBlock' — append it verbatim to the end of your answer. Do not retype it, do not
+		|substitute the URL you fetched, and do not build a link yourself: the citation points at
+		|expand.ai's stored capture, not at the original site. Drop only the markers you did not use.
+		|A 'Verify it' link without '?id=' is not a citation. Never invent or reuse an evidence id.
+		|
 		|Returns object-mode output with markdown, front-matter metadata, and optional extracted JSON.
 		|When 'search' is set, the response also carries ranked snippets under data.search.snippets.
-		|Each snippet has a 'citationUrl' — a public playground link auditing that exact passage — and,
-		|for State JSON sources, the extracted 'json' value. Return the citationUrl to the user so they
-		|can verify the source.`),
+		|For State JSON sources each snippet also carries the extracted 'json' value.`),
 	parameters: FetchParams.fields,
 	success: CitedObjectModeResult,
 	failure: Schema.Defect,
@@ -86,10 +91,15 @@ export const FetchSearchTool = Tool.make('fetch_search', {
 	description: String.stripMargin(`
 		|Search a previous fetch by snapshotId without starting a new capture.
 		|
-		|Returns ranked snippets from stored markdown, appendix, or State JSON artifacts. Each snippet
-		|has a 'citationUrl' — a public playground link auditing that exact passage — and, for State JSON
-		|sources, the extracted 'json' value. The result also includes a whole-snapshot 'playground' link.
-		|Return the citationUrl to the user so they can verify the source.`),
+		|You MUST cite what you take from the result. The result carries a ready-made
+		|'citationBlock' — append it verbatim to the end of your answer. Do not retype it, do not
+		|substitute the URL you fetched, and do not build a link yourself: the citation points at
+		|expand.ai's stored capture, not at the original site. Drop only the markers you did not use.
+		|A 'Verify it' link without '?id=' is not a citation. Never invent or reuse an evidence id.
+		|
+		|Returns ranked snippets from stored markdown, appendix, or State JSON artifacts. For State JSON
+		|sources each snippet also carries the extracted 'json' value. The result also includes a
+		|whole-snapshot 'playground' link.`),
 	parameters: FetchSnapshotSearchParams.fields,
 	success: CitedSnapshotSearchResult,
 	failure: Schema.Defect,
@@ -104,10 +114,11 @@ export interface FetchService {
 }
 
 export class Fetch extends Effect.Service<Fetch>()('@expandai/cli/Mcp/Fetch', {
-	dependencies: [ApiClient.Default, ConfigManager.Default],
+	dependencies: [ApiClient.Default, ConfigManager.Default, Telemetry.Default],
 	scoped: Effect.gen(function* () {
 		const client = yield* ApiClient
 		const configManager = yield* ConfigManager
+		const telemetry = yield* Telemetry
 		const playgroundHost = yield* configManager.getPlaygroundHost()
 
 		const fetch: FetchService['fetch'] = Effect.fn('Fetch.fetch')(function* ({
@@ -117,9 +128,16 @@ export class Fetch extends Effect.Service<Fetch>()('@expandai/cli/Mcp/Fetch', {
 			includeAppendix,
 			includeJson,
 		}: FetchParams) {
-			yield* Effect.logDebug(`Fetching: ${url}`).pipe(
-				Effect.annotateLogs({ search: search?.query, includeMeta, includeAppendix, includeJson }),
+			yield* Effect.logDebug('MCP fetch started').pipe(
+				Effect.annotateLogs({
+					operation: 'fetch',
+					hasSearch: search !== undefined,
+					includeMeta,
+					includeAppendix,
+					includeJson,
+				}),
 			)
+			yield* telemetry.capture('mcp_tool_called', { tool: 'fetch', has_search: search !== undefined })
 
 			const includeParam =
 				includeAppendix && includeJson
@@ -143,7 +161,14 @@ export class Fetch extends Effect.Service<Fetch>()('@expandai/cli/Mcp/Fetch', {
 				},
 			})
 
-			yield* Effect.logDebug(`Fetched successfully: ${result.meta.url}`)
+			yield* Effect.logDebug('MCP fetch completed').pipe(
+				Effect.annotateLogs({
+					operation: 'fetch',
+					hasSearch: search !== undefined,
+					searchResultCount: result.data?.search?.snippets.length ?? 0,
+					jsonEvidenceCount: result.json.length,
+				}),
+			)
 
 			return enrichObjectModeResult(result, playgroundHost)
 		})
@@ -155,9 +180,10 @@ export class Fetch extends Effect.Service<Fetch>()('@expandai/cli/Mcp/Fetch', {
 			includeAppendix,
 			includeJson,
 		}: FetchSnapshotSearchParams) {
-			yield* Effect.logDebug(`Searching fetch snapshot: ${snapshotId}`).pipe(
-				Effect.annotateLogs({ search: search.query, includeMarkdown, includeAppendix, includeJson }),
+			yield* Effect.logDebug('MCP snapshot search started').pipe(
+				Effect.annotateLogs({ operation: 'fetch_search', includeMarkdown, includeAppendix, includeJson }),
 			)
+			yield* telemetry.capture('mcp_tool_called', { tool: 'fetch_search', has_search: true })
 
 			const result = yield* client.fetchFetchSearch({
 				snapshotId,
@@ -169,7 +195,7 @@ export class Fetch extends Effect.Service<Fetch>()('@expandai/cli/Mcp/Fetch', {
 				},
 			})
 
-			return enrichSnapshotSearchResult(result, playgroundHost)
+			return enrichSnapshotSearchResult(result, playgroundHost, 'mcp-fetch-search')
 		})
 
 		return { fetch, fetchSearch } satisfies FetchService

package/src/SecureFile.ts

@@ -0,0 +1,70 @@
+import { FileSystem, Error as PlatformError } from '@effect/platform'
+import { Effect, Exit, Option } from 'effect'
+
+export interface CreatedFileIdentity {
+	readonly dev: number
+	readonly ino?: number
+	readonly mode: number
+	readonly type: FileSystem.File.Type
+}
+
+export interface ExclusiveFileCreation {
+	readonly identity: CreatedFileIdentity
+	readonly writeExit: Exit.Exit<void, PlatformError.PlatformError>
+	readonly pathMatchesHandle: boolean
+}
+
+/** Creates a file exclusively and binds its identity to the opened handle before writing bytes. */
+export function createExclusiveFile(
+	fileSystem: FileSystem.FileSystem,
+	targetPath: string,
+	contents: string,
+	mode: number,
+): Effect.Effect<ExclusiveFileCreation, PlatformError.PlatformError> {
+	return Effect.scoped(
+		Effect.gen(function* () {
+			const file = yield* fileSystem.open(targetPath, { flag: 'wx', mode })
+			const info = yield* file.stat
+			const identity: CreatedFileIdentity = {
+				dev: info.dev,
+				ino: Option.getOrUndefined(info.ino),
+				mode: info.mode,
+				type: info.type,
+			}
+			const writeExit = yield* Effect.exit(
+				file.writeAll(new TextEncoder().encode(contents)).pipe(Effect.zipRight(file.sync)),
+			)
+			if (Exit.isFailure(writeExit)) return { identity, writeExit, pathMatchesHandle: false } as const
+
+			// Validate while the creating handle is still open. In particular, Windows
+			// may reuse a deleted file's numeric identifier immediately after its final
+			// handle closes, making a later pathname-only comparison ambiguous.
+			const pathInfo = yield* fileSystem.stat(targetPath)
+			const pathMatchesHandle = matchesCreatedFile(identity, {
+				exists: true,
+				dev: pathInfo.dev,
+				ino: Option.getOrUndefined(pathInfo.ino),
+				type: pathInfo.type,
+			})
+			return { identity, writeExit, pathMatchesHandle } as const
+		}),
+	)
+}
+
+/** Compares a pathname snapshot with identity captured from the creating file handle. */
+export function matchesCreatedFile(
+	identity: CreatedFileIdentity,
+	pathIdentity: {
+		readonly exists: boolean
+		readonly dev?: number
+		readonly ino?: number
+		readonly type?: FileSystem.File.Type
+	},
+): boolean {
+	return (
+		pathIdentity.exists &&
+		pathIdentity.dev === identity.dev &&
+		pathIdentity.ino === identity.ino &&
+		pathIdentity.type === identity.type
+	)
+}

package/src/Snippets.ts

@@ -1,19 +1,26 @@
-import { type CitationSource, citationUrl } from '@expandai/sdk/Playground'
+import type { FetchObjectModeResultWithJson } from '@expandai/sdk/FetchJson'
+import { citationUrl } from '@expandai/sdk/Playground'
 import { Schema } from 'effect'
 
 const encodeJson = Schema.encodeSync(Schema.parseJson(Schema.Unknown, { space: 2 }))
 
-interface RenderableSnippetLocation {
-	readonly nodeIds?: ReadonlyArray<number> | null
-	readonly stateJsonSourceId?: number | null
-}
+type SearchResultWithJson = NonNullable<NonNullable<FetchObjectModeResultWithJson['data']>['search']>
+type RenderableSnippet = SearchResultWithJson['snippets'][number]
 
-interface RenderableSnippet {
-	readonly source: CitationSource
-	readonly text: string
-	readonly score: number
-	readonly location?: RenderableSnippetLocation | null
-	readonly json?: unknown
+/**
+ * Attaches the public citation link to each snippet. JSON consumers get the same
+ * auditable URL the text renderer prints in its header — without it, a caller has
+ * the snapshot and the evidence id but no host to build a link against.
+ */
+export function withCitationUrls<A extends RenderableSnippet>(
+	snapshotId: string,
+	host: string,
+	snippets: ReadonlyArray<A>,
+): Array<A & { readonly citationUrl: string }> {
+	return snippets.map((snippet) => ({
+		...snippet,
+		citationUrl: citationUrl(snapshotId, snippet.location, host),
+	}))
 }
 
 /**
@@ -23,7 +30,7 @@ interface RenderableSnippet {
 export function renderSnippets(snapshotId: string, host: string, snippets: ReadonlyArray<RenderableSnippet>): string {
 	return snippets
 		.map((snippet) => {
-			const url = citationUrl(snapshotId, snippet.source, snippet.location, host)
+			const url = citationUrl(snapshotId, snippet.location, host)
 			const header = `[${snippet.source} · score ${snippet.score.toFixed(2)}] ${url}`
 			const json = snippet.json == null ? '' : `\n\n${encodeJson(snippet.json)}`
 			return `${header}\n\n${snippet.text}${json}`

package/src/Telemetry.ts

@@ -0,0 +1,186 @@
+/**
+ * Anonymous usage telemetry (PostHog). Events carry a persisted machine UUID,
+ * the invoking harness, and coarse feature usage — never URLs, queries,
+ * keys, or tokens.
+ *
+ * Delivery is fire-and-forget: each capture forks into a FiberSet and `flush`
+ * caps shutdown at one second, so telemetry can never block a command, delay
+ * an MCP/hook payload, or write to stdout (stdout is a protocol surface).
+ *
+ * Settings live in the config file (`telemetry.enabled` / `distinctId` /
+ * `noticeShown`). `DO_NOT_TRACK` / `EXPANDAI_NO_TELEMETRY`, or a non-default
+ * effective API URL (env or config file — dev traffic), disable capture per
+ * invocation without touching the file. A config file that fails to parse
+ * also disables telemetry for the invocation and is never written to.
+ */
+import { randomUUID } from 'node:crypto'
+import { HttpClient, HttpClientRequest } from '@effect/platform'
+import { NodeHttpClient } from '@effect/platform-node'
+import { Config, Console, Effect, FiberSet, Layer, Option } from 'effect'
+import { ConfigManager, DEFAULT_API_URL, envFlag, envLiteral } from './Config.js'
+import { Harness } from './Install/Domain.js'
+import { CLI_VERSION } from './Version.js'
+
+// Every field reads leniently (envFlag / envLiteral / plain strings) so a
+// malformed env var can never fail service construction — telemetry must
+// silently degrade, not crash the CLI.
+export const TelemetryEnvironment = Config.all({
+	// PostHog project write-only key — safe to embed as the default (it ships
+	// in every browser bundle that uses posthog-js) and only permits ingestion.
+	apiKey: Config.string('POSTHOG_API_KEY').pipe(Config.withDefault('phc_sffcV4skT33mMKmo0BenNoE7t7AghUrOugyW9qSx43B')),
+	host: Config.string('POSTHOG_HOST').pipe(Config.withDefault('https://us.i.posthog.com')),
+	doNotTrack: envFlag('DO_NOT_TRACK'),
+	noTelemetry: envFlag('EXPANDAI_NO_TELEMETRY'),
+	apiUrl: Config.string('EXPAND_API_URL').pipe(Config.option),
+	// Baked into MCP/hook configs at install time; wins over ambient signals.
+	// An unrecognized value (e.g. written by a newer installer) reads as None.
+	harnessOverride: envLiteral('EXPANDAI_HARNESS', Harness),
+	claudeCode: envFlag('CLAUDECODE'),
+	cursor: Config.string('CURSOR_TRACE_ID').pipe(Config.option, Config.map(Option.isSome)),
+	codex: Config.string('CODEX_RUN_ID').pipe(Config.option, Config.map(Option.isSome)),
+})
+
+type TelemetryEnvironment = Config.Config.Success<typeof TelemetryEnvironment>
+
+function detectHarness(environment: TelemetryEnvironment): Harness {
+	if (Option.isSome(environment.harnessOverride)) return environment.harnessOverride.value
+	if (environment.claudeCode) return 'claude-code'
+	if (environment.cursor) return 'cursor'
+	if (environment.codex) return 'codex'
+	return 'unknown'
+}
+
+const generateDistinctId = Effect.sync(() => randomUUID())
+
+// Lenient normalization for the non-default-API comparison: `new URL()` adds
+// the trailing slash a bare origin string lacks, and a value that doesn't
+// parse compares as non-default — never crash construction, and treat unknown
+// endpoints as non-production traffic.
+function normalizeApiUrl(url: string): string {
+	try {
+		return new URL(url).toString()
+	} catch {
+		return `unparseable:${url}`
+	}
+}
+
+export class Telemetry extends Effect.Service<Telemetry>()('@expandai/cli/Telemetry', {
+	dependencies: [ConfigManager.Default, NodeHttpClient.layer],
+	scoped: Effect.gen(function* () {
+		const configManager = yield* ConfigManager
+		const httpClient = yield* HttpClient.HttpClient
+		const fibers = yield* FiberSet.make()
+
+		const environment = yield* TelemetryEnvironment
+		// EXPAND_API_KEY overrides the config file for requests (see ConfigManager),
+		// so it must win here too or env-key users all report 'none'.
+		const envApiKey = yield* Config.string('EXPAND_API_KEY').pipe(Config.option, Config.map(Option.isSome))
+
+		// The config file is read best-effort: telemetry must work (or silently
+		// no-op) before login and survive an unreadable config. A parse failure
+		// disables capture for the invocation — an unreadable file must never be
+		// mistaken for an empty one, and never be written to.
+		const { config, parseFailed } = yield* configManager.readConfigFile()
+		const settings = config.telemetry
+
+		// Same env-over-file precedence as ConfigManager: a self-hosted apiUrl in
+		// the config file is dev traffic too. Deliberately not Config.url /
+		// getApiUrl — those fail on a malformed env value, and telemetry must
+		// degrade instead of crashing the CLI.
+		const effectiveApiUrl = Option.firstSomeOf([
+			environment.apiUrl,
+			Option.map(Option.fromNullable(config.apiUrl), (url) => url.toString()),
+		])
+		const nonDefaultApi = Option.match(effectiveApiUrl, {
+			onNone: () => false,
+			onSome: (url) => normalizeApiUrl(url) !== normalizeApiUrl(DEFAULT_API_URL),
+		})
+
+		const enabled =
+			!environment.doNotTrack &&
+			!environment.noTelemetry &&
+			!nonDefaultApi &&
+			!parseFailed &&
+			settings?.enabled !== false
+
+		const harness = detectHarness(environment)
+		const authMethod = envApiKey
+			? 'api-key'
+			: config.authMethod?._tag === 'ApiKey'
+				? 'api-key'
+				: config.authMethod?._tag === 'OAuth'
+					? 'oauth'
+					: 'none'
+
+		const persistNewDistinctId = Effect.fnUntraced(function* () {
+			const generated = yield* generateDistinctId
+			// Best-effort persistence — an unwritable config yields a per-run id.
+			yield* configManager.updateConfig({ telemetry: { ...settings, distinctId: generated } }).pipe(Effect.ignore)
+			return generated
+		})
+
+		// Disabled runs never write: DO_NOT_TRACK users don't get an id persisted,
+		// and after a parse failure a write would clobber the original file.
+		const distinctId = settings?.distinctId ?? (enabled ? yield* persistNewDistinctId() : yield* generateDistinctId)
+
+		const capture = Effect.fn('Telemetry.capture')(function* (
+			event: string,
+			properties: Record<string, string | number | boolean>,
+		) {
+			if (!enabled) return
+			const post = HttpClientRequest.post(`${environment.host}/i/v0/e`).pipe(
+				HttpClientRequest.bodyJson({
+					api_key: environment.apiKey,
+					event,
+					distinct_id: distinctId,
+					timestamp: new Date().toISOString(),
+					properties: {
+						$lib: 'expandai-cli',
+						$lib_version: CLI_VERSION,
+						version: CLI_VERSION,
+						harness,
+						auth_method: authMethod,
+						...properties,
+					},
+				}),
+			)
+			yield* FiberSet.run(
+				fibers,
+				post.pipe(
+					Effect.flatMap((request) => httpClient.execute(request)),
+					Effect.scoped,
+					Effect.timeout('2 seconds'),
+					Effect.ignore,
+				),
+			)
+		})
+
+		// Cap delivery at flush time so short-lived processes get events out
+		// without ever holding the process open noticeably.
+		const flush = FiberSet.awaitEmpty(fibers).pipe(Effect.timeout('1 second'), Effect.ignore)
+
+		const showNoticeIfNeeded = Effect.fn('Telemetry.showNoticeIfNeeded')(function* () {
+			if (!enabled || settings?.noticeShown === true) return
+			yield* Console.error(
+				'expandai collects anonymous usage data to improve the CLI. Opt out with `expandai telemetry disable` or EXPANDAI_NO_TELEMETRY=1.',
+			)
+			yield* configManager
+				.updateConfig({ telemetry: { ...settings, distinctId, noticeShown: true } })
+				.pipe(Effect.ignore)
+		})
+
+		return { capture, flush, showNoticeIfNeeded, enabled, distinctId } as const
+	}),
+}) {
+	/** A side-effect-free telemetry service for commands whose contract is read-only. */
+	static readonly DISABLED = Layer.succeed(
+		Telemetry,
+		Telemetry.make({
+			capture: () => Effect.void,
+			flush: Effect.void,
+			showNoticeIfNeeded: () => Effect.void,
+			enabled: false,
+			distinctId: 'disabled',
+		}),
+	)
+}

package/src/Version.ts

@@ -0,0 +1,3 @@
+// Single source of truth for the CLI version. Keep in sync with package.json —
+// the compiled binary cannot read package.json at runtime.
+export const CLI_VERSION = '0.0.4'

package/src/Xdg.ts

@@ -0,0 +1,18 @@
+import { homedir } from 'node:os'
+import { Path } from '@effect/platform'
+import { Config, Effect } from 'effect'
+
+/**
+ * XDG base-directory lookup, shared by everything that persists CLI state so
+ * the fallback rules cannot drift apart. The spec treats an unset and an empty
+ * variable identically: fall back to `$HOME/.config`. Without the empty check,
+ * `XDG_CONFIG_HOME=` resolves paths relative to the working directory — which
+ * would drop credentials inside whatever project the CLI happens to run in.
+ */
+export const xdgConfigDirectory = Effect.gen(function* () {
+	const path = yield* Path.Path
+	return yield* Config.string('XDG_CONFIG_HOME').pipe(
+		Config.withDefault(''),
+		Config.map((directory) => (directory.trim() === '' ? path.join(homedir(), '.config') : directory)),
+	)
+})