@effect-x/ultimate-search 0.1.0 → 0.1.1

package/src/services/read-only-mcp.ts

@@ -0,0 +1,278 @@
+import { Effect, Layer, Option, Schema } from "effect";
+import { McpServer, Tool, Toolkit } from "effect/unstable/ai";
+import { UltimateSearchConfig } from "../config/settings";
+import {
+  GrokSearchInput,
+  GrokSearchResultSchema,
+} from "../providers/grok/schema";
+import { GrokProviderClient } from "../providers/grok/client";
+import {
+  FetchContentFormatSchema,
+  TavilyMapBreadthSchema,
+  TavilyMapDepthSchema,
+  TavilyMapInput,
+  TavilyMapLimitSchema,
+  TavilyMapResponseSchema,
+  TavilySearchDepthSchema,
+  TavilySearchInput,
+  TavilySearchResponseSchema,
+  TavilySearchTopicSchema,
+  TavilyTimeRangeSchema,
+} from "../providers/tavily/schema";
+import { TavilyProviderClient } from "../providers/tavily/client";
+import { FirecrawlProviderClient } from "../providers/firecrawl/client";
+import { DualSearch, DualSearchInput, DualSearchResultSchema } from "./dual-search";
+import { FirecrawlFetch } from "./firecrawl-fetch";
+import { GrokSearch } from "./grok-search";
+import { TavilyMap } from "./tavily-map";
+import { TavilyExtract } from "./tavily-extract";
+import { TavilySearch } from "./tavily-search";
+import { WebFetch } from "./web-fetch";
+import { WebFetchInput, WebFetchResultSchema } from "./web-fetch-schema";
+import { RenderedErrorSchema, renderStructuredError } from "../shared/render-error";
+import { absoluteUrlStringSchema, trimmedNonEmptyStringSchema } from "../shared/schema";
+
+const optionalTrimmedTextField = (message: string) =>
+  Schema.optional(trimmedNonEmptyStringSchema(message));
+
+const toOption = <A>(value: A | undefined) =>
+  value === undefined ? Option.none<A>() : Option.some(value);
+
+const SearchGrokParametersSchema = Schema.Struct({
+  query: trimmedNonEmptyStringSchema("query must be a non-empty string"),
+  platform: optionalTrimmedTextField("platform must be a non-empty string"),
+  model: optionalTrimmedTextField("model must be a non-empty string"),
+});
+
+const SearchTavilyParametersSchema = Schema.Struct({
+  query: trimmedNonEmptyStringSchema("query must be a non-empty string"),
+  depth: Schema.optional(TavilySearchDepthSchema),
+  maxResults: Schema.optional(
+    Schema.Int.pipe(
+      Schema.check(Schema.isBetween({ minimum: 1, maximum: 20 })),
+      Schema.annotate({
+        message: "maxResults must be an integer between 1 and 20",
+      }),
+    ),
+  ),
+  topic: Schema.optional(TavilySearchTopicSchema),
+  timeRange: Schema.optional(TavilyTimeRangeSchema),
+  includeAnswer: Schema.optional(Schema.Boolean),
+});
+
+const SearchDualParametersSchema = Schema.Struct({
+  query: trimmedNonEmptyStringSchema("query must be a non-empty string"),
+  platform: optionalTrimmedTextField("platform must be a non-empty string"),
+  model: optionalTrimmedTextField("model must be a non-empty string"),
+  depth: Schema.optional(TavilySearchDepthSchema),
+  maxResults: Schema.optional(
+    Schema.Int.pipe(
+      Schema.check(Schema.isBetween({ minimum: 1, maximum: 20 })),
+      Schema.annotate({
+        message: "maxResults must be an integer between 1 and 20",
+      }),
+    ),
+  ),
+  topic: Schema.optional(TavilySearchTopicSchema),
+  timeRange: Schema.optional(TavilyTimeRangeSchema),
+  includeAnswer: Schema.optional(Schema.Boolean),
+});
+
+const FetchParametersSchema = Schema.Struct({
+  url: absoluteUrlStringSchema("url must be an absolute URL"),
+  depth: Schema.optional(TavilySearchDepthSchema),
+  format: Schema.optional(FetchContentFormatSchema),
+});
+
+const MapParametersSchema = Schema.Struct({
+  url: absoluteUrlStringSchema("url must be an absolute URL"),
+  depth: Schema.optional(TavilyMapDepthSchema),
+  breadth: Schema.optional(TavilyMapBreadthSchema),
+  limit: Schema.optional(TavilyMapLimitSchema),
+  instructions: optionalTrimmedTextField("instructions must be a non-empty string"),
+});
+
+const searchGrokTool = Tool.make("search_grok", {
+  description: "Run the Grok-backed search flow used by the CLI.",
+  parameters: SearchGrokParametersSchema,
+  success: GrokSearchResultSchema,
+  failure: RenderedErrorSchema,
+  failureMode: "return",
+})
+  .annotate(Tool.Title, "Search Grok")
+  .annotate(Tool.Readonly, true)
+  .annotate(Tool.Destructive, false)
+  .annotate(Tool.Idempotent, true);
+
+const searchTavilyTool = Tool.make("search_tavily", {
+  description: "Run the Tavily-backed search flow used by the CLI.",
+  parameters: SearchTavilyParametersSchema,
+  success: TavilySearchResponseSchema,
+  failure: RenderedErrorSchema,
+  failureMode: "return",
+})
+  .annotate(Tool.Title, "Search Tavily")
+  .annotate(Tool.Readonly, true)
+  .annotate(Tool.Destructive, false)
+  .annotate(Tool.Idempotent, true);
+
+const searchDualTool = Tool.make("search_dual", {
+  description: "Run Grok and Tavily concurrently with the CLI's dual-search orchestration.",
+  parameters: SearchDualParametersSchema,
+  success: DualSearchResultSchema,
+  failure: RenderedErrorSchema,
+  failureMode: "return",
+})
+  .annotate(Tool.Title, "Search Dual")
+  .annotate(Tool.Readonly, true)
+  .annotate(Tool.Destructive, false)
+  .annotate(Tool.Idempotent, true);
+
+const fetchTool = Tool.make("fetch", {
+  description: "Fetch a page with Tavily-first and FireCrawl fallback, matching the CLI.",
+  parameters: FetchParametersSchema,
+  success: WebFetchResultSchema,
+  failure: RenderedErrorSchema,
+  failureMode: "return",
+})
+  .annotate(Tool.Title, "Fetch")
+  .annotate(Tool.Readonly, true)
+  .annotate(Tool.Destructive, false)
+  .annotate(Tool.Idempotent, true);
+
+const mapTool = Tool.make("map", {
+  description: "Map a site's reachable URLs with Tavily, matching the CLI.",
+  parameters: MapParametersSchema,
+  success: TavilyMapResponseSchema,
+  failure: RenderedErrorSchema,
+  failureMode: "return",
+})
+  .annotate(Tool.Title, "Map")
+  .annotate(Tool.Readonly, true)
+  .annotate(Tool.Destructive, false)
+  .annotate(Tool.Idempotent, true);
+
+export const readOnlyMcpToolkit = Toolkit.make(
+  searchGrokTool,
+  searchTavilyTool,
+  searchDualTool,
+  fetchTool,
+  mapTool,
+);
+
+export const readOnlyMcpToolNames = [
+  "search_grok",
+  "search_tavily",
+  "search_dual",
+  "fetch",
+  "map",
+] as const;
+
+const readOnlyMcpToolkitLayer = readOnlyMcpToolkit.toLayer(
+  Effect.gen(function* () {
+    const grokSearch = yield* GrokSearch;
+    const tavilySearch = yield* TavilySearch;
+    const dualSearch = yield* DualSearch;
+    const webFetch = yield* WebFetch;
+    const tavilyMap = yield* TavilyMap;
+
+    return readOnlyMcpToolkit.of({
+      search_grok: Effect.fn("ReadOnlyMcp.searchGrok")(function* (input) {
+        const request = yield* GrokSearchInput.decodeEffect({
+          query: input.query,
+          platform: toOption(input.platform),
+          model: toOption(input.model),
+        }).pipe(Effect.orDie);
+
+        return yield* grokSearch.search(request).pipe(Effect.mapError(renderStructuredError));
+      }),
+      search_tavily: Effect.fn("ReadOnlyMcp.searchTavily")(function* (input) {
+        const request = yield* TavilySearchInput.decodeEffect({
+          query: input.query,
+          searchDepth: toOption(input.depth),
+          topic: toOption(input.topic),
+          timeRange: toOption(input.timeRange),
+          maxResults: toOption(input.maxResults),
+          includeAnswer: input.includeAnswer ?? false,
+        }).pipe(Effect.orDie);
+
+        return yield* tavilySearch.search(request).pipe(Effect.mapError(renderStructuredError));
+      }),
+      search_dual: Effect.fn("ReadOnlyMcp.searchDual")(function* (input) {
+        const request = yield* DualSearchInput.decodeEffect({
+          query: input.query,
+          platform: toOption(input.platform),
+          model: toOption(input.model),
+          searchDepth: toOption(input.depth),
+          topic: toOption(input.topic),
+          timeRange: toOption(input.timeRange),
+          maxResults: toOption(input.maxResults),
+          includeAnswer: input.includeAnswer ?? false,
+        }).pipe(Effect.orDie);
+
+        return yield* dualSearch.search(request).pipe(Effect.mapError(renderStructuredError));
+      }),
+      fetch: Effect.fn("ReadOnlyMcp.fetch")(function* (input) {
+        const request = yield* WebFetchInput.decodeEffect({
+          urls: [input.url],
+          depth: input.depth ?? "basic",
+          format: input.format ?? "markdown",
+        }).pipe(Effect.orDie);
+
+        return yield* webFetch.fetch(request).pipe(
+          Effect.flatMap((result) => Schema.decodeUnknownEffect(WebFetchResultSchema)(result)),
+          Effect.mapError(renderStructuredError),
+        );
+      }),
+      map: Effect.fn("ReadOnlyMcp.map")(function* (input) {
+        const request = yield* TavilyMapInput.decodeEffect({
+          url: input.url,
+          depth: toOption(input.depth),
+          breadth: toOption(input.breadth),
+          limit: toOption(input.limit),
+          instructions: toOption(input.instructions),
+        }).pipe(Effect.orDie);
+
+        return yield* tavilyMap.map(request).pipe(Effect.mapError(renderStructuredError));
+      }),
+    });
+  }),
+);
+
+const readOnlyProviderLayer = Layer.mergeAll(
+  FirecrawlProviderClient.layer,
+  GrokProviderClient.layer,
+  TavilyProviderClient.layer,
+).pipe(Layer.provideMerge(UltimateSearchConfig.layer));
+
+const grokSearchLayer = GrokSearch.layer.pipe(Layer.provideMerge(readOnlyProviderLayer));
+
+const tavilySearchLayer = TavilySearch.layer.pipe(Layer.provideMerge(readOnlyProviderLayer));
+
+const tavilyMapLayer = TavilyMap.layer.pipe(Layer.provideMerge(readOnlyProviderLayer));
+
+const tavilyExtractLayer = TavilyExtract.layer.pipe(Layer.provideMerge(readOnlyProviderLayer));
+
+const firecrawlFetchLayer = FirecrawlFetch.layer.pipe(Layer.provideMerge(readOnlyProviderLayer));
+
+const webFetchLayer = WebFetch.layer.pipe(
+  Layer.provideMerge(firecrawlFetchLayer),
+  Layer.provideMerge(tavilyExtractLayer),
+);
+
+const dualSearchLayer = DualSearch.layer.pipe(
+  Layer.provideMerge(grokSearchLayer),
+  Layer.provideMerge(tavilySearchLayer),
+);
+
+export const readOnlyMcpServicesLayer = Layer.mergeAll(
+  grokSearchLayer,
+  tavilySearchLayer,
+  dualSearchLayer,
+  webFetchLayer,
+  tavilyMapLayer,
+);
+
+export const readOnlyMcpRegistrationLayer = Layer.effectDiscard(
+  McpServer.registerToolkit(readOnlyMcpToolkit),
+).pipe(Layer.provideMerge(readOnlyMcpToolkitLayer));

package/src/services/tavily-extract.ts

@@ -0,0 +1,66 @@
+import { Effect, Layer, ServiceMap } from "effect";
+import { TavilyProviderClient } from "../providers/tavily/client";
+import { buildTavilyExtractRequest } from "../providers/tavily/schema";
+import { ProviderContentError, type UltimateSearchError } from "../shared/errors";
+import type { ServicesReturns } from "../shared/effect";
+import type { FetchedPage, WebFetchInput } from "./web-fetch-schema";
+
+const normalizeContent = (content: string | null | undefined) => {
+  const text = content?.trim() ?? "";
+  return text.length > 0 ? text : null;
+};
+
+export class TavilyExtract extends ServiceMap.Service<
+  TavilyExtract,
+  {
+    readonly extract: (
+      input: WebFetchInput,
+    ) => Effect.Effect<ReadonlyArray<FetchedPage>, UltimateSearchError, never>;
+  }
+>()("TavilyExtract") {
+  static readonly layer = Layer.effect(
+    TavilyExtract,
+    Effect.gen(function* () {
+      const provider = yield* TavilyProviderClient;
+
+      const extract: TavilyExtract.Methods["extract"] = Effect.fn("TavilyExtract.extract")(
+        function* (input): TavilyExtract.Returns<"extract"> {
+          const response = yield* provider.extract(buildTavilyExtractRequest(input));
+          const results = response.results.flatMap((item) => {
+            const rawContent = normalizeContent(item.raw_content);
+
+            if (rawContent === null) {
+              return [];
+            }
+
+            return [
+              {
+                url: item.url,
+                title: item.title,
+                raw_content: rawContent,
+              } satisfies FetchedPage,
+            ];
+          });
+
+          if (results.length === 0) {
+            return yield* new ProviderContentError({
+              provider: "tavily",
+              message: "Tavily returned no extractable content.",
+            });
+          }
+
+          return results;
+        },
+      );
+
+      return TavilyExtract.of({
+        extract,
+      });
+    }),
+  );
+}
+
+export declare namespace TavilyExtract {
+  export type Methods = ServiceMap.Service.Shape<typeof TavilyExtract>;
+  export type Returns<key extends keyof Methods, R = never> = ServicesReturns<Methods[key], R>;
+}

package/src/services/tavily-map.ts

@@ -0,0 +1,38 @@
+import { Effect, Layer, ServiceMap } from "effect";
+import { TavilyProviderClient } from "../providers/tavily/client";
+import {
+  buildTavilyMapRequest,
+  TavilyMapInput,
+  type TavilyMapResponse,
+} from "../providers/tavily/schema";
+import type { UltimateSearchError } from "../shared/errors";
+import type { ServicesReturns } from "../shared/effect";
+
+export class TavilyMap extends ServiceMap.Service<
+  TavilyMap,
+  {
+    readonly map: (
+      input: TavilyMapInput,
+    ) => Effect.Effect<TavilyMapResponse, UltimateSearchError, never>;
+  }
+>()("TavilyMap") {
+  static readonly layer = Layer.effect(
+    TavilyMap,
+    Effect.gen(function* () {
+      const provider = yield* TavilyProviderClient;
+
+      const map: TavilyMap.Methods["map"] = Effect.fn("TavilyMap.map")(function* (input) {
+        return yield* provider.map(buildTavilyMapRequest(input));
+      });
+
+      return TavilyMap.of({
+        map,
+      });
+    }),
+  );
+}
+
+export declare namespace TavilyMap {
+  export type Methods = ServiceMap.Service.Shape<typeof TavilyMap>;
+  export type Returns<key extends keyof Methods, R = never> = ServicesReturns<Methods[key], R>;
+}

package/src/services/tavily-search.ts

@@ -0,0 +1,40 @@
+import { Effect, Layer, ServiceMap } from "effect";
+import { TavilyProviderClient } from "../providers/tavily/client";
+import {
+  buildTavilySearchRequest,
+  TavilySearchInput,
+  type TavilySearchResponse,
+} from "../providers/tavily/schema";
+import type { UltimateSearchError } from "../shared/errors";
+import type { ServicesReturns } from "../shared/effect";
+
+export class TavilySearch extends ServiceMap.Service<
+  TavilySearch,
+  {
+    readonly search: (
+      input: TavilySearchInput,
+    ) => Effect.Effect<TavilySearchResponse, UltimateSearchError, never>;
+  }
+>()("TavilySearch") {
+  static readonly layer = Layer.effect(
+    TavilySearch,
+    Effect.gen(function* () {
+      const provider = yield* TavilyProviderClient;
+
+      const search: TavilySearch.Methods["search"] = Effect.fn("TavilySearch.search")(
+        function* (input): TavilySearch.Returns<"search"> {
+          return yield* provider.search(buildTavilySearchRequest(input));
+        },
+      );
+
+      return TavilySearch.of({
+        search,
+      });
+    }),
+  );
+}
+
+export declare namespace TavilySearch {
+  export type Methods = ServiceMap.Service.Shape<typeof TavilySearch>;
+  export type Returns<key extends keyof Methods, R = never> = ServicesReturns<Methods[key], R>;
+}

package/src/services/web-fetch-schema.ts

@@ -0,0 +1,74 @@
+import { Schema } from "effect";
+import {
+  FetchContentFormatSchema,
+  TavilyExtractDepthSchema,
+  type FetchContentFormat,
+} from "../providers/tavily/schema";
+import { SearchProvider } from "../shared/errors";
+import { absoluteUrlStringSchema } from "../shared/schema";
+
+export const FetchBackendSchema = Schema.Literals(["tavily", "firecrawl"] as const);
+
+export type FetchBackend = typeof FetchBackendSchema.Type;
+
+export class WebFetchInput extends Schema.Class<WebFetchInput>("WebFetchInput")({
+  urls: Schema.NonEmptyArray(
+    absoluteUrlStringSchema("url must be an absolute URL"),
+  ),
+  depth: TavilyExtractDepthSchema,
+  format: FetchContentFormatSchema,
+}) {
+  static decodeEffect = Schema.decodeUnknownEffect(WebFetchInput);
+}
+
+export interface FetchedPage {
+  readonly url: string;
+  readonly title?: string | null | undefined;
+  readonly raw_content: string;
+}
+
+export const FetchedPageSchema = Schema.Struct({
+  url: Schema.String,
+  title: Schema.optional(Schema.NullOr(Schema.String)),
+  raw_content: Schema.String,
+});
+
+export interface FallbackReason {
+  readonly type: string;
+  readonly provider: typeof SearchProvider.Type;
+  readonly message: string;
+}
+
+export const FallbackReasonSchema = Schema.Struct({
+  type: Schema.String,
+  provider: SearchProvider,
+  message: Schema.String,
+});
+
+export interface FetchFallback {
+  readonly from: "tavily";
+  readonly to: "firecrawl";
+  readonly reason: FallbackReason;
+}
+
+export const FetchFallbackSchema = Schema.Struct({
+  from: Schema.Literal("tavily"),
+  to: Schema.Literal("firecrawl"),
+  reason: FallbackReasonSchema,
+});
+
+export interface WebFetchResult {
+  readonly backend: FetchBackend;
+  readonly format: FetchContentFormat;
+  readonly results: ReadonlyArray<FetchedPage>;
+  readonly fallback?: FetchFallback | undefined;
+}
+
+export const WebFetchResultSchema = Schema.Struct({
+  backend: FetchBackendSchema,
+  format: FetchContentFormatSchema,
+  results: Schema.NonEmptyArray(FetchedPageSchema),
+  fallback: Schema.optional(FetchFallbackSchema),
+});
+
+export type WebFetchFormat = FetchContentFormat;

package/src/services/web-fetch.ts

@@ -0,0 +1,105 @@
+import { Effect, Layer, Result, ServiceMap } from "effect";
+import {
+  ConfigValidationError,
+  ProviderContentError,
+  ProviderDecodeError,
+  ProviderRequestError,
+  ProviderResponseError,
+  type UltimateSearchError,
+} from "../shared/errors";
+import type { ServicesReturns } from "../shared/effect";
+import { FirecrawlFetch } from "./firecrawl-fetch";
+import type {
+  FallbackReason,
+  WebFetchInput,
+  WebFetchResult,
+} from "./web-fetch-schema";
+import { TavilyExtract } from "./tavily-extract";
+
+const summarizeFallbackReason = (error: UltimateSearchError): FallbackReason => {
+  if (
+    error instanceof ConfigValidationError ||
+    error instanceof ProviderRequestError ||
+    error instanceof ProviderResponseError ||
+    error instanceof ProviderDecodeError ||
+    error instanceof ProviderContentError
+  ) {
+    return {
+      type: error._tag,
+      provider: error.provider,
+      message: error.message,
+    };
+  }
+
+  return {
+    type: "UnknownError",
+    provider: "shared",
+    message: String(error),
+  };
+};
+
+const resolveDoubleFailure = (
+  primary: UltimateSearchError,
+  fallback: UltimateSearchError,
+): UltimateSearchError =>
+  fallback instanceof ConfigValidationError &&
+  !(primary instanceof ConfigValidationError)
+    ? primary
+    : fallback;
+
+export class WebFetch extends ServiceMap.Service<
+  WebFetch,
+  {
+    readonly fetch: (
+      input: WebFetchInput,
+    ) => Effect.Effect<WebFetchResult, UltimateSearchError, never>;
+  }
+>()("WebFetch") {
+  static readonly layer = Layer.effect(
+    WebFetch,
+    Effect.gen(function* () {
+      const tavilyExtract = yield* TavilyExtract;
+      const firecrawlFetch = yield* FirecrawlFetch;
+
+      const fetch: WebFetch.Methods["fetch"] = Effect.fn("WebFetch.fetch")(
+        function* (input): WebFetch.Returns<"fetch"> {
+          const tavilyAttempt = yield* Effect.result(tavilyExtract.extract(input));
+
+          if (Result.isSuccess(tavilyAttempt)) {
+            return {
+              backend: "tavily",
+              format: input.format,
+              results: tavilyAttempt.success,
+            } satisfies WebFetchResult;
+          }
+
+          const firecrawlAttempt = yield* Effect.result(firecrawlFetch.fetch(input));
+
+          if (Result.isSuccess(firecrawlAttempt)) {
+            return {
+              backend: "firecrawl",
+              format: input.format,
+              results: firecrawlAttempt.success,
+              fallback: {
+                from: "tavily",
+                to: "firecrawl",
+                reason: summarizeFallbackReason(tavilyAttempt.failure),
+              },
+            } satisfies WebFetchResult;
+          }
+
+          return yield* resolveDoubleFailure(tavilyAttempt.failure, firecrawlAttempt.failure);
+        },
+      );
+
+      return WebFetch.of({
+        fetch,
+      });
+    }),
+  );
+}
+
+export declare namespace WebFetch {
+  export type Methods = ServiceMap.Service.Shape<typeof WebFetch>;
+  export type Returns<key extends keyof Methods, R = never> = ServicesReturns<Methods[key], R>;
+}

package/src/shared/cli-flags.ts

@@ -0,0 +1,25 @@
+import { Option, Schema } from "effect";
+import { Flag } from "effect/unstable/cli";
+
+export const optionalTrimmedTextFlag = (name: string, description: string) =>
+  Flag.optional(
+    Flag.string(name).pipe(Flag.withSchema(Schema.Trim), Flag.withDescription(description)),
+  ).pipe(Flag.map((value) => Option.filter(value, (text) => text.length > 0)));
+
+export const optionalChoiceFlag = <A extends string>(
+  name: string,
+  choices: ReadonlyArray<A>,
+  description: string,
+) => Flag.optional(Flag.choice(name, choices).pipe(Flag.withDescription(description)));
+
+export const optionalIntegerFlag = (name: string, description: string) =>
+  Flag.optional(Flag.integer(name).pipe(Flag.withDescription(description)));
+
+export const optionalIntegerFlagWithSchema = <A>(
+  name: string,
+  schema: Schema.Codec<A, number>,
+  description: string,
+) =>
+  Flag.optional(
+    Flag.integer(name).pipe(Flag.withSchema(schema), Flag.withDescription(description)),
+  );

package/src/shared/command-output.ts

@@ -0,0 +1,21 @@
+import { Effect, Option } from "effect";
+import { CliOutput, type OutputMode, resolveOutputMode } from "./output";
+
+export interface CommandOutput {
+  readonly human: string;
+  readonly llm: unknown;
+}
+
+export const runCommandWithOutput = <E, R>(
+  selectedMode: Option.Option<OutputMode>,
+  buildOutput: (mode: OutputMode) => Effect.Effect<CommandOutput, E, R>,
+) =>
+  Effect.gen(function* () {
+    const cliOutput = yield* CliOutput;
+    const mode = resolveOutputMode(selectedMode, cliOutput.defaultMode);
+    const output = yield* buildOutput(mode).pipe(
+      Effect.tapError((error) => cliOutput.logError(error, mode)),
+    );
+
+    yield* cliOutput.writeOutput(output, mode);
+  });