@effect-x/ultimate-search 0.1.2 → 0.1.4

package/src/providers/tavily/schema.ts

@@ -0,0 +1,207 @@
+import { Option, Schema } from "effect";
+import { absoluteUrlStringSchema, trimmedNonEmptyStringSchema } from "../../shared/schema";
+
+export const TavilySearchDepthSchema = Schema.Literals(["basic", "advanced"] as const);
+
+export type TavilySearchDepth = typeof TavilySearchDepthSchema.Type;
+
+export const TavilyExtractDepthSchema = TavilySearchDepthSchema;
+
+export type TavilyExtractDepth = typeof TavilyExtractDepthSchema.Type;
+
+export const TavilySearchTopicSchema = Schema.Literals(["general", "news", "finance"] as const);
+
+export type TavilySearchTopic = typeof TavilySearchTopicSchema.Type;
+
+export const FetchContentFormatSchema = Schema.Literals(["markdown", "text"] as const);
+
+export type FetchContentFormat = typeof FetchContentFormatSchema.Type;
+
+export const TavilyTimeRangeSchema = Schema.Literals(["day", "week", "month", "year"] as const);
+
+export type TavilyTimeRange = typeof TavilyTimeRangeSchema.Type;
+
+const maxResultsSchema = Schema.Int.pipe(
+  Schema.check(Schema.isBetween({ minimum: 1, maximum: 20 })),
+  Schema.annotate({
+    message: "max-results must be an integer between 1 and 20",
+  }),
+);
+
+export const TavilyMapDepthSchema = Schema.Int.pipe(
+  Schema.check(Schema.isBetween({ minimum: 1, maximum: 5 })),
+  Schema.annotate({
+    message: "depth must be an integer between 1 and 5",
+  }),
+);
+
+export type TavilyMapDepth = typeof TavilyMapDepthSchema.Type;
+
+export const TavilyMapBreadthSchema = Schema.Int.pipe(
+  Schema.check(Schema.isBetween({ minimum: 1, maximum: 500 })),
+  Schema.annotate({
+    message: "breadth must be an integer between 1 and 500",
+  }),
+);
+
+export type TavilyMapBreadth = typeof TavilyMapBreadthSchema.Type;
+
+export const TavilyMapLimitSchema = Schema.Int.pipe(
+  Schema.check(Schema.isGreaterThanOrEqualTo(1)),
+  Schema.annotate({
+    message: "limit must be an integer greater than or equal to 1",
+  }),
+);
+
+export type TavilyMapLimit = typeof TavilyMapLimitSchema.Type;
+
+const responseTimeSchema = Schema.Union([Schema.Number, Schema.NumberFromString]);
+
+export const TavilySearchRequestSchema = Schema.Struct({
+  query: Schema.NonEmptyString,
+  search_depth: Schema.optional(TavilySearchDepthSchema),
+  topic: Schema.optional(TavilySearchTopicSchema),
+  time_range: Schema.optional(TavilyTimeRangeSchema),
+  max_results: Schema.optional(maxResultsSchema),
+  include_answer: Schema.optional(Schema.Boolean),
+});
+
+export type TavilySearchRequest = typeof TavilySearchRequestSchema.Type;
+
+export const TavilySearchResultItemSchema = Schema.Struct({
+  title: Schema.String,
+  url: Schema.String,
+  content: Schema.String,
+  score: Schema.Number,
+  raw_content: Schema.optional(Schema.NullOr(Schema.String)),
+});
+
+export type TavilySearchResultItem = typeof TavilySearchResultItemSchema.Type;
+
+export const TavilySearchResponseSchema = Schema.Struct({
+  query: Schema.String,
+  answer: Schema.optional(Schema.NullOr(Schema.String)),
+  images: Schema.optional(Schema.Array(Schema.String)),
+  response_time: Schema.optional(responseTimeSchema),
+  results: Schema.Array(TavilySearchResultItemSchema),
+});
+
+export type TavilySearchResponse = typeof TavilySearchResponseSchema.Type;
+
+export const TavilyExtractRequestSchema = Schema.Struct({
+  urls: Schema.NonEmptyArray(Schema.String),
+  extract_depth: Schema.optional(TavilyExtractDepthSchema),
+  format: Schema.optional(FetchContentFormatSchema),
+});
+
+export type TavilyExtractRequest = typeof TavilyExtractRequestSchema.Type;
+
+export const TavilyExtractResultItemSchema = Schema.Struct({
+  url: Schema.String,
+  title: Schema.optional(Schema.NullOr(Schema.String)),
+  raw_content: Schema.optional(Schema.NullOr(Schema.String)),
+});
+
+export type TavilyExtractResultItem = typeof TavilyExtractResultItemSchema.Type;
+
+export const TavilyExtractResponseSchema = Schema.Struct({
+  results: Schema.Array(TavilyExtractResultItemSchema),
+});
+
+export type TavilyExtractResponse = typeof TavilyExtractResponseSchema.Type;
+
+export class TavilySearchInput extends Schema.Class<TavilySearchInput>("TavilySearchInput")({
+  query: trimmedNonEmptyStringSchema("query must be a non-empty string"),
+  searchDepth: Schema.Option(TavilySearchDepthSchema),
+  topic: Schema.Option(TavilySearchTopicSchema),
+  timeRange: Schema.Option(TavilyTimeRangeSchema),
+  maxResults: Schema.Option(maxResultsSchema),
+  includeAnswer: Schema.Boolean,
+}) {
+  static decodeEffect = Schema.decodeUnknownEffect(TavilySearchInput);
+}
+
+export const buildTavilySearchRequest = (input: TavilySearchInput): TavilySearchRequest => ({
+  query: input.query,
+  ...(Option.isSome(input.searchDepth) && {
+    search_depth: input.searchDepth.value,
+  }),
+  ...(Option.isSome(input.topic) && {
+    topic: input.topic.value,
+  }),
+  ...(Option.isSome(input.timeRange) && {
+    time_range: input.timeRange.value,
+  }),
+  ...(Option.isSome(input.maxResults) && {
+    max_results: input.maxResults.value,
+  }),
+  ...(input.includeAnswer ? { include_answer: true } : {}),
+});
+
+export const TavilyMapRequestSchema = Schema.Struct({
+  url: Schema.String,
+  max_depth: Schema.optional(TavilyMapDepthSchema),
+  max_breadth: Schema.optional(TavilyMapBreadthSchema),
+  limit: Schema.optional(TavilyMapLimitSchema),
+  instructions: Schema.optional(Schema.NonEmptyString),
+});
+
+export type TavilyMapRequest = typeof TavilyMapRequestSchema.Type;
+
+export const TavilyMapUsageSchema = Schema.Struct({
+  credits_used: Schema.optional(Schema.Number),
+});
+
+export type TavilyMapUsage = typeof TavilyMapUsageSchema.Type;
+
+export const TavilyMapResponseSchema = Schema.Struct({
+  base_url: Schema.String,
+  results: Schema.Array(Schema.String),
+  response_time: Schema.optional(responseTimeSchema),
+  request_id: Schema.optional(Schema.String),
+  usage: Schema.optional(TavilyMapUsageSchema),
+});
+
+export type TavilyMapResponse = typeof TavilyMapResponseSchema.Type;
+
+export class TavilyMapInput extends Schema.Class<TavilyMapInput>("TavilyMapInput")({
+  url: absoluteUrlStringSchema("url must be an absolute URL"),
+  depth: Schema.Option(TavilyMapDepthSchema),
+  breadth: Schema.Option(TavilyMapBreadthSchema),
+  limit: Schema.Option(TavilyMapLimitSchema),
+  instructions: Schema.Option(
+    trimmedNonEmptyStringSchema("instructions must be a non-empty string"),
+  ),
+}) {
+  static decodeEffect = Schema.decodeUnknownEffect(TavilyMapInput);
+}
+
+export const buildTavilyMapRequest = (input: TavilyMapInput): TavilyMapRequest => ({
+  url: input.url,
+  ...(Option.isSome(input.depth) && {
+    max_depth: input.depth.value,
+  }),
+  ...(Option.isSome(input.breadth) && {
+    max_breadth: input.breadth.value,
+  }),
+  ...(Option.isSome(input.limit) && {
+    limit: input.limit.value,
+  }),
+  ...(Option.isSome(input.instructions) && {
+    instructions: input.instructions.value,
+  }),
+});
+
+export const buildTavilyExtractRequest = (input: {
+  readonly urls: readonly [string, ...Array<string>];
+  readonly depth: TavilyExtractDepth;
+  readonly format: FetchContentFormat;
+}): TavilyExtractRequest => {
+  const [firstUrl, ...remainingUrls] = input.urls;
+
+  return {
+    urls: [firstUrl, ...remainingUrls],
+    extract_depth: input.depth,
+    format: input.format,
+  };
+};

package/src/services/dual-search.ts

@@ -0,0 +1,150 @@
+import { Effect, Layer, Result, Schema, ServiceMap } from "effect";
+import {
+  GrokSearchInput,
+  GrokSearchResultSchema,
+  type GrokSearchResult,
+} from "../providers/grok/schema";
+import {
+  TavilySearchDepthSchema,
+  TavilySearchInput,
+  TavilySearchResponseSchema,
+  type TavilySearchResponse,
+  TavilySearchTopicSchema,
+  TavilyTimeRangeSchema,
+} from "../providers/tavily/schema";
+import type { ServicesReturns } from "../shared/effect";
+import type { RenderedError } from "../shared/render-error";
+import { RenderedErrorSchema, renderStructuredError } from "../shared/render-error";
+import { trimmedNonEmptyStringSchema } from "../shared/schema";
+import { GrokSearch } from "./grok-search";
+import { TavilySearch } from "./tavily-search";
+
+export class DualSearchInput extends Schema.Class<DualSearchInput>("DualSearchInput")({
+  query: trimmedNonEmptyStringSchema("query must be a non-empty string"),
+  platform: Schema.Option(Schema.NonEmptyString),
+  model: Schema.Option(Schema.NonEmptyString),
+  searchDepth: Schema.Option(TavilySearchDepthSchema),
+  topic: Schema.Option(TavilySearchTopicSchema),
+  timeRange: Schema.Option(TavilyTimeRangeSchema),
+  maxResults: Schema.Option(
+    Schema.Int.pipe(
+      Schema.check(Schema.isBetween({ minimum: 1, maximum: 20 })),
+      Schema.annotate({
+        message: "max-results must be an integer between 1 and 20",
+      }),
+    ),
+  ),
+  includeAnswer: Schema.Boolean,
+}) {
+  static decodeEffect = Schema.decodeUnknownEffect(DualSearchInput);
+}
+
+export interface DualSearchProviderSuccess<A> {
+  readonly status: "success";
+  readonly result: A;
+}
+
+export interface DualSearchProviderFailure {
+  readonly status: "error";
+  readonly error: RenderedError;
+}
+
+export type DualSearchProviderResult<A> = DualSearchProviderSuccess<A> | DualSearchProviderFailure;
+
+export interface DualSearchResult {
+  readonly grok: DualSearchProviderResult<GrokSearchResult>;
+  readonly tavily: DualSearchProviderResult<TavilySearchResponse>;
+}
+
+const dualSearchProviderSuccessSchema = <A extends Schema.Top>(resultSchema: A) =>
+  Schema.Struct({
+    status: Schema.Literal("success"),
+    result: resultSchema,
+  });
+
+const DualSearchProviderFailureSchema = Schema.Struct({
+  status: Schema.Literal("error"),
+  error: RenderedErrorSchema,
+});
+
+export const DualSearchResultSchema = Schema.Struct({
+  grok: Schema.Union([
+    dualSearchProviderSuccessSchema(GrokSearchResultSchema),
+    DualSearchProviderFailureSchema,
+  ]),
+  tavily: Schema.Union([
+    dualSearchProviderSuccessSchema(TavilySearchResponseSchema),
+    DualSearchProviderFailureSchema,
+  ]),
+});
+
+const toProviderResult = <A, E>(result: Result.Result<A, E>): DualSearchProviderResult<A> =>
+  Result.match(result, {
+    onSuccess: (value) => ({
+      status: "success",
+      result: value,
+    }),
+    onFailure: (error) => ({
+      status: "error",
+      error: renderStructuredError(error),
+    }),
+  });
+
+export class DualSearch extends ServiceMap.Service<
+  DualSearch,
+  {
+    readonly search: (input: DualSearchInput) => Effect.Effect<DualSearchResult>;
+  }
+>()("DualSearch") {
+  static readonly layer = Layer.effect(
+    DualSearch,
+    Effect.gen(function* () {
+      const grokSearch = yield* GrokSearch;
+      const tavilySearch = yield* TavilySearch;
+
+      const search: DualSearch.Methods["search"] = Effect.fn("DualSearch.search")(
+        function* (input): DualSearch.Returns<"search"> {
+          const results = yield* Effect.all(
+            {
+              grok: grokSearch.search(
+                yield* GrokSearchInput.decodeEffect({
+                  query: input.query,
+                  platform: input.platform,
+                  model: input.model,
+                }).pipe(Effect.orDie),
+              ),
+              tavily: tavilySearch.search(
+                yield* TavilySearchInput.decodeEffect({
+                  query: input.query,
+                  searchDepth: input.searchDepth,
+                  topic: input.topic,
+                  timeRange: input.timeRange,
+                  maxResults: input.maxResults,
+                  includeAnswer: input.includeAnswer,
+                }).pipe(Effect.orDie),
+              ),
+            },
+            {
+              concurrency: "unbounded",
+              mode: "result",
+            },
+          );
+
+          return {
+            grok: toProviderResult(results.grok),
+            tavily: toProviderResult(results.tavily),
+          };
+        },
+      );
+
+      return DualSearch.of({
+        search,
+      });
+    }),
+  );
+}
+
+export declare namespace DualSearch {
+  export type Methods = ServiceMap.Service.Shape<typeof DualSearch>;
+  export type Returns<key extends keyof Methods, R = never> = ServicesReturns<Methods[key], R>;
+}

package/src/services/firecrawl-fetch.ts

@@ -0,0 +1,101 @@
+import { Effect, Layer, Result, ServiceMap } from "effect";
+import { FirecrawlProviderClient } from "../providers/firecrawl/client";
+import type { FirecrawlScrapeResponse } from "../providers/firecrawl/schema";
+import { ProviderContentError, type UltimateSearchError } from "../shared/errors";
+import type { ServicesReturns } from "../shared/effect";
+import type { FetchedPage, WebFetchInput } from "./web-fetch-schema";
+
+const normalizeContent = (response: FirecrawlScrapeResponse, format: WebFetchInput["format"]) => {
+  const data = response.data;
+
+  if (data == null) {
+    return null;
+  }
+
+  const preferred = format === "text" ? data.content : data.markdown;
+  const fallback = format === "text" ? data.markdown : data.content;
+  const content = (preferred ?? fallback ?? "").trim();
+
+  return content.length > 0 ? content : null;
+};
+
+export class FirecrawlFetch extends ServiceMap.Service<
+  FirecrawlFetch,
+  {
+    readonly fetch: (
+      input: WebFetchInput,
+    ) => Effect.Effect<ReadonlyArray<FetchedPage>, UltimateSearchError, never>;
+  }
+>()("FirecrawlFetch") {
+  static readonly layer = Layer.effect(
+    FirecrawlFetch,
+    Effect.gen(function* () {
+      const provider = yield* FirecrawlProviderClient;
+
+      const fetch: FirecrawlFetch.Methods["fetch"] = Effect.fn("FirecrawlFetch.fetch")(
+        function* (input): FirecrawlFetch.Returns<"fetch"> {
+          const attempts = yield* Effect.forEach(
+            input.urls,
+            (url) =>
+              Effect.result(
+                provider.scrape({
+                  url,
+                  formats: ["markdown"],
+                }),
+              ).pipe(
+                Effect.map((result) => ({
+                  url,
+                  result,
+                })),
+              ),
+            { concurrency: "unbounded" },
+          );
+
+          const successes = attempts.flatMap(({ url, result }) => {
+            if (Result.isFailure(result)) {
+              return [];
+            }
+
+            const rawContent = normalizeContent(result.success, input.format);
+
+            if (rawContent === null) {
+              return [];
+            }
+
+            return [
+              {
+                url,
+                title: result.success.data?.metadata?.title,
+                raw_content: rawContent,
+              } satisfies FetchedPage,
+            ];
+          });
+
+          if (successes.length > 0) {
+            return successes;
+          }
+
+          const firstFailure = attempts.find(({ result }) => Result.isFailure(result));
+
+          if (firstFailure != null && Result.isFailure(firstFailure.result)) {
+            return yield* firstFailure.result.failure;
+          }
+
+          return yield* new ProviderContentError({
+            provider: "firecrawl",
+            message: "FireCrawl returned no extractable content.",
+          });
+        },
+      );
+
+      return FirecrawlFetch.of({
+        fetch,
+      });
+    }),
+  );
+}
+
+export declare namespace FirecrawlFetch {
+  export type Methods = ServiceMap.Service.Shape<typeof FirecrawlFetch>;
+  export type Returns<key extends keyof Methods, R = never> = ServicesReturns<Methods[key], R>;
+}

package/src/services/grok-search.ts

@@ -0,0 +1,68 @@
+import { Effect, Layer, Option, ServiceMap } from "effect";
+import { UltimateSearchConfig } from "../config/settings";
+import { GrokProviderClient } from "../providers/grok/client";
+import {
+  buildGrokUserMessage,
+  type GrokChatCompletionRequest,
+  GrokSearchInput,
+  type GrokSearchResult,
+  grokSystemPrompt,
+} from "../providers/grok/schema";
+import type { UltimateSearchError } from "../shared/errors";
+import type { ServicesReturns } from "../shared/effect";
+
+export class GrokSearch extends ServiceMap.Service<
+  GrokSearch,
+  {
+    readonly search: (
+      input: GrokSearchInput,
+    ) => Effect.Effect<GrokSearchResult, UltimateSearchError, never>;
+  }
+>()("GrokSearch") {
+  //  也可以单独定义成变量或者class上的静态方法
+  static readonly layer = Layer.effect(
+    GrokSearch,
+    Effect.gen(function* () {
+      const config = yield* UltimateSearchConfig;
+      const provider = yield* GrokProviderClient;
+
+      const search: GrokSearch.Methods["search"] = Effect.fn("search")(
+        function* (input): GrokSearch.Returns<"search"> {
+          const grok = yield* config.getGrokConfig();
+          const payload: GrokChatCompletionRequest = {
+            model: Option.getOrElse(input.model, () => grok.model),
+            stream: false,
+            messages: [
+              {
+                role: "system",
+                content: grokSystemPrompt,
+              },
+              {
+                role: "user",
+                content: buildGrokUserMessage(input),
+              },
+            ],
+          };
+          const response = yield* provider.createChatCompletion(payload);
+
+          return {
+            content: response.choices[0].message.content,
+            model: response.model,
+            usage: response.usage,
+          } satisfies GrokSearchResult;
+        },
+      );
+
+      return GrokSearch.of({
+        search,
+      });
+    }),
+  );
+
+  // 也可以再定义 live = Layer.provide(xxx.layer, deps) // 无依赖的 Layer<A, E, never>
+}
+
+export declare namespace GrokSearch {
+  export type Methods = ServiceMap.Service.Shape<typeof GrokSearch>;
+  export type Returns<key extends keyof Methods, R = never> = ServicesReturns<Methods[key], R>;
+}