@effect-x/ultimate-search 0.1.0 → 0.1.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@effect-x/ultimate-search",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "description": "CLI-first web search and MCP server for agents, built with Effect.",
   "keywords": [
     "agent",
@@ -12,12 +12,23 @@
     "tavily"
   ],
   "license": "MIT",
-  "author": "effect-x contributors",
+  "repository": {
+     "type": "git",
+     "url": "https://github.com/effect-anything/ultimate-search"
+   },
+   "homepage": "https://github.com/effect-anything/ultimate-search#readme",
+   "bugs": {
+     "url": "https://github.com/effect-anything/ultimate-search/issues"
+   },
+  "contributors": [
+    "xesrevinu"
+  ],
   "bin": {
-    "ultimate-search": "./dist/cli.js"
+    "ultimate-search": "dist/cli.js"
   },
   "files": [
     "dist",
+    "src",
     "README.md",
     "LICENSE",
     "SKILL.md"
@@ -33,13 +44,7 @@
     "madge": "madge --circular --ts-config ./tsconfig.json --no-color --no-spinner --extensions 'ts,tsx' ./src",
     "check": "concurrently \"oxlint\" \"tsgo --noEmit\" \"bun run test --watch false\" \"bun run madge\" \"effect-language-service diagnostics --project tsconfig.json\"",
     "fix": "concurrently \"oxlint --fix\" \"oxfmt\"",
-    "build": "node ./tools/build-cli.mjs",
-    "prepack": "bun run build",
-    "pack:check": "npm pack --dry-run",
-    "release:check": "bun run check && bun run pack:check",
-    "release:version": "changeset version",
-    "release:publish": "changeset publish",
-    "prepublishOnly": "bun run release:check"
+    "build": "node ./tools/build-cli.mjs"
   },
   "dependencies": {
     "@effect/platform-node": "4.0.0-beta.31"
@@ -60,7 +65,6 @@
     "typescript": "^5.9.3"
   },
   "engines": {
-    "node": ">=20"
-  },
-  "packageManager": "bun@1.3.10"
+    "node": ">=24"
+  }
 }

package/src/cli.ts

@@ -0,0 +1,21 @@
+#!/usr/bin/env node
+import { NodeHttpClient, NodeRuntime, NodeServices } from "@effect/platform-node";
+import { ConfigProvider, Effect, Layer, Logger } from "effect";
+import { Command } from "effect/unstable/cli";
+import PackageJson from "../package.json" with { type: "json" };
+import { commandRoot } from "./commands/root";
+import { CliOutput, cliLoggerLayer } from "./shared/output";
+import { TracingLayer } from "./shared/tracing";
+
+const Live = Layer.mergeAll(
+  NodeServices.layer,
+  NodeHttpClient.layerFetch,
+  cliLoggerLayer,
+  CliOutput.layer,
+  Layer.succeed(Logger.LogToStderr, true),
+).pipe(Layer.provide([TracingLayer, ConfigProvider.layer(ConfigProvider.fromEnv())]));
+
+NodeRuntime.runMain(
+  Command.run(commandRoot, { version: PackageJson.version }).pipe(Effect.provide(Live)),
+  { disableErrorReporting: true },
+);

package/src/commands/fetch.ts

@@ -0,0 +1,98 @@
+import { Effect, Layer } from "effect";
+import { Command, Flag } from "effect/unstable/cli";
+import { UltimateSearchConfig } from "../config/settings";
+import { FirecrawlProviderClient } from "../providers/firecrawl/client";
+import {
+  FetchContentFormatSchema,
+  TavilyExtractDepthSchema,
+} from "../providers/tavily/schema";
+import { TavilyProviderClient } from "../providers/tavily/client";
+import { FirecrawlFetch } from "../services/firecrawl-fetch";
+import { TavilyExtract } from "../services/tavily-extract";
+import { WebFetch } from "../services/web-fetch";
+import { WebFetchInput, type WebFetchResult } from "../services/web-fetch-schema";
+import { runCommandWithOutput } from "../shared/command-output";
+import { outputFlag } from "../shared/output";
+import { absoluteUrlStringSchema } from "../shared/schema";
+
+const fetchCommandLayer = WebFetch.layer.pipe(
+  Layer.provideMerge(FirecrawlFetch.layer),
+  Layer.provideMerge(TavilyExtract.layer),
+  Layer.provideMerge(FirecrawlProviderClient.layer),
+  Layer.provideMerge(TavilyProviderClient.layer),
+  Layer.provideMerge(UltimateSearchConfig.layer),
+);
+
+const renderHumanFetchResult = (result: WebFetchResult) => {
+  const lines = [`Backend: ${result.backend}`];
+
+  if (result.fallback != null) {
+    lines.push(
+      `Fallback: ${result.fallback.from} -> ${result.fallback.to} (${result.fallback.reason.message})`,
+    );
+  }
+
+  for (const [index, page] of result.results.entries()) {
+    lines.push("");
+    lines.push(`URL: ${page.url}`);
+
+    if (page.title != null && page.title.trim().length > 0) {
+      lines.push(`Title: ${page.title.trim()}`);
+    }
+
+    lines.push("");
+    lines.push(page.raw_content);
+
+    if (index < result.results.length - 1) {
+      lines.push("", "---");
+    }
+  }
+
+  return lines.join("\n");
+};
+
+export const commandFetch = Command.make(
+  "fetch",
+  {
+    url: Flag.string("url").pipe(
+      Flag.withSchema(absoluteUrlStringSchema("url must be an absolute URL")),
+      Flag.withDescription("Target page URL."),
+    ),
+    depth: Flag.choice("depth", TavilyExtractDepthSchema.literals).pipe(
+      Flag.withDefault("basic"),
+      Flag.withDescription("Optional Tavily extract depth."),
+    ),
+    format: Flag.choice("format", FetchContentFormatSchema.literals).pipe(
+      Flag.withDefault("markdown"),
+      Flag.withDescription("Normalized content format to return."),
+    ),
+    output: outputFlag,
+  },
+  Effect.fn(function* (input) {
+    yield* runCommandWithOutput(input.output, (mode) =>
+      Effect.gen(function* () {
+        const request = yield* WebFetchInput.decodeEffect({
+          urls: [input.url],
+          depth: input.depth,
+          format: input.format,
+        });
+        const webFetch = yield* WebFetch;
+        const result = yield* webFetch.fetch(request);
+
+        return {
+          human: mode === "human" ? renderHumanFetchResult(result) : "",
+          llm: result,
+        };
+      }),
+    );
+  }),
+).pipe(
+  Command.withDescription("Fetch and normalize page content from a URL."),
+  Command.withExamples([
+    {
+      command: 'ultimate-search fetch --url "https://example.com"',
+      description: "Fetch a page with Tavily first and FireCrawl as fallback.",
+    },
+  ]),
+  Command.provide(fetchCommandLayer),
+);

package/src/commands/map.ts

@@ -0,0 +1,103 @@
+import { Effect, Layer } from "effect";
+import { Command, Flag } from "effect/unstable/cli";
+import { UltimateSearchConfig } from "../config/settings";
+import {
+  TavilyMapBreadthSchema,
+  TavilyMapDepthSchema,
+  TavilyMapInput,
+  TavilyMapLimitSchema,
+  type TavilyMapResponse,
+} from "../providers/tavily/schema";
+import { TavilyProviderClient } from "../providers/tavily/client";
+import { TavilyMap } from "../services/tavily-map";
+import {
+  optionalIntegerFlagWithSchema,
+  optionalTrimmedTextFlag,
+} from "../shared/cli-flags";
+import { runCommandWithOutput } from "../shared/command-output";
+import { outputFlag } from "../shared/output";
+import { absoluteUrlStringSchema } from "../shared/schema";
+
+const mapCommandLayer = TavilyMap.layer.pipe(
+  Layer.provideMerge(TavilyProviderClient.layer),
+  Layer.provideMerge(UltimateSearchConfig.layer),
+);
+
+const renderHumanMapResult = (result: TavilyMapResponse) => {
+  const lines = [
+    `Base URL: ${result.base_url}`,
+    `Discovered URLs: ${result.results.length}`,
+  ];
+
+  if (result.response_time !== undefined) {
+    lines.push(`Response time: ${result.response_time}s`);
+  }
+
+  if (result.usage?.credits_used !== undefined) {
+    lines.push(`Credits used: ${result.usage.credits_used}`);
+  }
+
+  if (result.results.length > 0) {
+    lines.push("", ...result.results.map((url) => `- ${url}`));
+  }
+
+  return lines.join("\n");
+};
+
+export const commandMap = Command.make(
+  "map",
+  {
+    url: Flag.string("url").pipe(
+      Flag.withSchema(absoluteUrlStringSchema("url must be an absolute URL")),
+      Flag.withDescription("Root URL to map with Tavily."),
+    ),
+    depth: optionalIntegerFlagWithSchema(
+      "depth",
+      TavilyMapDepthSchema,
+      "Optional crawl depth between 1 and 5.",
+    ),
+    breadth: optionalIntegerFlagWithSchema(
+      "breadth",
+      TavilyMapBreadthSchema,
+      "Optional crawl breadth between 1 and 500.",
+    ),
+    limit: optionalIntegerFlagWithSchema(
+      "limit",
+      TavilyMapLimitSchema,
+      "Optional maximum number of discovered URLs to return.",
+    ),
+    instructions: optionalTrimmedTextFlag(
+      "instructions",
+      "Optional guidance for how Tavily should explore the site.",
+    ),
+    output: outputFlag,
+  },
+  Effect.fn(function* (input) {
+    yield* runCommandWithOutput(input.output, () =>
+      Effect.gen(function* () {
+        const request = yield* TavilyMapInput.decodeEffect(input);
+        const tavilyMap = yield* TavilyMap;
+        const result = yield* tavilyMap.map(request);
+
+        return {
+          human: renderHumanMapResult(result),
+          llm: result,
+        };
+      }),
+    );
+  }),
+).pipe(
+  Command.withDescription("Map a site's reachable URLs with Tavily."),
+  Command.withExamples([
+    {
+      command:
+        'ultimate-search map --url "https://fastapi.tiangolo.com" --depth 2 --breadth 20 --limit 50',
+      description: "Discover reachable URLs for a site with Tavily map.",
+    },
+    {
+      command: 'ultimate-search map --url "https://fastapi.tiangolo.com" --output llm',
+      description: "Emit structured JSON for agent-driven workflows.",
+    },
+  ]),
+  Command.provide(mapCommandLayer),
+);

package/src/commands/mcp/stdio.ts

@@ -0,0 +1,27 @@
+import { Cause, Effect, Layer } from "effect";
+import { McpServer } from "effect/unstable/ai";
+import { Command } from "effect/unstable/cli";
+import PackageJson from "../../../package.json" with { type: "json" };
+import {
+  readOnlyMcpRegistrationLayer,
+  readOnlyMcpServicesLayer,
+} from "../../services/read-only-mcp";
+
+const mcpStdioServerLayer = readOnlyMcpRegistrationLayer.pipe(
+  Layer.provideMerge(
+    McpServer.layerStdio({
+      name: "ultimate-search",
+      version: PackageJson.version,
+    }),
+  ),
+  Layer.provideMerge(readOnlyMcpServicesLayer),
+);
+
+export const commandMcpStdio = Command.make("stdio").pipe(
+  Command.withDescription("Serve the MCP protocol over stdio."),
+  Command.withHandler(() =>
+    Layer.launch(mcpStdioServerLayer).pipe(
+      Effect.catchCauseIf(Cause.hasInterruptsOnly, () => Effect.void),
+    ),
+  ),
+);

package/src/commands/mcp.ts

@@ -0,0 +1,7 @@
+import { Command } from "effect/unstable/cli";
+import { commandMcpStdio } from "./mcp/stdio";
+
+export const commandMcp = Command.make("mcp").pipe(
+  Command.withDescription("Expose ultimate-search over MCP transports."),
+  Command.withSubcommands([commandMcpStdio]),
+);

package/src/commands/root.ts

@@ -0,0 +1,10 @@
+import { Command } from "effect/unstable/cli";
+import { commandFetch } from "./fetch";
+import { commandMap } from "./map";
+import { commandMcp } from "./mcp";
+import { commandSearch } from "./search";
+
+export const commandRoot = Command.make("ultimate-search").pipe(
+  Command.withDescription("CLI entrypoint for search, fetch, map, and MCP workflows."),
+  Command.withSubcommands([commandSearch, commandFetch, commandMap, commandMcp]),
+);

package/src/commands/search/dual.ts

@@ -0,0 +1,91 @@
+import { Effect, Layer } from "effect";
+import { Command, Flag } from "effect/unstable/cli";
+import { UltimateSearchConfig } from "../../config/settings";
+import { GrokProviderClient } from "../../providers/grok/client";
+import {
+  TavilySearchDepthSchema,
+  TavilySearchTopicSchema,
+  TavilyTimeRangeSchema,
+} from "../../providers/tavily/schema";
+import { TavilyProviderClient } from "../../providers/tavily/client";
+import { DualSearch, DualSearchInput } from "../../services/dual-search";
+import { GrokSearch } from "../../services/grok-search";
+import { TavilySearch } from "../../services/tavily-search";
+import {
+  optionalChoiceFlag,
+  optionalIntegerFlag,
+  optionalTrimmedTextFlag,
+} from "../../shared/cli-flags";
+import { runCommandWithOutput } from "../../shared/command-output";
+import { outputFlag, renderJsonText } from "../../shared/output";
+import { trimmedNonEmptyStringSchema } from "../../shared/schema";
+
+const dualCommandLayer = DualSearch.layer.pipe(
+  Layer.provideMerge(GrokSearch.layer),
+  Layer.provideMerge(TavilySearch.layer),
+  Layer.provideMerge(GrokProviderClient.layer),
+  Layer.provideMerge(TavilyProviderClient.layer),
+  Layer.provideMerge(UltimateSearchConfig.layer),
+);
+
+export const commandSearchDual = Command.make(
+  "dual",
+  {
+    query: Flag.string("query").pipe(
+      Flag.withSchema(trimmedNonEmptyStringSchema("query must be a non-empty string")),
+      Flag.withDescription("Search query to send to both Grok and Tavily."),
+    ),
+    platform: optionalTrimmedTextFlag(
+      "platform",
+      "Optional platform focus for the Grok branch such as GitHub or Reddit.",
+    ),
+    model: optionalTrimmedTextFlag("model", "Override the configured Grok model."),
+    searchDepth: optionalChoiceFlag(
+      "depth",
+      TavilySearchDepthSchema.literals,
+      "Optional Tavily search depth.",
+    ),
+    maxResults: optionalIntegerFlag(
+      "max-results",
+      "Optional number of Tavily results to return.",
+    ),
+    topic: optionalChoiceFlag(
+      "topic",
+      TavilySearchTopicSchema.literals,
+      "Optional Tavily topic focus.",
+    ),
+    timeRange: optionalChoiceFlag(
+      "time-range",
+      TavilyTimeRangeSchema.literals,
+      "Optional Tavily recency window.",
+    ),
+    includeAnswer: Flag.boolean("include-answer").pipe(
+      Flag.withDescription("Request a synthesized Tavily answer in the response."),
+    ),
+    output: outputFlag,
+  },
+  Effect.fn(function* (input) {
+    yield* runCommandWithOutput(input.output, () =>
+      Effect.gen(function* () {
+        const request = yield* DualSearchInput.decodeEffect(input);
+        const dualSearch = yield* DualSearch;
+        const result = yield* dualSearch.search(request);
+
+        return {
+          human: renderJsonText(result),
+          llm: result,
+        };
+      }),
+    );
+  }),
+).pipe(
+  Command.withDescription("Run Grok and Tavily search concurrently."),
+  Command.withExamples([
+    {
+      command:
+        'ultimate-search search dual --query "FastAPI latest features" --depth advanced --include-answer',
+      description: "Run both providers and merge their status-tagged results into one JSON object.",
+    },
+  ]),
+  Command.provide(dualCommandLayer),
+);

package/src/commands/search/grok.ts

@@ -0,0 +1,70 @@
+import { Effect, Layer } from "effect";
+import { Command, Flag } from "effect/unstable/cli";
+import { UltimateSearchConfig } from "../../config/settings";
+import { GrokProviderClient } from "../../providers/grok/client";
+import { GrokSearchInput, type GrokSearchResult } from "../../providers/grok/schema";
+import { GrokSearch } from "../../services/grok-search";
+import { optionalTrimmedTextFlag } from "../../shared/cli-flags";
+import { runCommandWithOutput } from "../../shared/command-output";
+import { outputFlag } from "../../shared/output";
+import { trimmedNonEmptyStringSchema } from "../../shared/schema";
+
+const grokCommandLayer = GrokSearch.layer.pipe(
+  Layer.provideMerge(GrokProviderClient.layer),
+  Layer.provideMerge(UltimateSearchConfig.layer),
+);
+
+const renderHumanGrokResult = (result: GrokSearchResult) =>
+  [
+    result.content.trim().length > 0 ? result.content.trim() : "Grok returned an empty response.",
+    "",
+    `Model: ${result.model}`,
+    `Tokens: ${result.usage.total_tokens} total (${result.usage.prompt_tokens} prompt, ${result.usage.completion_tokens} completion)`,
+  ].join("\n");
+
+export const commandSearchGrok = Command.make(
+  "grok",
+  {
+    query: Flag.string("query").pipe(
+      Flag.withSchema(trimmedNonEmptyStringSchema("query must be a non-empty string")),
+      Flag.withDescription("Search query to send to Grok."),
+    ),
+    platform: optionalTrimmedTextFlag(
+      "platform",
+      "Optional platform focus such as GitHub or Reddit.",
+    ),
+    model: optionalTrimmedTextFlag("model", "Override the configured Grok model."),
+    output: outputFlag,
+  },
+  Effect.fn(function* (input) {
+    yield* runCommandWithOutput(input.output, () =>
+      Effect.gen(function* () {
+        const request = yield* GrokSearchInput.decodeEffect({
+          query: input.query,
+          platform: input.platform,
+          model: input.model,
+        });
+        const grokSearch = yield* GrokSearch;
+        const result = yield* grokSearch.search(request);
+
+        return {
+          human: renderHumanGrokResult(result),
+          llm: result,
+        };
+      }),
+    );
+  }),
+).pipe(
+  Command.withDescription("Run Grok-backed search."),
+  Command.withExamples([
+    {
+      command: 'ultimate-search search grok --query "FastAPI latest features"',
+      description: "Run a Grok-backed web search with the configured model.",
+    },
+    {
+      command: 'ultimate-search search grok --query "FastAPI latest features" --output llm',
+      description: "Emit structured JSON for agent-driven workflows.",
+    },
+  ]),
+  Command.provide(grokCommandLayer),
+);

package/src/commands/search/tavily.ts

@@ -0,0 +1,102 @@
+import { Effect, Layer } from "effect";
+import { Command, Flag } from "effect/unstable/cli";
+import { UltimateSearchConfig } from "../../config/settings";
+import {
+  TavilySearchDepthSchema,
+  TavilySearchInput,
+  type TavilySearchResponse,
+  TavilySearchTopicSchema,
+  TavilyTimeRangeSchema,
+} from "../../providers/tavily/schema";
+import { TavilyProviderClient } from "../../providers/tavily/client";
+import { TavilySearch } from "../../services/tavily-search";
+import { optionalChoiceFlag, optionalIntegerFlag } from "../../shared/cli-flags";
+import { runCommandWithOutput } from "../../shared/command-output";
+import { outputFlag } from "../../shared/output";
+import { trimmedNonEmptyStringSchema } from "../../shared/schema";
+
+const tavilyCommandLayer = TavilySearch.layer.pipe(
+  Layer.provideMerge(TavilyProviderClient.layer),
+  Layer.provideMerge(UltimateSearchConfig.layer),
+);
+
+const renderHumanTavilyResult = (result: TavilySearchResponse) => {
+  const lines: Array<string> = [];
+
+  if (result.answer != null && result.answer.trim().length > 0) {
+    lines.push(result.answer.trim(), "");
+  }
+
+  if (result.response_time != null) {
+    lines.push(`Response time: ${result.response_time}s`, "");
+  }
+
+  for (const [index, item] of result.results.entries()) {
+    lines.push(`${index + 1}. ${item.title}`);
+    lines.push(item.url);
+    lines.push(item.content.trim());
+
+    if (index < result.results.length - 1) {
+      lines.push("");
+    }
+  }
+
+  return lines.join("\n").trim();
+};
+
+export const commandSearchTavily = Command.make(
+  "tavily",
+  {
+    query: Flag.string("query").pipe(
+      Flag.withSchema(trimmedNonEmptyStringSchema("query must be a non-empty string")),
+      Flag.withDescription("Search query to send to Tavily."),
+    ),
+    searchDepth: optionalChoiceFlag(
+      "depth",
+      TavilySearchDepthSchema.literals,
+      "Optional Tavily search depth.",
+    ),
+    maxResults: optionalIntegerFlag(
+      "max-results",
+      "Optional number of Tavily results to return.",
+    ),
+    topic: optionalChoiceFlag(
+      "topic",
+      TavilySearchTopicSchema.literals,
+      "Optional Tavily topic focus.",
+    ),
+    timeRange: optionalChoiceFlag(
+      "time-range",
+      TavilyTimeRangeSchema.literals,
+      "Optional Tavily recency window.",
+    ),
+    includeAnswer: Flag.boolean("include-answer").pipe(
+      Flag.withDescription("Request a synthesized Tavily answer in the response."),
+    ),
+    output: outputFlag,
+  },
+  Effect.fn(function* (input) {
+    yield* runCommandWithOutput(input.output, (mode) =>
+      Effect.gen(function* () {
+        const request = yield* TavilySearchInput.decodeEffect(input);
+        const tavilySearch = yield* TavilySearch;
+        const result = yield* tavilySearch.search(request);
+
+        return {
+          human: mode === "human" ? renderHumanTavilyResult(result) : "",
+          llm: result,
+        };
+      }),
+    );
+  }),
+).pipe(
+  Command.withDescription("Run Tavily-backed search."),
+  Command.withExamples([
+    {
+      command:
+        'ultimate-search search tavily --query "FastAPI latest features" --depth advanced --max-results 5',
+      description: "Run a Tavily-backed web search with optional search tuning.",
+    },
+  ]),
+  Command.provide(tavilyCommandLayer),
+);

package/src/commands/search.ts

@@ -0,0 +1,9 @@
+import { Command } from "effect/unstable/cli";
+import { commandSearchDual } from "./search/dual";
+import { commandSearchGrok } from "./search/grok";
+import { commandSearchTavily } from "./search/tavily";
+
+export const commandSearch = Command.make("search").pipe(
+  Command.withDescription("Search the web with one or more providers."),
+  Command.withSubcommands([commandSearchGrok, commandSearchTavily, commandSearchDual]),
+);