overai 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +248 -0
- package/dist/agent/context.d.ts +74 -0
- package/dist/agent/context.js +140 -0
- package/dist/agent/enhanced.d.ts +98 -0
- package/dist/agent/enhanced.js +293 -0
- package/dist/agent/handoff.d.ts +82 -0
- package/dist/agent/handoff.js +124 -0
- package/dist/agent/image.d.ts +57 -0
- package/dist/agent/image.js +116 -0
- package/dist/agent/index.d.ts +19 -0
- package/dist/agent/index.js +38 -0
- package/dist/agent/prompt-expander.d.ts +43 -0
- package/dist/agent/prompt-expander.js +99 -0
- package/dist/agent/proxy.d.ts +26 -0
- package/dist/agent/proxy.js +173 -0
- package/dist/agent/query-rewriter.d.ts +41 -0
- package/dist/agent/query-rewriter.js +94 -0
- package/dist/agent/research.d.ts +55 -0
- package/dist/agent/research.js +134 -0
- package/dist/agent/router.d.ts +141 -0
- package/dist/agent/router.js +209 -0
- package/dist/agent/simple.d.ts +289 -0
- package/dist/agent/simple.js +696 -0
- package/dist/agent/types.d.ts +56 -0
- package/dist/agent/types.js +165 -0
- package/dist/ai/agent-loop.d.ts +177 -0
- package/dist/ai/agent-loop.js +313 -0
- package/dist/ai/devtools.d.ts +74 -0
- package/dist/ai/devtools.js +216 -0
- package/dist/ai/embed.d.ts +71 -0
- package/dist/ai/embed.js +154 -0
- package/dist/ai/generate-image.d.ts +66 -0
- package/dist/ai/generate-image.js +138 -0
- package/dist/ai/generate-object.d.ts +132 -0
- package/dist/ai/generate-object.js +206 -0
- package/dist/ai/generate-text.d.ts +201 -0
- package/dist/ai/generate-text.js +253 -0
- package/dist/ai/index.d.ts +27 -0
- package/dist/ai/index.js +140 -0
- package/dist/ai/mcp.d.ts +181 -0
- package/dist/ai/mcp.js +234 -0
- package/dist/ai/middleware.d.ts +110 -0
- package/dist/ai/middleware.js +186 -0
- package/dist/ai/models.d.ts +67 -0
- package/dist/ai/models.js +249 -0
- package/dist/ai/multimodal.d.ts +125 -0
- package/dist/ai/multimodal.js +270 -0
- package/dist/ai/nextjs.d.ts +122 -0
- package/dist/ai/nextjs.js +218 -0
- package/dist/ai/server.d.ts +126 -0
- package/dist/ai/server.js +336 -0
- package/dist/ai/speech.d.ts +135 -0
- package/dist/ai/speech.js +296 -0
- package/dist/ai/telemetry.d.ts +162 -0
- package/dist/ai/telemetry.js +311 -0
- package/dist/ai/tool-approval.d.ts +225 -0
- package/dist/ai/tool-approval.js +407 -0
- package/dist/ai/tools.d.ts +93 -0
- package/dist/ai/tools.js +89 -0
- package/dist/ai/types.d.ts +47 -0
- package/dist/ai/types.js +5 -0
- package/dist/ai/ui-message.d.ts +218 -0
- package/dist/ai/ui-message.js +291 -0
- package/dist/auto/index.d.ts +62 -0
- package/dist/auto/index.js +163 -0
- package/dist/cache/index.d.ts +78 -0
- package/dist/cache/index.js +235 -0
- package/dist/cli/commands/agent.d.ts +30 -0
- package/dist/cli/commands/agent.js +151 -0
- package/dist/cli/commands/agents.d.ts +24 -0
- package/dist/cli/commands/agents.js +166 -0
- package/dist/cli/commands/approval.d.ts +10 -0
- package/dist/cli/commands/approval.js +253 -0
- package/dist/cli/commands/auto.d.ts +12 -0
- package/dist/cli/commands/auto.js +109 -0
- package/dist/cli/commands/autonomy.d.ts +5 -0
- package/dist/cli/commands/autonomy.js +152 -0
- package/dist/cli/commands/benchmark.d.ts +19 -0
- package/dist/cli/commands/benchmark.js +484 -0
- package/dist/cli/commands/cache.d.ts +9 -0
- package/dist/cli/commands/cache.js +143 -0
- package/dist/cli/commands/chat.d.ts +14 -0
- package/dist/cli/commands/chat.js +127 -0
- package/dist/cli/commands/checkpoints.d.ts +5 -0
- package/dist/cli/commands/checkpoints.js +236 -0
- package/dist/cli/commands/context.d.ts +11 -0
- package/dist/cli/commands/context.js +182 -0
- package/dist/cli/commands/cost.d.ts +5 -0
- package/dist/cli/commands/cost.js +146 -0
- package/dist/cli/commands/db.d.ts +9 -0
- package/dist/cli/commands/db.js +149 -0
- package/dist/cli/commands/embed.d.ts +20 -0
- package/dist/cli/commands/embed.js +375 -0
- package/dist/cli/commands/eval.d.ts +17 -0
- package/dist/cli/commands/eval.js +247 -0
- package/dist/cli/commands/external-agents.d.ts +5 -0
- package/dist/cli/commands/external-agents.js +170 -0
- package/dist/cli/commands/fast-context.d.ts +5 -0
- package/dist/cli/commands/fast-context.js +126 -0
- package/dist/cli/commands/flow.d.ts +5 -0
- package/dist/cli/commands/flow.js +141 -0
- package/dist/cli/commands/git.d.ts +5 -0
- package/dist/cli/commands/git.js +178 -0
- package/dist/cli/commands/graph-rag.d.ts +9 -0
- package/dist/cli/commands/graph-rag.js +131 -0
- package/dist/cli/commands/guardrail.d.ts +11 -0
- package/dist/cli/commands/guardrail.js +156 -0
- package/dist/cli/commands/handoff.d.ts +9 -0
- package/dist/cli/commands/handoff.js +133 -0
- package/dist/cli/commands/help.d.ts +8 -0
- package/dist/cli/commands/help.js +139 -0
- package/dist/cli/commands/image.d.ts +13 -0
- package/dist/cli/commands/image.js +192 -0
- package/dist/cli/commands/interactive.d.ts +6 -0
- package/dist/cli/commands/interactive.js +79 -0
- package/dist/cli/commands/jobs.d.ts +5 -0
- package/dist/cli/commands/jobs.js +282 -0
- package/dist/cli/commands/knowledge.d.ts +9 -0
- package/dist/cli/commands/knowledge.js +226 -0
- package/dist/cli/commands/llm.d.ts +21 -0
- package/dist/cli/commands/llm.js +748 -0
- package/dist/cli/commands/mcp.d.ts +9 -0
- package/dist/cli/commands/mcp.js +243 -0
- package/dist/cli/commands/memory.d.ts +10 -0
- package/dist/cli/commands/memory.js +198 -0
- package/dist/cli/commands/n8n.d.ts +5 -0
- package/dist/cli/commands/n8n.js +137 -0
- package/dist/cli/commands/observability.d.ts +16 -0
- package/dist/cli/commands/observability.js +335 -0
- package/dist/cli/commands/planning.d.ts +9 -0
- package/dist/cli/commands/planning.js +167 -0
- package/dist/cli/commands/prompt-expand.d.ts +11 -0
- package/dist/cli/commands/prompt-expand.js +100 -0
- package/dist/cli/commands/providers.d.ts +17 -0
- package/dist/cli/commands/providers.js +386 -0
- package/dist/cli/commands/query-rewrite.d.ts +12 -0
- package/dist/cli/commands/query-rewrite.js +102 -0
- package/dist/cli/commands/repo-map.d.ts +5 -0
- package/dist/cli/commands/repo-map.js +151 -0
- package/dist/cli/commands/reranker.d.ts +10 -0
- package/dist/cli/commands/reranker.js +144 -0
- package/dist/cli/commands/research.d.ts +12 -0
- package/dist/cli/commands/research.js +108 -0
- package/dist/cli/commands/router.d.ts +11 -0
- package/dist/cli/commands/router.js +142 -0
- package/dist/cli/commands/run.d.ts +14 -0
- package/dist/cli/commands/run.js +111 -0
- package/dist/cli/commands/sandbox.d.ts +5 -0
- package/dist/cli/commands/sandbox.js +135 -0
- package/dist/cli/commands/scheduler.d.ts +5 -0
- package/dist/cli/commands/scheduler.js +260 -0
- package/dist/cli/commands/session.d.ts +9 -0
- package/dist/cli/commands/session.js +238 -0
- package/dist/cli/commands/skills.d.ts +9 -0
- package/dist/cli/commands/skills.js +256 -0
- package/dist/cli/commands/telemetry.d.ts +9 -0
- package/dist/cli/commands/telemetry.js +146 -0
- package/dist/cli/commands/tools.d.ts +19 -0
- package/dist/cli/commands/tools.js +576 -0
- package/dist/cli/commands/vector.d.ts +10 -0
- package/dist/cli/commands/vector.js +171 -0
- package/dist/cli/commands/version.d.ts +8 -0
- package/dist/cli/commands/version.js +40 -0
- package/dist/cli/commands/voice.d.ts +10 -0
- package/dist/cli/commands/voice.js +162 -0
- package/dist/cli/commands/workflow.d.ts +13 -0
- package/dist/cli/commands/workflow.js +184 -0
- package/dist/cli/config/index.d.ts +6 -0
- package/dist/cli/config/index.js +22 -0
- package/dist/cli/config/load.d.ts +20 -0
- package/dist/cli/config/load.js +229 -0
- package/dist/cli/config/resolve.d.ts +28 -0
- package/dist/cli/config/resolve.js +70 -0
- package/dist/cli/config/schema.d.ts +15 -0
- package/dist/cli/config/schema.js +65 -0
- package/dist/cli/features/autonomy-mode.d.ts +98 -0
- package/dist/cli/features/autonomy-mode.js +266 -0
- package/dist/cli/features/background-jobs.d.ts +155 -0
- package/dist/cli/features/background-jobs.js +416 -0
- package/dist/cli/features/checkpoints.d.ts +126 -0
- package/dist/cli/features/checkpoints.js +288 -0
- package/dist/cli/features/cost-tracker.d.ts +101 -0
- package/dist/cli/features/cost-tracker.js +212 -0
- package/dist/cli/features/external-agents.d.ts +117 -0
- package/dist/cli/features/external-agents.js +296 -0
- package/dist/cli/features/fast-context.d.ts +126 -0
- package/dist/cli/features/fast-context.js +310 -0
- package/dist/cli/features/flow-display.d.ts +100 -0
- package/dist/cli/features/flow-display.js +254 -0
- package/dist/cli/features/git-integration.d.ts +138 -0
- package/dist/cli/features/git-integration.js +374 -0
- package/dist/cli/features/index.d.ts +17 -0
- package/dist/cli/features/index.js +102 -0
- package/dist/cli/features/interactive-tui.d.ts +114 -0
- package/dist/cli/features/interactive-tui.js +326 -0
- package/dist/cli/features/n8n-integration.d.ts +111 -0
- package/dist/cli/features/n8n-integration.js +299 -0
- package/dist/cli/features/repo-map.d.ts +101 -0
- package/dist/cli/features/repo-map.js +350 -0
- package/dist/cli/features/sandbox-executor.d.ts +89 -0
- package/dist/cli/features/sandbox-executor.js +314 -0
- package/dist/cli/features/scheduler.d.ts +111 -0
- package/dist/cli/features/scheduler.js +298 -0
- package/dist/cli/features/slash-commands.d.ts +77 -0
- package/dist/cli/features/slash-commands.js +316 -0
- package/dist/cli/index.d.ts +24 -0
- package/dist/cli/index.js +214 -0
- package/dist/cli/output/errors.d.ts +23 -0
- package/dist/cli/output/errors.js +36 -0
- package/dist/cli/output/json.d.ts +27 -0
- package/dist/cli/output/json.js +31 -0
- package/dist/cli/output/pretty.d.ts +13 -0
- package/dist/cli/output/pretty.js +31 -0
- package/dist/cli/runtime/env.d.ts +12 -0
- package/dist/cli/runtime/env.js +55 -0
- package/dist/cli/runtime/exit.d.ts +11 -0
- package/dist/cli/runtime/exit.js +49 -0
- package/dist/cli/runtime/index.d.ts +6 -0
- package/dist/cli/runtime/index.js +22 -0
- package/dist/cli/runtime/lazy.d.ts +18 -0
- package/dist/cli/runtime/lazy.js +85 -0
- package/dist/cli/spec/cli-spec.d.ts +87 -0
- package/dist/cli/spec/cli-spec.js +551 -0
- package/dist/cli/spec/index.d.ts +4 -0
- package/dist/cli/spec/index.js +20 -0
- package/dist/db/index.d.ts +6 -0
- package/dist/db/index.js +75 -0
- package/dist/db/types.d.ts +42 -0
- package/dist/db/types.js +2 -0
- package/dist/eval/index.d.ts +61 -0
- package/dist/eval/index.js +157 -0
- package/dist/events/index.d.ts +84 -0
- package/dist/events/index.js +153 -0
- package/dist/guardrails/index.d.ts +82 -0
- package/dist/guardrails/index.js +202 -0
- package/dist/guardrails/llm-guardrail.d.ts +46 -0
- package/dist/guardrails/llm-guardrail.js +112 -0
- package/dist/index.d.ts +92 -0
- package/dist/index.js +608 -0
- package/dist/integrations/computer-use.d.ts +182 -0
- package/dist/integrations/computer-use.js +356 -0
- package/dist/integrations/index.d.ts +7 -0
- package/dist/integrations/index.js +26 -0
- package/dist/integrations/observability/base.d.ts +123 -0
- package/dist/integrations/observability/base.js +183 -0
- package/dist/integrations/observability/index.d.ts +8 -0
- package/dist/integrations/observability/index.js +29 -0
- package/dist/integrations/observability/langfuse.d.ts +32 -0
- package/dist/integrations/observability/langfuse.js +174 -0
- package/dist/integrations/postgres.d.ts +162 -0
- package/dist/integrations/postgres.js +396 -0
- package/dist/integrations/slack.d.ts +145 -0
- package/dist/integrations/slack.js +270 -0
- package/dist/integrations/vector/base.d.ts +110 -0
- package/dist/integrations/vector/base.js +158 -0
- package/dist/integrations/vector/chroma.d.ts +25 -0
- package/dist/integrations/vector/chroma.js +143 -0
- package/dist/integrations/vector/index.d.ts +14 -0
- package/dist/integrations/vector/index.js +37 -0
- package/dist/integrations/vector/pinecone.d.ts +28 -0
- package/dist/integrations/vector/pinecone.js +172 -0
- package/dist/integrations/vector/qdrant.d.ts +25 -0
- package/dist/integrations/vector/qdrant.js +146 -0
- package/dist/integrations/vector/weaviate.d.ts +30 -0
- package/dist/integrations/vector/weaviate.js +206 -0
- package/dist/integrations/voice/base.d.ts +76 -0
- package/dist/integrations/voice/base.js +168 -0
- package/dist/integrations/voice/index.d.ts +6 -0
- package/dist/integrations/voice/index.js +26 -0
- package/dist/knowledge/chunking.d.ts +55 -0
- package/dist/knowledge/chunking.js +158 -0
- package/dist/knowledge/graph-rag.d.ts +125 -0
- package/dist/knowledge/graph-rag.js +289 -0
- package/dist/knowledge/index.d.ts +24 -0
- package/dist/knowledge/index.js +47 -0
- package/dist/knowledge/knowledge.d.ts +0 -0
- package/dist/knowledge/knowledge.js +1 -0
- package/dist/knowledge/rag.d.ts +80 -0
- package/dist/knowledge/rag.js +147 -0
- package/dist/knowledge/reranker.d.ts +86 -0
- package/dist/knowledge/reranker.js +196 -0
- package/dist/llm/backend-resolver.d.ts +77 -0
- package/dist/llm/backend-resolver.js +242 -0
- package/dist/llm/embeddings.d.ts +66 -0
- package/dist/llm/embeddings.js +294 -0
- package/dist/llm/index.d.ts +29 -0
- package/dist/llm/index.js +33 -0
- package/dist/llm/llm.d.ts +0 -0
- package/dist/llm/llm.js +1 -0
- package/dist/llm/openai.d.ts +39 -0
- package/dist/llm/openai.js +302 -0
- package/dist/llm/providers/ai-sdk/adapter.d.ts +121 -0
- package/dist/llm/providers/ai-sdk/adapter.js +399 -0
- package/dist/llm/providers/ai-sdk/backend.d.ts +49 -0
- package/dist/llm/providers/ai-sdk/backend.js +421 -0
- package/dist/llm/providers/ai-sdk/index.d.ts +69 -0
- package/dist/llm/providers/ai-sdk/index.js +200 -0
- package/dist/llm/providers/ai-sdk/middleware.d.ts +60 -0
- package/dist/llm/providers/ai-sdk/middleware.js +262 -0
- package/dist/llm/providers/ai-sdk/provider-map.d.ts +75 -0
- package/dist/llm/providers/ai-sdk/provider-map.js +258 -0
- package/dist/llm/providers/ai-sdk/types.d.ts +217 -0
- package/dist/llm/providers/ai-sdk/types.js +222 -0
- package/dist/llm/providers/anthropic.d.ts +33 -0
- package/dist/llm/providers/anthropic.js +291 -0
- package/dist/llm/providers/base.d.ts +25 -0
- package/dist/llm/providers/base.js +43 -0
- package/dist/llm/providers/google.d.ts +27 -0
- package/dist/llm/providers/google.js +275 -0
- package/dist/llm/providers/index.d.ts +107 -0
- package/dist/llm/providers/index.js +216 -0
- package/dist/llm/providers/openai.d.ts +18 -0
- package/dist/llm/providers/openai.js +203 -0
- package/dist/llm/providers/registry.d.ts +159 -0
- package/dist/llm/providers/registry.js +279 -0
- package/dist/llm/providers/types.d.ts +94 -0
- package/dist/llm/providers/types.js +5 -0
- package/dist/main.d.ts +0 -0
- package/dist/main.js +1 -0
- package/dist/memory/auto-memory.d.ts +136 -0
- package/dist/memory/auto-memory.js +301 -0
- package/dist/memory/file-memory.d.ts +88 -0
- package/dist/memory/file-memory.js +287 -0
- package/dist/memory/index.d.ts +26 -0
- package/dist/memory/index.js +41 -0
- package/dist/memory/memory.d.ts +92 -0
- package/dist/memory/memory.js +170 -0
- package/dist/observability/adapters/console.d.ts +33 -0
- package/dist/observability/adapters/console.js +71 -0
- package/dist/observability/adapters/external/arize.d.ts +20 -0
- package/dist/observability/adapters/external/arize.js +22 -0
- package/dist/observability/adapters/external/axiom.d.ts +20 -0
- package/dist/observability/adapters/external/axiom.js +22 -0
- package/dist/observability/adapters/external/braintrust.d.ts +20 -0
- package/dist/observability/adapters/external/braintrust.js +22 -0
- package/dist/observability/adapters/external/helicone.d.ts +20 -0
- package/dist/observability/adapters/external/helicone.js +22 -0
- package/dist/observability/adapters/external/laminar.d.ts +20 -0
- package/dist/observability/adapters/external/laminar.js +22 -0
- package/dist/observability/adapters/external/langfuse.d.ts +28 -0
- package/dist/observability/adapters/external/langfuse.js +186 -0
- package/dist/observability/adapters/external/langsmith.d.ts +21 -0
- package/dist/observability/adapters/external/langsmith.js +83 -0
- package/dist/observability/adapters/external/langwatch.d.ts +20 -0
- package/dist/observability/adapters/external/langwatch.js +22 -0
- package/dist/observability/adapters/external/maxim.d.ts +20 -0
- package/dist/observability/adapters/external/maxim.js +22 -0
- package/dist/observability/adapters/external/patronus.d.ts +20 -0
- package/dist/observability/adapters/external/patronus.js +22 -0
- package/dist/observability/adapters/external/scorecard.d.ts +20 -0
- package/dist/observability/adapters/external/scorecard.js +22 -0
- package/dist/observability/adapters/external/signoz.d.ts +20 -0
- package/dist/observability/adapters/external/signoz.js +22 -0
- package/dist/observability/adapters/external/traceloop.d.ts +20 -0
- package/dist/observability/adapters/external/traceloop.js +22 -0
- package/dist/observability/adapters/external/weave.d.ts +20 -0
- package/dist/observability/adapters/external/weave.js +22 -0
- package/dist/observability/adapters/index.d.ts +19 -0
- package/dist/observability/adapters/index.js +289 -0
- package/dist/observability/adapters/memory.d.ts +41 -0
- package/dist/observability/adapters/memory.js +190 -0
- package/dist/observability/adapters/noop.d.ts +25 -0
- package/dist/observability/adapters/noop.js +64 -0
- package/dist/observability/index.d.ts +47 -0
- package/dist/observability/index.js +115 -0
- package/dist/observability/types.d.ts +165 -0
- package/dist/observability/types.js +165 -0
- package/dist/planning/index.d.ts +253 -0
- package/dist/planning/index.js +475 -0
- package/dist/process/index.d.ts +25 -0
- package/dist/process/index.js +37 -0
- package/dist/process/process.d.ts +0 -0
- package/dist/process/process.js +1 -0
- package/dist/session/index.d.ts +111 -0
- package/dist/session/index.js +250 -0
- package/dist/skills/index.d.ts +70 -0
- package/dist/skills/index.js +233 -0
- package/dist/task/index.d.ts +25 -0
- package/dist/task/index.js +32 -0
- package/dist/task/task.d.ts +0 -0
- package/dist/task/task.js +1 -0
- package/dist/telemetry/index.d.ts +175 -0
- package/dist/telemetry/index.js +287 -0
- package/dist/tools/arxivTools.d.ts +32 -0
- package/dist/tools/arxivTools.js +81 -0
- package/dist/tools/base.d.ts +97 -0
- package/dist/tools/base.js +147 -0
- package/dist/tools/builtins/airweave.d.ts +32 -0
- package/dist/tools/builtins/airweave.js +119 -0
- package/dist/tools/builtins/bedrock-agentcore.d.ts +62 -0
- package/dist/tools/builtins/bedrock-agentcore.js +233 -0
- package/dist/tools/builtins/code-execution.d.ts +36 -0
- package/dist/tools/builtins/code-execution.js +136 -0
- package/dist/tools/builtins/code-mode.d.ts +54 -0
- package/dist/tools/builtins/code-mode.js +167 -0
- package/dist/tools/builtins/custom.d.ts +42 -0
- package/dist/tools/builtins/custom.js +195 -0
- package/dist/tools/builtins/exa.d.ts +53 -0
- package/dist/tools/builtins/exa.js +112 -0
- package/dist/tools/builtins/firecrawl.d.ts +56 -0
- package/dist/tools/builtins/firecrawl.js +158 -0
- package/dist/tools/builtins/index.d.ts +45 -0
- package/dist/tools/builtins/index.js +180 -0
- package/dist/tools/builtins/parallel.d.ts +30 -0
- package/dist/tools/builtins/parallel.js +113 -0
- package/dist/tools/builtins/perplexity.d.ts +32 -0
- package/dist/tools/builtins/perplexity.js +113 -0
- package/dist/tools/builtins/superagent.d.ts +66 -0
- package/dist/tools/builtins/superagent.js +187 -0
- package/dist/tools/builtins/tavily.d.ts +99 -0
- package/dist/tools/builtins/tavily.js +285 -0
- package/dist/tools/builtins/valyu.d.ts +65 -0
- package/dist/tools/builtins/valyu.js +193 -0
- package/dist/tools/decorator.d.ts +91 -0
- package/dist/tools/decorator.js +165 -0
- package/dist/tools/index.d.ts +8 -0
- package/dist/tools/index.js +36 -0
- package/dist/tools/mcpSse.d.ts +43 -0
- package/dist/tools/mcpSse.js +110 -0
- package/dist/tools/registry/index.d.ts +9 -0
- package/dist/tools/registry/index.js +28 -0
- package/dist/tools/registry/middleware.d.ts +43 -0
- package/dist/tools/registry/middleware.js +220 -0
- package/dist/tools/registry/registry.d.ts +92 -0
- package/dist/tools/registry/registry.js +280 -0
- package/dist/tools/registry/types.d.ts +217 -0
- package/dist/tools/registry/types.js +49 -0
- package/dist/tools/test.d.ts +0 -0
- package/dist/tools/test.js +1 -0
- package/dist/tools/tools.d.ts +74 -0
- package/dist/tools/tools.js +133 -0
- package/dist/utils/logger.d.ts +25 -0
- package/dist/utils/logger.js +149 -0
- package/dist/utils/pretty-logger.d.ts +21 -0
- package/dist/utils/pretty-logger.js +248 -0
- package/dist/workflows/index.d.ts +118 -0
- package/dist/workflows/index.js +247 -0
- package/dist/workflows/loop.d.ts +0 -0
- package/dist/workflows/loop.js +1 -0
- package/dist/workflows/yaml-parser.d.ts +48 -0
- package/dist/workflows/yaml-parser.js +304 -0
- package/package.json +119 -0
|
@@ -0,0 +1,375 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Embed Command - Generate embeddings using AI SDK (preferred) or native provider
|
|
4
|
+
*
|
|
5
|
+
* Subcommands:
|
|
6
|
+
* - text: Embed a single text or multiple texts
|
|
7
|
+
* - file: Embed contents of a file
|
|
8
|
+
* - query: Find similar texts using embeddings
|
|
9
|
+
*/
|
|
10
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
11
|
+
if (k2 === undefined) k2 = k;
|
|
12
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
13
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
14
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
15
|
+
}
|
|
16
|
+
Object.defineProperty(o, k2, desc);
|
|
17
|
+
}) : (function(o, m, k, k2) {
|
|
18
|
+
if (k2 === undefined) k2 = k;
|
|
19
|
+
o[k2] = m[k];
|
|
20
|
+
}));
|
|
21
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
22
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
23
|
+
}) : function(o, v) {
|
|
24
|
+
o["default"] = v;
|
|
25
|
+
});
|
|
26
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
27
|
+
var ownKeys = function(o) {
|
|
28
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
29
|
+
var ar = [];
|
|
30
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
31
|
+
return ar;
|
|
32
|
+
};
|
|
33
|
+
return ownKeys(o);
|
|
34
|
+
};
|
|
35
|
+
return function (mod) {
|
|
36
|
+
if (mod && mod.__esModule) return mod;
|
|
37
|
+
var result = {};
|
|
38
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
39
|
+
__setModuleDefault(result, mod);
|
|
40
|
+
return result;
|
|
41
|
+
};
|
|
42
|
+
})();
|
|
43
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
44
|
+
exports.execute = execute;
|
|
45
|
+
const pretty_logger_1 = require("../../utils/pretty-logger");
|
|
46
|
+
function outputJson(data) {
|
|
47
|
+
console.log(JSON.stringify(data, null, 2));
|
|
48
|
+
}
|
|
49
|
+
function formatError(code, message) {
|
|
50
|
+
return { success: false, error: { code, message } };
|
|
51
|
+
}
|
|
52
|
+
const ERROR_CODES = {
|
|
53
|
+
INVALID_ARGS: 'INVALID_ARGS',
|
|
54
|
+
MISSING_ARG: 'MISSING_ARG',
|
|
55
|
+
UNKNOWN: 'UNKNOWN_ERROR',
|
|
56
|
+
};
|
|
57
|
+
// Simple success formatter for embed command
|
|
58
|
+
function formatEmbedSuccess(data, meta) {
|
|
59
|
+
return {
|
|
60
|
+
success: true,
|
|
61
|
+
data,
|
|
62
|
+
meta,
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
const fs = __importStar(require("fs"));
|
|
66
|
+
const path = __importStar(require("path"));
|
|
67
|
+
const EXIT_CODES = {
|
|
68
|
+
SUCCESS: 0,
|
|
69
|
+
GENERAL_ERROR: 1,
|
|
70
|
+
INVALID_ARGS: 2,
|
|
71
|
+
};
|
|
72
|
+
async function execute(args, options) {
|
|
73
|
+
const subcommand = args[0] || 'text';
|
|
74
|
+
const outputFormat = options.json ? 'json' : (options.output || 'pretty');
|
|
75
|
+
switch (subcommand) {
|
|
76
|
+
case 'text':
|
|
77
|
+
await textCommand(args.slice(1), options, outputFormat);
|
|
78
|
+
break;
|
|
79
|
+
case 'file':
|
|
80
|
+
await fileCommand(args.slice(1), options, outputFormat);
|
|
81
|
+
break;
|
|
82
|
+
case 'query':
|
|
83
|
+
await queryCommand(args.slice(1), options, outputFormat);
|
|
84
|
+
break;
|
|
85
|
+
case 'models':
|
|
86
|
+
await modelsCommand(args.slice(1), options, outputFormat);
|
|
87
|
+
break;
|
|
88
|
+
default:
|
|
89
|
+
// If no subcommand, treat args[0] as text to embed
|
|
90
|
+
if (args[0] && !['text', 'file', 'query', 'models'].includes(args[0])) {
|
|
91
|
+
await textCommand(args, options, outputFormat);
|
|
92
|
+
}
|
|
93
|
+
else {
|
|
94
|
+
if (outputFormat === 'json') {
|
|
95
|
+
outputJson(formatError(ERROR_CODES.INVALID_ARGS, `Unknown subcommand: ${subcommand}`));
|
|
96
|
+
}
|
|
97
|
+
else {
|
|
98
|
+
await pretty_logger_1.PrettyLogger.error(`Unknown subcommand: ${subcommand}`);
|
|
99
|
+
await pretty_logger_1.PrettyLogger.info('Available subcommands: text, file, query, models');
|
|
100
|
+
}
|
|
101
|
+
process.exit(EXIT_CODES.INVALID_ARGS);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Embed text(s)
|
|
107
|
+
*/
|
|
108
|
+
async function textCommand(args, options, outputFormat) {
|
|
109
|
+
const texts = args.filter(a => !a.startsWith('-'));
|
|
110
|
+
if (texts.length === 0) {
|
|
111
|
+
if (outputFormat === 'json') {
|
|
112
|
+
outputJson(formatError(ERROR_CODES.MISSING_ARG, 'Please provide text to embed'));
|
|
113
|
+
}
|
|
114
|
+
else {
|
|
115
|
+
await pretty_logger_1.PrettyLogger.error('Please provide text to embed');
|
|
116
|
+
await pretty_logger_1.PrettyLogger.info('Usage: praisonai-ts embed text "Hello world"');
|
|
117
|
+
}
|
|
118
|
+
process.exit(EXIT_CODES.INVALID_ARGS);
|
|
119
|
+
}
|
|
120
|
+
try {
|
|
121
|
+
const { embed, embedMany, getDefaultEmbeddingModel, isAISDKAvailable } = await Promise.resolve().then(() => __importStar(require('../../llm')));
|
|
122
|
+
const startTime = Date.now();
|
|
123
|
+
const model = options.model || getDefaultEmbeddingModel(options.provider);
|
|
124
|
+
const aiSdkAvailable = await isAISDKAvailable();
|
|
125
|
+
let result;
|
|
126
|
+
if (texts.length === 1) {
|
|
127
|
+
const single = await embed(texts[0], {
|
|
128
|
+
model,
|
|
129
|
+
backend: options.backend
|
|
130
|
+
});
|
|
131
|
+
result = {
|
|
132
|
+
embeddings: [single.embedding],
|
|
133
|
+
usage: single.usage
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
else {
|
|
137
|
+
result = await embedMany(texts, {
|
|
138
|
+
model,
|
|
139
|
+
backend: options.backend
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
const duration = Date.now() - startTime;
|
|
143
|
+
// Save to file if requested
|
|
144
|
+
if (options.save) {
|
|
145
|
+
const saveData = {
|
|
146
|
+
texts,
|
|
147
|
+
embeddings: result.embeddings,
|
|
148
|
+
model,
|
|
149
|
+
backend: aiSdkAvailable ? 'ai-sdk' : 'native',
|
|
150
|
+
timestamp: new Date().toISOString(),
|
|
151
|
+
};
|
|
152
|
+
fs.writeFileSync(options.save, JSON.stringify(saveData, null, 2));
|
|
153
|
+
}
|
|
154
|
+
if (outputFormat === 'json') {
|
|
155
|
+
outputJson(formatEmbedSuccess({
|
|
156
|
+
texts,
|
|
157
|
+
embeddings: result.embeddings,
|
|
158
|
+
dimensions: result.embeddings[0]?.length || 0,
|
|
159
|
+
count: result.embeddings.length,
|
|
160
|
+
}, {
|
|
161
|
+
model,
|
|
162
|
+
backend: aiSdkAvailable ? 'ai-sdk' : 'native',
|
|
163
|
+
duration_ms: duration,
|
|
164
|
+
tokens: result.usage?.tokens,
|
|
165
|
+
saved_to: options.save,
|
|
166
|
+
}));
|
|
167
|
+
}
|
|
168
|
+
else {
|
|
169
|
+
await pretty_logger_1.PrettyLogger.success(`Embedded ${texts.length} text(s)`);
|
|
170
|
+
await pretty_logger_1.PrettyLogger.info(`Model: ${model}`);
|
|
171
|
+
await pretty_logger_1.PrettyLogger.info(`Backend: ${aiSdkAvailable ? 'AI SDK' : 'Native'}`);
|
|
172
|
+
await pretty_logger_1.PrettyLogger.info(`Dimensions: ${result.embeddings[0]?.length || 0}`);
|
|
173
|
+
await pretty_logger_1.PrettyLogger.info(`Duration: ${duration}ms`);
|
|
174
|
+
if (result.usage?.tokens) {
|
|
175
|
+
await pretty_logger_1.PrettyLogger.info(`Tokens: ${result.usage.tokens}`);
|
|
176
|
+
}
|
|
177
|
+
if (options.save) {
|
|
178
|
+
await pretty_logger_1.PrettyLogger.info(`Saved to: ${options.save}`);
|
|
179
|
+
}
|
|
180
|
+
if (options.verbose) {
|
|
181
|
+
console.log('\nEmbeddings (first 5 values each):');
|
|
182
|
+
result.embeddings.forEach((emb, i) => {
|
|
183
|
+
console.log(` [${i}]: [${emb.slice(0, 5).join(', ')}...]`);
|
|
184
|
+
});
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
catch (error) {
|
|
189
|
+
if (outputFormat === 'json') {
|
|
190
|
+
outputJson(formatError(ERROR_CODES.UNKNOWN, error.message));
|
|
191
|
+
}
|
|
192
|
+
else {
|
|
193
|
+
await pretty_logger_1.PrettyLogger.error(`Embedding failed: ${error.message}`);
|
|
194
|
+
}
|
|
195
|
+
process.exit(EXIT_CODES.GENERAL_ERROR);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
/**
|
|
199
|
+
* Embed file contents
|
|
200
|
+
*/
|
|
201
|
+
async function fileCommand(args, options, outputFormat) {
|
|
202
|
+
const filePath = args[0];
|
|
203
|
+
if (!filePath) {
|
|
204
|
+
if (outputFormat === 'json') {
|
|
205
|
+
outputJson(formatError(ERROR_CODES.MISSING_ARG, 'Please provide a file path'));
|
|
206
|
+
}
|
|
207
|
+
else {
|
|
208
|
+
await pretty_logger_1.PrettyLogger.error('Please provide a file path');
|
|
209
|
+
await pretty_logger_1.PrettyLogger.info('Usage: praisonai-ts embed file ./document.txt');
|
|
210
|
+
}
|
|
211
|
+
process.exit(EXIT_CODES.INVALID_ARGS);
|
|
212
|
+
}
|
|
213
|
+
try {
|
|
214
|
+
const absolutePath = path.resolve(filePath);
|
|
215
|
+
if (!fs.existsSync(absolutePath)) {
|
|
216
|
+
throw new Error(`File not found: ${absolutePath}`);
|
|
217
|
+
}
|
|
218
|
+
const content = fs.readFileSync(absolutePath, 'utf-8');
|
|
219
|
+
const { embed, getDefaultEmbeddingModel, isAISDKAvailable } = await Promise.resolve().then(() => __importStar(require('../../llm')));
|
|
220
|
+
const startTime = Date.now();
|
|
221
|
+
const model = options.model || getDefaultEmbeddingModel(options.provider);
|
|
222
|
+
const aiSdkAvailable = await isAISDKAvailable();
|
|
223
|
+
const result = await embed(content, {
|
|
224
|
+
model,
|
|
225
|
+
backend: options.backend
|
|
226
|
+
});
|
|
227
|
+
const duration = Date.now() - startTime;
|
|
228
|
+
// Save to file if requested
|
|
229
|
+
if (options.save) {
|
|
230
|
+
const saveData = {
|
|
231
|
+
file: absolutePath,
|
|
232
|
+
embedding: result.embedding,
|
|
233
|
+
model,
|
|
234
|
+
backend: aiSdkAvailable ? 'ai-sdk' : 'native',
|
|
235
|
+
timestamp: new Date().toISOString(),
|
|
236
|
+
};
|
|
237
|
+
fs.writeFileSync(options.save, JSON.stringify(saveData, null, 2));
|
|
238
|
+
}
|
|
239
|
+
if (outputFormat === 'json') {
|
|
240
|
+
outputJson(formatEmbedSuccess({
|
|
241
|
+
file: absolutePath,
|
|
242
|
+
embedding: result.embedding,
|
|
243
|
+
dimensions: result.embedding.length,
|
|
244
|
+
content_length: content.length,
|
|
245
|
+
}, {
|
|
246
|
+
model,
|
|
247
|
+
backend: aiSdkAvailable ? 'ai-sdk' : 'native',
|
|
248
|
+
duration_ms: duration,
|
|
249
|
+
tokens: result.usage?.tokens,
|
|
250
|
+
saved_to: options.save,
|
|
251
|
+
}));
|
|
252
|
+
}
|
|
253
|
+
else {
|
|
254
|
+
await pretty_logger_1.PrettyLogger.success(`Embedded file: ${filePath}`);
|
|
255
|
+
await pretty_logger_1.PrettyLogger.info(`Model: ${model}`);
|
|
256
|
+
await pretty_logger_1.PrettyLogger.info(`Backend: ${aiSdkAvailable ? 'AI SDK' : 'Native'}`);
|
|
257
|
+
await pretty_logger_1.PrettyLogger.info(`Dimensions: ${result.embedding.length}`);
|
|
258
|
+
await pretty_logger_1.PrettyLogger.info(`Content length: ${content.length} chars`);
|
|
259
|
+
await pretty_logger_1.PrettyLogger.info(`Duration: ${duration}ms`);
|
|
260
|
+
if (options.save) {
|
|
261
|
+
await pretty_logger_1.PrettyLogger.info(`Saved to: ${options.save}`);
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
catch (error) {
|
|
266
|
+
if (outputFormat === 'json') {
|
|
267
|
+
outputJson(formatError(ERROR_CODES.UNKNOWN, error.message));
|
|
268
|
+
}
|
|
269
|
+
else {
|
|
270
|
+
await pretty_logger_1.PrettyLogger.error(`Embedding failed: ${error.message}`);
|
|
271
|
+
}
|
|
272
|
+
process.exit(EXIT_CODES.GENERAL_ERROR);
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
/**
|
|
276
|
+
* Query similar texts
|
|
277
|
+
*/
|
|
278
|
+
async function queryCommand(args, options, outputFormat) {
|
|
279
|
+
const query = args[0];
|
|
280
|
+
const dataFile = options.file;
|
|
281
|
+
if (!query) {
|
|
282
|
+
if (outputFormat === 'json') {
|
|
283
|
+
outputJson(formatError(ERROR_CODES.MISSING_ARG, 'Please provide a query'));
|
|
284
|
+
}
|
|
285
|
+
else {
|
|
286
|
+
await pretty_logger_1.PrettyLogger.error('Please provide a query');
|
|
287
|
+
await pretty_logger_1.PrettyLogger.info('Usage: overai embed query "search text" --file embeddings.json');
|
|
288
|
+
}
|
|
289
|
+
process.exit(EXIT_CODES.INVALID_ARGS);
|
|
290
|
+
}
|
|
291
|
+
if (!dataFile) {
|
|
292
|
+
if (outputFormat === 'json') {
|
|
293
|
+
outputJson(formatError(ERROR_CODES.MISSING_ARG, 'Please provide --file with embeddings data'));
|
|
294
|
+
}
|
|
295
|
+
else {
|
|
296
|
+
await pretty_logger_1.PrettyLogger.error('Please provide --file with embeddings data');
|
|
297
|
+
}
|
|
298
|
+
process.exit(EXIT_CODES.INVALID_ARGS);
|
|
299
|
+
}
|
|
300
|
+
try {
|
|
301
|
+
const { embed, cosineSimilarity, getDefaultEmbeddingModel } = await Promise.resolve().then(() => __importStar(require('../../llm')));
|
|
302
|
+
// Load embeddings data
|
|
303
|
+
const data = JSON.parse(fs.readFileSync(dataFile, 'utf-8'));
|
|
304
|
+
const texts = data.texts || [];
|
|
305
|
+
const embeddings = data.embeddings || [];
|
|
306
|
+
if (embeddings.length === 0) {
|
|
307
|
+
throw new Error('No embeddings found in data file');
|
|
308
|
+
}
|
|
309
|
+
// Embed the query
|
|
310
|
+
const model = options.model || data.model || getDefaultEmbeddingModel(options.provider);
|
|
311
|
+
const queryResult = await embed(query, { model, backend: options.backend });
|
|
312
|
+
// Calculate similarities
|
|
313
|
+
const similarities = embeddings.map((emb, i) => ({
|
|
314
|
+
index: i,
|
|
315
|
+
text: texts[i] || `[${i}]`,
|
|
316
|
+
score: cosineSimilarity(queryResult.embedding, emb),
|
|
317
|
+
}));
|
|
318
|
+
// Sort by similarity (descending)
|
|
319
|
+
similarities.sort((a, b) => b.score - a.score);
|
|
320
|
+
// Take top 5
|
|
321
|
+
const topResults = similarities.slice(0, 5);
|
|
322
|
+
if (outputFormat === 'json') {
|
|
323
|
+
outputJson(formatEmbedSuccess({
|
|
324
|
+
query,
|
|
325
|
+
results: topResults,
|
|
326
|
+
}, {
|
|
327
|
+
model,
|
|
328
|
+
total_embeddings: embeddings.length,
|
|
329
|
+
}));
|
|
330
|
+
}
|
|
331
|
+
else {
|
|
332
|
+
await pretty_logger_1.PrettyLogger.success(`Query: "${query}"`);
|
|
333
|
+
console.log('\nTop results:');
|
|
334
|
+
topResults.forEach((r, i) => {
|
|
335
|
+
console.log(` ${i + 1}. [${(r.score * 100).toFixed(1)}%] ${r.text.substring(0, 80)}${r.text.length > 80 ? '...' : ''}`);
|
|
336
|
+
});
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
catch (error) {
|
|
340
|
+
if (outputFormat === 'json') {
|
|
341
|
+
outputJson(formatError(ERROR_CODES.UNKNOWN, error.message));
|
|
342
|
+
}
|
|
343
|
+
else {
|
|
344
|
+
await pretty_logger_1.PrettyLogger.error(`Query failed: ${error.message}`);
|
|
345
|
+
}
|
|
346
|
+
process.exit(EXIT_CODES.GENERAL_ERROR);
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
/**
|
|
350
|
+
* List available embedding models
|
|
351
|
+
*/
|
|
352
|
+
async function modelsCommand(args, options, outputFormat) {
|
|
353
|
+
const models = [
|
|
354
|
+
{ provider: 'openai', model: 'text-embedding-3-small', dimensions: 1536, description: 'Fast, cost-effective' },
|
|
355
|
+
{ provider: 'openai', model: 'text-embedding-3-large', dimensions: 3072, description: 'Higher quality' },
|
|
356
|
+
{ provider: 'openai', model: 'text-embedding-ada-002', dimensions: 1536, description: 'Legacy model' },
|
|
357
|
+
{ provider: 'google', model: 'text-embedding-004', dimensions: 768, description: 'Google embedding' },
|
|
358
|
+
{ provider: 'cohere', model: 'embed-english-v3.0', dimensions: 1024, description: 'Cohere English' },
|
|
359
|
+
{ provider: 'cohere', model: 'embed-multilingual-v3.0', dimensions: 1024, description: 'Cohere Multilingual' },
|
|
360
|
+
];
|
|
361
|
+
if (outputFormat === 'json') {
|
|
362
|
+
outputJson(formatEmbedSuccess({ models }));
|
|
363
|
+
}
|
|
364
|
+
else {
|
|
365
|
+
await pretty_logger_1.PrettyLogger.success('Available Embedding Models');
|
|
366
|
+
console.log('');
|
|
367
|
+
console.log('Provider Model Dimensions Description');
|
|
368
|
+
console.log('─'.repeat(75));
|
|
369
|
+
models.forEach(m => {
|
|
370
|
+
console.log(`${m.provider.padEnd(12)}${m.model.padEnd(27)}${String(m.dimensions).padEnd(12)}${m.description}`);
|
|
371
|
+
});
|
|
372
|
+
console.log('');
|
|
373
|
+
await pretty_logger_1.PrettyLogger.info('Usage: praisonai-ts embed text "Hello" --model openai/text-embedding-3-small');
|
|
374
|
+
}
|
|
375
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Eval command - Evaluate agent performance
|
|
3
|
+
*/
|
|
4
|
+
export interface EvalOptions {
|
|
5
|
+
input?: string;
|
|
6
|
+
expected?: string;
|
|
7
|
+
iterations?: number;
|
|
8
|
+
warmup?: number;
|
|
9
|
+
'expected-tools'?: string;
|
|
10
|
+
model?: string;
|
|
11
|
+
verbose?: boolean;
|
|
12
|
+
profile?: string;
|
|
13
|
+
config?: string;
|
|
14
|
+
output?: 'json' | 'text' | 'pretty';
|
|
15
|
+
json?: boolean;
|
|
16
|
+
}
|
|
17
|
+
export declare function execute(args: string[], options: EvalOptions): Promise<void>;
|
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Eval command - Evaluate agent performance
|
|
4
|
+
*/
|
|
5
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
6
|
+
if (k2 === undefined) k2 = k;
|
|
7
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
8
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
9
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
10
|
+
}
|
|
11
|
+
Object.defineProperty(o, k2, desc);
|
|
12
|
+
}) : (function(o, m, k, k2) {
|
|
13
|
+
if (k2 === undefined) k2 = k;
|
|
14
|
+
o[k2] = m[k];
|
|
15
|
+
}));
|
|
16
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
17
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
18
|
+
}) : function(o, v) {
|
|
19
|
+
o["default"] = v;
|
|
20
|
+
});
|
|
21
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
22
|
+
var ownKeys = function(o) {
|
|
23
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
24
|
+
var ar = [];
|
|
25
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
26
|
+
return ar;
|
|
27
|
+
};
|
|
28
|
+
return ownKeys(o);
|
|
29
|
+
};
|
|
30
|
+
return function (mod) {
|
|
31
|
+
if (mod && mod.__esModule) return mod;
|
|
32
|
+
var result = {};
|
|
33
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
34
|
+
__setModuleDefault(result, mod);
|
|
35
|
+
return result;
|
|
36
|
+
};
|
|
37
|
+
})();
|
|
38
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
+
exports.execute = execute;
|
|
40
|
+
const agent_1 = require("../../agent");
|
|
41
|
+
const resolve_1 = require("../config/resolve");
|
|
42
|
+
const json_1 = require("../output/json");
|
|
43
|
+
const pretty = __importStar(require("../output/pretty"));
|
|
44
|
+
const cli_spec_1 = require("../spec/cli-spec");
|
|
45
|
+
const errors_1 = require("../output/errors");
|
|
46
|
+
async function execute(args, options) {
|
|
47
|
+
const subcommand = args[0];
|
|
48
|
+
if (!subcommand || !['accuracy', 'performance', 'reliability'].includes(subcommand)) {
|
|
49
|
+
if (options.json || options.output === 'json') {
|
|
50
|
+
(0, json_1.printError)(errors_1.ERROR_CODES.INVALID_ARGS, 'Please specify a subcommand: accuracy, performance, or reliability');
|
|
51
|
+
}
|
|
52
|
+
else {
|
|
53
|
+
await pretty.error('Please specify a subcommand: accuracy, performance, or reliability');
|
|
54
|
+
}
|
|
55
|
+
process.exit(cli_spec_1.EXIT_CODES.INVALID_ARGUMENTS);
|
|
56
|
+
}
|
|
57
|
+
const config = (0, resolve_1.resolveConfig)({
|
|
58
|
+
configPath: options.config,
|
|
59
|
+
profile: options.profile,
|
|
60
|
+
model: options.model,
|
|
61
|
+
verbose: options.verbose
|
|
62
|
+
});
|
|
63
|
+
const outputFormat = options.json ? 'json' : (options.output || 'pretty');
|
|
64
|
+
try {
|
|
65
|
+
switch (subcommand) {
|
|
66
|
+
case 'accuracy':
|
|
67
|
+
await runAccuracyEval(options, config, outputFormat);
|
|
68
|
+
break;
|
|
69
|
+
case 'performance':
|
|
70
|
+
await runPerformanceEval(options, config, outputFormat);
|
|
71
|
+
break;
|
|
72
|
+
case 'reliability':
|
|
73
|
+
await runReliabilityEval(options, config, outputFormat);
|
|
74
|
+
break;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
catch (error) {
|
|
78
|
+
const cliError = (0, errors_1.normalizeError)(error);
|
|
79
|
+
if (outputFormat === 'json') {
|
|
80
|
+
(0, json_1.outputJson)((0, json_1.formatError)(cliError.code, cliError.message, cliError.details));
|
|
81
|
+
}
|
|
82
|
+
else {
|
|
83
|
+
await pretty.error(cliError.message);
|
|
84
|
+
if (config.verbose && error instanceof Error && error.stack) {
|
|
85
|
+
await pretty.dim(error.stack);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
process.exit(cliError.exitCode);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
async function runAccuracyEval(options, config, outputFormat) {
|
|
92
|
+
if (!options.input) {
|
|
93
|
+
throw new Error('--input is required for accuracy evaluation');
|
|
94
|
+
}
|
|
95
|
+
if (!options.expected) {
|
|
96
|
+
throw new Error('--expected is required for accuracy evaluation');
|
|
97
|
+
}
|
|
98
|
+
const iterations = options.iterations || 1;
|
|
99
|
+
const results = [];
|
|
100
|
+
const startTime = Date.now();
|
|
101
|
+
const agent = new agent_1.Agent({
|
|
102
|
+
name: 'Eval Agent',
|
|
103
|
+
instructions: 'You are a helpful AI assistant.',
|
|
104
|
+
llm: config.model,
|
|
105
|
+
verbose: config.verbose
|
|
106
|
+
});
|
|
107
|
+
for (let i = 0; i < iterations; i++) {
|
|
108
|
+
const output = await agent.start(options.input);
|
|
109
|
+
// Simple similarity check (in production, use LLM-as-judge)
|
|
110
|
+
const similarity = calculateSimilarity(output, options.expected);
|
|
111
|
+
const score = Math.round(similarity * 10);
|
|
112
|
+
results.push({
|
|
113
|
+
iteration: i + 1,
|
|
114
|
+
output,
|
|
115
|
+
score,
|
|
116
|
+
passed: score >= 7
|
|
117
|
+
});
|
|
118
|
+
}
|
|
119
|
+
const duration = Date.now() - startTime;
|
|
120
|
+
const avgScore = results.reduce((sum, r) => sum + r.score, 0) / results.length;
|
|
121
|
+
const passRate = results.filter(r => r.passed).length / results.length;
|
|
122
|
+
if (outputFormat === 'json') {
|
|
123
|
+
(0, json_1.outputJson)((0, json_1.formatSuccess)({
|
|
124
|
+
type: 'accuracy',
|
|
125
|
+
results,
|
|
126
|
+
summary: {
|
|
127
|
+
avg_score: avgScore,
|
|
128
|
+
pass_rate: passRate,
|
|
129
|
+
iterations
|
|
130
|
+
}
|
|
131
|
+
}, { duration_ms: duration, model: config.model }));
|
|
132
|
+
}
|
|
133
|
+
else {
|
|
134
|
+
await pretty.heading('Accuracy Evaluation Results');
|
|
135
|
+
await pretty.keyValue({
|
|
136
|
+
'Iterations': iterations,
|
|
137
|
+
'Average Score': avgScore.toFixed(2),
|
|
138
|
+
'Pass Rate': `${(passRate * 100).toFixed(1)}%`,
|
|
139
|
+
'Duration': `${duration}ms`
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
async function runPerformanceEval(options, config, outputFormat) {
|
|
144
|
+
const iterations = options.iterations || 10;
|
|
145
|
+
const warmup = options.warmup || 2;
|
|
146
|
+
const results = [];
|
|
147
|
+
const agent = new agent_1.Agent({
|
|
148
|
+
name: 'Perf Agent',
|
|
149
|
+
instructions: 'You are a helpful AI assistant.',
|
|
150
|
+
llm: config.model,
|
|
151
|
+
verbose: false // Disable verbose for performance testing
|
|
152
|
+
});
|
|
153
|
+
const testPrompt = 'Say "hello" and nothing else.';
|
|
154
|
+
// Warmup runs
|
|
155
|
+
for (let i = 0; i < warmup; i++) {
|
|
156
|
+
await agent.start(testPrompt);
|
|
157
|
+
}
|
|
158
|
+
// Actual runs
|
|
159
|
+
for (let i = 0; i < iterations; i++) {
|
|
160
|
+
const start = Date.now();
|
|
161
|
+
await agent.start(testPrompt);
|
|
162
|
+
const duration = Date.now() - start;
|
|
163
|
+
results.push({
|
|
164
|
+
iteration: i + 1,
|
|
165
|
+
duration_ms: duration
|
|
166
|
+
});
|
|
167
|
+
}
|
|
168
|
+
const durations = results.map(r => r.duration_ms);
|
|
169
|
+
const avg = durations.reduce((a, b) => a + b, 0) / durations.length;
|
|
170
|
+
const min = Math.min(...durations);
|
|
171
|
+
const max = Math.max(...durations);
|
|
172
|
+
const sorted = [...durations].sort((a, b) => a - b);
|
|
173
|
+
const p95 = sorted[Math.floor(sorted.length * 0.95)];
|
|
174
|
+
if (outputFormat === 'json') {
|
|
175
|
+
(0, json_1.outputJson)((0, json_1.formatSuccess)({
|
|
176
|
+
type: 'performance',
|
|
177
|
+
results,
|
|
178
|
+
summary: {
|
|
179
|
+
avg_ms: avg,
|
|
180
|
+
min_ms: min,
|
|
181
|
+
max_ms: max,
|
|
182
|
+
p95_ms: p95,
|
|
183
|
+
iterations,
|
|
184
|
+
warmup_runs: warmup
|
|
185
|
+
}
|
|
186
|
+
}, { model: config.model }));
|
|
187
|
+
}
|
|
188
|
+
else {
|
|
189
|
+
await pretty.heading('Performance Evaluation Results');
|
|
190
|
+
await pretty.keyValue({
|
|
191
|
+
'Iterations': iterations,
|
|
192
|
+
'Warmup Runs': warmup,
|
|
193
|
+
'Avg Duration': `${avg.toFixed(2)}ms`,
|
|
194
|
+
'Min Duration': `${min}ms`,
|
|
195
|
+
'Max Duration': `${max}ms`,
|
|
196
|
+
'P95 Duration': `${p95}ms`
|
|
197
|
+
});
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
async function runReliabilityEval(options, config, outputFormat) {
|
|
201
|
+
const expectedTools = options['expected-tools']?.split(',').map(t => t.trim()) || [];
|
|
202
|
+
if (expectedTools.length === 0) {
|
|
203
|
+
throw new Error('--expected-tools is required for reliability evaluation');
|
|
204
|
+
}
|
|
205
|
+
// For now, simulate reliability check
|
|
206
|
+
// In production, this would track actual tool calls
|
|
207
|
+
const results = {
|
|
208
|
+
expected: expectedTools,
|
|
209
|
+
called: expectedTools, // Simulated
|
|
210
|
+
missing: [],
|
|
211
|
+
unexpected: []
|
|
212
|
+
};
|
|
213
|
+
const passed = results.missing.length === 0;
|
|
214
|
+
if (outputFormat === 'json') {
|
|
215
|
+
(0, json_1.outputJson)((0, json_1.formatSuccess)({
|
|
216
|
+
type: 'reliability',
|
|
217
|
+
status: passed ? 'PASSED' : 'FAILED',
|
|
218
|
+
results
|
|
219
|
+
}));
|
|
220
|
+
}
|
|
221
|
+
else {
|
|
222
|
+
await pretty.heading('Reliability Evaluation Results');
|
|
223
|
+
if (passed) {
|
|
224
|
+
await pretty.success('All expected tools were called');
|
|
225
|
+
}
|
|
226
|
+
else {
|
|
227
|
+
await pretty.error(`Missing tool calls: ${results.missing.join(', ')}`);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
/**
|
|
232
|
+
* Simple string similarity calculation
|
|
233
|
+
*/
|
|
234
|
+
function calculateSimilarity(str1, str2) {
|
|
235
|
+
const s1 = str1.toLowerCase().trim();
|
|
236
|
+
const s2 = str2.toLowerCase().trim();
|
|
237
|
+
if (s1 === s2)
|
|
238
|
+
return 1;
|
|
239
|
+
if (s1.includes(s2) || s2.includes(s1))
|
|
240
|
+
return 0.9;
|
|
241
|
+
// Simple word overlap
|
|
242
|
+
const words1 = new Set(s1.split(/\s+/));
|
|
243
|
+
const words2 = new Set(s2.split(/\s+/));
|
|
244
|
+
const intersection = [...words1].filter(w => words2.has(w));
|
|
245
|
+
const union = new Set([...words1, ...words2]);
|
|
246
|
+
return intersection.length / union.size;
|
|
247
|
+
}
|