@causyne/agent 0.0.0-stage → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +102 -0
- package/README.md +31 -2
- package/SECURITY.md +24 -0
- package/bin/causyne.js +13 -0
- package/docs/ARCHITECTURE.md +63 -0
- package/docs/PROVENANCE.md +29 -0
- package/docs/SECURITY_REVIEW.md +89 -0
- package/docs/THIRD_PARTY_NOTICES.md +49 -0
- package/native-skills/ai-engineering/context-window-management/SKILL.md +43 -0
- package/native-skills/ai-engineering/llm-evaluation/SKILL.md +46 -0
- package/native-skills/ai-engineering/model-cost-and-latency/SKILL.md +46 -0
- package/native-skills/ai-engineering/prompt-construction/SKILL.md +42 -0
- package/native-skills/ai-engineering/retrieval-augmented-generation/SKILL.md +42 -0
- package/native-skills/ai-engineering/structured-output-and-schemas/SKILL.md +45 -0
- package/native-skills/ai-engineering/tool-calling-agents/SKILL.md +43 -0
- package/native-skills/android/app-performance-profiling/SKILL.md +42 -0
- package/native-skills/android/background-work-scheduling/SKILL.md +47 -0
- package/native-skills/android/compose-state-and-recomposition/SKILL.md +43 -0
- package/native-skills/android/gradle-build-configuration/SKILL.md +41 -0
- package/native-skills/android/lifecycle-and-leaks/SKILL.md +46 -0
- package/native-skills/android/play-store-release/SKILL.md +43 -0
- package/native-skills/android/room-and-persistence/SKILL.md +40 -0
- package/native-skills/backend/authentication-and-sessions/SKILL.md +45 -0
- package/native-skills/backend/background-jobs-and-queues/SKILL.md +42 -0
- package/native-skills/backend/caching-strategy/SKILL.md +44 -0
- package/native-skills/backend/graceful-shutdown/SKILL.md +42 -0
- package/native-skills/backend/rate-limiting-and-quotas/SKILL.md +40 -0
- package/native-skills/backend/rest-api-design/SKILL.md +40 -0
- package/native-skills/backend/webhook-delivery-and-receipt/SKILL.md +43 -0
- package/native-skills/cloud/cost-optimisation/SKILL.md +46 -0
- package/native-skills/cloud/iam-least-privilege/SKILL.md +43 -0
- package/native-skills/cloud/infrastructure-as-code/SKILL.md +43 -0
- package/native-skills/cloud/multi-region-and-failover/SKILL.md +43 -0
- package/native-skills/cloud/serverless-patterns/SKILL.md +47 -0
- package/native-skills/data/data-quality-monitoring/SKILL.md +51 -0
- package/native-skills/data/etl-pipeline-reliability/SKILL.md +43 -0
- package/native-skills/data/exploratory-analysis/SKILL.md +43 -0
- package/native-skills/data/metric-definition/SKILL.md +42 -0
- package/native-skills/data/notebook-to-production/SKILL.md +43 -0
- package/native-skills/database/backup-and-recovery/SKILL.md +43 -0
- package/native-skills/database/connection-pooling/SKILL.md +42 -0
- package/native-skills/database/data-integrity-and-constraints/SKILL.md +42 -0
- package/native-skills/database/query-optimisation/SKILL.md +38 -0
- package/native-skills/database/safe-migrations/SKILL.md +47 -0
- package/native-skills/database/schema-design/SKILL.md +38 -0
- package/native-skills/database/transactions-and-isolation/SKILL.md +41 -0
- package/native-skills/design/colour-system-construction/SKILL.md +38 -0
- package/native-skills/design/design-direction-brief/SKILL.md +41 -0
- package/native-skills/design/motion-and-transitions/SKILL.md +38 -0
- package/native-skills/design/responsive-strategy/SKILL.md +39 -0
- package/native-skills/design/spacing-and-rhythm/SKILL.md +38 -0
- package/native-skills/design/type-scale-and-hierarchy/SKILL.md +39 -0
- package/native-skills/design/visual-qa-loop/SKILL.md +42 -0
- package/native-skills/devops/docker-multi-stage-builds/SKILL.md +60 -0
- package/native-skills/devops/github-actions-ci/SKILL.md +60 -0
- package/native-skills/devops/incident-triage/SKILL.md +38 -0
- package/native-skills/devops/kubernetes-deployment-hardening/SKILL.md +64 -0
- package/native-skills/devops/observability-baseline/SKILL.md +38 -0
- package/native-skills/devops/secret-management/SKILL.md +40 -0
- package/native-skills/devops/terraform-state-and-modules/SKILL.md +36 -0
- package/native-skills/devops/zero-downtime-deploys/SKILL.md +40 -0
- package/native-skills/git/bisect-regression-hunt/SKILL.md +43 -0
- package/native-skills/git/branch-strategy/SKILL.md +39 -0
- package/native-skills/git/commit-hygiene/SKILL.md +41 -0
- package/native-skills/git/history-recovery/SKILL.md +44 -0
- package/native-skills/git/large-repository-surgery/SKILL.md +46 -0
- package/native-skills/git/merge-conflict-resolution/SKILL.md +42 -0
- package/native-skills/media/cinematic-3d-shot/SKILL.md +104 -0
- package/native-skills/mobile/app-startup-optimisation/SKILL.md +41 -0
- package/native-skills/mobile/cross-platform-tradeoffs/SKILL.md +42 -0
- package/native-skills/mobile/deep-linking/SKILL.md +39 -0
- package/native-skills/mobile/mobile-storage-security/SKILL.md +45 -0
- package/native-skills/mobile/offline-first-sync/SKILL.md +42 -0
- package/native-skills/mobile/push-notifications/SKILL.md +38 -0
- package/native-skills/performance/algorithmic-complexity/SKILL.md +43 -0
- package/native-skills/performance/concurrency-and-parallelism/SKILL.md +42 -0
- package/native-skills/performance/load-testing/SKILL.md +42 -0
- package/native-skills/performance/memory-and-leak-hunting/SKILL.md +41 -0
- package/native-skills/performance/network-and-payload-efficiency/SKILL.md +42 -0
- package/native-skills/performance/profiling-methodology/SKILL.md +46 -0
- package/native-skills/python/async-correctness/SKILL.md +40 -0
- package/native-skills/python/cli-tool-construction/SKILL.md +44 -0
- package/native-skills/python/data-processing-pipelines/SKILL.md +38 -0
- package/native-skills/python/environment-isolation/SKILL.md +41 -0
- package/native-skills/python/packaging-and-distribution/SKILL.md +56 -0
- package/native-skills/python/pytest-suite-design/SKILL.md +39 -0
- package/native-skills/python/type-hints-and-static-checking/SKILL.md +42 -0
- package/native-skills/security/authorization-model/SKILL.md +42 -0
- package/native-skills/security/crypto-usage/SKILL.md +48 -0
- package/native-skills/security/dependency-supply-chain/SKILL.md +42 -0
- package/native-skills/security/input-validation-and-injection/SKILL.md +44 -0
- package/native-skills/security/secure-file-uploads/SKILL.md +46 -0
- package/native-skills/security/security-headers-and-csp/SKILL.md +44 -0
- package/native-skills/security/threat-modelling/SKILL.md +46 -0
- package/native-skills/testing/end-to-end-test-stability/SKILL.md +42 -0
- package/native-skills/testing/property-based-testing/SKILL.md +46 -0
- package/native-skills/testing/regression-test-from-a-bug/SKILL.md +41 -0
- package/native-skills/testing/snapshot-and-visual-regression/SKILL.md +42 -0
- package/native-skills/testing/test-doubles-and-mocking/SKILL.md +43 -0
- package/native-skills/testing/test-strategy-and-pyramid/SKILL.md +39 -0
- package/native-skills/testing/writing-good-assertions/SKILL.md +42 -0
- package/native-skills/web/accessible-components/SKILL.md +45 -0
- package/native-skills/web/api-client-resilience/SKILL.md +39 -0
- package/native-skills/web/bundle-size-reduction/SKILL.md +38 -0
- package/native-skills/web/core-web-vitals/SKILL.md +40 -0
- package/native-skills/web/css-layout-systems/SKILL.md +41 -0
- package/native-skills/web/form-validation-ux/SKILL.md +39 -0
- package/native-skills/web/react-state-architecture/SKILL.md +41 -0
- package/native-skills/web/ssr-hydration-correctness/SKILL.md +45 -0
- package/package.json +68 -4
- package/src/agentLoop.js +2458 -0
- package/src/appChannel.js +109 -0
- package/src/approvalGrants.js +66 -0
- package/src/auditLog.js +52 -0
- package/src/autoUpdate.js +12 -0
- package/src/capabilities.js +36 -0
- package/src/checkpoints.js +1199 -0
- package/src/cliMode.js +638 -0
- package/src/commands.js +805 -0
- package/src/compiledSkills.js +540 -0
- package/src/config.js +130 -0
- package/src/contextManager.js +498 -0
- package/src/cortex/autoCondenser.js +192 -0
- package/src/cortex/provisioner.js +27 -0
- package/src/cortex/vectorDb.js +447 -0
- package/src/designPalettes.js +99 -0
- package/src/devMode.js +6 -0
- package/src/embodimentSession.js +180 -0
- package/src/entrypoint.js +194 -0
- package/src/execSafe.js +21 -0
- package/src/gitSnapshot.js +378 -0
- package/src/mcp/jsonRpc.js +165 -0
- package/src/mcp/mcpClient.js +136 -0
- package/src/mcp/mcpRegistry.js +207 -0
- package/src/memory.js +62 -0
- package/src/memoryManager.js +124 -0
- package/src/modelCatalogue.js +132 -0
- package/src/modelIds.js +42 -0
- package/src/modelRouter.js +145 -0
- package/src/outputStore.js +246 -0
- package/src/perception.js +124 -0
- package/src/planCache.js +437 -0
- package/src/procSignal.js +32 -0
- package/src/providers/anthropic.js +408 -0
- package/src/providers/base.js +328 -0
- package/src/providers/dashscope-byok.js +86 -0
- package/src/providers/deepseek.js +80 -0
- package/src/providers/factory.js +37 -0
- package/src/providers/gemini.js +346 -0
- package/src/providers/moonshot.js +73 -0
- package/src/providers/muse.js +169 -0
- package/src/providers/ollama.js +104 -0
- package/src/providers/openai-compatible.js +84 -0
- package/src/providers/openai.js +20 -0
- package/src/providers/qwen-byok.js +76 -0
- package/src/providers/xai.js +73 -0
- package/src/provisioning/capabilityConsent.js +24 -0
- package/src/provisioning/capabilityProvisioner.js +271 -0
- package/src/provisioning/capabilityTools.js +29 -0
- package/src/reporting.js +373 -0
- package/src/sandbox.js +529 -0
- package/src/sessionStore.js +279 -0
- package/src/skillsManager.js +288 -0
- package/src/slopDetector.js +307 -0
- package/src/taskBuffer.js +45 -0
- package/src/taskShape.js +241 -0
- package/src/textDiff.js +150 -0
- package/src/tokenTracker.js +560 -0
- package/src/tool-scripts/ax_snapshot.applescript +39 -0
- package/src/tool-scripts/ax_snapshot.ps1 +10 -0
- package/src/tool-scripts/desktop_act.applescript +58 -0
- package/src/tool-scripts/resolve_automator.py +119 -0
- package/src/toolPacks.js +524 -0
- package/src/tools/archive.js +63 -0
- package/src/tools/astryxOracle.js +78 -0
- package/src/tools/axSnapshot.js +165 -0
- package/src/tools/blenderAutomator.js +349 -0
- package/src/tools/browserAutomator.js +252 -0
- package/src/tools/checkSystemSpecs.js +64 -0
- package/src/tools/clipboard.js +36 -0
- package/src/tools/codeSearchAst.js +101 -0
- package/src/tools/codeSearcher.js +230 -0
- package/src/tools/codeTools.js +262 -0
- package/src/tools/codebaseSearch.js +75 -0
- package/src/tools/createSkill.js +129 -0
- package/src/tools/deferredMcpSearch.js +246 -0
- package/src/tools/desktopAct.js +208 -0
- package/src/tools/desktopOpen.js +34 -0
- package/src/tools/dynamicLoader.js +44 -0
- package/src/tools/executeProceduralMedia.js +93 -0
- package/src/tools/fsDelete.js +51 -0
- package/src/tools/fsEdit.js +105 -0
- package/src/tools/fsFind.js +128 -0
- package/src/tools/fsList.js +77 -0
- package/src/tools/fsMove.js +57 -0
- package/src/tools/fsRead.js +87 -0
- package/src/tools/fsSearch.js +198 -0
- package/src/tools/fsWrite.js +41 -0
- package/src/tools/gameAiBehaviorTree.js +208 -0
- package/src/tools/gameAnimationRig.js +194 -0
- package/src/tools/gameAssetFetcher.js +372 -0
- package/src/tools/gameAssetHarvester.js +210 -0
- package/src/tools/gameAudioSynth.js +267 -0
- package/src/tools/gameBuilder.js +886 -0
- package/src/tools/gameCanvasAudit.js +101 -0
- package/src/tools/gameCinematicSequencer.js +151 -0
- package/src/tools/gameDataForge.js +124 -0
- package/src/tools/gameDialogueTreeGen.js +166 -0
- package/src/tools/gameEconomyBalancer.js +161 -0
- package/src/tools/gameEntitySpawner.js +207 -0
- package/src/tools/gameLoreForge.js +153 -0
- package/src/tools/gameQuestArcBuilder.js +152 -0
- package/src/tools/gameSceneBuilder.js +158 -0
- package/src/tools/gameShaderForge.js +181 -0
- package/src/tools/gameTilemapGenerator.js +164 -0
- package/src/tools/gameUiHudBuilder.js +292 -0
- package/src/tools/gameVfxEmitterGen.js +108 -0
- package/src/tools/gameWebRuntimeInspector.js +91 -0
- package/src/tools/generateMediaViaApi.js +358 -0
- package/src/tools/gitExec.js +54 -0
- package/src/tools/godotAutomator.js +437 -0
- package/src/tools/manageLocalAiDependencies.js +220 -0
- package/src/tools/memoryWrite.js +99 -0
- package/src/tools/multiEdit.js +107 -0
- package/src/tools/palmierAutomator.js +104 -0
- package/src/tools/pdfReader.js +112 -0
- package/src/tools/persistentShell.js +148 -0
- package/src/tools/printer.js +316 -0
- package/src/tools/procTerm.js +91 -0
- package/src/tools/processList.js +52 -0
- package/src/tools/readNativeSkill.js +39 -0
- package/src/tools/registry.js +277 -0
- package/src/tools/resolveAutomator.js +102 -0
- package/src/tools/screenshot.js +75 -0
- package/src/tools/subagentOrchestrator.js +305 -0
- package/src/tools/sysRun.js +125 -0
- package/src/tools/todoWrite.js +75 -0
- package/src/tools/toolDispatch.js +127 -0
- package/src/tools/toolForge.js +124 -0
- package/src/tools/updatePlan.js +88 -0
- package/src/tools/viewImage.js +82 -0
- package/src/tools/visionInspector.js +154 -0
- package/src/tools/webFetch.js +74 -0
- package/src/tools/webSearch.js +82 -0
- package/src/tui/ask.js +164 -0
- package/src/tui/banner.js +158 -0
- package/src/tui/block.js +152 -0
- package/src/tui/chatRenderer.js +953 -0
- package/src/tui/completion.js +424 -0
- package/src/tui/dialogs.js +606 -0
- package/src/tui/diff.js +296 -0
- package/src/tui/history.js +135 -0
- package/src/tui/home.js +217 -0
- package/src/tui/invocation.js +105 -0
- package/src/tui/keys.js +443 -0
- package/src/tui/lineEditor.js +256 -0
- package/src/tui/overlay.js +256 -0
- package/src/tui/prompt.js +1258 -0
- package/src/tui/selectList.js +335 -0
- package/src/tui/stage.js +330 -0
- package/src/tui/stageSurface.js +609 -0
- package/src/tui/statusLine.js +254 -0
- package/src/tui/text.js +345 -0
- package/src/tui/theme.js +393 -0
- package/src/tui/toolDisplay.js +176 -0
- package/src/tui/transcript.js +235 -0
- package/src/tui/viewport.js +337 -0
- package/src/uncertainty.js +271 -0
- package/src/userSkills.js +265 -0
- package/src/utils/cli_ui.js +0 -0
- package/src/utils/inquirer.js +12 -0
- package/src/utils/paths.js +37 -0
- package/src/version.js +38 -0
- package/src/vfeTracker.js +220 -0
|
@@ -0,0 +1,346 @@
|
|
|
1
|
+
import { canonicalModelId } from '../modelIds.js';
|
|
2
|
+
import { DEFAULT_MODEL_BY_PROVIDER } from '../modelCatalogue.js';
|
|
3
|
+
import { LLMProvider, mimeTypeForAudioFormat } from './base.js';
|
|
4
|
+
|
|
5
|
+
// Gemini's FunctionDeclaration schema is an OpenAPI subset, and it is strict in
|
|
6
|
+
// two ways that matter here:
|
|
7
|
+
// 1. `additionalProperties` is not part of the subset.
|
|
8
|
+
// 2. An OBJECT that declares no properties is rejected.
|
|
9
|
+
// Crucially, one bad declaration fails the *entire* request — so a single
|
|
10
|
+
// free-form object anywhere in the 69-tool catalogue takes every tool down with
|
|
11
|
+
// it. `web_fetch.headers` is exactly that shape and web_fetch is a CORE tool,
|
|
12
|
+
// which meant Gemini could not complete step one of any task at all.
|
|
13
|
+
//
|
|
14
|
+
// Rather than delete those parameters and lose the capability, free-form maps
|
|
15
|
+
// travel as JSON strings and are decoded again on the way back, so the tool
|
|
16
|
+
// still receives a real object. The paths that were rewritten are recorded per
|
|
17
|
+
// tool so the decode knows exactly where to look.
|
|
18
|
+
|
|
19
|
+
const FREE_FORM_NOTE = ' Pass this as a JSON object encoded in a string, for example {"key":"value"}.';
|
|
20
|
+
|
|
21
|
+
function isEmptyObjectSchema(schema) {
|
|
22
|
+
return schema
|
|
23
|
+
&& schema.type === 'object'
|
|
24
|
+
&& (!schema.properties || Object.keys(schema.properties).length === 0);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Rewrite one JSON Schema into Gemini's subset.
|
|
29
|
+
*
|
|
30
|
+
* @param {object} schema The source schema.
|
|
31
|
+
* @param {Array} path Segments walked so far; '[]' denotes array items.
|
|
32
|
+
* @param {Array} rewritten Collects the paths converted to JSON strings.
|
|
33
|
+
* @returns {object} A schema Gemini will accept.
|
|
34
|
+
*/
|
|
35
|
+
function adaptSchemaForGemini(schema, path, rewritten) {
|
|
36
|
+
if (Array.isArray(schema)) {
|
|
37
|
+
return schema.map(entry => adaptSchemaForGemini(entry, path, rewritten));
|
|
38
|
+
}
|
|
39
|
+
if (schema === null || typeof schema !== 'object') {
|
|
40
|
+
return schema;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
const adapted = {};
|
|
44
|
+
for (const [key, value] of Object.entries(schema)) {
|
|
45
|
+
if (key === 'additionalProperties') continue;
|
|
46
|
+
|
|
47
|
+
if (key === 'properties' && value && typeof value === 'object') {
|
|
48
|
+
const properties = {};
|
|
49
|
+
for (const [name, child] of Object.entries(value)) {
|
|
50
|
+
const childPath = [...path, name];
|
|
51
|
+
if (isEmptyObjectSchema(child)) {
|
|
52
|
+
rewritten.push(childPath);
|
|
53
|
+
properties[name] = {
|
|
54
|
+
type: 'string',
|
|
55
|
+
description: (child.description || 'Free-form object.') + FREE_FORM_NOTE
|
|
56
|
+
};
|
|
57
|
+
} else {
|
|
58
|
+
properties[name] = adaptSchemaForGemini(child, childPath, rewritten);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
adapted.properties = properties;
|
|
62
|
+
continue;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
if (key === 'items') {
|
|
66
|
+
adapted.items = adaptSchemaForGemini(value, [...path, '[]'], rewritten);
|
|
67
|
+
continue;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
adapted[key] = adaptSchemaForGemini(value, path, rewritten);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
return adapted;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** Walk a decoded argument object and JSON.parse the fields that were stringified. */
|
|
77
|
+
function decodeFreeFormArgs(args, paths) {
|
|
78
|
+
if (!args || paths.length === 0) return args;
|
|
79
|
+
|
|
80
|
+
for (const path of paths) {
|
|
81
|
+
applyAtPath(args, path, value => {
|
|
82
|
+
if (typeof value !== 'string') return value;
|
|
83
|
+
try {
|
|
84
|
+
return JSON.parse(value);
|
|
85
|
+
} catch (e) {
|
|
86
|
+
// The model sent something that is not JSON. Hand it through untouched
|
|
87
|
+
// rather than destroying the argument.
|
|
88
|
+
return value;
|
|
89
|
+
}
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
return args;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function applyAtPath(node, path, transform) {
|
|
97
|
+
if (node === null || typeof node !== 'object') return;
|
|
98
|
+
|
|
99
|
+
const [head, ...rest] = path;
|
|
100
|
+
|
|
101
|
+
if (head === '[]') {
|
|
102
|
+
if (!Array.isArray(node)) return;
|
|
103
|
+
for (const entry of node) applyAtPath(entry, rest, transform);
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
if (rest.length === 0) {
|
|
108
|
+
if (head in node) node[head] = transform(node[head]);
|
|
109
|
+
return;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
applyAtPath(node[head], rest, transform);
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* Translate Gemini's `usageMetadata` into the OpenAI-shaped object that
|
|
117
|
+
* `LLMProvider.normalizeUsage()` consumes, so every provider ends up on one
|
|
118
|
+
* normalisation path instead of Gemini hand-rolling its own.
|
|
119
|
+
*
|
|
120
|
+
* TWO BUGS THIS FIXES, both in the cost the user is shown:
|
|
121
|
+
*
|
|
122
|
+
* 1. `cachedContentTokenCount` was dropped entirely. Gemini caches implicitly
|
|
123
|
+
* on a stable prefix, so the cache WAS working and `calculateCost()` IS
|
|
124
|
+
* cache-aware — it was simply fed a zero, and every cached token was
|
|
125
|
+
* displayed at the full input rate instead of 0.5x. Overstating the bill is
|
|
126
|
+
* how a user concludes Causyne is expensive and goes direct instead.
|
|
127
|
+
*
|
|
128
|
+
* 2. `thoughtsTokenCount` was dropped, and thinking tokens ARE billed as
|
|
129
|
+
* output ("Response pricing is the sum of output tokens and thinking
|
|
130
|
+
* tokens"). That understated the bill — the worse direction, because the
|
|
131
|
+
* real invoice then exceeds what Causyne promised.
|
|
132
|
+
*
|
|
133
|
+
* WHY `totalTokenCount` IS THE ANCHOR: whether `thoughtsTokenCount` is already
|
|
134
|
+
* folded into `candidatesTokenCount` DIFFERS BY BACKEND — it is on the Gemini
|
|
135
|
+
* API and is not on Vertex AI, and reports exist of the Gemini API behaving
|
|
136
|
+
* both ways across model versions. Deriving completion as `total - prompt` is
|
|
137
|
+
* correct under either convention; adding thoughts to candidates is correct
|
|
138
|
+
* under only one and double-counts under the other. The sum is used only when
|
|
139
|
+
* the total is missing or incoherent, where overstating is the safer error.
|
|
140
|
+
*
|
|
141
|
+
* @param {object} usageMetadata Gemini's usage block, possibly undefined.
|
|
142
|
+
* @returns {object} OpenAI-shaped usage for normalizeUsage().
|
|
143
|
+
*/
|
|
144
|
+
export function geminiUsageToOpenAIShape(usageMetadata) {
|
|
145
|
+
const prompt = usageMetadata?.promptTokenCount || 0;
|
|
146
|
+
const candidates = usageMetadata?.candidatesTokenCount || 0;
|
|
147
|
+
const thoughts = usageMetadata?.thoughtsTokenCount || 0;
|
|
148
|
+
const total = usageMetadata?.totalTokenCount || 0;
|
|
149
|
+
|
|
150
|
+
const completion = total > prompt ? total - prompt : candidates + thoughts;
|
|
151
|
+
|
|
152
|
+
return {
|
|
153
|
+
prompt_tokens: prompt,
|
|
154
|
+
completion_tokens: completion,
|
|
155
|
+
// normalizeUsage() clamps this to prompt_tokens, which also protects
|
|
156
|
+
// against a backend that reports the cached count OUTSIDE the prompt total.
|
|
157
|
+
prompt_tokens_details: { cached_tokens: usageMetadata?.cachedContentTokenCount || 0 }
|
|
158
|
+
};
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
export class GeminiProvider extends LLMProvider {
|
|
162
|
+
constructor(config) {
|
|
163
|
+
super(config);
|
|
164
|
+
this.model = canonicalModelId(config.model || DEFAULT_MODEL_BY_PROVIDER['gemini']);
|
|
165
|
+
this.baseUrl = `https://generativelanguage.googleapis.com/v1beta/models/${this.model}:generateContent?key=${this.apiKey}`;
|
|
166
|
+
// Every Gemini model from 1.5 onward accepts image input.
|
|
167
|
+
this.visionModels = ['gemini-'];
|
|
168
|
+
this.imagePayloadFormat = 'gemini';
|
|
169
|
+
// Gemini takes audio the same way it takes images — inlineData with a real
|
|
170
|
+
// MIME type — across the whole gemini- family, and accepts m4a/aac natively,
|
|
171
|
+
|
|
172
|
+
// smaller than WAV, so it clears the transport ceiling far more often.
|
|
173
|
+
this.audioModels = ['gemini-'];
|
|
174
|
+
this.audioPayloadFormat = 'gemini';
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
async chat(messages, options = {}) {
|
|
178
|
+
messages = this.prepareMessages(messages);
|
|
179
|
+
// Map OpenAI messages to Gemini contents.
|
|
180
|
+
// System blocks ACCUMULATE. Assigning instead of appending meant that with
|
|
181
|
+
// the agent's three-block system stack only the last block survived: the
|
|
182
|
+
// core SYSTEM_PROMPT, the capability-pack catalogue and every playbook were
|
|
183
|
+
// silently discarded on every single Gemini request.
|
|
184
|
+
const systemParts = [];
|
|
185
|
+
const contents = [];
|
|
186
|
+
|
|
187
|
+
for (const msg of messages) {
|
|
188
|
+
if (msg.role === 'system') {
|
|
189
|
+
const text = typeof msg.content === 'string'
|
|
190
|
+
? msg.content
|
|
191
|
+
: JSON.stringify(msg.content);
|
|
192
|
+
if (text) systemParts.push({ text });
|
|
193
|
+
continue;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
const role = msg.role === 'assistant' ? 'model' : 'user';
|
|
197
|
+
|
|
198
|
+
if (msg.tool_calls) {
|
|
199
|
+
// The assistant's own words travel with its calls. Dropping the text
|
|
200
|
+
// meant the model's stated reasoning never re-entered the history.
|
|
201
|
+
const parts = [];
|
|
202
|
+
if (msg.content) parts.push({ text: String(msg.content) });
|
|
203
|
+
for (const tc of msg.tool_calls) {
|
|
204
|
+
parts.push({
|
|
205
|
+
functionCall: {
|
|
206
|
+
name: tc.function.name,
|
|
207
|
+
args: typeof tc.function.arguments === 'string'
|
|
208
|
+
? JSON.parse(tc.function.arguments || '{}')
|
|
209
|
+
: (tc.function.arguments || {})
|
|
210
|
+
}
|
|
211
|
+
});
|
|
212
|
+
}
|
|
213
|
+
contents.push({ role, parts });
|
|
214
|
+
} else if (msg.role === 'tool') {
|
|
215
|
+
// Add tool response
|
|
216
|
+
contents.push({
|
|
217
|
+
role: 'user', // Gemini expects tool responses to come from 'user' or 'function'
|
|
218
|
+
parts: [{
|
|
219
|
+
functionResponse: {
|
|
220
|
+
name: msg.name,
|
|
221
|
+
response: { result: msg.content }
|
|
222
|
+
}
|
|
223
|
+
}]
|
|
224
|
+
});
|
|
225
|
+
} else {
|
|
226
|
+
if (Array.isArray(msg.content)) {
|
|
227
|
+
const mappedParts = msg.content.map(part => {
|
|
228
|
+
if (part.type === 'text') return { text: part.text };
|
|
229
|
+
if (part.type === 'image_url') {
|
|
230
|
+
const b64 = part.image_url.url.split('base64,')[1];
|
|
231
|
+
const mime = part.image_url.url.split(';')[0].replace('data:', '');
|
|
232
|
+
return { inlineData: { mimeType: mime, data: b64 } };
|
|
233
|
+
}
|
|
234
|
+
// Anthropic-shaped image block. Without this branch it fell through
|
|
235
|
+
// to the empty-text default and the image vanished silently.
|
|
236
|
+
if (part.type === 'image' && part.source?.type === 'base64') {
|
|
237
|
+
return { inlineData: { mimeType: part.source.media_type, data: part.source.data } };
|
|
238
|
+
}
|
|
239
|
+
// Audio normalised by prepareMessages already arrives as inlineData;
|
|
240
|
+
// this branch covers an input_audio block replayed from history that
|
|
241
|
+
// predates the normaliser. The MIME type is derived from the clip's
|
|
242
|
+
// real container instead of the old hardcoded audio/mp4, which
|
|
243
|
+
|
|
244
|
+
if (part.type === 'input_audio') {
|
|
245
|
+
return {
|
|
246
|
+
inlineData: {
|
|
247
|
+
mimeType: mimeTypeForAudioFormat(part.input_audio?.format),
|
|
248
|
+
data: part.input_audio?.data
|
|
249
|
+
}
|
|
250
|
+
};
|
|
251
|
+
}
|
|
252
|
+
if (part.inlineData) return part;
|
|
253
|
+
return { text: '' };
|
|
254
|
+
});
|
|
255
|
+
contents.push({ role, parts: mappedParts });
|
|
256
|
+
} else {
|
|
257
|
+
contents.push({ role, parts: [{ text: msg.content }] });
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
const payload = {
|
|
263
|
+
contents,
|
|
264
|
+
generationConfig: {
|
|
265
|
+
temperature: options.temperature || 0.1,
|
|
266
|
+
}
|
|
267
|
+
};
|
|
268
|
+
|
|
269
|
+
if (systemParts.length > 0) {
|
|
270
|
+
payload.systemInstruction = { parts: systemParts };
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
// Map tools. freeFormPaths remembers which arguments were stringified so the
|
|
274
|
+
// response handler can turn them back into objects.
|
|
275
|
+
const freeFormPaths = new Map();
|
|
276
|
+
|
|
277
|
+
if (options.tools && options.tools.length > 0) {
|
|
278
|
+
payload.tools = [{
|
|
279
|
+
functionDeclarations: options.tools.map(t => {
|
|
280
|
+
const toolObj = t.function ? t.function : t;
|
|
281
|
+
const declaration = {
|
|
282
|
+
name: toolObj.name,
|
|
283
|
+
description: toolObj.description || ''
|
|
284
|
+
};
|
|
285
|
+
|
|
286
|
+
const source = toolObj.parameters || toolObj.input_schema;
|
|
287
|
+
// A tool that takes no arguments must omit `parameters` entirely —
|
|
288
|
+
// an OBJECT with empty properties is rejected outright.
|
|
289
|
+
if (source && !isEmptyObjectSchema(source)) {
|
|
290
|
+
const rewritten = [];
|
|
291
|
+
declaration.parameters = adaptSchemaForGemini(source, [], rewritten);
|
|
292
|
+
if (rewritten.length > 0) freeFormPaths.set(toolObj.name, rewritten);
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
return declaration;
|
|
296
|
+
})
|
|
297
|
+
}];
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
const response = await fetch(this.baseUrl, {
|
|
301
|
+
method: 'POST',
|
|
302
|
+
headers: { 'Content-Type': 'application/json' },
|
|
303
|
+
body: JSON.stringify(payload)
|
|
304
|
+
});
|
|
305
|
+
|
|
306
|
+
if (!response.ok) {
|
|
307
|
+
const errorData = await response.text();
|
|
308
|
+
throw new Error(`Gemini API Error (${response.status}): ${errorData}`);
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
const data = await response.json();
|
|
312
|
+
|
|
313
|
+
const firstCandidate = data.candidates?.[0];
|
|
314
|
+
if (!firstCandidate) {
|
|
315
|
+
throw new Error('No candidate returned from Gemini');
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
let content = '';
|
|
319
|
+
let toolCalls = [];
|
|
320
|
+
|
|
321
|
+
for (const part of firstCandidate.content.parts) {
|
|
322
|
+
if (part.text) {
|
|
323
|
+
content += part.text;
|
|
324
|
+
} else if (part.functionCall) {
|
|
325
|
+
const args = decodeFreeFormArgs(
|
|
326
|
+
part.functionCall.args || {},
|
|
327
|
+
freeFormPaths.get(part.functionCall.name) || []
|
|
328
|
+
);
|
|
329
|
+
toolCalls.push({
|
|
330
|
+
id: `call_${Math.random().toString(36).substr(2, 9)}`,
|
|
331
|
+
type: 'function',
|
|
332
|
+
function: {
|
|
333
|
+
name: part.functionCall.name,
|
|
334
|
+
arguments: JSON.stringify(args)
|
|
335
|
+
}
|
|
336
|
+
});
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
return {
|
|
341
|
+
content: content.trim(),
|
|
342
|
+
toolCalls: toolCalls.length > 0 ? toolCalls : undefined,
|
|
343
|
+
usage: this.normalizeUsage(geminiUsageToOpenAIShape(data.usageMetadata))
|
|
344
|
+
};
|
|
345
|
+
}
|
|
346
|
+
}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import { canonicalModelId } from '../modelIds.js';
|
|
2
|
+
import { DEFAULT_MODEL_BY_PROVIDER } from '../modelCatalogue.js';
|
|
3
|
+
import { LLMProvider } from './base.js';
|
|
4
|
+
|
|
5
|
+
export class MoonshotProvider extends LLMProvider {
|
|
6
|
+
constructor(config) {
|
|
7
|
+
super(config);
|
|
8
|
+
this.baseUrl = 'https://api.moonshot.ai/v1/chat/completions';
|
|
9
|
+
this.model = canonicalModelId(config.model || DEFAULT_MODEL_BY_PROVIDER['moonshot']);
|
|
10
|
+
this.visionModels = ['kimi-k3', 'kimi-k2.5', 'vision'];
|
|
11
|
+
// Kimi exposes no audio input on the chat completions API.
|
|
12
|
+
this.audioModels = [];
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
async chat(messages, options = {}) {
|
|
16
|
+
const sanitizedMessages = this.prepareMessages(messages);
|
|
17
|
+
|
|
18
|
+
const payload = {
|
|
19
|
+
model: this.model,
|
|
20
|
+
messages: sanitizedMessages,
|
|
21
|
+
temperature: options.temperature || 0.1,
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
if (options.jsonMode) {
|
|
25
|
+
payload.response_format = { type: 'json_object' };
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
let headers = {
|
|
29
|
+
'Content-Type': 'application/json'
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
if (this.apiKey) {
|
|
33
|
+
headers['Authorization'] = `Bearer ${this.apiKey}`;
|
|
34
|
+
} else {
|
|
35
|
+
// EXPLICIT BYOK CONSTRAINT
|
|
36
|
+
throw new Error(`Moonshot AI (Kimi) API requires a valid API key (BYOK mode). Please set it in your configuration.`);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
if (options.tools) {
|
|
40
|
+
payload.tools = options.tools.map(tool => ({
|
|
41
|
+
type: 'function',
|
|
42
|
+
function: tool.function ? tool.function : tool
|
|
43
|
+
}));
|
|
44
|
+
payload.tool_choice = "auto";
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const response = await fetch(this.baseUrl, {
|
|
48
|
+
method: 'POST',
|
|
49
|
+
headers: headers,
|
|
50
|
+
body: JSON.stringify(payload)
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
if (!response.ok) {
|
|
54
|
+
const errorData = await response.text();
|
|
55
|
+
throw new Error(`Moonshot AI API Error (${response.status}): ${errorData}`);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
const data = await response.json();
|
|
59
|
+
let content = '';
|
|
60
|
+
let toolCalls = [];
|
|
61
|
+
|
|
62
|
+
if (data.choices && data.choices[0] && data.choices[0].message) {
|
|
63
|
+
content = data.choices[0].message.content || '';
|
|
64
|
+
toolCalls = data.choices[0].message.tool_calls || [];
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
return {
|
|
68
|
+
content: content,
|
|
69
|
+
toolCalls: (toolCalls && toolCalls.length > 0) ? toolCalls : undefined,
|
|
70
|
+
usage: this.normalizeUsage(data.usage || { prompt_tokens: 0, completion_tokens: 0 })
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
}
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
import { canonicalModelId } from '../modelIds.js';
|
|
2
|
+
import { DEFAULT_MODEL_BY_PROVIDER } from '../modelCatalogue.js';
|
|
3
|
+
import { OpenAICompatibleProvider } from './openai-compatible.js';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Meta Model API — the Muse family.
|
|
7
|
+
*
|
|
8
|
+
* Drop-in OpenAI-compatible, so this extends the compatible provider rather
|
|
9
|
+
* than reimplementing the wire format. Everything below is the part that is
|
|
10
|
+
* NOT generic: which models can see, which can hear, which reasoning levels a
|
|
11
|
+
* given tier will actually accept, and how wide the context is.
|
|
12
|
+
*
|
|
13
|
+
* Model IDs verified against the live status page (all seven operational):
|
|
14
|
+
* muse-spark-1.3, muse-spark-1.3-contributor, muse-spark-1.2,
|
|
15
|
+
* muse-spark-1.2-contributor, muse-spark-1.1, muse-image-1.0,
|
|
16
|
+
* muse-voice-transcribe-1.0.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
export const MUSE_BASE_URL = 'https://api.meta.ai/v1';
|
|
20
|
+
|
|
21
|
+
/** Every Muse Spark model shares this. Not a guess — it is the published spec. */
|
|
22
|
+
export const MUSE_CONTEXT_TOKENS = 1_048_576;
|
|
23
|
+
|
|
24
|
+
/** The image and speech models, which are not chat models and are reached elsewhere. */
|
|
25
|
+
export const MUSE_IMAGE_MODEL = 'muse-image-1.0';
|
|
26
|
+
export const MUSE_TRANSCRIBE_MODEL = 'muse-voice-transcribe-1.0';
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Chat models, newest first, with what each one can actually do.
|
|
30
|
+
*
|
|
31
|
+
* `audio: false` on 1.3 is not caution, it is the documented state: "Audio
|
|
32
|
+
* understanding in Muse Spark 1.3 is currently not fully supported, and
|
|
33
|
+
* response quality for requests including audio content may be degraded."
|
|
34
|
+
* Advertising it would send a voice clip into a model that answers badly and
|
|
35
|
+
* charges for it — worse than declining, because the user cannot tell a
|
|
36
|
+
* degraded answer from a wrong one.
|
|
37
|
+
*/
|
|
38
|
+
export const MUSE_CHAT_MODELS = [
|
|
39
|
+
{ id: 'muse-spark-1.3', tier: 'standard', audio: false, recommended: true },
|
|
40
|
+
{ id: 'muse-spark-1.3-contributor', tier: 'contributor', audio: false },
|
|
41
|
+
{ id: 'muse-spark-1.2', tier: 'standard', audio: true },
|
|
42
|
+
{ id: 'muse-spark-1.2-contributor', tier: 'contributor', audio: true },
|
|
43
|
+
{ id: 'muse-spark-1.1', tier: 'standard', audio: true }
|
|
44
|
+
];
|
|
45
|
+
|
|
46
|
+
/** Models that accept audio, as bare id prefixes for LLMProvider.supportsAudio. */
|
|
47
|
+
const AUDIO_CAPABLE = MUSE_CHAT_MODELS.filter((m) => m.audio).map((m) => m.id);
|
|
48
|
+
|
|
49
|
+
/** Is this a Contributor-tier model? Its inputs and outputs train Meta's models. */
|
|
50
|
+
export function isContributorModel(model) {
|
|
51
|
+
return String(model || '').toLowerCase().endsWith('-contributor');
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Reasoning effort the model will actually accept.
|
|
56
|
+
*
|
|
57
|
+
* `"max"` is Standard tier only. Sending it to a `-contributor` model is a
|
|
58
|
+
* rejected request, and a rejected request in the middle of an agent loop looks
|
|
59
|
+
* to the user like the agent broke — so it is stepped down here, once, where
|
|
60
|
+
* the reason can be written down.
|
|
61
|
+
*/
|
|
62
|
+
export function clampReasoningEffort(effort, model) {
|
|
63
|
+
const requested = String(effort || '').toLowerCase();
|
|
64
|
+
if (!requested || requested === 'off') return requested || null;
|
|
65
|
+
if (requested === 'max' && isContributorModel(model)) return 'high';
|
|
66
|
+
return requested;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export class MuseProvider extends OpenAICompatibleProvider {
|
|
70
|
+
constructor(config) {
|
|
71
|
+
super(config);
|
|
72
|
+
|
|
73
|
+
this.baseUrl = `${MUSE_BASE_URL}/chat/completions`;
|
|
74
|
+
this.model = canonicalModelId(config.model || DEFAULT_MODEL_BY_PROVIDER['muse']);
|
|
75
|
+
|
|
76
|
+
// Every Spark model takes text, image, video and PDF. The prefix covers all
|
|
77
|
+
// five ids including the -contributor variants, which share the exact same
|
|
78
|
+
// modalities as their Standard twins.
|
|
79
|
+
this.visionModels = ['muse-spark'];
|
|
80
|
+
this.imagePayloadFormat = 'openai';
|
|
81
|
+
|
|
82
|
+
// Audio is the one capability that is NOT uniform across the family.
|
|
83
|
+
this.audioModels = AUDIO_CAPABLE;
|
|
84
|
+
this.audioPayloadFormat = 'openai';
|
|
85
|
+
|
|
86
|
+
// Meta publishes no per-clip ceiling for Muse Spark, so no vendor clamp is
|
|
87
|
+
|
|
88
|
+
// "this vendor adds no limit of its own", not "unlimited".
|
|
89
|
+
this.maxAudioSeconds = null;
|
|
90
|
+
this.maxAudioBytes = null;
|
|
91
|
+
|
|
92
|
+
this.contextTokens = MUSE_CONTEXT_TOKENS;
|
|
93
|
+
this.reasoningEffort = clampReasoningEffort(config.reasoningEffort, this.model);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* The transcription model exists for exactly the case 1.3 creates.
|
|
98
|
+
*
|
|
99
|
+
* The recommended chat model cannot hear, and Meta's answer to that is a
|
|
100
|
+
* dedicated speech-to-text model on the same key and the same base URL. This
|
|
101
|
+
* is what lets the agent loop offer the user a real route instead of only a
|
|
102
|
+
* refusal.
|
|
103
|
+
*/
|
|
104
|
+
audioAlternative() {
|
|
105
|
+
if (this.supportsAudio) return null;
|
|
106
|
+
// Keep the tier the user chose: someone on Contributor is there for the
|
|
107
|
+
// price, and answering "switch to Standard" would quietly multiply it by
|
|
108
|
+
// twelve.
|
|
109
|
+
return isContributorModel(this.model) ? 'muse-spark-1.2-contributor' : 'muse-spark-1.2';
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
get transcribeModel() {
|
|
113
|
+
return MUSE_TRANSCRIBE_MODEL;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
get transcribeUrl() {
|
|
117
|
+
return `${MUSE_BASE_URL}/asr/transcribe`;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Transcribe a clip with Muse Voice Transcribe.
|
|
122
|
+
*
|
|
123
|
+
* THE REQUEST SHAPE IS INFERRED, and that is stated rather than hidden. Meta
|
|
124
|
+
* documents the endpoint (`POST /v1/asr/transcribe`) and states that Model API
|
|
125
|
+
* is drop-in compatible with the OpenAI SDK, so this sends the multipart form
|
|
126
|
+
* that OpenAI's transcription endpoint takes. If that inference is wrong the
|
|
127
|
+
* request fails, the caller catches it, and the user lands exactly where they
|
|
128
|
+
* would have without this method — never worse, and told why.
|
|
129
|
+
*
|
|
130
|
+
* Returns the transcript, or null when there is nothing to send.
|
|
131
|
+
*/
|
|
132
|
+
|
|
133
|
+
get canTranscribe() {
|
|
134
|
+
return Boolean(this.apiKey) && process.env.CAUSYNE_MUSE_ASR === '1';
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
async transcribeAudio(base64, mimeType = 'audio/wav') {
|
|
138
|
+
if (!base64 || !this.apiKey) return null;
|
|
139
|
+
|
|
140
|
+
const extension = String(mimeType).includes('mp3') ? 'mp3'
|
|
141
|
+
: String(mimeType).includes('mp4') || String(mimeType).includes('m4a') ? 'm4a'
|
|
142
|
+
: String(mimeType).includes('ogg') ? 'ogg'
|
|
143
|
+
: String(mimeType).includes('webm') ? 'webm'
|
|
144
|
+
: 'wav';
|
|
145
|
+
|
|
146
|
+
const form = new FormData();
|
|
147
|
+
form.append('model', MUSE_TRANSCRIBE_MODEL);
|
|
148
|
+
form.append('file', new Blob([Buffer.from(base64, 'base64')], { type: mimeType }), `clip.${extension}`);
|
|
149
|
+
|
|
150
|
+
const response = await fetch(this.transcribeUrl, {
|
|
151
|
+
method: 'POST',
|
|
152
|
+
headers: { Authorization: `Bearer ${this.apiKey}` },
|
|
153
|
+
body: form
|
|
154
|
+
});
|
|
155
|
+
|
|
156
|
+
if (!response.ok) {
|
|
157
|
+
const detail = await response.text();
|
|
158
|
+
throw new Error(`Muse Voice Transcribe (${response.status}): ${detail.slice(0, 400)}`);
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
const data = await response.json();
|
|
162
|
+
// Accept the shapes a transcription endpoint plausibly answers in, rather
|
|
163
|
+
// than betting the feature on one field name.
|
|
164
|
+
const text = data.text
|
|
165
|
+
|| data.transcript
|
|
166
|
+
|| (Array.isArray(data.turns) ? data.turns.map((t) => t.text).filter(Boolean).join(' ') : '');
|
|
167
|
+
return String(text || '').trim() || null;
|
|
168
|
+
}
|
|
169
|
+
}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import { canonicalModelId } from '../modelIds.js';
|
|
2
|
+
import { DEFAULT_MODEL_BY_PROVIDER } from '../modelCatalogue.js';
|
|
3
|
+
import { LLMProvider } from './base.js';
|
|
4
|
+
|
|
5
|
+
export class OllamaProvider extends LLMProvider {
|
|
6
|
+
constructor(config) {
|
|
7
|
+
super(config);
|
|
8
|
+
this.baseUrl = config.baseUrl || 'http://localhost:11434/api/chat';
|
|
9
|
+
this.model = canonicalModelId(config.model || DEFAULT_MODEL_BY_PROVIDER['ollama']);
|
|
10
|
+
// Locally installable vision models. Anything else stays text-only, and a
|
|
11
|
+
// user on an unlisted build can set config.supportsVision explicitly.
|
|
12
|
+
this.visionModels = ['llava', 'vision', 'bakllava', 'moondream', 'minicpm-v', 'gemma3', 'qwen2.5vl', 'qwen3-vl'];
|
|
13
|
+
// No local audio model is common enough to pattern-match honestly, and a
|
|
14
|
+
// wrong guess here fails as an opaque vendor error on the user's own
|
|
15
|
+
// machine. Left empty: config.supportsAudio is the deliberate opt-in,
|
|
16
|
+
// mirroring how supportsVision works for unlisted local builds.
|
|
17
|
+
this.audioModels = [];
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
normalizeImageBlocks(msg) {
|
|
22
|
+
if (!Array.isArray(msg.content)) return msg;
|
|
23
|
+
|
|
24
|
+
const text = [];
|
|
25
|
+
const images = [];
|
|
26
|
+
|
|
27
|
+
for (const block of msg.content) {
|
|
28
|
+
if (block.type === 'text') {
|
|
29
|
+
text.push(block.text);
|
|
30
|
+
} else if (block.type === 'image' && block.source?.type === 'base64') {
|
|
31
|
+
images.push(block.source.data);
|
|
32
|
+
} else if (block.type === 'image_url' && typeof block.image_url?.url === 'string') {
|
|
33
|
+
const encoded = block.image_url.url.split('base64,')[1];
|
|
34
|
+
if (encoded) images.push(encoded);
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const normalized = { ...msg, content: text.join('\n') };
|
|
39
|
+
if (images.length > 0) normalized.images = images;
|
|
40
|
+
return normalized;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
async chat(messages, options = {}) {
|
|
44
|
+
const sanitizedMessages = this.prepareMessages(messages);
|
|
45
|
+
|
|
46
|
+
const payload = {
|
|
47
|
+
model: this.model,
|
|
48
|
+
messages: sanitizedMessages,
|
|
49
|
+
stream: false,
|
|
50
|
+
options: {
|
|
51
|
+
temperature: options.temperature || 0.1
|
|
52
|
+
}
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
if (options.tools) {
|
|
56
|
+
payload.tools = options.tools.map(tool => ({
|
|
57
|
+
type: 'function',
|
|
58
|
+
function: tool.function ? tool.function : tool
|
|
59
|
+
}));
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
const response = await fetch(this.baseUrl, {
|
|
63
|
+
method: 'POST',
|
|
64
|
+
headers: { 'Content-Type': 'application/json' },
|
|
65
|
+
body: JSON.stringify(payload)
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
if (!response.ok) {
|
|
69
|
+
const errorData = await response.text();
|
|
70
|
+
throw new Error(`Ollama API Error (${response.status}): ${errorData}`);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
const data = await response.json();
|
|
74
|
+
|
|
75
|
+
// Ollama returns tool calls with no id. Every stage downstream — the tool
|
|
76
|
+
// result messages, the approval map, and sanitizeSequences, which filters an
|
|
77
|
+
// assistant turn's tool_calls against the ids of the results that follow it —
|
|
78
|
+
// keys on that id. Without one, sanitizeSequences matched nothing and
|
|
79
|
+
// stripped every tool call out of the history on the very next step, so a
|
|
80
|
+
// local model lost all memory of the actions it had just taken.
|
|
81
|
+
const rawToolCalls = data.message?.tool_calls || [];
|
|
82
|
+
const toolCalls = rawToolCalls.map((call, index) => ({
|
|
83
|
+
id: call.id || `call_ollama_${Date.now()}_${index}`,
|
|
84
|
+
type: 'function',
|
|
85
|
+
function: {
|
|
86
|
+
name: call.function?.name,
|
|
87
|
+
// Ollama hands back a parsed object where the OpenAI shape carries a
|
|
88
|
+
// JSON string. Normalise so the whole pipeline sees one contract.
|
|
89
|
+
arguments: typeof call.function?.arguments === 'string'
|
|
90
|
+
? call.function.arguments
|
|
91
|
+
: JSON.stringify(call.function?.arguments || {})
|
|
92
|
+
}
|
|
93
|
+
}));
|
|
94
|
+
|
|
95
|
+
return {
|
|
96
|
+
content: data.message?.content || '',
|
|
97
|
+
toolCalls: toolCalls.length > 0 ? toolCalls : undefined,
|
|
98
|
+
usage: {
|
|
99
|
+
promptTokens: data.prompt_eval_count || 0,
|
|
100
|
+
completionTokens: data.eval_count || 0
|
|
101
|
+
}
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
}
|