@mmnto/cli 1.101.2 → 1.102.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/init-detect.js +1 -1
- package/dist/commands/init-detect.js.map +1 -1
- package/dist/commands/init-detect.test.js +2 -1
- package/dist/commands/init-detect.test.js.map +1 -1
- package/dist/commands/review-fan.d.ts.map +1 -1
- package/dist/commands/review-fan.js +32 -8
- package/dist/commands/review-fan.js.map +1 -1
- package/dist/commands/review-fan.test.js +153 -7
- package/dist/commands/review-fan.test.js.map +1 -1
- package/dist/commands/shield.d.ts +25 -3
- package/dist/commands/shield.d.ts.map +1 -1
- package/dist/commands/shield.js +67 -37
- package/dist/commands/shield.js.map +1 -1
- package/dist/commands/shield.test.js +69 -2
- package/dist/commands/shield.test.js.map +1 -1
- package/dist/hook/schema.d.ts +2 -2
- package/dist/orchestrators/orchestrator.d.ts +86 -1
- package/dist/orchestrators/orchestrator.d.ts.map +1 -1
- package/dist/orchestrators/orchestrator.js +277 -9
- package/dist/orchestrators/orchestrator.js.map +1 -1
- package/dist/orchestrators/orchestrator.test.js +240 -4
- package/dist/orchestrators/orchestrator.test.js.map +1 -1
- package/dist/orchestrators/shell-orchestrator.d.ts +12 -1
- package/dist/orchestrators/shell-orchestrator.d.ts.map +1 -1
- package/dist/orchestrators/shell-orchestrator.js +256 -49
- package/dist/orchestrators/shell-orchestrator.js.map +1 -1
- package/dist/orchestrators/shell-orchestrator.test.js +252 -4
- package/dist/orchestrators/shell-orchestrator.test.js.map +1 -1
- package/dist/utils.d.ts +18 -1
- package/dist/utils.d.ts.map +1 -1
- package/dist/utils.js +389 -108
- package/dist/utils.js.map +1 -1
- package/dist/utils.test.js +406 -14
- package/dist/utils.test.js.map +1 -1
- package/package.json +2 -2
package/dist/utils.js
CHANGED
|
@@ -3,8 +3,8 @@ import * as fs from 'node:fs';
|
|
|
3
3
|
import * as os from 'node:os';
|
|
4
4
|
import * as path from 'node:path';
|
|
5
5
|
import dotenv from 'dotenv';
|
|
6
|
-
import { ADMISSION_COMPLETION_ONLY, buildGroundingBundle, calculateDeterministicHash, CONFIG_FILES, maskSecrets, RUN_ARTIFACT_SCHEMA_VERSION, saveRunArtifact, TotemConfigError, TotemConfigSchema, TotemOrchestratorError, } from '@mmnto/totem';
|
|
7
|
-
import { createOrchestrator, resolveOrchestrator } from './orchestrators/orchestrator.js';
|
|
6
|
+
import { ADMISSION_COMPLETION_ONLY, buildGroundingBundle, calculateDeterministicHash, CONFIG_FILES, INVOCATION_FAILURE_ARTIFACT_SCHEMA_VERSION, INVOKE_MESSAGE_EVIDENCE_LIMIT_BYTES, INVOKE_STREAM_EVIDENCE_LIMIT_BYTES, maskSecrets, RUN_ARTIFACT_SCHEMA_VERSION, SAFE_PROVIDER_CODE_RE, sanitizeForTerminal, saveInvocationFailureArtifact, saveRunArtifact, TotemConfigError, TotemConfigSchema, TotemOrchestratorError, } from '@mmnto/totem';
|
|
7
|
+
import { classifyInvokeFailure, createOrchestrator, OrchestratorInvokeError, resolveOrchestrator, toOrchestratorInvokeError, } from './orchestrators/orchestrator.js';
|
|
8
8
|
import { bold, log } from './ui.js';
|
|
9
9
|
// ─── Shared constants ────────────────────────────────────
|
|
10
10
|
const TELEMETRY_FILE = 'telemetry.jsonl';
|
|
@@ -322,6 +322,231 @@ function buildResponseCacheHash(prompt, systemPrompt, qualifiedModel, contract)
|
|
|
322
322
|
}
|
|
323
323
|
return hash.digest('hex').slice(0, 16);
|
|
324
324
|
}
|
|
325
|
+
function takeUtf8Prefix(text, limitBytes) {
|
|
326
|
+
let retained = '';
|
|
327
|
+
let bytes = 0;
|
|
328
|
+
for (const character of text) {
|
|
329
|
+
const characterBytes = Buffer.byteLength(character, 'utf-8');
|
|
330
|
+
if (bytes + characterBytes > limitBytes)
|
|
331
|
+
break;
|
|
332
|
+
retained += character;
|
|
333
|
+
bytes += characterBytes;
|
|
334
|
+
}
|
|
335
|
+
return retained;
|
|
336
|
+
}
|
|
337
|
+
function takeUtf8Tail(text, limitBytes) {
|
|
338
|
+
const characters = Array.from(text);
|
|
339
|
+
const retainedReversed = [];
|
|
340
|
+
let bytes = 0;
|
|
341
|
+
for (let index = characters.length - 1; index >= 0; index--) {
|
|
342
|
+
const character = characters[index];
|
|
343
|
+
const characterBytes = Buffer.byteLength(character, 'utf-8');
|
|
344
|
+
if (bytes + characterBytes > limitBytes)
|
|
345
|
+
break;
|
|
346
|
+
retainedReversed.push(character);
|
|
347
|
+
bytes += characterBytes;
|
|
348
|
+
}
|
|
349
|
+
return retainedReversed.reverse().join('');
|
|
350
|
+
}
|
|
351
|
+
/**
|
|
352
|
+
* Convert bounded raw runtime text into persisted evidence. Masking and
|
|
353
|
+
* terminal sanitization happen before a second UTF-8 byte bound because a
|
|
354
|
+
* replacement may grow or shrink the retained text. A masking exception
|
|
355
|
+
* fails closed: no raw bytes cross the artifact boundary.
|
|
356
|
+
*/
|
|
357
|
+
export function persistRuntimeTextEvidence(runtime, customSecrets, limitBytes = INVOKE_STREAM_EVIDENCE_LIMIT_BYTES, masker = maskSecrets) {
|
|
358
|
+
let safeHead;
|
|
359
|
+
let safeTail;
|
|
360
|
+
try {
|
|
361
|
+
safeHead = sanitizeForTerminal(masker(sanitizeForTerminal(runtime.head), customSecrets));
|
|
362
|
+
safeTail =
|
|
363
|
+
runtime.tail === undefined
|
|
364
|
+
? undefined
|
|
365
|
+
: sanitizeForTerminal(masker(sanitizeForTerminal(runtime.tail), customSecrets));
|
|
366
|
+
// totem-context: evidence masking fails closed by intentionally omitting all raw text and recording the typed omission marker below.
|
|
367
|
+
}
|
|
368
|
+
catch {
|
|
369
|
+
return {
|
|
370
|
+
encoding: 'utf-8',
|
|
371
|
+
head: '',
|
|
372
|
+
observedBytes: runtime.observedBytes,
|
|
373
|
+
retainedBytes: 0,
|
|
374
|
+
limitBytes,
|
|
375
|
+
truncated: false,
|
|
376
|
+
dlp: 'omitted-on-mask-failure',
|
|
377
|
+
};
|
|
378
|
+
}
|
|
379
|
+
const hasTail = safeTail !== undefined;
|
|
380
|
+
const headLimit = hasTail ? Math.floor(limitBytes / 2) : limitBytes;
|
|
381
|
+
const tailLimit = limitBytes - headLimit;
|
|
382
|
+
const head = takeUtf8Prefix(safeHead, headLimit);
|
|
383
|
+
const tail = safeTail === undefined ? undefined : takeUtf8Tail(safeTail, tailLimit);
|
|
384
|
+
const retainedBytes = Buffer.byteLength(head, 'utf-8') + Buffer.byteLength(tail ?? '', 'utf-8');
|
|
385
|
+
const postMaskTruncated = Buffer.byteLength(safeHead, 'utf-8') > headLimit ||
|
|
386
|
+
(safeTail !== undefined && Buffer.byteLength(safeTail, 'utf-8') > tailLimit);
|
|
387
|
+
return {
|
|
388
|
+
encoding: 'utf-8',
|
|
389
|
+
head,
|
|
390
|
+
...(tail !== undefined ? { tail } : {}),
|
|
391
|
+
observedBytes: runtime.observedBytes,
|
|
392
|
+
retainedBytes,
|
|
393
|
+
limitBytes,
|
|
394
|
+
truncated: runtime.truncated || postMaskTruncated,
|
|
395
|
+
dlp: 'masked',
|
|
396
|
+
};
|
|
397
|
+
}
|
|
398
|
+
function persistRuntimeAttempts(attempts, customSecrets) {
|
|
399
|
+
return attempts.map((attempt, index) => {
|
|
400
|
+
const providerCode = persistProviderCode(attempt.providerCode, customSecrets);
|
|
401
|
+
return {
|
|
402
|
+
sequence: index + 1,
|
|
403
|
+
route: attempt.route,
|
|
404
|
+
provider: attempt.provider,
|
|
405
|
+
model: attempt.model,
|
|
406
|
+
status: attempt.status,
|
|
407
|
+
durationMs: attempt.durationMs,
|
|
408
|
+
...(attempt.failureKind !== undefined ? { failureKind: attempt.failureKind } : {}),
|
|
409
|
+
...(attempt.providerStatus !== undefined ? { providerStatus: attempt.providerStatus } : {}),
|
|
410
|
+
...(providerCode !== undefined ? { providerCode } : {}),
|
|
411
|
+
...(attempt.process !== undefined
|
|
412
|
+
? {
|
|
413
|
+
process: {
|
|
414
|
+
exitCode: attempt.process.exitCode,
|
|
415
|
+
signal: attempt.process.signal,
|
|
416
|
+
timedOut: attempt.process.timedOut,
|
|
417
|
+
...(attempt.process.timeoutMs !== undefined
|
|
418
|
+
? { timeoutMs: attempt.process.timeoutMs }
|
|
419
|
+
: {}),
|
|
420
|
+
...(attempt.process.stdout !== undefined
|
|
421
|
+
? { stdout: persistRuntimeTextEvidence(attempt.process.stdout, customSecrets) }
|
|
422
|
+
: {}),
|
|
423
|
+
...(attempt.process.stderr !== undefined
|
|
424
|
+
? { stderr: persistRuntimeTextEvidence(attempt.process.stderr, customSecrets) }
|
|
425
|
+
: {}),
|
|
426
|
+
},
|
|
427
|
+
}
|
|
428
|
+
: {}),
|
|
429
|
+
};
|
|
430
|
+
});
|
|
431
|
+
}
|
|
432
|
+
function persistProviderCode(providerCode, customSecrets) {
|
|
433
|
+
if (providerCode === undefined || !SAFE_PROVIDER_CODE_RE.test(providerCode))
|
|
434
|
+
return undefined;
|
|
435
|
+
try {
|
|
436
|
+
return maskSecrets(providerCode, customSecrets) === providerCode ? providerCode : undefined;
|
|
437
|
+
// totem-context: provider codes are optional diagnostics; a masking failure intentionally omits the code rather than persisting an unsafe token.
|
|
438
|
+
}
|
|
439
|
+
catch {
|
|
440
|
+
return undefined;
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
/**
|
|
444
|
+
* Pre-bound provider-controlled terminal prose before DLP. Secret masking can
|
|
445
|
+
* expand retained text, so `persistRuntimeTextEvidence` applies the same cap a
|
|
446
|
+
* second time after masking; this first bound prevents unbounded regex work.
|
|
447
|
+
*/
|
|
448
|
+
export function runtimeMessageEvidence(message) {
|
|
449
|
+
const observedBytes = Buffer.byteLength(message, 'utf-8');
|
|
450
|
+
if (observedBytes <= INVOKE_MESSAGE_EVIDENCE_LIMIT_BYTES) {
|
|
451
|
+
return {
|
|
452
|
+
encoding: 'utf-8',
|
|
453
|
+
head: message,
|
|
454
|
+
observedBytes,
|
|
455
|
+
retainedBytes: observedBytes,
|
|
456
|
+
limitBytes: INVOKE_MESSAGE_EVIDENCE_LIMIT_BYTES,
|
|
457
|
+
truncated: false,
|
|
458
|
+
};
|
|
459
|
+
}
|
|
460
|
+
const headLimit = Math.floor(INVOKE_MESSAGE_EVIDENCE_LIMIT_BYTES / 2);
|
|
461
|
+
const tailLimit = INVOKE_MESSAGE_EVIDENCE_LIMIT_BYTES - headLimit;
|
|
462
|
+
const head = takeUtf8Prefix(message, headLimit);
|
|
463
|
+
const tail = takeUtf8Tail(message, tailLimit);
|
|
464
|
+
const retainedBytes = Buffer.byteLength(head, 'utf-8') + Buffer.byteLength(tail, 'utf-8');
|
|
465
|
+
return {
|
|
466
|
+
encoding: 'utf-8',
|
|
467
|
+
head,
|
|
468
|
+
tail,
|
|
469
|
+
observedBytes,
|
|
470
|
+
retainedBytes,
|
|
471
|
+
limitBytes: INVOKE_MESSAGE_EVIDENCE_LIMIT_BYTES,
|
|
472
|
+
truncated: true,
|
|
473
|
+
};
|
|
474
|
+
}
|
|
475
|
+
function buildArtifactSharedFields(args) {
|
|
476
|
+
const inputBundle = {
|
|
477
|
+
maskedPrompt: args.safePrompt,
|
|
478
|
+
...(args.safeSystemPrompt !== undefined && args.safeSystemPrompt.length > 0
|
|
479
|
+
? { maskedSystemPrompt: args.safeSystemPrompt }
|
|
480
|
+
: {}),
|
|
481
|
+
...(args.artifact.diffScope !== undefined ? { diffScope: args.artifact.diffScope } : {}),
|
|
482
|
+
...(args.artifact.specContract !== undefined
|
|
483
|
+
? { specContract: args.artifact.specContract }
|
|
484
|
+
: {}),
|
|
485
|
+
};
|
|
486
|
+
return {
|
|
487
|
+
inputBundle,
|
|
488
|
+
inputHash: calculateDeterministicHash(inputBundle),
|
|
489
|
+
grounding: {
|
|
490
|
+
hash: args.artifact.groundingHash,
|
|
491
|
+
provenanceSummary: args.artifact.provenanceSummary,
|
|
492
|
+
...(args.groundingBundle !== undefined ? { bundle: args.groundingBundle } : {}),
|
|
493
|
+
},
|
|
494
|
+
...(args.outputContract !== undefined ||
|
|
495
|
+
args.contextPolicy !== undefined ||
|
|
496
|
+
args.runMetadata !== undefined
|
|
497
|
+
? {
|
|
498
|
+
admission: {
|
|
499
|
+
...(args.outputContract !== undefined ? { outputContract: args.outputContract } : {}),
|
|
500
|
+
...(args.contextPolicy !== undefined ? { contextPolicy: args.contextPolicy } : {}),
|
|
501
|
+
...(args.runMetadata !== undefined ? { runMetadata: args.runMetadata } : {}),
|
|
502
|
+
},
|
|
503
|
+
}
|
|
504
|
+
: {}),
|
|
505
|
+
};
|
|
506
|
+
}
|
|
507
|
+
function buildArtifactBackend(args) {
|
|
508
|
+
return {
|
|
509
|
+
provider: args.provider,
|
|
510
|
+
model: args.model,
|
|
511
|
+
qualifiedModel: args.qualifiedModel,
|
|
512
|
+
admissionClass: args.admissionClass,
|
|
513
|
+
taskProfile: args.taskProfile,
|
|
514
|
+
...(args.temperature !== undefined ? { temperature: args.temperature } : {}),
|
|
515
|
+
};
|
|
516
|
+
}
|
|
517
|
+
function runtimeAttemptForError(args) {
|
|
518
|
+
return {
|
|
519
|
+
sequence: 1,
|
|
520
|
+
route: args.route,
|
|
521
|
+
provider: args.provider,
|
|
522
|
+
model: args.model,
|
|
523
|
+
status: 'failed',
|
|
524
|
+
durationMs: 0,
|
|
525
|
+
failureKind: classifyInvokeFailure(args.err),
|
|
526
|
+
...(args.err instanceof Error &&
|
|
527
|
+
'status' in args.err &&
|
|
528
|
+
typeof args.err.status === 'number'
|
|
529
|
+
? { providerStatus: args.err.status }
|
|
530
|
+
: {}),
|
|
531
|
+
...(args.err instanceof Error &&
|
|
532
|
+
'code' in args.err &&
|
|
533
|
+
typeof args.err.code === 'string'
|
|
534
|
+
? { providerCode: args.err.code }
|
|
535
|
+
: {}),
|
|
536
|
+
};
|
|
537
|
+
}
|
|
538
|
+
function runtimeAttemptsForError(err, provider, model, route) {
|
|
539
|
+
if (err instanceof OrchestratorInvokeError && err.attempts.length > 0) {
|
|
540
|
+
return err.attempts.map((attempt) => ({ ...attempt }));
|
|
541
|
+
}
|
|
542
|
+
return [runtimeAttemptForError({ err, provider, model, route })];
|
|
543
|
+
}
|
|
544
|
+
function resequenceRuntimeAttempts(attempts) {
|
|
545
|
+
return attempts.map((attempt, index) => ({ ...attempt, sequence: index + 1 }));
|
|
546
|
+
}
|
|
547
|
+
function quotaFallbackAttempts(attempts) {
|
|
548
|
+
return attempts.map((attempt) => ({ ...attempt, route: 'quota-model-fallback' }));
|
|
549
|
+
}
|
|
325
550
|
/**
|
|
326
551
|
* Assemble the grounding bundle for the spec/review retrieval shape
|
|
327
552
|
* (mmnto-ai/totem#2101): every partition's items enter under their partition
|
|
@@ -502,7 +727,7 @@ export async function runOrchestrator(opts) {
|
|
|
502
727
|
let resolved = resolveOrchestrator(rawModel, baseProvider, baseInvoke);
|
|
503
728
|
let model = resolved.parsed.model;
|
|
504
729
|
let qualifiedModel = resolved.qualifiedModel;
|
|
505
|
-
|
|
730
|
+
const invoke = resolved.invoke;
|
|
506
731
|
// ── Admission gate, primary path (mmnto-ai/totem#2102) ──
|
|
507
732
|
// Decided per RESOLVED backend BEFORE the invoke (and before the response
|
|
508
733
|
// cache: a denied class must not be served a replay either). No tokens are
|
|
@@ -598,78 +823,156 @@ export async function runOrchestrator(opts) {
|
|
|
598
823
|
...(opts.outputContract !== undefined ? { outputContract: opts.outputContract } : {}),
|
|
599
824
|
...(opts.runMetadata !== undefined ? { runMetadata: opts.runMetadata } : {}),
|
|
600
825
|
};
|
|
826
|
+
const requestedBackend = buildArtifactBackend({
|
|
827
|
+
provider: resolved.parsed.provider,
|
|
828
|
+
model,
|
|
829
|
+
qualifiedModel,
|
|
830
|
+
admissionClass: requestedAdmissionClass,
|
|
831
|
+
taskProfile,
|
|
832
|
+
...(opts.temperature !== undefined ? { temperature: opts.temperature } : {}),
|
|
833
|
+
});
|
|
601
834
|
let result;
|
|
835
|
+
const primaryStartMs = Date.now();
|
|
602
836
|
try {
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
837
|
+
try {
|
|
838
|
+
result = await invoke({
|
|
839
|
+
prompt: safePrompt,
|
|
840
|
+
...(safeSystemPrompt !== undefined ? { systemPrompt: safeSystemPrompt } : {}),
|
|
841
|
+
model,
|
|
842
|
+
cwd,
|
|
843
|
+
tag,
|
|
844
|
+
totemDir: config.totemDir,
|
|
845
|
+
temperature: opts.temperature,
|
|
846
|
+
...(enableContextCaching !== undefined ? { enableContextCaching } : {}),
|
|
847
|
+
...(cacheTTL !== undefined ? { cacheTTL } : {}),
|
|
848
|
+
...admissionTransport,
|
|
849
|
+
});
|
|
850
|
+
}
|
|
851
|
+
catch (err) {
|
|
852
|
+
const primaryErr = toOrchestratorInvokeError({
|
|
853
|
+
err,
|
|
854
|
+
provider: resolved.parsed.provider,
|
|
855
|
+
model,
|
|
856
|
+
route: 'sdk',
|
|
857
|
+
durationMs: Date.now() - primaryStartMs,
|
|
858
|
+
});
|
|
859
|
+
if (primaryErr.kind !== 'quota')
|
|
860
|
+
throw primaryErr;
|
|
861
|
+
const primaryAttempts = runtimeAttemptsForError(primaryErr, resolved.parsed.provider, model, 'sdk');
|
|
618
862
|
const rawFallback = config.orchestrator.fallbackModel;
|
|
619
|
-
if (rawFallback
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
863
|
+
if (!rawFallback || rawModel === rawFallback) {
|
|
864
|
+
if (err instanceof OrchestratorInvokeError)
|
|
865
|
+
throw primaryErr;
|
|
866
|
+
throw new OrchestratorInvokeError(`Quota exhausted for ${model}.`, 'quota', resequenceRuntimeAttempts(primaryAttempts), {
|
|
867
|
+
cause: err,
|
|
868
|
+
recoveryHint: 'Wait for quota to reset, configure orchestrator.fallbackModel, or select another model.',
|
|
869
|
+
});
|
|
870
|
+
}
|
|
871
|
+
log.warn(tag, `Quota exhausted for ${rawModel}. Retrying with fallback model: ${bold(rawFallback)}...`);
|
|
872
|
+
const fallbackResolved = resolveOrchestrator(rawFallback, baseProvider, baseInvoke);
|
|
873
|
+
// ── Admission gate, fallback path (mmnto-ai/totem#2102) ──
|
|
874
|
+
// `resolveOrchestrator` can route a provider-qualified fallbackModel
|
|
875
|
+
// to a DIFFERENT provider, and a single config-level declaration
|
|
876
|
+
// cannot honestly cover backends with different real capabilities.
|
|
877
|
+
// Slice-3 rule, conservative and deterministic: an elevated class
|
|
878
|
+
// admits the fallback only when it resolves to the SAME provider as
|
|
879
|
+
// the primary — cross-provider fails loud BEFORE the fallback invoke.
|
|
880
|
+
if (requestedAdmissionClass !== ADMISSION_COMPLETION_ONLY &&
|
|
881
|
+
fallbackResolved.parsed.provider !== resolved.parsed.provider) {
|
|
882
|
+
throw new OrchestratorInvokeError(`Primary model '${rawModel}' failed and the quota fallback '${rawFallback}' was denied admission.\n\n` +
|
|
883
|
+
`Primary error:\n${primaryErr.message}\n\n` +
|
|
884
|
+
`Admission error:\nfallback resolves to provider '${fallbackResolved.parsed.provider}' (primary: '${resolved.parsed.provider}') while admission class '${requestedAdmissionClass}' is requested — a cross-provider fallback is not admitted above '${ADMISSION_COMPLETION_ONLY}'.`, 'quota', resequenceRuntimeAttempts(primaryAttempts), {
|
|
885
|
+
cause: primaryErr,
|
|
886
|
+
recoveryHint: 'Use a same-provider fallbackModel, or drop the elevated backendAdmissionClass request.',
|
|
887
|
+
});
|
|
888
|
+
}
|
|
889
|
+
const fallbackStartMs = Date.now();
|
|
890
|
+
try {
|
|
891
|
+
const fallbackResult = await fallbackResolved.invoke({
|
|
892
|
+
prompt: safePrompt,
|
|
893
|
+
...(safeSystemPrompt !== undefined ? { systemPrompt: safeSystemPrompt } : {}),
|
|
894
|
+
model: fallbackResolved.parsed.model,
|
|
895
|
+
cwd,
|
|
896
|
+
tag,
|
|
897
|
+
totemDir: config.totemDir,
|
|
898
|
+
temperature: opts.temperature,
|
|
899
|
+
...(enableContextCaching !== undefined ? { enableContextCaching } : {}),
|
|
900
|
+
...(cacheTTL !== undefined ? { cacheTTL } : {}),
|
|
901
|
+
...admissionTransport,
|
|
902
|
+
});
|
|
903
|
+
const fallbackAttempts = quotaFallbackAttempts(fallbackResult.attempts ?? [
|
|
904
|
+
{
|
|
905
|
+
sequence: 1,
|
|
906
|
+
route: 'quota-model-fallback',
|
|
907
|
+
provider: fallbackResolved.parsed.provider,
|
|
641
908
|
model: fallbackResolved.parsed.model,
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
}
|
|
656
|
-
catch (fallbackErr) {
|
|
657
|
-
const originalMsg = err.message;
|
|
658
|
-
const fallbackMsg = fallbackErr instanceof Error ? fallbackErr.message : String(fallbackErr);
|
|
659
|
-
throw new TotemOrchestratorError(`Primary model '${rawModel}' failed and fallback model '${rawFallback}' also failed.\n\n` +
|
|
660
|
-
`Primary error:\n${originalMsg}\n\nFallback error:\n${fallbackMsg}`, 'Check API quotas and model availability, or try a different model with --model.', fallbackErr);
|
|
661
|
-
}
|
|
909
|
+
status: 'succeeded',
|
|
910
|
+
durationMs: fallbackResult.durationMs,
|
|
911
|
+
},
|
|
912
|
+
]);
|
|
913
|
+
result = {
|
|
914
|
+
...fallbackResult,
|
|
915
|
+
attempts: resequenceRuntimeAttempts([...primaryAttempts, ...fallbackAttempts]),
|
|
916
|
+
};
|
|
917
|
+
// Update the resolved backend identity so telemetry, cache, and success
|
|
918
|
+
// artifacts log the backend that actually produced the semantic output.
|
|
919
|
+
model = fallbackResolved.parsed.model;
|
|
920
|
+
qualifiedModel = fallbackResolved.qualifiedModel;
|
|
921
|
+
resolved = fallbackResolved;
|
|
662
922
|
}
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
923
|
+
catch (fallbackErr) {
|
|
924
|
+
const normalizedFallbackErr = toOrchestratorInvokeError({
|
|
925
|
+
err: fallbackErr,
|
|
926
|
+
provider: fallbackResolved.parsed.provider,
|
|
927
|
+
model: fallbackResolved.parsed.model,
|
|
928
|
+
route: 'quota-model-fallback',
|
|
929
|
+
durationMs: Date.now() - fallbackStartMs,
|
|
930
|
+
});
|
|
931
|
+
const fallbackAttempts = quotaFallbackAttempts(runtimeAttemptsForError(normalizedFallbackErr, fallbackResolved.parsed.provider, fallbackResolved.parsed.model, 'quota-model-fallback'));
|
|
932
|
+
const attempts = resequenceRuntimeAttempts([...primaryAttempts, ...fallbackAttempts]);
|
|
933
|
+
const kind = normalizedFallbackErr.kind;
|
|
934
|
+
const fallbackMsg = normalizedFallbackErr.message;
|
|
935
|
+
throw new OrchestratorInvokeError(`Primary model '${rawModel}' failed and fallback model '${rawFallback}' also failed.\n\n` +
|
|
936
|
+
`Primary error:\n${primaryErr.message}\n\nFallback error:\n${fallbackMsg}`, kind, attempts, { cause: fallbackErr });
|
|
668
937
|
}
|
|
669
938
|
}
|
|
670
|
-
|
|
671
|
-
|
|
939
|
+
}
|
|
940
|
+
catch (err) {
|
|
941
|
+
if (opts.artifact !== undefined && err instanceof OrchestratorInvokeError) {
|
|
942
|
+
try {
|
|
943
|
+
const attempts = persistRuntimeAttempts(err.attempts, opts.customSecrets);
|
|
944
|
+
const shared = buildArtifactSharedFields({
|
|
945
|
+
artifact: opts.artifact,
|
|
946
|
+
safePrompt,
|
|
947
|
+
...(safeSystemPrompt !== undefined ? { safeSystemPrompt } : {}),
|
|
948
|
+
...(groundingBundle !== undefined ? { groundingBundle } : {}),
|
|
949
|
+
...(opts.outputContract !== undefined ? { outputContract: opts.outputContract } : {}),
|
|
950
|
+
...(opts.contextPolicy !== undefined ? { contextPolicy: opts.contextPolicy } : {}),
|
|
951
|
+
...(opts.runMetadata !== undefined ? { runMetadata: opts.runMetadata } : {}),
|
|
952
|
+
});
|
|
953
|
+
const failureArtifact = {
|
|
954
|
+
schemaVersion: INVOCATION_FAILURE_ARTIFACT_SCHEMA_VERSION,
|
|
955
|
+
...shared,
|
|
956
|
+
requestedBackend,
|
|
957
|
+
attempts,
|
|
958
|
+
terminal: {
|
|
959
|
+
kind: err.kind,
|
|
960
|
+
attempt: attempts.at(-1)?.sequence ?? 1,
|
|
961
|
+
message: persistRuntimeTextEvidence(runtimeMessageEvidence(err.message), opts.customSecrets, INVOKE_MESSAGE_EVIDENCE_LIMIT_BYTES),
|
|
962
|
+
},
|
|
963
|
+
createdAt: new Date().toISOString(),
|
|
964
|
+
};
|
|
965
|
+
const saved = saveInvocationFailureArtifact(path.join(configRoot, config.totemDir), failureArtifact);
|
|
966
|
+
err.failureArtifactHash = saved.hash;
|
|
967
|
+
log.dim(tag, `Invocation failure artifact ${saved.existed ? 'already recorded' : 'recorded'}: ${saved.hash.slice(0, 12)}…`);
|
|
968
|
+
opts.artifact.onFailureEmitted?.(saved.hash, saved.path);
|
|
969
|
+
// totem-context: companion failure evidence is warn-only by contract; preserve and rethrow the original invocation error after surfacing this emission failure.
|
|
970
|
+
}
|
|
971
|
+
catch (artifactErr) {
|
|
972
|
+
log.warn(tag, `Invocation failure artifact emission failed (original error preserved): ${artifactErr instanceof Error ? artifactErr.message : String(artifactErr)}`);
|
|
973
|
+
}
|
|
672
974
|
}
|
|
975
|
+
throw err;
|
|
673
976
|
}
|
|
674
977
|
if (useCache && result.content && result.durationMs > 0) {
|
|
675
978
|
try {
|
|
@@ -699,39 +1002,26 @@ export async function runOrchestrator(opts) {
|
|
|
699
1002
|
// degradation is warned, never silent).
|
|
700
1003
|
if (opts.artifact !== undefined) {
|
|
701
1004
|
try {
|
|
702
|
-
const
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
...(opts.
|
|
708
|
-
...(opts.
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
};
|
|
1005
|
+
const shared = buildArtifactSharedFields({
|
|
1006
|
+
artifact: opts.artifact,
|
|
1007
|
+
safePrompt,
|
|
1008
|
+
...(safeSystemPrompt !== undefined ? { safeSystemPrompt } : {}),
|
|
1009
|
+
...(groundingBundle !== undefined ? { groundingBundle } : {}),
|
|
1010
|
+
...(opts.outputContract !== undefined ? { outputContract: opts.outputContract } : {}),
|
|
1011
|
+
...(opts.contextPolicy !== undefined ? { contextPolicy: opts.contextPolicy } : {}),
|
|
1012
|
+
...(opts.runMetadata !== undefined ? { runMetadata: opts.runMetadata } : {}),
|
|
1013
|
+
});
|
|
712
1014
|
const runArtifact = {
|
|
713
1015
|
schemaVersion: RUN_ARTIFACT_SCHEMA_VERSION,
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
grounding: {
|
|
717
|
-
hash: opts.artifact.groundingHash,
|
|
718
|
-
provenanceSummary: opts.artifact.provenanceSummary,
|
|
719
|
-
// Verbatim passthrough — the bundle was assembled and hashed by the
|
|
720
|
-
// caller (mmnto-ai/totem#2101); this seam records, never re-derives
|
|
721
|
-
// or upgrades. Post-reconciliation (#2102): a caller-supplied
|
|
722
|
-
// `groundingBundle` serves this role when `artifact.bundle` is absent.
|
|
723
|
-
...(groundingBundle !== undefined ? { bundle: groundingBundle } : {}),
|
|
724
|
-
},
|
|
725
|
-
backend: {
|
|
1016
|
+
...shared,
|
|
1017
|
+
backend: buildArtifactBackend({
|
|
726
1018
|
provider: resolved.parsed.provider,
|
|
727
1019
|
model,
|
|
728
1020
|
qualifiedModel,
|
|
729
|
-
// #2102: caller-supplied wins; the default reproduces the slice-1
|
|
730
|
-
// constant — every undeclared backend is factually completion-only.
|
|
731
1021
|
admissionClass: requestedAdmissionClass,
|
|
732
1022
|
taskProfile,
|
|
733
1023
|
...(opts.temperature !== undefined ? { temperature: opts.temperature } : {}),
|
|
734
|
-
},
|
|
1024
|
+
}),
|
|
735
1025
|
output: {
|
|
736
1026
|
content: result.content,
|
|
737
1027
|
metrics: {
|
|
@@ -743,23 +1033,14 @@ export async function runOrchestrator(opts) {
|
|
|
743
1033
|
durationMs: result.durationMs,
|
|
744
1034
|
...(result.finishReason !== undefined ? { finishReason: result.finishReason } : {}),
|
|
745
1035
|
},
|
|
1036
|
+
...(result.attempts !== undefined
|
|
1037
|
+
? {
|
|
1038
|
+
execution: {
|
|
1039
|
+
attempts: persistRuntimeAttempts(result.attempts, opts.customSecrets),
|
|
1040
|
+
},
|
|
1041
|
+
}
|
|
1042
|
+
: {}),
|
|
746
1043
|
},
|
|
747
|
-
// #2102: the admitted contract group is recorded ONLY when the caller
|
|
748
|
-
// supplied at least one member — an omitted contract stays an omitted
|
|
749
|
-
// key, never an empty object (additive 1.x, slice-1 artifacts unchanged).
|
|
750
|
-
...(opts.outputContract !== undefined ||
|
|
751
|
-
opts.contextPolicy !== undefined ||
|
|
752
|
-
opts.runMetadata !== undefined
|
|
753
|
-
? {
|
|
754
|
-
admission: {
|
|
755
|
-
...(opts.outputContract !== undefined
|
|
756
|
-
? { outputContract: opts.outputContract }
|
|
757
|
-
: {}),
|
|
758
|
-
...(opts.contextPolicy !== undefined ? { contextPolicy: opts.contextPolicy } : {}),
|
|
759
|
-
...(opts.runMetadata !== undefined ? { runMetadata: opts.runMetadata } : {}),
|
|
760
|
-
},
|
|
761
|
-
}
|
|
762
|
-
: {}),
|
|
763
1044
|
createdAt: new Date().toISOString(),
|
|
764
1045
|
};
|
|
765
1046
|
const saved = saveRunArtifact(path.join(configRoot, config.totemDir), runArtifact);
|