@marinoscar/platform-api 0.1.0-next.20 → 0.1.0-next.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/dist/ai/features/ai-feature.registry.d.ts +7 -0
- package/dist/ai/features/ai-feature.registry.d.ts.map +1 -1
- package/dist/ai/features/ai-feature.registry.js +3 -0
- package/dist/ai/features/ai-feature.registry.js.map +1 -1
- package/dist/ai/keys/ai-key-resolver.service.d.ts +16 -0
- package/dist/ai/keys/ai-key-resolver.service.d.ts.map +1 -1
- package/dist/ai/keys/ai-key-resolver.service.js +44 -0
- package/dist/ai/keys/ai-key-resolver.service.js.map +1 -1
- package/dist/ai/keys/usable-models.service.d.ts +13 -1
- package/dist/ai/keys/usable-models.service.d.ts.map +1 -1
- package/dist/ai/keys/usable-models.service.js +37 -12
- package/dist/ai/keys/usable-models.service.js.map +1 -1
- package/dist/ai/runtime/ai-limits.service.d.ts +2 -1
- package/dist/ai/runtime/ai-limits.service.d.ts.map +1 -1
- package/dist/ai/runtime/ai-limits.service.js +7 -2
- package/dist/ai/runtime/ai-limits.service.js.map +1 -1
- package/dist/ai/runtime/ai-runtime.types.d.ts +46 -1
- package/dist/ai/runtime/ai-runtime.types.d.ts.map +1 -1
- package/dist/ai/runtime/ai-runtime.types.js.map +1 -1
- package/dist/ai/runtime/ai.service.d.ts +142 -5
- package/dist/ai/runtime/ai.service.d.ts.map +1 -1
- package/dist/ai/runtime/ai.service.js +353 -46
- package/dist/ai/runtime/ai.service.js.map +1 -1
- package/dist/ai/runtime/index.d.ts +1 -1
- package/dist/ai/runtime/index.d.ts.map +1 -1
- package/dist/ai/runtime/target-resolver.d.ts +2 -2
- package/dist/ai/runtime/target-resolver.d.ts.map +1 -1
- package/dist/ai/runtime/target-resolver.js +2 -1
- package/dist/ai/runtime/target-resolver.js.map +1 -1
- package/dist/ai/testing/conformance/ai-key-policy.suite.d.ts.map +1 -1
- package/dist/ai/testing/conformance/ai-key-policy.suite.js +54 -0
- package/dist/ai/testing/conformance/ai-key-policy.suite.js.map +1 -1
- package/dist/ai/testing/conformance/ai-secret-egress.suite.d.ts.map +1 -1
- package/dist/ai/testing/conformance/ai-secret-egress.suite.js +59 -0
- package/dist/ai/testing/conformance/ai-secret-egress.suite.js.map +1 -1
- package/dist/identity/device-auth/device-auth.service.d.ts +9 -2
- package/dist/identity/device-auth/device-auth.service.d.ts.map +1 -1
- package/dist/identity/device-auth/device-auth.service.js +44 -4
- package/dist/identity/device-auth/device-auth.service.js.map +1 -1
- package/dist/identity/device-auth/dto/device-code-request.dto.d.ts +12 -0
- package/dist/identity/device-auth/dto/device-code-request.dto.d.ts.map +1 -1
- package/dist/identity/identity.options.d.ts +25 -0
- package/dist/identity/identity.options.d.ts.map +1 -1
- package/dist/identity/identity.options.js +11 -0
- package/dist/identity/identity.options.js.map +1 -1
- package/dist/identity/index.d.ts +1 -1
- package/dist/identity/index.d.ts.map +1 -1
- package/dist/nodes/current-node.decorator.d.ts +18 -0
- package/dist/nodes/current-node.decorator.d.ts.map +1 -0
- package/dist/nodes/current-node.decorator.js +28 -0
- package/dist/nodes/current-node.decorator.js.map +1 -0
- package/dist/nodes/index.d.ts +8 -0
- package/dist/nodes/index.d.ts.map +1 -1
- package/dist/nodes/index.js +21 -2
- package/dist/nodes/index.js.map +1 -1
- package/dist/nodes/node-ownership.d.ts +34 -0
- package/dist/nodes/node-ownership.d.ts.map +1 -0
- package/dist/nodes/node-ownership.js +92 -0
- package/dist/nodes/node-ownership.js.map +1 -0
- package/dist/nodes/node-route.decorator.d.ts +31 -0
- package/dist/nodes/node-route.decorator.d.ts.map +1 -0
- package/dist/nodes/node-route.decorator.js +54 -0
- package/dist/nodes/node-route.decorator.js.map +1 -0
- package/dist/nodes/node-route.guard.d.ts +22 -0
- package/dist/nodes/node-route.guard.d.ts.map +1 -0
- package/dist/nodes/node-route.guard.js +74 -0
- package/dist/nodes/node-route.guard.js.map +1 -0
- package/dist/nodes/node-route.module.d.ts +21 -0
- package/dist/nodes/node-route.module.d.ts.map +1 -0
- package/dist/nodes/node-route.module.js +99 -0
- package/dist/nodes/node-route.module.js.map +1 -0
- package/dist/nodes/node-route.options.d.ts +59 -0
- package/dist/nodes/node-route.options.d.ts.map +1 -0
- package/dist/nodes/node-route.options.js +29 -0
- package/dist/nodes/node-route.options.js.map +1 -0
- package/dist/nodes/node-route.paths.d.ts +47 -0
- package/dist/nodes/node-route.paths.d.ts.map +1 -0
- package/dist/nodes/node-route.paths.js +129 -0
- package/dist/nodes/node-route.paths.js.map +1 -0
- package/dist/nodes/nodes.service.d.ts.map +1 -1
- package/dist/nodes/nodes.service.js +3 -15
- package/dist/nodes/nodes.service.js.map +1 -1
- package/dist/nodes/testing/conformance.d.ts +86 -0
- package/dist/nodes/testing/conformance.d.ts.map +1 -0
- package/dist/nodes/testing/conformance.js +230 -0
- package/dist/nodes/testing/conformance.js.map +1 -0
- package/dist/nodes/testing/discover.d.ts +13 -0
- package/dist/nodes/testing/discover.d.ts.map +1 -0
- package/dist/nodes/testing/discover.js +42 -0
- package/dist/nodes/testing/discover.js.map +1 -0
- package/dist/nodes/testing/index.d.ts +3 -0
- package/dist/nodes/testing/index.d.ts.map +1 -0
- package/dist/nodes/testing/index.js +13 -0
- package/dist/nodes/testing/index.js.map +1 -0
- package/dist/webhooks/handlers/webhook-deliver.handler.d.ts.map +1 -1
- package/dist/webhooks/handlers/webhook-deliver.handler.js +3 -2
- package/dist/webhooks/handlers/webhook-deliver.handler.js.map +1 -1
- package/package.json +7 -3
|
@@ -277,15 +277,71 @@ let AiService = AiService_1 = class AiService {
|
|
|
277
277
|
speak: (req) => this.startSpeechRun(bound, req),
|
|
278
278
|
createRealtimeSession: (req, opts) => this.createRealtimeSession(bound, req, opts),
|
|
279
279
|
run: (operationId, input, opts) => this.runOperation(bound, operationId, input, opts),
|
|
280
|
+
editImageInline: (req, opts) => this.editImageInline(bound, req, opts),
|
|
281
|
+
transcribeInline: (req, opts) => this.transcribeInline(bound, req, opts),
|
|
280
282
|
};
|
|
281
283
|
}
|
|
284
|
+
/**
|
|
285
|
+
* The AI client for work with NO acting user: a per-organization job
|
|
286
|
+
* (memory titles, a digest), an importer's tagging. It runs the same gate
|
|
287
|
+
* pipeline as `forUser` and records one usage row per round-trip with
|
|
288
|
+
* `userId: null` and `scope.feature`, but:
|
|
289
|
+
*
|
|
290
|
+
* - the feature must be registered with `systemCalls: true`, else every call
|
|
291
|
+
* is refused with `AI_KEY_REQUIRED` (an app whose rule is "AI keys are per
|
|
292
|
+
* user" registers none, and system calls stay impossible for it);
|
|
293
|
+
* - the key is the organization's own, else the deployment's, NEVER a
|
|
294
|
+
* user's, and `ai.keyPolicy` does not apply (no person is borrowing);
|
|
295
|
+
* - only the per-organization limits (`limits.perOrg`) apply;
|
|
296
|
+
* - the kill switches (`ai.enabled`, the organization's) still apply first.
|
|
297
|
+
*
|
|
298
|
+
* Server-side code only: it is not reachable over HTTP and must never be
|
|
299
|
+
* handed to a worker node. See `AiSystemClient` for what it offers.
|
|
300
|
+
*
|
|
301
|
+
* @example
|
|
302
|
+
* ```ts
|
|
303
|
+
* const titles = await this.ai
|
|
304
|
+
* .forSystem({ orgId, feature: 'example_titles' })
|
|
305
|
+
* .respondStructured({ schema, input });
|
|
306
|
+
* ```
|
|
307
|
+
*
|
|
308
|
+
* @stability experimental
|
|
309
|
+
*/
|
|
310
|
+
forSystem(scope) {
|
|
311
|
+
const bound = {
|
|
312
|
+
userId: null,
|
|
313
|
+
...(scope.orgId ? { orgId: scope.orgId } : {}),
|
|
314
|
+
feature: scope.feature,
|
|
315
|
+
...(scope.jobId ? { jobId: scope.jobId } : {}),
|
|
316
|
+
};
|
|
317
|
+
return {
|
|
318
|
+
respond: (req, opts) => this.respond(bound, req, opts),
|
|
319
|
+
stream: (req, opts) => this.lazyStream(bound, req, opts),
|
|
320
|
+
openStream: (req, opts) => this.openStream(bound, req, opts),
|
|
321
|
+
respondStructured: (req, opts) => this.respondStructured(bound, req, opts),
|
|
322
|
+
embed: (req, opts) => this.embed(bound, req, opts),
|
|
323
|
+
editImageInline: (req, opts) => this.editImageInline(bound, req, opts),
|
|
324
|
+
transcribeInline: (req, opts) => this.transcribeInline(bound, req, opts),
|
|
325
|
+
};
|
|
326
|
+
}
|
|
327
|
+
/** The acting user of a call only a user can make (queued runs, realtime, custom operations). */
|
|
328
|
+
userOf(scope) {
|
|
329
|
+
if (scope.userId === null) {
|
|
330
|
+
throw new ai_error_1.AiError('AI_INVALID_REQUEST', 'This call needs an acting user; a system client cannot make it.');
|
|
331
|
+
}
|
|
332
|
+
return scope.userId;
|
|
333
|
+
}
|
|
282
334
|
/**
|
|
283
335
|
* The organization this call runs in: the scope's own, else (single tenancy)
|
|
284
336
|
* the default organization, else a `MissingOrgScopeError` (multi tenancy).
|
|
285
337
|
* Memoised on the scope, so one call resolves it once.
|
|
286
338
|
*/
|
|
287
339
|
async orgOf(scope) {
|
|
288
|
-
scope.orgId ??= await (0, index_3.resolveOrgId)(this.prisma, scope.orgId, scope.jobId
|
|
340
|
+
scope.orgId ??= await (0, index_3.resolveOrgId)(this.prisma, scope.orgId, scope.jobId
|
|
341
|
+
? `AI call under job ${scope.jobId}`
|
|
342
|
+
: scope.userId === null
|
|
343
|
+
? 'AI system call'
|
|
344
|
+
: `AI call for user ${scope.userId}`);
|
|
289
345
|
return scope.orgId;
|
|
290
346
|
}
|
|
291
347
|
// ---- respond ----------------------------------------------------------------
|
|
@@ -304,9 +360,10 @@ let AiService = AiService_1 = class AiService {
|
|
|
304
360
|
if (!(await this.aiConfig.resolveForOrg(orgId)).defaults.allowBackgroundRuns) {
|
|
305
361
|
throw new ai_error_1.AiError('AI_INVALID_REQUEST', 'Background AI runs are disabled in this deployment.');
|
|
306
362
|
}
|
|
307
|
-
const
|
|
363
|
+
const userId = this.userOf(scope);
|
|
364
|
+
const call = await this.prepare(userId, req, { streaming: false, orgId, feature: scope.feature });
|
|
308
365
|
return this.runs.create({
|
|
309
|
-
userId
|
|
366
|
+
userId,
|
|
310
367
|
orgId,
|
|
311
368
|
provider: call.provider,
|
|
312
369
|
modelId: call.request.model,
|
|
@@ -351,7 +408,9 @@ let AiService = AiService_1 = class AiService {
|
|
|
351
408
|
/** The settler every result of `call` leaves through (see `ai-hosted-outputs.ts`). */
|
|
352
409
|
hostedOutputs(scope, call) {
|
|
353
410
|
return new ai_hosted_outputs_1.AiHostedOutputSettler({
|
|
354
|
-
|
|
411
|
+
// A system call has no user and `prepare` refuses its hosted tools, so
|
|
412
|
+
// nothing is ever stored under this owner.
|
|
413
|
+
userId: scope.userId ?? '',
|
|
355
414
|
...(scope.orgId ? { orgId: scope.orgId } : {}),
|
|
356
415
|
...(scope.jobId ? { jobId: scope.jobId } : {}),
|
|
357
416
|
...(scope.runId ? { runId: scope.runId } : {}),
|
|
@@ -418,7 +477,7 @@ let AiService = AiService_1 = class AiService {
|
|
|
418
477
|
}
|
|
419
478
|
// ---- realtime ---------------------------------------------------------------------
|
|
420
479
|
async createRealtimeSession(scope, req, opts = {}) {
|
|
421
|
-
const call = await this.prepareRealtime(scope
|
|
480
|
+
const call = await this.prepareRealtime(this.userOf(scope), req, scope.feature, await this.orgOf(scope));
|
|
422
481
|
const { ctx, keySource } = await this.context(scope, call, opts, () => ({
|
|
423
482
|
instructions: call.request.instructions,
|
|
424
483
|
}));
|
|
@@ -454,7 +513,7 @@ let AiService = AiService_1 = class AiService {
|
|
|
454
513
|
* row and span as every built-in port. See `AiUserClient.run`.
|
|
455
514
|
*/
|
|
456
515
|
async runOperation(scope, operationId, input, opts = {}) {
|
|
457
|
-
const call = await this.prepareOperation(scope
|
|
516
|
+
const call = await this.prepareOperation(this.userOf(scope), operationId, input, opts, await this.orgOf(scope), scope.feature);
|
|
458
517
|
// ⚠ The input is NOT passed as the debug prompt: an operation's input is
|
|
459
518
|
// whatever the app defined (a transcript, a document) and only the
|
|
460
519
|
// operation id is safe to name.
|
|
@@ -556,9 +615,10 @@ let AiService = AiService_1 = class AiService {
|
|
|
556
615
|
// ---- images ---------------------------------------------------------------------
|
|
557
616
|
async startImageRun(scope, operation, req) {
|
|
558
617
|
const orgId = await this.orgOf(scope);
|
|
559
|
-
const
|
|
618
|
+
const userId = this.userOf(scope);
|
|
619
|
+
const call = await this.prepareImage(userId, operation, req, orgId, scope.feature);
|
|
560
620
|
return this.runs.create({
|
|
561
|
-
userId
|
|
621
|
+
userId,
|
|
562
622
|
orgId,
|
|
563
623
|
provider: call.provider,
|
|
564
624
|
modelId: call.modelId,
|
|
@@ -597,33 +657,153 @@ let AiService = AiService_1 = class AiService {
|
|
|
597
657
|
? await this.inputs.read(call.inputs.mask, { maxBytes: media_types_1.AI_IMAGE_INPUT_MAX_BYTES, label: 'mask' })
|
|
598
658
|
: undefined;
|
|
599
659
|
const request = (0, ai_image_run_request_1.toImageGenerationRequest)(call.stored);
|
|
600
|
-
|
|
601
|
-
const tracker = this.track(scope, call, keySource, call.operation);
|
|
602
|
-
let result;
|
|
603
|
-
try {
|
|
660
|
+
return this.imageRoundTrip(scope, call, call.operation, opts, request.prompt, (ctx) => {
|
|
604
661
|
if (edit) {
|
|
605
662
|
// `prepareImage` refused a port without `edit`; this narrows the type.
|
|
606
663
|
const editPort = call.port.edit;
|
|
607
|
-
|
|
608
|
-
}
|
|
609
|
-
else {
|
|
610
|
-
result = await call.port.generate(request, ctx);
|
|
664
|
+
return editPort({ ...request, images, ...(mask ? { mask } : {}) }, ctx);
|
|
611
665
|
}
|
|
666
|
+
return call.port.generate(request, ctx);
|
|
667
|
+
});
|
|
668
|
+
}
|
|
669
|
+
/**
|
|
670
|
+
* Steps 6-9 of one image round-trip, shared by the queued run
|
|
671
|
+
* (`executeImageRun`) and the inline edit (`editImageInline`): resolve the
|
|
672
|
+
* key, apply the limits, call the port, record ONE usage row (with the image count as its units) and end the span. The error never carries the key.
|
|
673
|
+
*/
|
|
674
|
+
async imageRoundTrip(scope, call, operation, opts, prompt, send) {
|
|
675
|
+
const { ctx, keySource } = await this.context(scope, call, opts, () => ({ prompt }));
|
|
676
|
+
const tracker = this.track(scope, call, keySource, operation);
|
|
677
|
+
let result;
|
|
678
|
+
try {
|
|
679
|
+
result = await send(ctx);
|
|
612
680
|
}
|
|
613
681
|
catch (err) {
|
|
614
|
-
const error = toAiError(err, opts.signal);
|
|
682
|
+
const error = scrubKey(toAiError(err, opts.signal), ctx.apiKey);
|
|
615
683
|
await tracker.finish(failure(error, opts.signal));
|
|
616
684
|
throw error;
|
|
617
685
|
}
|
|
618
686
|
await tracker.finish({ status: 'succeeded', result, units: { images: result.images.length } });
|
|
619
687
|
return result;
|
|
620
688
|
}
|
|
689
|
+
// ---- inline media (a caller already inside its own job) ----------------------------
|
|
690
|
+
async editImageInline(scope, req, opts = {}) {
|
|
691
|
+
const call = await this.prepareImageInline(scope.userId, req, await this.orgOf(scope), scope.feature);
|
|
692
|
+
return this.imageRoundTrip(scope, call, 'images.edit', opts, call.request.prompt, (ctx) => call.edit(call.request, ctx));
|
|
693
|
+
}
|
|
694
|
+
async transcribeInline(scope, req, opts = {}) {
|
|
695
|
+
const call = await this.prepareTranscriptionInline(scope.userId, req, await this.orgOf(scope), scope.feature);
|
|
696
|
+
return this.transcriptionRoundTrip(scope, call, opts, { language: call.request.language, prompt: call.request.prompt }, call.request, call.exceeded);
|
|
697
|
+
}
|
|
698
|
+
/**
|
|
699
|
+
* The gate pipeline for an inline image edit: kill switch, the system
|
|
700
|
+
* caller's opt-in, request shape (prompt, `n`, the images' count, size and
|
|
701
|
+
* type, the mask), target, provider, model with `image_edit` (model AND port
|
|
702
|
+
* with `edit`). Decrypts nothing; the bytes are already the caller's.
|
|
703
|
+
*
|
|
704
|
+
* @internal
|
|
705
|
+
*/
|
|
706
|
+
async prepareImageInline(userId, req, orgId, featureId) {
|
|
707
|
+
// 1. Kill switch — before anything else is read.
|
|
708
|
+
await this.aiConfig.assertEnabled(orgId);
|
|
709
|
+
const feature = this.featureOf(featureId);
|
|
710
|
+
this.assertSystemCaller(userId, feature);
|
|
711
|
+
assertImageShape(req, false);
|
|
712
|
+
assertInlineImages(req);
|
|
713
|
+
if (req.providerOptions !== undefined)
|
|
714
|
+
assertProviderOptionsShape(req.providerOptions);
|
|
715
|
+
if (!req.model?.trim()) {
|
|
716
|
+
throw new ai_error_1.AiError('AI_INVALID_REQUEST', 'Name the image model explicitly.');
|
|
717
|
+
}
|
|
718
|
+
const { provider, model } = await this.resolveTarget(userId, req, { orgId, feature });
|
|
719
|
+
// 2. Provider enabled in settings AND registered in this process.
|
|
720
|
+
const slot = await this.aiConfig.assertProviderEnabled(provider, orgId);
|
|
721
|
+
const port = this.registry.get(provider)?.images;
|
|
722
|
+
// 3. Model enabled, `image_edit` declared (model AND provider port), key reach.
|
|
723
|
+
const { model: usableImage } = await this.usableModels.assertUsable(userId, provider, model, ['image_edit'], { orgId });
|
|
724
|
+
assertFeatureFits(feature, provider, model, usableImage.capabilities);
|
|
725
|
+
if (!port?.edit) {
|
|
726
|
+
throw capabilityUnsupported(provider, model, 'image_edit');
|
|
727
|
+
}
|
|
728
|
+
const policy = await this.aiConfig.resolveForOrg(orgId);
|
|
729
|
+
// Named fields only: whatever else the caller's object carried stays here.
|
|
730
|
+
const request = {
|
|
731
|
+
model,
|
|
732
|
+
prompt: req.prompt,
|
|
733
|
+
images: req.images.map(inlinePayload),
|
|
734
|
+
...(req.mask ? { mask: inlinePayload(req.mask) } : {}),
|
|
735
|
+
...(req.size !== undefined ? { size: req.size } : {}),
|
|
736
|
+
...(req.quality !== undefined ? { quality: req.quality } : {}),
|
|
737
|
+
...(req.background !== undefined ? { background: req.background } : {}),
|
|
738
|
+
...(req.outputFormat !== undefined ? { outputFormat: req.outputFormat } : {}),
|
|
739
|
+
...(req.n !== undefined ? { n: req.n } : {}),
|
|
740
|
+
...(req.providerOptions !== undefined ? { providerOptions: req.providerOptions } : {}),
|
|
741
|
+
};
|
|
742
|
+
return {
|
|
743
|
+
provider,
|
|
744
|
+
modelId: model,
|
|
745
|
+
edit: port.edit.bind(port),
|
|
746
|
+
request,
|
|
747
|
+
...(0, ai_config_service_1.providerCallSettings)(slot),
|
|
748
|
+
logPromptContent: policy.logPromptContent,
|
|
749
|
+
};
|
|
750
|
+
}
|
|
751
|
+
/**
|
|
752
|
+
* The gate pipeline for an inline transcription: kill switch, the system
|
|
753
|
+
* caller's opt-in, request shape, target (an omitted model is the first
|
|
754
|
+
* usable `audio_transcription` model), provider, model with
|
|
755
|
+
* `audio_transcription` (model AND port), then the recording itself: type
|
|
756
|
+
* and the provider's size limit (a stream is counted as it is sent).
|
|
757
|
+
* Decrypts nothing.
|
|
758
|
+
*
|
|
759
|
+
* @internal
|
|
760
|
+
*/
|
|
761
|
+
async prepareTranscriptionInline(userId, req, orgId, featureId) {
|
|
762
|
+
// 1. Kill switch — before anything else is read.
|
|
763
|
+
await this.aiConfig.assertEnabled(orgId);
|
|
764
|
+
const feature = this.featureOf(featureId);
|
|
765
|
+
this.assertSystemCaller(userId, feature);
|
|
766
|
+
assertTranscriptionOptions(req);
|
|
767
|
+
const audio = req.audio;
|
|
768
|
+
if (!audio || typeof audio.mimeType !== 'string' || !mimeAllowed(audio.mimeType, media_types_1.AI_TRANSCRIPTION_INPUT_MIME_TYPES)) {
|
|
769
|
+
throw new ai_error_1.AiError('AI_INVALID_REQUEST', 'A transcription needs an audio recording (audio/*, video/mp4 or video/webm).', {
|
|
770
|
+
details: { mimeType: audio?.mimeType },
|
|
771
|
+
});
|
|
772
|
+
}
|
|
773
|
+
const { provider, model } = req.model?.trim()
|
|
774
|
+
? await this.resolveTarget(userId, req, { orgId, feature })
|
|
775
|
+
: await this.firstUsableModel(userId, 'audio_transcription', req.provider, orgId);
|
|
776
|
+
// 2. Provider enabled in settings AND registered in this process.
|
|
777
|
+
const slot = await this.aiConfig.assertProviderEnabled(provider, orgId);
|
|
778
|
+
const port = this.registry.get(provider)?.audio;
|
|
779
|
+
// 3. Model enabled, `audio_transcription` declared (model AND port), key reach.
|
|
780
|
+
const { model: usableTranscription } = await this.usableModels.assertUsable(userId, provider, model, ['audio_transcription'], { orgId });
|
|
781
|
+
assertFeatureFits(feature, provider, model, usableTranscription.capabilities);
|
|
782
|
+
const transcribe = port?.transcribe?.bind(port);
|
|
783
|
+
if (!port || !transcribe) {
|
|
784
|
+
throw capabilityUnsupported(provider, model, 'audio_transcription');
|
|
785
|
+
}
|
|
786
|
+
// 4. The recording within the provider's limit.
|
|
787
|
+
const maxBytes = port.transcriptionMaxBytes ?? media_types_1.AI_TRANSCRIPTION_DEFAULT_MAX_BYTES;
|
|
788
|
+
const capped = capInlineAudio(audio, maxBytes);
|
|
789
|
+
const policy = await this.aiConfig.resolveForOrg(orgId);
|
|
790
|
+
return {
|
|
791
|
+
provider,
|
|
792
|
+
modelId: model,
|
|
793
|
+
transcribe,
|
|
794
|
+
request: { model, audio: capped.audio, ...transcriptionFields(req) },
|
|
795
|
+
exceeded: capped.exceeded,
|
|
796
|
+
...(0, ai_config_service_1.providerCallSettings)(slot),
|
|
797
|
+
logPromptContent: policy.logPromptContent,
|
|
798
|
+
};
|
|
799
|
+
}
|
|
621
800
|
// ---- transcription ------------------------------------------------------------------
|
|
622
801
|
async startTranscriptionRun(scope, req) {
|
|
623
802
|
const orgId = await this.orgOf(scope);
|
|
624
|
-
const
|
|
803
|
+
const userId = this.userOf(scope);
|
|
804
|
+
const call = await this.prepareTranscription(userId, req, orgId, scope.feature);
|
|
625
805
|
return this.runs.create({
|
|
626
|
-
userId
|
|
806
|
+
userId,
|
|
627
807
|
orgId,
|
|
628
808
|
provider: call.provider,
|
|
629
809
|
modelId: call.modelId,
|
|
@@ -652,35 +832,41 @@ let AiService = AiService_1 = class AiService {
|
|
|
652
832
|
const audio = await this.openRecording(call);
|
|
653
833
|
let result;
|
|
654
834
|
try {
|
|
655
|
-
const { ctx, keySource } = await this.context(scope, call, opts, () => ({
|
|
656
|
-
language: call.stored.language,
|
|
657
|
-
prompt: call.stored.prompt,
|
|
658
|
-
}));
|
|
659
|
-
const tracker = this.track(scope, call, keySource, 'audio.transcribe');
|
|
660
835
|
const request = { model: call.modelId, audio: audio.payload, ...transcriptionFields(call.stored) };
|
|
661
|
-
|
|
662
|
-
result = await call.transcribe(request, ctx);
|
|
663
|
-
}
|
|
664
|
-
catch (err) {
|
|
665
|
-
// A recording that turned out larger than its row said: the cap
|
|
666
|
-
// stopped the upload, and THAT is the answer, not the transport
|
|
667
|
-
// failure the SDK saw.
|
|
668
|
-
const error = audio.exceeded() ?? toAiError(err, opts.signal);
|
|
669
|
-
await tracker.finish(failure(error, opts.signal));
|
|
670
|
-
throw error;
|
|
671
|
-
}
|
|
672
|
-
result = this.boundExtras(result);
|
|
673
|
-
await tracker.finish({
|
|
674
|
-
status: 'succeeded',
|
|
675
|
-
result,
|
|
676
|
-
...(result.durationSeconds !== undefined ? { units: { audioSeconds: result.durationSeconds } } : {}),
|
|
677
|
-
});
|
|
836
|
+
result = await this.transcriptionRoundTrip(scope, call, opts, { language: call.stored.language, prompt: call.stored.prompt }, request, audio.exceeded);
|
|
678
837
|
}
|
|
679
838
|
finally {
|
|
680
839
|
audio.close();
|
|
681
840
|
}
|
|
682
841
|
return result;
|
|
683
842
|
}
|
|
843
|
+
/**
|
|
844
|
+
* Steps 6-9 of one transcription round-trip, shared by the queued run and
|
|
845
|
+
* the inline call: key, limits, the port, ONE usage row (with the audio seconds as units, when reported) and the span. The error never carries the key.
|
|
846
|
+
*/
|
|
847
|
+
async transcriptionRoundTrip(scope, call, opts, promptInfo, request, exceeded) {
|
|
848
|
+
const { ctx, keySource } = await this.context(scope, call, opts, () => promptInfo);
|
|
849
|
+
const tracker = this.track(scope, call, keySource, 'audio.transcribe');
|
|
850
|
+
let result;
|
|
851
|
+
try {
|
|
852
|
+
result = await call.transcribe(request, ctx);
|
|
853
|
+
}
|
|
854
|
+
catch (err) {
|
|
855
|
+
// A recording that turned out larger than its row said: the cap
|
|
856
|
+
// stopped the upload, and THAT is the answer, not the transport
|
|
857
|
+
// failure the SDK saw.
|
|
858
|
+
const error = scrubKey(exceeded() ?? toAiError(err, opts.signal), ctx.apiKey);
|
|
859
|
+
await tracker.finish(failure(error, opts.signal));
|
|
860
|
+
throw error;
|
|
861
|
+
}
|
|
862
|
+
result = this.boundExtras(result);
|
|
863
|
+
await tracker.finish({
|
|
864
|
+
status: 'succeeded',
|
|
865
|
+
result,
|
|
866
|
+
...(result.durationSeconds !== undefined ? { units: { audioSeconds: result.durationSeconds } } : {}),
|
|
867
|
+
});
|
|
868
|
+
return result;
|
|
869
|
+
}
|
|
684
870
|
/**
|
|
685
871
|
* The result with its `extras` checked (PP-14.17, #935): kept when bounded
|
|
686
872
|
* and well-formed, dropped otherwise. The transcript is never lost to a bad
|
|
@@ -721,9 +907,10 @@ let AiService = AiService_1 = class AiService {
|
|
|
721
907
|
// ---- speech -------------------------------------------------------------------------
|
|
722
908
|
async startSpeechRun(scope, req) {
|
|
723
909
|
const orgId = await this.orgOf(scope);
|
|
724
|
-
const
|
|
910
|
+
const userId = this.userOf(scope);
|
|
911
|
+
const call = await this.prepareSpeech(userId, req, scope.feature, orgId);
|
|
725
912
|
return this.runs.create({
|
|
726
|
-
userId
|
|
913
|
+
userId,
|
|
727
914
|
orgId,
|
|
728
915
|
provider: call.provider,
|
|
729
916
|
modelId: call.modelId,
|
|
@@ -917,6 +1104,7 @@ let AiService = AiService_1 = class AiService {
|
|
|
917
1104
|
// 1. Kill switch — before anything else is read.
|
|
918
1105
|
await this.aiConfig.assertEnabled(opts.orgId);
|
|
919
1106
|
const feature = this.featureOf(opts.feature);
|
|
1107
|
+
this.assertSystemCaller(userId, feature);
|
|
920
1108
|
const { provider, model } = await this.resolveTarget(userId, req, { orgId: opts.orgId, feature });
|
|
921
1109
|
// 2. Provider enabled in settings AND registered in this process.
|
|
922
1110
|
const slot = await this.aiConfig.assertProviderEnabled(provider, opts.orgId);
|
|
@@ -932,6 +1120,12 @@ let AiService = AiService_1 = class AiService {
|
|
|
932
1120
|
// and an MCP server on an allowed host.
|
|
933
1121
|
(0, hosted_tools_1.assertHostedToolShapes)(req.tools);
|
|
934
1122
|
(0, hosted_tools_1.assertHostedToolsAllowed)(req.tools, policy.hostedTools);
|
|
1123
|
+
if (userId === null && (0, hosted_tools_1.hostedToolsOf)(req.tools).length > 0) {
|
|
1124
|
+
// A hosted tool's output (a generated image) is stored as the user's own object.
|
|
1125
|
+
throw new ai_error_1.AiError('AI_INVALID_REQUEST', 'Hosted tools are not available to system calls.', {
|
|
1126
|
+
details: { provider, model, capability: 'hosted_tools' },
|
|
1127
|
+
});
|
|
1128
|
+
}
|
|
935
1129
|
// 3. Model enabled, capabilities (model AND provider port), key reach.
|
|
936
1130
|
const needed = requiredCapabilities(req, opts.streaming);
|
|
937
1131
|
const { model: usable } = await this.usableModels.assertUsable(userId, provider, model, needed, { orgId: opts.orgId });
|
|
@@ -948,7 +1142,13 @@ let AiService = AiService_1 = class AiService {
|
|
|
948
1142
|
throw new ai_error_1.AiError('AI_CAPABILITY_UNSUPPORTED', `Model "${model}" does not offer reasoning effort "${effort}".`, { details: { provider, model, capability: 'reasoning', effort } });
|
|
949
1143
|
}
|
|
950
1144
|
// 4b. Storage-object inputs: ownership, readiness, modality, size, strategy.
|
|
951
|
-
|
|
1145
|
+
if (userId === null && storageParts(req.input).length > 0) {
|
|
1146
|
+
// A stored file is authorised by its owner; a system call has none. Pass bytes or a url instead.
|
|
1147
|
+
throw new ai_error_1.AiError('AI_INVALID_REQUEST', 'System calls cannot reference storage objects; pass a url or the content.', {
|
|
1148
|
+
details: { provider, model, capability: 'file_input' },
|
|
1149
|
+
});
|
|
1150
|
+
}
|
|
1151
|
+
const storageInputs = userId === null ? [] : await this.planStorageInputs(userId, provider, model, usable, req.input, opts.orgId);
|
|
952
1152
|
// 5. Clamp output tokens: the deployment cap and the model's own
|
|
953
1153
|
// `ai.limits.perModel` cap (#450) combine — the smaller wins — and bound
|
|
954
1154
|
// the call even when the request named no limit of its own.
|
|
@@ -986,6 +1186,7 @@ let AiService = AiService_1 = class AiService {
|
|
|
986
1186
|
throw new ai_error_1.AiError('AI_INVALID_REQUEST', 'Name the embedding model explicitly: vectors are only comparable within one model.');
|
|
987
1187
|
}
|
|
988
1188
|
const feature = this.featureOf(featureId);
|
|
1189
|
+
this.assertSystemCaller(userId, feature);
|
|
989
1190
|
const { provider, model } = await this.resolveTarget(userId, req, { feature });
|
|
990
1191
|
// 2. Provider enabled in settings AND registered in this process.
|
|
991
1192
|
const slot = await this.aiConfig.assertProviderEnabled(provider, orgId);
|
|
@@ -1388,7 +1589,9 @@ let AiService = AiService_1 = class AiService {
|
|
|
1388
1589
|
* operation whose model is never the chat `ai.defaultModel`.
|
|
1389
1590
|
*/
|
|
1390
1591
|
async firstUsableModel(userId, capability, provider, orgId, operation) {
|
|
1391
|
-
const usable =
|
|
1592
|
+
const usable = userId === null
|
|
1593
|
+
? await this.usableModels.listForSystem(orgId ? { orgId } : {})
|
|
1594
|
+
: await this.usableModels.listForUser(userId, orgId ? { orgId } : {});
|
|
1392
1595
|
const match = usable.find((m) => (provider === undefined || m.provider === provider) &&
|
|
1393
1596
|
m.capabilities.capabilities.includes(capability) &&
|
|
1394
1597
|
this.registry.supports(m.provider, capability) &&
|
|
@@ -1409,7 +1612,10 @@ let AiService = AiService_1 = class AiService {
|
|
|
1409
1612
|
}
|
|
1410
1613
|
const orgId = await this.orgOf(scope);
|
|
1411
1614
|
// #739: the org tier needs the call's organization.
|
|
1412
|
-
|
|
1615
|
+
// A system call (no user) is served by the organization's or the deployment's key, never a user's.
|
|
1616
|
+
const { apiKey, keySource, tier } = scope.userId === null
|
|
1617
|
+
? await this.keyResolver.resolveForSystem(call.provider, { orgId })
|
|
1618
|
+
: await this.keyResolver.resolve(scope.userId, call.provider, { orgId });
|
|
1413
1619
|
call.keyTier = tier;
|
|
1414
1620
|
// 6b. Rate limits (#450) — here because the org-key limits need to know
|
|
1415
1621
|
// whose key pays. A refusal records no usage row: nothing was sent.
|
|
@@ -1458,6 +1664,18 @@ let AiService = AiService_1 = class AiService {
|
|
|
1458
1664
|
}
|
|
1459
1665
|
return { provider: target.provider, model: target.model };
|
|
1460
1666
|
}
|
|
1667
|
+
/**
|
|
1668
|
+
* A system call (no acting user) must name a registered feature that opted in
|
|
1669
|
+
* with `systemCalls: true` (checked right after the kill switch). Refused with
|
|
1670
|
+
* `AI_KEY_REQUIRED`: without the opt-in there is no key a system call may use.
|
|
1671
|
+
*/
|
|
1672
|
+
assertSystemCaller(userId, feature) {
|
|
1673
|
+
if (userId !== null)
|
|
1674
|
+
return;
|
|
1675
|
+
if (!feature?.systemCalls) {
|
|
1676
|
+
throw new ai_error_1.AiError('AI_KEY_REQUIRED', 'System AI calls are not enabled for this feature; register it with systemCalls: true.', { details: { ...(feature ? { feature: feature.id } : {}), reason: 'system_calls_not_enabled' } });
|
|
1677
|
+
}
|
|
1678
|
+
}
|
|
1461
1679
|
/** The registered feature a call names, or `undefined`; an unknown id is AI_INVALID_REQUEST. */
|
|
1462
1680
|
featureOf(id) {
|
|
1463
1681
|
if (id === undefined)
|
|
@@ -1645,6 +1863,10 @@ function assertTranscriptionShape(req) {
|
|
|
1645
1863
|
if (typeof req.storageObjectId !== 'string' || req.storageObjectId.trim().length === 0) {
|
|
1646
1864
|
throw new ai_error_1.AiError('AI_INVALID_REQUEST', 'A transcription needs the recording\'s storageObjectId.');
|
|
1647
1865
|
}
|
|
1866
|
+
assertTranscriptionOptions(req);
|
|
1867
|
+
}
|
|
1868
|
+
/** The recording-independent part of a transcription request's shape (shared with the inline form). */
|
|
1869
|
+
function assertTranscriptionOptions(req) {
|
|
1648
1870
|
if (req.language !== undefined && !ai_audio_run_request_1.AI_LANGUAGE_CODE.test(req.language)) {
|
|
1649
1871
|
throw new ai_error_1.AiError('AI_INVALID_REQUEST', 'language must be an ISO-639-1 code such as "en".', {
|
|
1650
1872
|
details: { language: req.language },
|
|
@@ -1686,6 +1908,91 @@ function assertProviderOptionsShape(options) {
|
|
|
1686
1908
|
throw new ai_error_1.AiError('AI_INVALID_REQUEST', `providerOptions must be a JSON object of at most ${media_types_1.AI_PROVIDER_OPTIONS_MAX_BYTES} bytes.`, { details: { max: media_types_1.AI_PROVIDER_OPTIONS_MAX_BYTES } });
|
|
1687
1909
|
}
|
|
1688
1910
|
}
|
|
1911
|
+
/** A MIME type (parameters such as `;codecs=opus` ignored) against an allow-list that may use `type/*`. */
|
|
1912
|
+
function mimeAllowed(mimeType, allowed) {
|
|
1913
|
+
const base = mimeType.split(';')[0].trim().toLowerCase();
|
|
1914
|
+
return allowed.some((pattern) => pattern.endsWith('/*') ? base.startsWith(pattern.slice(0, -1)) && base.length > pattern.length - 1 : base === pattern);
|
|
1915
|
+
}
|
|
1916
|
+
/** The bytes of an inline payload, copied by reference into the named fields the adapter takes. */
|
|
1917
|
+
function inlinePayload(payload) {
|
|
1918
|
+
return {
|
|
1919
|
+
data: payload.data,
|
|
1920
|
+
mimeType: payload.mimeType,
|
|
1921
|
+
...(payload.filename !== undefined ? { filename: payload.filename } : {}),
|
|
1922
|
+
};
|
|
1923
|
+
}
|
|
1924
|
+
/**
|
|
1925
|
+
* An inline edit's images and mask, checked before any gate reads a table: 1
|
|
1926
|
+
* to `AI_IMAGE_EDIT_MAX_INPUTS` non-empty images of an allowed type and at
|
|
1927
|
+
* most `AI_IMAGE_INPUT_MAX_BYTES` each; a PNG mask within the same size.
|
|
1928
|
+
* Errors name the index and the limit, never the bytes.
|
|
1929
|
+
*/
|
|
1930
|
+
function assertInlineImages(req) {
|
|
1931
|
+
const { images, mask } = req;
|
|
1932
|
+
if (!Array.isArray(images) || images.length === 0 || images.length > media_types_1.AI_IMAGE_EDIT_MAX_INPUTS) {
|
|
1933
|
+
throw new ai_error_1.AiError('AI_INVALID_REQUEST', `An image edit needs 1 to ${media_types_1.AI_IMAGE_EDIT_MAX_INPUTS} source images.`, { details: { max: media_types_1.AI_IMAGE_EDIT_MAX_INPUTS } });
|
|
1934
|
+
}
|
|
1935
|
+
const check = (payload, label, allowed, index) => {
|
|
1936
|
+
const where = index === undefined ? {} : { index };
|
|
1937
|
+
if (!payload ||
|
|
1938
|
+
!(payload.data instanceof Uint8Array) ||
|
|
1939
|
+
payload.data.byteLength === 0 ||
|
|
1940
|
+
typeof payload.mimeType !== 'string') {
|
|
1941
|
+
throw new ai_error_1.AiError('AI_INVALID_REQUEST', `The ${label} must be non-empty bytes with a MIME type.`, { details: where });
|
|
1942
|
+
}
|
|
1943
|
+
if (!mimeAllowed(payload.mimeType, allowed)) {
|
|
1944
|
+
throw new ai_error_1.AiError('AI_INVALID_REQUEST', `The ${label} must be ${allowed.join(', ')}.`, {
|
|
1945
|
+
details: { ...where, mimeType: payload.mimeType },
|
|
1946
|
+
});
|
|
1947
|
+
}
|
|
1948
|
+
if (payload.data.byteLength > media_types_1.AI_IMAGE_INPUT_MAX_BYTES) {
|
|
1949
|
+
throw new ai_error_1.AiError('AI_INVALID_REQUEST', `The ${label} is larger than ${media_types_1.AI_IMAGE_INPUT_MAX_BYTES} bytes.`, {
|
|
1950
|
+
details: { ...where, size: payload.data.byteLength, maxBytes: media_types_1.AI_IMAGE_INPUT_MAX_BYTES },
|
|
1951
|
+
});
|
|
1952
|
+
}
|
|
1953
|
+
};
|
|
1954
|
+
images.forEach((image, index) => check(image, 'image', media_types_1.AI_IMAGE_INPUT_MIME_TYPES, index));
|
|
1955
|
+
if (mask !== undefined)
|
|
1956
|
+
check(mask, 'mask', media_types_1.AI_IMAGE_MASK_MIME_TYPES);
|
|
1957
|
+
}
|
|
1958
|
+
/**
|
|
1959
|
+
* An inline recording within `maxBytes`: bytes are checked now; a stream is
|
|
1960
|
+
* refused now when it declares a larger `size`, and otherwise counted as it is
|
|
1961
|
+
* sent and cut off past the cap (`exceeded()` then names the cause, as for a
|
|
1962
|
+
* stored recording that outgrows its row).
|
|
1963
|
+
*/
|
|
1964
|
+
function capInlineAudio(audio, maxBytes) {
|
|
1965
|
+
const tooLarge = (size) => new ai_error_1.AiError('AI_INVALID_REQUEST', `The audio is larger than ${maxBytes} bytes (${maxBytes / 1024 / 1024} MiB).`, {
|
|
1966
|
+
details: { size, maxBytes },
|
|
1967
|
+
});
|
|
1968
|
+
if (!('stream' in audio) || audio.stream === undefined) {
|
|
1969
|
+
const bytes = audio;
|
|
1970
|
+
if (!(bytes.data instanceof Uint8Array) || bytes.data.byteLength === 0) {
|
|
1971
|
+
throw new ai_error_1.AiError('AI_INVALID_REQUEST', 'The audio must be non-empty bytes.');
|
|
1972
|
+
}
|
|
1973
|
+
if (bytes.data.byteLength > maxBytes)
|
|
1974
|
+
throw tooLarge(bytes.data.byteLength);
|
|
1975
|
+
return { audio: bytes, exceeded: () => undefined };
|
|
1976
|
+
}
|
|
1977
|
+
if (audio.size !== undefined && audio.size > maxBytes)
|
|
1978
|
+
throw tooLarge(audio.size);
|
|
1979
|
+
let seen = 0;
|
|
1980
|
+
let overflow;
|
|
1981
|
+
const source = audio.stream;
|
|
1982
|
+
const counted = {
|
|
1983
|
+
async *[Symbol.asyncIterator]() {
|
|
1984
|
+
for await (const chunk of source) {
|
|
1985
|
+
seen += chunk.byteLength;
|
|
1986
|
+
if (seen > maxBytes) {
|
|
1987
|
+
overflow = tooLarge(seen);
|
|
1988
|
+
throw overflow;
|
|
1989
|
+
}
|
|
1990
|
+
yield chunk;
|
|
1991
|
+
}
|
|
1992
|
+
},
|
|
1993
|
+
};
|
|
1994
|
+
return { audio: { ...audio, stream: counted }, exceeded: () => overflow };
|
|
1995
|
+
}
|
|
1689
1996
|
/** The optional, provider-facing fields of a transcription request (named fields only). */
|
|
1690
1997
|
function transcriptionFields(req) {
|
|
1691
1998
|
return {
|