nexus-ai-pro 2.20.2 → 2.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/API_STABILITY.md +93 -0
- package/CHANGELOG.md +208 -1
- package/README.md +11 -6
- package/SECURITY.md +16 -2
- package/dist/adapter-kit/gather.d.ts +76 -0
- package/dist/adapter-kit/gather.js +147 -0
- package/dist/adapter-kit/index.d.ts +1 -0
- package/dist/adapter-kit/index.js +1 -0
- package/dist/adapter-kit/record.d.ts +3 -1
- package/dist/adapter-kit/record.js +20 -5
- package/dist/agent/openapi.d.ts +94 -8
- package/dist/agent/openapi.js +528 -32
- package/dist/cli/adapters.d.ts +4 -3
- package/dist/cli/adapters.js +71 -2
- package/dist/cli/doctor.js +3 -0
- package/dist/cli.js +3 -1
- package/dist/core/config-builder.d.ts +1 -0
- package/dist/core/config-builder.js +1 -0
- package/dist/core/defaults.d.ts +34 -0
- package/dist/core/defaults.js +39 -0
- package/dist/core/secure-stream.js +4 -1
- package/dist/decisions/completion.d.ts +10 -7
- package/dist/decisions/completion.js +82 -8
- package/dist/decisions/core.d.ts +88 -43
- package/dist/decisions/core.js +115 -28
- package/dist/decisions/errors.d.ts +25 -0
- package/dist/decisions/errors.js +12 -0
- package/dist/decisions/index.d.ts +6 -5
- package/dist/decisions/index.js +2 -1
- package/dist/decisions/integrations.d.ts +14 -5
- package/dist/decisions/integrations.js +16 -1
- package/dist/decisions/mock.d.ts +5 -3
- package/dist/decisions/mock.js +4 -1
- package/dist/decisions/pictures.d.ts +43 -0
- package/dist/decisions/pictures.js +103 -0
- package/dist/decisions/typesafe.d.ts +33 -5
- package/dist/decisions/typesafe.js +121 -8
- package/dist/doctor/index.d.ts +6 -9
- package/dist/doctor/index.js +41 -0
- package/dist/embeddings/adapters.d.ts +1 -1
- package/dist/embeddings/adapters.js +3 -3
- package/dist/embeddings/batch-google.d.ts +27 -8
- package/dist/embeddings/batch-google.js +112 -36
- package/dist/embeddings/batch-mock.js +1 -1
- package/dist/embeddings/batch.js +10 -4
- package/dist/embeddings/engine.js +14 -13
- package/dist/embeddings/index.d.ts +1 -1
- package/dist/embeddings/inputs.d.ts +26 -3
- package/dist/embeddings/inputs.js +84 -5
- package/dist/embeddings/mock.js +1 -1
- package/dist/embeddings/models.js +39 -4
- package/dist/loaders/csv.d.ts +23 -1
- package/dist/loaders/csv.js +125 -72
- package/dist/loaders/index.d.ts +7 -2
- package/dist/loaders/input.d.ts +6 -0
- package/dist/loaders/input.js +42 -1
- package/dist/loaders/json.d.ts +14 -3
- package/dist/loaders/json.js +171 -20
- package/dist/loaders/web.d.ts +55 -0
- package/dist/loaders/web.js +20 -1
- package/dist/mcp/client.d.ts +54 -5
- package/dist/mcp/client.js +171 -20
- package/dist/mcp/http.d.ts +45 -0
- package/dist/mcp/http.js +233 -0
- package/dist/mcp/index.d.ts +4 -2
- package/dist/mcp/index.js +3 -1
- package/dist/mcp/protocol.d.ts +107 -0
- package/dist/mcp/sampling.d.ts +41 -0
- package/dist/mcp/sampling.js +52 -0
- package/dist/mcp/server.d.ts +96 -14
- package/dist/mcp/server.js +274 -59
- package/dist/models/data.js +20 -1
- package/dist/operations/adapters.d.ts +1 -0
- package/dist/operations/adapters.js +3 -1
- package/dist/protocols/a2a-grpc.js +10 -3
- package/dist/protocols/a2a-signing.d.ts +65 -0
- package/dist/protocols/a2a-signing.js +133 -0
- package/dist/protocols/a2a.d.ts +14 -0
- package/dist/protocols/a2a.js +60 -5
- package/dist/providers/cohere.js +16 -2
- package/dist/providers/google.js +7 -1
- package/dist/providers/ollama.d.ts +1 -1
- package/dist/providers/ollama.js +35 -27
- package/dist/providers/openai.js +19 -2
- package/dist/rag/pinecone.d.ts +63 -2
- package/dist/rag/pinecone.js +80 -0
- package/dist/realtime/conversation.js +18 -0
- package/dist/realtime/gemini-live.transport.d.ts +28 -3
- package/dist/realtime/gemini-live.transport.js +170 -32
- package/dist/realtime/session.d.ts +2 -0
- package/dist/realtime/session.js +19 -4
- package/dist/realtime/types.d.ts +35 -3
- package/dist/security/output-guard.js +4 -1
- package/dist/server/auth.d.ts +263 -2
- package/dist/server/auth.js +432 -5
- package/dist/server/deployments.d.ts +3 -2
- package/dist/server/deployments.js +6 -1
- package/dist/telephony/index.d.ts +1 -1
- package/dist/telephony/manager.d.ts +8 -1
- package/dist/telephony/manager.js +13 -0
- package/dist/telephony/providers/telnyx.d.ts +18 -5
- package/dist/telephony/providers/telnyx.js +44 -7
- package/dist/telephony/providers/twilio.d.ts +3 -1
- package/dist/telephony/providers/twilio.js +16 -0
- package/dist/tracing/stores.d.ts +2 -0
- package/dist/tracing/stores.js +1 -0
- package/dist/tracing/tracer.d.ts +5 -2
- package/dist/tracing/tracer.js +6 -3
- package/dist/types/embeddings.d.ts +47 -5
- package/dist/types/messages.d.ts +7 -0
- package/dist/types/principal.d.ts +1 -1
- package/dist/types/response.d.ts +5 -0
- package/dist/types/telephony.d.ts +38 -0
- package/dist/types/tracing.d.ts +6 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/voice/providers/google.d.ts +44 -0
- package/dist/voice/providers/google.js +215 -0
- package/dist-cjs/adapter-kit/gather.d.ts +76 -0
- package/dist-cjs/adapter-kit/gather.js +152 -0
- package/dist-cjs/adapter-kit/index.d.ts +1 -0
- package/dist-cjs/adapter-kit/index.js +4 -1
- package/dist-cjs/adapter-kit/record.d.ts +3 -1
- package/dist-cjs/adapter-kit/record.js +20 -5
- package/dist-cjs/agent/openapi.d.ts +94 -8
- package/dist-cjs/agent/openapi.js +530 -32
- package/dist-cjs/cli/adapters.d.ts +4 -3
- package/dist-cjs/cli/adapters.js +70 -1
- package/dist-cjs/cli/doctor.js +3 -0
- package/dist-cjs/core/config-builder.d.ts +1 -0
- package/dist-cjs/core/config-builder.js +5 -1
- package/dist-cjs/core/defaults.d.ts +34 -0
- package/dist-cjs/core/defaults.js +44 -0
- package/dist-cjs/core/secure-stream.js +4 -1
- package/dist-cjs/decisions/completion.d.ts +10 -7
- package/dist-cjs/decisions/completion.js +81 -7
- package/dist-cjs/decisions/core.d.ts +88 -43
- package/dist-cjs/decisions/core.js +134 -46
- package/dist-cjs/decisions/errors.d.ts +25 -0
- package/dist-cjs/decisions/errors.js +16 -0
- package/dist-cjs/decisions/index.d.ts +6 -5
- package/dist-cjs/decisions/index.js +5 -1
- package/dist-cjs/decisions/integrations.d.ts +14 -5
- package/dist-cjs/decisions/integrations.js +16 -1
- package/dist-cjs/decisions/mock.d.ts +5 -3
- package/dist-cjs/decisions/mock.js +4 -1
- package/dist-cjs/decisions/pictures.d.ts +43 -0
- package/dist-cjs/decisions/pictures.js +109 -0
- package/dist-cjs/decisions/typesafe.d.ts +33 -5
- package/dist-cjs/decisions/typesafe.js +126 -13
- package/dist-cjs/doctor/index.d.ts +6 -9
- package/dist-cjs/doctor/index.js +41 -0
- package/dist-cjs/embeddings/adapters.d.ts +1 -1
- package/dist-cjs/embeddings/adapters.js +3 -3
- package/dist-cjs/embeddings/batch-google.d.ts +27 -8
- package/dist-cjs/embeddings/batch-google.js +112 -36
- package/dist-cjs/embeddings/batch-mock.js +1 -1
- package/dist-cjs/embeddings/batch.js +9 -3
- package/dist-cjs/embeddings/engine.js +13 -12
- package/dist-cjs/embeddings/index.d.ts +1 -1
- package/dist-cjs/embeddings/inputs.d.ts +26 -3
- package/dist-cjs/embeddings/inputs.js +89 -5
- package/dist-cjs/embeddings/mock.js +1 -1
- package/dist-cjs/embeddings/models.js +39 -4
- package/dist-cjs/loaders/csv.d.ts +23 -1
- package/dist-cjs/loaders/csv.js +126 -71
- package/dist-cjs/loaders/index.d.ts +7 -2
- package/dist-cjs/loaders/input.d.ts +6 -0
- package/dist-cjs/loaders/input.js +43 -1
- package/dist-cjs/loaders/json.d.ts +14 -3
- package/dist-cjs/loaders/json.js +170 -19
- package/dist-cjs/loaders/web.d.ts +55 -0
- package/dist-cjs/loaders/web.js +21 -1
- package/dist-cjs/mcp/client.d.ts +54 -5
- package/dist-cjs/mcp/client.js +171 -20
- package/dist-cjs/mcp/http.d.ts +45 -0
- package/dist-cjs/mcp/http.js +236 -0
- package/dist-cjs/mcp/index.d.ts +4 -2
- package/dist-cjs/mcp/index.js +5 -1
- package/dist-cjs/mcp/protocol.d.ts +107 -0
- package/dist-cjs/mcp/sampling.d.ts +41 -0
- package/dist-cjs/mcp/sampling.js +55 -0
- package/dist-cjs/mcp/server.d.ts +96 -14
- package/dist-cjs/mcp/server.js +273 -58
- package/dist-cjs/models/data.js +20 -1
- package/dist-cjs/operations/adapters.d.ts +1 -0
- package/dist-cjs/operations/adapters.js +3 -1
- package/dist-cjs/protocols/a2a-grpc.js +10 -3
- package/dist-cjs/protocols/a2a-signing.d.ts +65 -0
- package/dist-cjs/protocols/a2a-signing.js +140 -0
- package/dist-cjs/protocols/a2a.d.ts +14 -0
- package/dist-cjs/protocols/a2a.js +65 -6
- package/dist-cjs/providers/cohere.js +16 -2
- package/dist-cjs/providers/google.js +7 -1
- package/dist-cjs/providers/ollama.d.ts +1 -1
- package/dist-cjs/providers/ollama.js +35 -27
- package/dist-cjs/providers/openai.js +19 -2
- package/dist-cjs/rag/pinecone.d.ts +63 -2
- package/dist-cjs/rag/pinecone.js +81 -0
- package/dist-cjs/realtime/conversation.js +18 -0
- package/dist-cjs/realtime/gemini-live.transport.d.ts +28 -3
- package/dist-cjs/realtime/gemini-live.transport.js +170 -32
- package/dist-cjs/realtime/session.d.ts +2 -0
- package/dist-cjs/realtime/session.js +19 -4
- package/dist-cjs/realtime/types.d.ts +35 -3
- package/dist-cjs/security/output-guard.js +4 -1
- package/dist-cjs/server/auth.d.ts +263 -2
- package/dist-cjs/server/auth.js +437 -5
- package/dist-cjs/server/deployments.d.ts +3 -2
- package/dist-cjs/server/deployments.js +6 -1
- package/dist-cjs/telephony/index.d.ts +1 -1
- package/dist-cjs/telephony/manager.d.ts +8 -1
- package/dist-cjs/telephony/manager.js +13 -0
- package/dist-cjs/telephony/providers/telnyx.d.ts +18 -5
- package/dist-cjs/telephony/providers/telnyx.js +44 -7
- package/dist-cjs/telephony/providers/twilio.d.ts +3 -1
- package/dist-cjs/telephony/providers/twilio.js +16 -0
- package/dist-cjs/tracing/stores.d.ts +2 -0
- package/dist-cjs/tracing/stores.js +1 -0
- package/dist-cjs/tracing/tracer.d.ts +5 -2
- package/dist-cjs/tracing/tracer.js +6 -3
- package/dist-cjs/types/embeddings.d.ts +47 -5
- package/dist-cjs/types/messages.d.ts +7 -0
- package/dist-cjs/types/principal.d.ts +1 -1
- package/dist-cjs/types/response.d.ts +5 -0
- package/dist-cjs/types/telephony.d.ts +38 -0
- package/dist-cjs/types/tracing.d.ts +6 -0
- package/dist-cjs/version.d.ts +1 -1
- package/dist-cjs/version.js +1 -1
- package/dist-cjs/voice/providers/google.d.ts +44 -0
- package/dist-cjs/voice/providers/google.js +252 -0
- package/llms.txt +1 -1
- package/package.json +17 -1
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.DecisionError = void 0;
|
|
4
|
+
class DecisionError extends Error {
|
|
5
|
+
code;
|
|
6
|
+
retryable;
|
|
7
|
+
status;
|
|
8
|
+
constructor(message, code, retryable = false, status) {
|
|
9
|
+
super(message);
|
|
10
|
+
this.code = code;
|
|
11
|
+
this.retryable = retryable;
|
|
12
|
+
this.status = status;
|
|
13
|
+
this.name = 'DecisionError';
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
exports.DecisionError = DecisionError;
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Typed decisions: questions a model answers with a yes or no, a choice,
|
|
3
|
-
*
|
|
4
|
-
* The contract, its manager, gates, and calibration;
|
|
5
|
-
* decisions in graphs, agents, and evaluations.
|
|
2
|
+
* Typed decisions: questions a model answers with a yes or no, a choice, a score, or a box around
|
|
3
|
+
* something in a picture, each with a confidence read alike from any provider, so software acts on an
|
|
4
|
+
* answer without parsing text. The contract, its manager, gates, and calibration; the pictures a state
|
|
5
|
+
* holds; decisions from any completion model; and decisions in graphs, agents, and evaluations.
|
|
6
6
|
*/
|
|
7
7
|
export { type CompletionDecisionMethod, completionDecisions, type CompletionDecisionsOptions, type DecisionCompletionClient, } from './completion.js';
|
|
8
8
|
export { decisionEvaluator, type DecisionEvaluatorOptions, decisionGuard, type DecisionGuardOptions, type DecisionGuardReview, type DecisionManagerLike, decisionModel, type DecisionModelOptions, decisionPermissions, type DecisionPermissionsOptions, type DecisionRequestOptions, decisionRoute, type DecisionRouteOptions, reviewUnsure, } from './integrations.js';
|
|
9
|
-
export { answeredRight, type AnswerOf, answerProbability, type CachedDecision, calibrate, type CalibrateOptions, type CalibrationBin, type CalibrationReport, type CalibrationSample, type ChoiceAnswer, choiceConfidence, type ChoiceQuestion, decide, DECISION_PRICES, decisionAnswer, type DecisionAnswer, type DecisionAnswers, type DecisionCallContext, type DecisionConfig, type DecisionConformanceCase, type DecisionConformanceOptions, type DecisionConformanceReport, type DecisionDistribution, DecisionError, type DecisionErrorCode, DecisionManager, type DecisionPrice, type DecisionProvider, type DecisionProviderCapabilities, type DecisionProviderInfo, type DecisionProviderQuestion, type DecisionProviderRequest, type DecisionProviderResult, type DecisionQuestion, type DecisionQuestions, type DecisionQuestionType, type DecisionRequest, type DecisionResult, type DecisionRunLike, type DecisionRuntime, type DecisionState, type DecisionTracerLike, type DecisionUsage, gate, type GateOutcome, type GateThresholds, priceDecision, runDecisionConformance, type ScoreAnswer, scoreConfidence, type ScoreQuestion, type YesNoAnswer, yesNoConfidence, type YesNoQuestion, } from './core.js';
|
|
9
|
+
export { answeredRight, type AnswerOf, answerProbability, type BoxAnswer, boxOverlap, type BoxQuestion, type CachedDecision, calibrate, type CalibrateOptions, type CalibrationBin, type CalibrationReport, type CalibrationSample, type ChoiceAnswer, choiceConfidence, type ChoiceQuestion, decide, DECISION_PRICES, decisionAnswer, type DecisionAnswer, type DecisionAnswers, type DecisionBox, type DecisionCallContext, type DecisionConfig, type DecisionConformanceCase, type DecisionConformanceOptions, type DecisionConformanceReport, type DecisionDistribution, DecisionError, type DecisionErrorCode, DecisionManager, type DecisionPrice, type DecisionProvider, type DecisionProviderCapabilities, type DecisionProviderInfo, type DecisionProviderQuestion, type DecisionProviderRequest, type DecisionProviderResult, type DecisionQuestion, type DecisionQuestions, type DecisionQuestionType, type DecisionRequest, type DecisionResult, type DecisionRunLike, type DecisionRuntime, type DecisionState, type DecisionTracerLike, type DecisionUsage, gate, type GateOutcome, type GateThresholds, priceDecision, runDecisionConformance, type ScoreAnswer, scoreConfidence, type ScoreQuestion, type YesNoAnswer, yesNoConfidence, type YesNoQuestion, } from './core.js';
|
|
10
|
+
export { type DecisionPicture, type ReadDecisionPicture, readPicture, statePictures } from './pictures.js';
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.yesNoConfidence = exports.scoreConfidence = exports.runDecisionConformance = exports.priceDecision = exports.gate = exports.DecisionManager = exports.DecisionError = exports.decisionAnswer = exports.DECISION_PRICES = exports.decide = exports.choiceConfidence = exports.calibrate = exports.answerProbability = exports.answeredRight = exports.reviewUnsure = exports.decisionRoute = exports.decisionPermissions = exports.decisionModel = exports.decisionGuard = exports.decisionEvaluator = exports.completionDecisions = void 0;
|
|
3
|
+
exports.statePictures = exports.readPicture = exports.yesNoConfidence = exports.scoreConfidence = exports.runDecisionConformance = exports.priceDecision = exports.gate = exports.DecisionManager = exports.DecisionError = exports.decisionAnswer = exports.DECISION_PRICES = exports.decide = exports.choiceConfidence = exports.calibrate = exports.boxOverlap = exports.answerProbability = exports.answeredRight = exports.reviewUnsure = exports.decisionRoute = exports.decisionPermissions = exports.decisionModel = exports.decisionGuard = exports.decisionEvaluator = exports.completionDecisions = void 0;
|
|
4
4
|
var completion_js_1 = require("./completion.js");
|
|
5
5
|
Object.defineProperty(exports, "completionDecisions", { enumerable: true, get: function () { return completion_js_1.completionDecisions; } });
|
|
6
6
|
var integrations_js_1 = require("./integrations.js");
|
|
@@ -13,6 +13,7 @@ Object.defineProperty(exports, "reviewUnsure", { enumerable: true, get: function
|
|
|
13
13
|
var core_js_1 = require("./core.js");
|
|
14
14
|
Object.defineProperty(exports, "answeredRight", { enumerable: true, get: function () { return core_js_1.answeredRight; } });
|
|
15
15
|
Object.defineProperty(exports, "answerProbability", { enumerable: true, get: function () { return core_js_1.answerProbability; } });
|
|
16
|
+
Object.defineProperty(exports, "boxOverlap", { enumerable: true, get: function () { return core_js_1.boxOverlap; } });
|
|
16
17
|
Object.defineProperty(exports, "calibrate", { enumerable: true, get: function () { return core_js_1.calibrate; } });
|
|
17
18
|
Object.defineProperty(exports, "choiceConfidence", { enumerable: true, get: function () { return core_js_1.choiceConfidence; } });
|
|
18
19
|
Object.defineProperty(exports, "decide", { enumerable: true, get: function () { return core_js_1.decide; } });
|
|
@@ -25,3 +26,6 @@ Object.defineProperty(exports, "priceDecision", { enumerable: true, get: functio
|
|
|
25
26
|
Object.defineProperty(exports, "runDecisionConformance", { enumerable: true, get: function () { return core_js_1.runDecisionConformance; } });
|
|
26
27
|
Object.defineProperty(exports, "scoreConfidence", { enumerable: true, get: function () { return core_js_1.scoreConfidence; } });
|
|
27
28
|
Object.defineProperty(exports, "yesNoConfidence", { enumerable: true, get: function () { return core_js_1.yesNoConfidence; } });
|
|
29
|
+
var pictures_js_1 = require("./pictures.js");
|
|
30
|
+
Object.defineProperty(exports, "readPicture", { enumerable: true, get: function () { return pictures_js_1.readPicture; } });
|
|
31
|
+
Object.defineProperty(exports, "statePictures", { enumerable: true, get: function () { return pictures_js_1.statePictures; } });
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
import type { AgentMiddleware } from '../agent/create-agent.js';
|
|
7
7
|
import type { PermissionPolicyLike, PermissionRequest } from '../agent/permissions.js';
|
|
8
8
|
import type { EvaluationContext, EvaluationScore, Evaluator } from '../types/evaluate.js';
|
|
9
|
-
import type
|
|
9
|
+
import { type BoxQuestion, type ChoiceAnswer, type ChoiceQuestion, type DecisionBox, type DecisionQuestions, type DecisionRequest, type DecisionResult, type DecisionState, type ScoreQuestion, type YesNoAnswer, type YesNoQuestion } from './core.js';
|
|
10
10
|
/** Anything that answers decisions, such as a `DecisionManager`. */
|
|
11
11
|
export interface DecisionManagerLike {
|
|
12
12
|
/** Answers every question of a decision. */
|
|
@@ -117,8 +117,8 @@ export declare function decisionModel<const K extends string>(decisions: Decisio
|
|
|
117
117
|
export interface DecisionEvaluatorOptions<I = unknown, O = unknown> {
|
|
118
118
|
/** The score's key. */
|
|
119
119
|
key: string;
|
|
120
|
-
/** The question: a yes or no, a score, or a
|
|
121
|
-
question: YesNoQuestion | ScoreQuestion | ChoiceQuestion;
|
|
120
|
+
/** The question: a yes or no, a score, a choice, or a box about a picture the state holds. */
|
|
121
|
+
question: YesNoQuestion | ScoreQuestion | ChoiceQuestion | BoxQuestion;
|
|
122
122
|
/** What the question is about. Defaults to the example's inputs, the output, and what was expected. */
|
|
123
123
|
state?: (context: EvaluationContext<I, O>) => DecisionState;
|
|
124
124
|
/**
|
|
@@ -126,13 +126,22 @@ export interface DecisionEvaluatorOptions<I = unknown, O = unknown> {
|
|
|
126
126
|
* passes (the top one by default); for a choice, the option that does.
|
|
127
127
|
*/
|
|
128
128
|
passing?: boolean | number | string;
|
|
129
|
+
/**
|
|
130
|
+
* For a box, where the thing is, such as from the example's expected output: the score is how far
|
|
131
|
+
* the box overlaps it, `boxOverlap()`, and it passes at `overlap` or more. Without it, the score is
|
|
132
|
+
* the box's confidence.
|
|
133
|
+
*/
|
|
134
|
+
where?: (context: EvaluationContext<I, O>) => DecisionBox | undefined;
|
|
135
|
+
/** The overlap a box passes at, from 0 to 1. Defaults to one half. */
|
|
136
|
+
overlap?: number;
|
|
129
137
|
/** Whom and what to ask. */
|
|
130
138
|
request?: DecisionRequestOptions;
|
|
131
139
|
}
|
|
132
140
|
/**
|
|
133
141
|
* An evaluator that asks a decision: the score is the probability of yes, the score's level as a
|
|
134
|
-
* share of the top one,
|
|
135
|
-
*
|
|
142
|
+
* share of the top one, the passing option's probability, or how far a box overlaps where the thing
|
|
143
|
+
* is, and its metadata carries the answer's confidence and probabilities, or its box, so an uncertain
|
|
144
|
+
* score can go to people with `reviewUnsure()`.
|
|
136
145
|
*/
|
|
137
146
|
export declare function decisionEvaluator<I = unknown, O = unknown>(decisions: DecisionManagerLike, options: DecisionEvaluatorOptions<I, O>): Evaluator<I, O>;
|
|
138
147
|
/**
|
|
@@ -6,6 +6,7 @@ exports.decisionGuard = decisionGuard;
|
|
|
6
6
|
exports.decisionModel = decisionModel;
|
|
7
7
|
exports.decisionEvaluator = decisionEvaluator;
|
|
8
8
|
exports.reviewUnsure = reviewUnsure;
|
|
9
|
+
const core_js_1 = require("./core.js");
|
|
9
10
|
function decisionRoute(decisions, options) {
|
|
10
11
|
return async (graph) => {
|
|
11
12
|
const { answers } = await decisions.decide({
|
|
@@ -136,9 +137,23 @@ function decisionEvaluator(decisions, options) {
|
|
|
136
137
|
const answer = answers.judged;
|
|
137
138
|
const metadata = {
|
|
138
139
|
confidence: answer.confidence,
|
|
139
|
-
|
|
140
|
+
...(answer.type === 'box'
|
|
141
|
+
? { box: answer.box }
|
|
142
|
+
: { probabilities: answer.type === 'yes-no' ? { yes: answer.probability } : answer.probabilities }),
|
|
140
143
|
model,
|
|
141
144
|
};
|
|
145
|
+
if (answer.type === 'box') {
|
|
146
|
+
const where = options.where?.(context);
|
|
147
|
+
if (!where)
|
|
148
|
+
return { key: options.key, score: answer.confidence, metadata };
|
|
149
|
+
const overlap = (0, core_js_1.boxOverlap)(answer.box, where);
|
|
150
|
+
return {
|
|
151
|
+
key: options.key,
|
|
152
|
+
score: overlap,
|
|
153
|
+
passed: overlap >= (options.overlap ?? 0.5),
|
|
154
|
+
metadata,
|
|
155
|
+
};
|
|
156
|
+
}
|
|
142
157
|
if (answer.type === 'yes-no') {
|
|
143
158
|
const passWhen = options.passing !== false;
|
|
144
159
|
return {
|
|
@@ -7,8 +7,9 @@ export interface MockDecisionProviderOptions {
|
|
|
7
7
|
/** The model it reports. Defaults to `mock-decision`. */
|
|
8
8
|
model?: string;
|
|
9
9
|
/**
|
|
10
|
-
* Answers each question with its probabilities, from the question's name, the question,
|
|
11
|
-
* state. Returning `undefined`, or leaving it out, answers with an even
|
|
10
|
+
* Answers each question with its probabilities, or a box, from the question's name, the question,
|
|
11
|
+
* and the state, pictures and all. Returning `undefined`, or leaving it out, answers with an even
|
|
12
|
+
* spread, and a box with the whole picture: confidence 0.
|
|
12
13
|
*/
|
|
13
14
|
answer?: (name: string, question: DecisionProviderQuestion, state: DecisionState) => DecisionDistribution | undefined;
|
|
14
15
|
/** Fails every call with this error, or the error the function returns. */
|
|
@@ -18,7 +19,8 @@ export interface MockDecisionProviderOptions {
|
|
|
18
19
|
}
|
|
19
20
|
/**
|
|
20
21
|
* Answers decisions as `answer` says, without a model, and keeps every request it was asked in
|
|
21
|
-
* `requests`. It takes every question type, 255 options, and 10 levels, as TypeSafe AI's Jev
|
|
22
|
+
* `requests`. It takes every question type, pictures, 255 options, and 10 levels, as TypeSafe AI's Jev
|
|
23
|
+
* does.
|
|
22
24
|
*/
|
|
23
25
|
export declare class MockDecisionProvider implements DecisionProvider {
|
|
24
26
|
private readonly options;
|
|
@@ -10,9 +10,10 @@ class MockDecisionProvider {
|
|
|
10
10
|
this.info = {
|
|
11
11
|
name: options.name ?? 'mock',
|
|
12
12
|
capabilities: {
|
|
13
|
-
types: ['yes-no', 'choice', 'score'],
|
|
13
|
+
types: ['yes-no', 'choice', 'score', 'box'],
|
|
14
14
|
maxOptions: 255,
|
|
15
15
|
maxLevels: 10,
|
|
16
|
+
images: true,
|
|
16
17
|
defaultModel: options.model ?? 'mock-decision',
|
|
17
18
|
},
|
|
18
19
|
};
|
|
@@ -41,6 +42,8 @@ exports.MockDecisionProvider = MockDecisionProvider;
|
|
|
41
42
|
function evenly(question) {
|
|
42
43
|
if (question.type === 'yes-no')
|
|
43
44
|
return { type: 'yes-no', yes: 0.5 };
|
|
45
|
+
if (question.type === 'box')
|
|
46
|
+
return { type: 'box', box: { top: 0, bottom: 1, left: 0, right: 1 }, confidence: 0 };
|
|
44
47
|
if (question.type === 'choice') {
|
|
45
48
|
return {
|
|
46
49
|
type: 'choice',
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The pictures a decision's state holds: image parts, or image assets, anywhere in an object or a
|
|
3
|
+
* list, each named by its path, which a question writes in backticks, such as `photo`, `faces.happy`,
|
|
4
|
+
* or `frames[2].frame`.
|
|
5
|
+
*/
|
|
6
|
+
import { type MediaResolveOptions } from '../evaluate/media.js';
|
|
7
|
+
import type { MediaPart } from '../types/evaluate.js';
|
|
8
|
+
/** A picture in a decision's state, and where it is. */
|
|
9
|
+
export interface DecisionPicture {
|
|
10
|
+
/** Its path in the state, as a question names it in backticks: `photo`, `faces.happy`, or `frames[2].frame`. */
|
|
11
|
+
path: string;
|
|
12
|
+
/** The picture: an image part, `{ type: 'image', source }`, or an image asset. */
|
|
13
|
+
part: MediaPart;
|
|
14
|
+
}
|
|
15
|
+
/** A picture's bytes, read for a provider that sends them. */
|
|
16
|
+
export interface ReadDecisionPicture extends DecisionPicture {
|
|
17
|
+
/** Its bytes. */
|
|
18
|
+
bytes: Uint8Array;
|
|
19
|
+
/** Its MIME type, as its first bytes say, whatever it was declared as. */
|
|
20
|
+
mimeType: string;
|
|
21
|
+
/** Its width in pixels, from its header. */
|
|
22
|
+
width?: number;
|
|
23
|
+
/** Its height in pixels. */
|
|
24
|
+
height?: number;
|
|
25
|
+
}
|
|
26
|
+
/** Every picture in a state, in the order they appear, each with its path. A picture used twice is listed at both. */
|
|
27
|
+
export declare function statePictures(state: unknown): DecisionPicture[];
|
|
28
|
+
/**
|
|
29
|
+
* The state with each picture replaced by what `replace` returns for it, copying only the objects and
|
|
30
|
+
* lists on the way to one, so a state without pictures comes back as it was given.
|
|
31
|
+
*/
|
|
32
|
+
export declare function replacePictures(state: unknown, replace: (picture: DecisionPicture) => unknown): unknown;
|
|
33
|
+
/**
|
|
34
|
+
* Identifies a picture without its bytes, for a cache key or a trace: a digest of what it holds, or
|
|
35
|
+
* where it is when it is a file or at a URL, so a picture changed at the same path is keyed the same.
|
|
36
|
+
*/
|
|
37
|
+
export declare function pictureDigest(part: MediaPart): string;
|
|
38
|
+
/**
|
|
39
|
+
* Reads a picture's bytes, with the media family's readers: bytes and base64 as they are, a file with
|
|
40
|
+
* `readFile`, a URL with `fetch`, and a stored asset from `assets`. Throws `DecisionError`
|
|
41
|
+
* `DECISION_INVALID`, naming the picture's path, for one that cannot be read or is not a picture.
|
|
42
|
+
*/
|
|
43
|
+
export declare function readPicture(picture: DecisionPicture, options?: MediaResolveOptions): Promise<ReadDecisionPicture>;
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.statePictures = statePictures;
|
|
4
|
+
exports.replacePictures = replacePictures;
|
|
5
|
+
exports.pictureDigest = pictureDigest;
|
|
6
|
+
exports.readPicture = readPicture;
|
|
7
|
+
const media_js_1 = require("../evaluate/media.js");
|
|
8
|
+
const trace_js_1 = require("../evaluate/trace.js");
|
|
9
|
+
const sha256_js_1 = require("../utils/sha256.js");
|
|
10
|
+
const errors_js_1 = require("./errors.js");
|
|
11
|
+
const MAX_DEPTH = 256;
|
|
12
|
+
function statePictures(state) {
|
|
13
|
+
const found = [];
|
|
14
|
+
replacePictures(state, (picture) => {
|
|
15
|
+
found.push(picture);
|
|
16
|
+
return picture.part;
|
|
17
|
+
});
|
|
18
|
+
return found;
|
|
19
|
+
}
|
|
20
|
+
function replacePictures(state, replace) {
|
|
21
|
+
const above = new Set();
|
|
22
|
+
const visit = (item, path, depth) => {
|
|
23
|
+
if (item === null || typeof item !== 'object' || depth > MAX_DEPTH || above.has(item))
|
|
24
|
+
return item;
|
|
25
|
+
if (ArrayBuffer.isView(item) || item instanceof ArrayBuffer)
|
|
26
|
+
return item;
|
|
27
|
+
const part = pictureOf(item);
|
|
28
|
+
if (part)
|
|
29
|
+
return replace({ path, part });
|
|
30
|
+
above.add(item);
|
|
31
|
+
try {
|
|
32
|
+
if (Array.isArray(item)) {
|
|
33
|
+
let changed = false;
|
|
34
|
+
const copy = item.map((entry, index) => {
|
|
35
|
+
const next = visit(entry, `${path}[${index}]`, depth + 1);
|
|
36
|
+
if (next !== entry)
|
|
37
|
+
changed = true;
|
|
38
|
+
return next;
|
|
39
|
+
});
|
|
40
|
+
return changed ? copy : item;
|
|
41
|
+
}
|
|
42
|
+
const prototype = Object.getPrototypeOf(item);
|
|
43
|
+
if (prototype !== Object.prototype && prototype !== null)
|
|
44
|
+
return item;
|
|
45
|
+
let changed = false;
|
|
46
|
+
const copy = {};
|
|
47
|
+
for (const [key, entry] of Object.entries(item)) {
|
|
48
|
+
const next = visit(entry, path ? `${path}.${key}` : key, depth + 1);
|
|
49
|
+
if (next !== entry)
|
|
50
|
+
changed = true;
|
|
51
|
+
copy[key] = next;
|
|
52
|
+
}
|
|
53
|
+
return changed ? copy : item;
|
|
54
|
+
}
|
|
55
|
+
finally {
|
|
56
|
+
above.delete(item);
|
|
57
|
+
}
|
|
58
|
+
};
|
|
59
|
+
return visit(state, '', 0);
|
|
60
|
+
}
|
|
61
|
+
function pictureDigest(part) {
|
|
62
|
+
const source = (part.type === 'asset' ? part.asset.location : part.source);
|
|
63
|
+
if (typeof source.base64 === 'string')
|
|
64
|
+
return `sha256:${(0, sha256_js_1.sha256Hex)(source.base64)}`;
|
|
65
|
+
const bytes = source.buffer ?? source.data;
|
|
66
|
+
if (bytes instanceof Uint8Array)
|
|
67
|
+
return `sha256:${(0, sha256_js_1.sha256Hex)(bytes)}`;
|
|
68
|
+
if (typeof source.path === 'string')
|
|
69
|
+
return `path:${source.path}`;
|
|
70
|
+
if (typeof source.url === 'string')
|
|
71
|
+
return `url:${source.url}`;
|
|
72
|
+
if (typeof source.assetId === 'string')
|
|
73
|
+
return `asset:${source.assetId}`;
|
|
74
|
+
if (source.asset && typeof source.asset === 'object')
|
|
75
|
+
return pictureDigest({ type: 'asset', asset: source.asset });
|
|
76
|
+
return `sha256:${(0, sha256_js_1.sha256Hex)(JSON.stringify(source) ?? '')}`;
|
|
77
|
+
}
|
|
78
|
+
async function readPicture(picture, options = {}) {
|
|
79
|
+
const media = await (0, media_js_1.resolveMedia)(picture.part, options);
|
|
80
|
+
if (!media.bytes) {
|
|
81
|
+
const why = media.unreadable ??
|
|
82
|
+
(media.path !== undefined
|
|
83
|
+
? `it is the file ${media.path}, and no readFile was given to read it`
|
|
84
|
+
: media.url !== undefined
|
|
85
|
+
? `it is at ${media.url}, and no fetch was given to fetch it`
|
|
86
|
+
: 'it holds no bytes');
|
|
87
|
+
throw new errors_js_1.DecisionError(`The picture at \`${picture.path}\` could not be read: ${why}`, 'DECISION_INVALID');
|
|
88
|
+
}
|
|
89
|
+
if (media.kind !== 'image' || !media.mimeType?.startsWith('image/')) {
|
|
90
|
+
throw new errors_js_1.DecisionError(`The picture at \`${picture.path}\` is not an image its bytes name${media.mimeType ? `, but ${media.mimeType}` : ''}`, 'DECISION_INVALID');
|
|
91
|
+
}
|
|
92
|
+
return {
|
|
93
|
+
...picture,
|
|
94
|
+
bytes: media.bytes,
|
|
95
|
+
mimeType: media.mimeType,
|
|
96
|
+
...(media.width !== undefined ? { width: media.width } : {}),
|
|
97
|
+
...(media.height !== undefined ? { height: media.height } : {}),
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
function pictureOf(item) {
|
|
101
|
+
const part = (0, trace_js_1.asMediaPart)(item);
|
|
102
|
+
if (!part)
|
|
103
|
+
return undefined;
|
|
104
|
+
if (part.type === 'image')
|
|
105
|
+
return part;
|
|
106
|
+
if (part.type === 'asset' && part.asset.mimeType.startsWith('image/'))
|
|
107
|
+
return part;
|
|
108
|
+
return undefined;
|
|
109
|
+
}
|
|
@@ -1,4 +1,10 @@
|
|
|
1
|
-
|
|
1
|
+
/**
|
|
2
|
+
* TypeSafe AI's decision models, Jev first, over its System One API: every question of a decision in
|
|
3
|
+
* one call, each answered with its probabilities, or with a box around something in a picture, and no
|
|
4
|
+
* text to parse.
|
|
5
|
+
*/
|
|
6
|
+
import type { MediaResolveOptions } from '../evaluate/media.js';
|
|
7
|
+
import type { DecisionCallContext, DecisionProvider, DecisionProviderInfo, DecisionProviderRequest, DecisionProviderResult } from './core.js';
|
|
2
8
|
/** Options for `TypeSafeDecisionProvider`. */
|
|
3
9
|
export interface TypeSafeDecisionProviderOptions {
|
|
4
10
|
/** A TypeSafe AI API key, from its console. A tenant's resolved credential replaces it per call. */
|
|
@@ -15,6 +21,18 @@ export interface TypeSafeDecisionProviderOptions {
|
|
|
15
21
|
* than its usual context, and marks as experimental.
|
|
16
22
|
*/
|
|
17
23
|
longContext?: boolean;
|
|
24
|
+
/**
|
|
25
|
+
* Reads the pictures a state holds by reference, with the media family's readers: `readFile` for a
|
|
26
|
+
* file, `fetch` for a URL, and `assets` for a stored asset. A picture held as bytes or base64 needs
|
|
27
|
+
* none of them. Without them, a picture by reference fails the decision, naming it.
|
|
28
|
+
*/
|
|
29
|
+
media?: MediaResolveOptions;
|
|
30
|
+
/**
|
|
31
|
+
* How a decision's pictures travel: `multipart`, the default, each as a part of its own beside the
|
|
32
|
+
* request, as bytes; or `base64`, within the JSON request, a third larger. Either way a request
|
|
33
|
+
* holds at most 32 MiB, so multipart carries more.
|
|
34
|
+
*/
|
|
35
|
+
pictures?: 'multipart' | 'base64';
|
|
18
36
|
/** How long a call may take before it fails, in milliseconds. Defaults to 30 seconds. */
|
|
19
37
|
timeoutMs?: number;
|
|
20
38
|
/** Headers added to every request. */
|
|
@@ -24,9 +42,12 @@ export interface TypeSafeDecisionProviderOptions {
|
|
|
24
42
|
}
|
|
25
43
|
/**
|
|
26
44
|
* TypeSafe AI's System One API (`POST /v1/systemone`): Jev, a model that answers only typed
|
|
27
|
-
* questions, a yes or no ("noul"), a choice among up to 255 options,
|
|
28
|
-
*
|
|
29
|
-
* one call and is answered in isolation, against the same state.
|
|
45
|
+
* questions, a yes or no ("noul"), a choice among up to 255 options, a score on up to ten levels, or a
|
|
46
|
+
* box around something in a picture, and bills for input tokens alone. Every question of a decision
|
|
47
|
+
* goes in one call and is answered in isolation, against the same state. The pictures a state holds
|
|
48
|
+
* are sent as the API's assets, each as a part of a multipart request unless `pictures` says
|
|
49
|
+
* `base64`, with the `TypeSafe-Danger-Zone-Images` header it requires for them and for a box, both of
|
|
50
|
+
* which it marks as experimental; a box needs Jev 1.14 or later.
|
|
30
51
|
*
|
|
31
52
|
* ```ts
|
|
32
53
|
* const decisions = new DecisionManager({ providers: { jev: new TypeSafeDecisionProvider({ apiKey: process.env.TYPESAFE_API_KEY! }) } });
|
|
@@ -34,12 +55,19 @@ export interface TypeSafeDecisionProviderOptions {
|
|
|
34
55
|
*/
|
|
35
56
|
export declare class TypeSafeDecisionProvider implements DecisionProvider {
|
|
36
57
|
private readonly options;
|
|
37
|
-
/** Its name and what it takes: every question type, 255 options, and 10 levels. */
|
|
58
|
+
/** Its name and what it takes: every question type, pictures, 255 options, and 10 levels. */
|
|
38
59
|
readonly info: DecisionProviderInfo;
|
|
39
60
|
constructor(options: TypeSafeDecisionProviderOptions);
|
|
40
61
|
/** Answers every question of a decision in one call. */
|
|
41
62
|
decide(request: DecisionProviderRequest, context: DecisionCallContext): Promise<DecisionProviderResult>;
|
|
42
63
|
/** The models the key may use, from `GET /v1/models`. */
|
|
43
64
|
listModels(context?: Partial<DecisionCallContext>): Promise<string[]>;
|
|
65
|
+
/**
|
|
66
|
+
* The state with each picture replaced by a reference of its own, and the pictures as assets by those
|
|
67
|
+
* references; a state without pictures as it was. A new reference per picture per request, as the
|
|
68
|
+
* API advises, since it matches a reference wherever its text appears. Sent multipart, each picture's
|
|
69
|
+
* part is named by its reference too.
|
|
70
|
+
*/
|
|
71
|
+
private withAssets;
|
|
44
72
|
private call;
|
|
45
73
|
}
|
|
@@ -1,8 +1,14 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.TypeSafeDecisionProvider = void 0;
|
|
4
|
+
const base64_js_1 = require("../utils/base64.js");
|
|
4
5
|
const signals_js_1 = require("../utils/signals.js");
|
|
5
|
-
const
|
|
6
|
+
const errors_js_1 = require("./errors.js");
|
|
7
|
+
const pictures_js_1 = require("./pictures.js");
|
|
8
|
+
const PICTURE_BYTES = 16 * 1024 * 1024;
|
|
9
|
+
const PICTURE_SIDE = { min: 280, max: 1568 };
|
|
10
|
+
const REQUEST_BYTES = 32 * 1024 * 1024;
|
|
11
|
+
const PART_OVERHEAD = 256;
|
|
6
12
|
class TypeSafeDecisionProvider {
|
|
7
13
|
options;
|
|
8
14
|
info;
|
|
@@ -11,21 +17,45 @@ class TypeSafeDecisionProvider {
|
|
|
11
17
|
this.info = {
|
|
12
18
|
name: 'typesafe',
|
|
13
19
|
capabilities: {
|
|
14
|
-
types: ['yes-no', 'choice', 'score'],
|
|
20
|
+
types: ['yes-no', 'choice', 'score', 'box'],
|
|
15
21
|
maxOptions: 255,
|
|
16
22
|
maxLevels: 10,
|
|
23
|
+
images: true,
|
|
17
24
|
defaultModel: options.model ?? 'jev-latest',
|
|
18
25
|
},
|
|
19
26
|
};
|
|
20
27
|
}
|
|
21
28
|
async decide(request, context) {
|
|
29
|
+
context.signal.throwIfAborted();
|
|
30
|
+
const boxes = Object.keys(request.questions).filter((name) => request.questions[name]?.type === 'box');
|
|
31
|
+
const before = boxes.length ? beforeBoxes(request.model) : undefined;
|
|
32
|
+
if (before) {
|
|
33
|
+
throw new errors_js_1.DecisionError(`TypeSafe AI answers a box, such as "${boxes[0]}", on Jev 1.14 or later, not ${before}`, 'DECISION_UNSUPPORTED');
|
|
34
|
+
}
|
|
35
|
+
const multipart = (this.options.pictures ?? 'multipart') === 'multipart';
|
|
36
|
+
const { state, assets, parts } = await this.withAssets(request.state, context, multipart);
|
|
22
37
|
const questions = Object.fromEntries(Object.entries(request.questions).map(([name, question]) => [name, wireQuestion(question)]));
|
|
23
|
-
const
|
|
38
|
+
const json = JSON.stringify({ model: request.model, state, questions, ...(assets ? { assets } : {}) });
|
|
39
|
+
const size = new TextEncoder().encode(json).byteLength +
|
|
40
|
+
(parts.length ? parts.reduce((total, part) => total + part.bytes.byteLength + PART_OVERHEAD, PART_OVERHEAD) : 0);
|
|
41
|
+
if (assets && size > REQUEST_BYTES) {
|
|
42
|
+
throw new errors_js_1.DecisionError(`The decision is ${size} bytes with its pictures, and TypeSafe AI takes at most ${REQUEST_BYTES}`, 'DECISION_INVALID');
|
|
43
|
+
}
|
|
44
|
+
let body = json;
|
|
45
|
+
if (parts.length) {
|
|
46
|
+
const form = new FormData();
|
|
47
|
+
form.append('request', json);
|
|
48
|
+
for (const part of parts) {
|
|
49
|
+
form.append(part.name, new Blob([part.bytes], { type: `image/${part.format}` }), `${part.name}.${part.format === 'png' ? 'png' : 'jpg'}`);
|
|
50
|
+
}
|
|
51
|
+
body = form;
|
|
52
|
+
}
|
|
53
|
+
const payload = await this.call('/v1/systemone', { method: 'POST', body }, context, assets || boxes.length ? { 'TypeSafe-Danger-Zone-Images': 'true' } : {});
|
|
24
54
|
const distributions = {};
|
|
25
55
|
for (const [name, question] of Object.entries(request.questions)) {
|
|
26
56
|
const answer = payload.answers?.[name];
|
|
27
57
|
if (!answer)
|
|
28
|
-
throw new
|
|
58
|
+
throw new errors_js_1.DecisionError(`TypeSafe AI did not answer "${name}"`, 'DECISION_FAILED');
|
|
29
59
|
distributions[name] = distributionOf(name, question, answer);
|
|
30
60
|
}
|
|
31
61
|
return {
|
|
@@ -54,7 +84,29 @@ class TypeSafeDecisionProvider {
|
|
|
54
84
|
: String(item?.id ?? item?.name ?? ''))
|
|
55
85
|
.filter(Boolean);
|
|
56
86
|
}
|
|
57
|
-
async
|
|
87
|
+
async withAssets(state, context, multipart) {
|
|
88
|
+
const pictures = typeof state === 'string' ? [] : (0, pictures_js_1.statePictures)(state);
|
|
89
|
+
if (pictures.length === 0)
|
|
90
|
+
return { state, parts: [] };
|
|
91
|
+
const read = await Promise.all(pictures.map((picture) => (0, pictures_js_1.readPicture)(picture, this.options.media)));
|
|
92
|
+
context.signal.throwIfAborted();
|
|
93
|
+
const references = new Map();
|
|
94
|
+
const assets = {};
|
|
95
|
+
const parts = [];
|
|
96
|
+
for (const picture of read) {
|
|
97
|
+
const format = fitting(picture);
|
|
98
|
+
const reference = globalThis.crypto.randomUUID();
|
|
99
|
+
references.set(picture.path, reference);
|
|
100
|
+
if (multipart) {
|
|
101
|
+
assets[reference] = { type: 'image', format, part: reference };
|
|
102
|
+
parts.push({ name: reference, format, bytes: picture.bytes });
|
|
103
|
+
}
|
|
104
|
+
else
|
|
105
|
+
assets[reference] = { type: 'image', format, data_base64: (0, base64_js_1.encodeBase64)(picture.bytes) };
|
|
106
|
+
}
|
|
107
|
+
return { state: (0, pictures_js_1.replacePictures)(state, (picture) => references.get(picture.path)), assets, parts };
|
|
108
|
+
}
|
|
109
|
+
async call(path, init, context, extra = {}) {
|
|
58
110
|
const credential = await context.credential?.(this.info.name);
|
|
59
111
|
const linked = (0, signals_js_1.linkSignals)([context.signal, AbortSignal.timeout(this.options.timeoutMs ?? 30_000)]);
|
|
60
112
|
const signal = linked.signal;
|
|
@@ -64,9 +116,10 @@ class TypeSafeDecisionProvider {
|
|
|
64
116
|
response = await (this.options.fetch ?? fetch)(`${(this.options.baseUrl ?? 'https://api.typesafe.ai').replace(/\/+$/, '')}${path}`, {
|
|
65
117
|
...init,
|
|
66
118
|
headers: {
|
|
67
|
-
...(init.body ? { 'content-type': 'application/json' } : {}),
|
|
119
|
+
...(typeof init.body === 'string' ? { 'content-type': 'application/json' } : {}),
|
|
68
120
|
authorization: `Bearer ${credential?.apiKey ?? this.options.apiKey}`,
|
|
69
121
|
...(this.options.longContext ? { 'TypeSafe-Danger-Zone-Long-Context': 'true' } : {}),
|
|
122
|
+
...extra,
|
|
70
123
|
...this.options.headers,
|
|
71
124
|
...credential?.headers,
|
|
72
125
|
},
|
|
@@ -77,7 +130,7 @@ class TypeSafeDecisionProvider {
|
|
|
77
130
|
catch (error) {
|
|
78
131
|
if (context.signal?.aborted)
|
|
79
132
|
throw error;
|
|
80
|
-
throw new
|
|
133
|
+
throw new errors_js_1.DecisionError(`TypeSafe AI could not be reached: ${error instanceof Error ? error.message : String(error)}`, 'DECISION_FAILED', true);
|
|
81
134
|
}
|
|
82
135
|
finally {
|
|
83
136
|
linked.dispose();
|
|
@@ -88,12 +141,56 @@ class TypeSafeDecisionProvider {
|
|
|
88
141
|
return JSON.parse(text);
|
|
89
142
|
}
|
|
90
143
|
catch {
|
|
91
|
-
throw new
|
|
144
|
+
throw new errors_js_1.DecisionError(`TypeSafe AI answered with something other than JSON: ${text.slice(0, 200)}`, 'DECISION_FAILED', true, response.status);
|
|
92
145
|
}
|
|
93
146
|
}
|
|
94
147
|
}
|
|
95
148
|
exports.TypeSafeDecisionProvider = TypeSafeDecisionProvider;
|
|
149
|
+
function beforeBoxes(model) {
|
|
150
|
+
const version = /^jev-(\d+)\.(\d+)/.exec(model);
|
|
151
|
+
if (!version)
|
|
152
|
+
return undefined;
|
|
153
|
+
const [major, minor] = [Number(version[1]), Number(version[2])];
|
|
154
|
+
return major < 1 || (major === 1 && minor < 14) ? model : undefined;
|
|
155
|
+
}
|
|
156
|
+
function fitting(picture) {
|
|
157
|
+
const at = `The picture at \`${picture.path}\``;
|
|
158
|
+
const refuse = (why) => new errors_js_1.DecisionError(`${at} ${why}; TypeSafe AI refuses it rather than convert or resize it`, 'DECISION_INVALID');
|
|
159
|
+
if (picture.mimeType !== 'image/png' && picture.mimeType !== 'image/jpeg') {
|
|
160
|
+
throw refuse(`is ${picture.mimeType}, and TypeSafe AI takes PNG and JPEG`);
|
|
161
|
+
}
|
|
162
|
+
if (picture.mimeType === 'image/png' && animated(picture.bytes))
|
|
163
|
+
throw refuse('is an animated PNG');
|
|
164
|
+
if (picture.bytes.length > PICTURE_BYTES) {
|
|
165
|
+
throw refuse(`is ${picture.bytes.length} bytes, over TypeSafe AI's ${PICTURE_BYTES}`);
|
|
166
|
+
}
|
|
167
|
+
const { width, height } = picture;
|
|
168
|
+
if (width === undefined || height === undefined)
|
|
169
|
+
throw refuse('has a header its size cannot be read from');
|
|
170
|
+
if ([width, height].some((side) => side < PICTURE_SIDE.min || side > PICTURE_SIDE.max)) {
|
|
171
|
+
throw refuse(`is ${width} by ${height} pixels, and TypeSafe AI takes ${PICTURE_SIDE.min} to ${PICTURE_SIDE.max} a side`);
|
|
172
|
+
}
|
|
173
|
+
return picture.mimeType === 'image/png' ? 'png' : 'jpeg';
|
|
174
|
+
}
|
|
175
|
+
function animated(bytes) {
|
|
176
|
+
let offset = 8;
|
|
177
|
+
while (offset + 8 <= bytes.length) {
|
|
178
|
+
const length = ((bytes[offset] ?? 0) << 24) |
|
|
179
|
+
((bytes[offset + 1] ?? 0) << 16) |
|
|
180
|
+
((bytes[offset + 2] ?? 0) << 8) |
|
|
181
|
+
(bytes[offset + 3] ?? 0);
|
|
182
|
+
const type = String.fromCharCode(...bytes.subarray(offset + 4, offset + 8));
|
|
183
|
+
if (type === 'acTL')
|
|
184
|
+
return true;
|
|
185
|
+
if (type === 'IDAT' || type === 'IEND' || length < 0)
|
|
186
|
+
return false;
|
|
187
|
+
offset += 12 + length;
|
|
188
|
+
}
|
|
189
|
+
return false;
|
|
190
|
+
}
|
|
96
191
|
function wireQuestion(question) {
|
|
192
|
+
if (question.type === 'box')
|
|
193
|
+
return { type: 'box', instructions: question.question };
|
|
97
194
|
const instructions = question.question ? { instructions: question.question } : {};
|
|
98
195
|
if (question.type === 'yes-no') {
|
|
99
196
|
const criteria = {
|
|
@@ -122,27 +219,43 @@ function distributionOf(name, question, answer) {
|
|
|
122
219
|
throw invalidAnswer(name, 'choice', answer);
|
|
123
220
|
return { type: 'choice', probabilities: answer.probabilities };
|
|
124
221
|
}
|
|
222
|
+
if (question.type === 'box') {
|
|
223
|
+
const box = answer.box;
|
|
224
|
+
if (!box || typeof answer.confidence !== 'number')
|
|
225
|
+
throw invalidAnswer(name, 'box', answer);
|
|
226
|
+
const edges = box;
|
|
227
|
+
return {
|
|
228
|
+
type: 'box',
|
|
229
|
+
box: { top: edges.top, bottom: edges.bottom, left: edges.left, right: edges.right },
|
|
230
|
+
confidence: answer.confidence,
|
|
231
|
+
};
|
|
232
|
+
}
|
|
125
233
|
if (!answer.probabilities)
|
|
126
234
|
throw invalidAnswer(name, 'score', answer);
|
|
127
235
|
const probabilities = answer.probabilities;
|
|
128
236
|
return { type: 'score', probabilities: question.levels.map((_, level) => probabilities[String(level)] ?? 0) };
|
|
129
237
|
}
|
|
130
238
|
function invalidAnswer(name, type, answer) {
|
|
131
|
-
return new
|
|
239
|
+
return new errors_js_1.DecisionError(`TypeSafe AI answered "${name}" without the ${type} it asks for: ${JSON.stringify(answer).slice(0, 200)}`, 'DECISION_FAILED');
|
|
132
240
|
}
|
|
133
241
|
function failure(status, text) {
|
|
134
|
-
if (status === 422) {
|
|
242
|
+
if (status === 422 || status === 400 || status === 413) {
|
|
135
243
|
let detail = text.slice(0, 300);
|
|
136
244
|
try {
|
|
137
245
|
const parsed = JSON.parse(text);
|
|
138
246
|
if (Array.isArray(parsed.detail)) {
|
|
139
|
-
detail = parsed.detail
|
|
247
|
+
detail = parsed.detail
|
|
248
|
+
.map((item) => `${(item.loc ?? []).join('.')}: ${item.msg ?? 'invalid'}`)
|
|
249
|
+
.join('; ');
|
|
250
|
+
}
|
|
251
|
+
else if (typeof parsed.detail === 'string') {
|
|
252
|
+
detail = parsed.detail;
|
|
140
253
|
}
|
|
141
254
|
}
|
|
142
255
|
catch {
|
|
143
256
|
}
|
|
144
|
-
return new
|
|
257
|
+
return new errors_js_1.DecisionError(`TypeSafe AI refused the decision: ${detail}`, 'DECISION_INVALID', false, status);
|
|
145
258
|
}
|
|
146
259
|
const retryable = status === 408 || status === 429 || status >= 500;
|
|
147
|
-
return new
|
|
260
|
+
return new errors_js_1.DecisionError(`TypeSafe AI answered ${status}: ${text.slice(0, 300)}`, 'DECISION_FAILED', retryable, status);
|
|
148
261
|
}
|