@terminals-tech/sdk 1.0.0-rc.1 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -19
- package/dist/WebContainerManager-4LIOGRVM.js +22 -0
- package/dist/browser-http-client-ZQLDWZMU.js +317 -0
- package/dist/cache-VKYSQRXX.js +45 -0
- package/dist/capabilities-MIPUMBLL.js +96 -0
- package/dist/chunk-2ESYSVXG.js +48 -0
- package/dist/chunk-2WTYE4SW.js +190 -0
- package/dist/chunk-3LFMIVJM.js +40 -0
- package/dist/chunk-ABCK4FWN.js +136 -0
- package/dist/chunk-AFDUOYHD.js +2060 -0
- package/dist/chunk-BCOQMFKT.js +265 -0
- package/dist/chunk-BYXBJQAS.js +0 -0
- package/dist/chunk-DKFJIILR.js +9798 -0
- package/dist/chunk-EXI3LJVJ.js +51 -0
- package/dist/chunk-FOXUEYWK.js +42 -0
- package/dist/chunk-GJWAJAX3.js +173 -0
- package/dist/chunk-H3POJCFA.js +333 -0
- package/dist/chunk-KASHT6C5.js +784 -0
- package/dist/chunk-KHR7ZYCX.js +4034 -0
- package/dist/chunk-L45BSQDJ.js +296 -0
- package/dist/chunk-LLGZTP3G.js +5521 -0
- package/dist/chunk-NTMBOESX.js +152 -0
- package/dist/chunk-OCLSAUCD.js +474 -0
- package/dist/chunk-OSSRZOGC.js +190 -0
- package/dist/chunk-PPFTKJDB.js +497 -0
- package/dist/chunk-PWAHFID5.js +381 -0
- package/dist/chunk-Q2VI6ICE.js +188 -0
- package/dist/chunk-QJFKEQHF.js +6460 -0
- package/dist/chunk-QWXPVB2L.js +320 -0
- package/dist/chunk-QWZRZKLZ.js +896 -0
- package/dist/chunk-STMI72WH.js +1005 -0
- package/dist/chunk-TSQ3BGLA.js +11945 -0
- package/dist/chunk-UJDUQNE2.js +79 -0
- package/dist/chunk-VZA2NUH3.js +118 -0
- package/dist/chunk-WGBCRNMB.js +1817 -0
- package/dist/chunk-WU4OTGJE.js +752 -0
- package/dist/chunk-XPJ63Y6T.js +70 -0
- package/dist/chunk-Y2EULKA2.js +172 -0
- package/dist/chunk-YJEZWCYV.js +94 -0
- package/dist/chunk-ZVO47SQV.js +150 -0
- package/dist/container-lite-QD3CRLS4.js +327 -0
- package/dist/core-H2UUDATO.js +146 -0
- package/dist/crypto-D4LMI2RN.js +45 -0
- package/dist/db-BWC2GGBN.js +50 -0
- package/dist/demo-T655Z5S4.js +87 -0
- package/dist/diagnostics-6RQTBR6I.js +113 -0
- package/dist/dist-OPDCWARF.js +727 -0
- package/dist/dist-VXJEKX3T.js +2441 -0
- package/dist/dist-VYGJXGUS.js +1008 -0
- package/dist/embeddings-7QXTXUMC.js +15 -0
- package/dist/embeddings-MAEWWUHW.js +9 -0
- package/dist/graph-RKMNE2X5.js +36 -0
- package/dist/hvm-CBEP3M4F.js +126 -0
- package/dist/index.cjs +49874 -8001
- package/dist/index.d.cts +1629 -1363
- package/dist/index.d.ts +1629 -1363
- package/dist/index.js +2462 -8130
- package/dist/mcp-NK34ZNM5.js +101 -0
- package/dist/mcp-client-service-browser-SGB2K3VZ.js +14 -0
- package/dist/neuro-state-XHRGIRVO.js +498 -0
- package/dist/nodes-K6GKI2FM.js +364 -0
- package/dist/package-EXUIU2RL.js +93 -0
- package/dist/package-VGL7HYTO.js +106 -0
- package/dist/package-XHMLOAQ4.js +98 -0
- package/dist/pg-events-QJAM2HIP.js +15 -0
- package/dist/pglite-adapter-43IOUBMV.js +50 -0
- package/dist/pgliteService-IUGNNOVU.js +258 -0
- package/dist/policy-IRJCM6FS.js +13 -0
- package/dist/registry-5WTDYQVQ.js +26 -0
- package/dist/registry-FW63E7FE.js +16 -0
- package/dist/registry-ZQ2IBLF6.js +9 -0
- package/dist/resolver-ALOJSOK5.js +24 -0
- package/dist/scheduler-B5CEYKWT.js +127 -0
- package/dist/secret-store-H7273UIT.js +18 -0
- package/dist/server-VW6DYDLH.js +18 -0
- package/dist/skills-VN7IN7SJ.js +6375 -0
- package/dist/stack-4KWCQQP7.js +103 -0
- package/dist/storage-L7MWNSPG.js +13 -0
- package/dist/supabaseService-6AYP2VY3.js +476 -0
- package/dist/topology-CIWWNVAN.js +13 -0
- package/dist/webcontainer-XWCE56F3.js +281 -0
- package/package.json +9 -3
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
// ../../lib/terminals-tech/machines/core/adapters/embeddings.ts
|
|
2
|
+
async function emb() {
|
|
3
|
+
return await import("./dist-OPDCWARF.js");
|
|
4
|
+
}
|
|
5
|
+
async function encodePoints(points, opts = {}) {
|
|
6
|
+
try {
|
|
7
|
+
const e = await emb();
|
|
8
|
+
if (e?.encode) {
|
|
9
|
+
const inputs = points.map((p) => [p.x, p.y, p.z, p.t || 0]);
|
|
10
|
+
const out = await e.encode(inputs, { model: opts.model || "default" });
|
|
11
|
+
if (out && typeof out === "object" && "vectors" in out) {
|
|
12
|
+
return out.vectors || out;
|
|
13
|
+
}
|
|
14
|
+
return out;
|
|
15
|
+
}
|
|
16
|
+
} catch {
|
|
17
|
+
}
|
|
18
|
+
const vectors = points.map((p) => {
|
|
19
|
+
const h = Math.sin(p.x * 0.1) + Math.cos(p.y * 0.13) + Math.sin(p.z * 0.07);
|
|
20
|
+
const base = [h, h * 0.5, -h];
|
|
21
|
+
return base;
|
|
22
|
+
});
|
|
23
|
+
return vectors;
|
|
24
|
+
}
|
|
25
|
+
async function encodeTextBatch(texts, opts = {}) {
|
|
26
|
+
if (typeof window !== "undefined") {
|
|
27
|
+
try {
|
|
28
|
+
const { embedText } = await import("./embeddings-7QXTXUMC.js");
|
|
29
|
+
return await Promise.all(texts.map((t) => embedText(t, opts.model)));
|
|
30
|
+
} catch (e) {
|
|
31
|
+
console.warn(
|
|
32
|
+
"Browser-side embedding failed. This may happen if modules are not found or WebGPU is unavailable. Fallback will be used.",
|
|
33
|
+
e
|
|
34
|
+
);
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
try {
|
|
38
|
+
const e = await emb();
|
|
39
|
+
if (e?.encodeText) {
|
|
40
|
+
const out = await e.encodeText(texts, { model: opts.model || "default" });
|
|
41
|
+
if (out && typeof out === "object" && "vectors" in out) {
|
|
42
|
+
const vectors = out.vectors;
|
|
43
|
+
return vectors ?? [];
|
|
44
|
+
}
|
|
45
|
+
return Array.isArray(out) ? out : [];
|
|
46
|
+
}
|
|
47
|
+
} catch (e) {
|
|
48
|
+
console.warn(
|
|
49
|
+
"@terminals-tech/embeddings failed. This is expected in a browser-only environment. Fallback will be used.",
|
|
50
|
+
e
|
|
51
|
+
);
|
|
52
|
+
}
|
|
53
|
+
console.log("Using fallback hashing for text embeddings.");
|
|
54
|
+
const toVec = (s) => {
|
|
55
|
+
let h1 = 0, h2 = 0, h3 = 0;
|
|
56
|
+
for (let i = 0; i < s.length; i++) {
|
|
57
|
+
const c = s.charCodeAt(i);
|
|
58
|
+
h1 = (h1 * 31 + c) % 997;
|
|
59
|
+
h2 = (h2 * 29 + c) % 991;
|
|
60
|
+
h3 = (h3 * 23 + c) % 983;
|
|
61
|
+
}
|
|
62
|
+
return [h1 / 997, h2 / 991, h3 / 983];
|
|
63
|
+
};
|
|
64
|
+
return texts.map(toVec);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export {
|
|
68
|
+
encodePoints,
|
|
69
|
+
encodeTextBatch
|
|
70
|
+
};
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
import {
|
|
2
|
+
robustJsonParse
|
|
3
|
+
} from "./chunk-WGBCRNMB.js";
|
|
4
|
+
import {
|
|
5
|
+
getDB
|
|
6
|
+
} from "./chunk-AFDUOYHD.js";
|
|
7
|
+
|
|
8
|
+
// ../../lib/terminals-tech/mesh/trace-path.ts
|
|
9
|
+
var LTREE_PATTERN = /^[a-zA-Z_][a-zA-Z0-9_]*(\.[a-zA-Z_][a-zA-Z0-9_]*)*$/;
|
|
10
|
+
function isValidTracePath(path) {
|
|
11
|
+
return LTREE_PATTERN.test(path) && path.length <= 256;
|
|
12
|
+
}
|
|
13
|
+
function assertTracePath(path) {
|
|
14
|
+
if (!isValidTracePath(path)) {
|
|
15
|
+
throw new Error(`Invalid trace_path: "${path}"`);
|
|
16
|
+
}
|
|
17
|
+
return path;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
// ../../lib/terminals-tech/mesh/pg-events.ts
|
|
21
|
+
async function appendMeshEvent(tracePath, payload, embeddingOrOpts, span, metadata) {
|
|
22
|
+
assertTracePath(tracePath);
|
|
23
|
+
let embedding;
|
|
24
|
+
let interactionId;
|
|
25
|
+
let runId;
|
|
26
|
+
let stackId;
|
|
27
|
+
let userId;
|
|
28
|
+
if (Array.isArray(embeddingOrOpts) || embeddingOrOpts === void 0) {
|
|
29
|
+
embedding = embeddingOrOpts;
|
|
30
|
+
} else {
|
|
31
|
+
embedding = embeddingOrOpts.embedding;
|
|
32
|
+
span = embeddingOrOpts.span;
|
|
33
|
+
metadata = embeddingOrOpts.metadata;
|
|
34
|
+
interactionId = embeddingOrOpts.interactionId;
|
|
35
|
+
runId = embeddingOrOpts.runId;
|
|
36
|
+
stackId = embeddingOrOpts.stackId;
|
|
37
|
+
userId = embeddingOrOpts.userId;
|
|
38
|
+
}
|
|
39
|
+
const db = await getDB();
|
|
40
|
+
const prevResult = await db.query(
|
|
41
|
+
`SELECT id, md5(id::text || COALESCE(prev_hash, '') || payload::text) as hash FROM mesh_events ORDER BY created_at DESC LIMIT 1`
|
|
42
|
+
);
|
|
43
|
+
const prevHash = prevResult.rows[0]?.hash || "00000000000000000000000000000000";
|
|
44
|
+
const type = tracePath.split(".").pop() || "event";
|
|
45
|
+
const spanVal = span ? JSON.stringify(span) : null;
|
|
46
|
+
const result = await db.query(
|
|
47
|
+
`INSERT INTO mesh_events (
|
|
48
|
+
prev_hash,
|
|
49
|
+
trace_path,
|
|
50
|
+
type,
|
|
51
|
+
payload,
|
|
52
|
+
embedding,
|
|
53
|
+
span,
|
|
54
|
+
metadata,
|
|
55
|
+
interaction_id,
|
|
56
|
+
run_id,
|
|
57
|
+
stack_id,
|
|
58
|
+
user_id
|
|
59
|
+
) VALUES ($1, $2::ltree, $3, $4, $5, $6::jsonb, $7, $8, $9, $10, $11)
|
|
60
|
+
RETURNING id, created_at;`,
|
|
61
|
+
[
|
|
62
|
+
prevHash,
|
|
63
|
+
tracePath,
|
|
64
|
+
type,
|
|
65
|
+
JSON.stringify(payload),
|
|
66
|
+
embedding ? `[${embedding.join(",")}]` : null,
|
|
67
|
+
spanVal,
|
|
68
|
+
metadata ? JSON.stringify(metadata) : null,
|
|
69
|
+
interactionId || null,
|
|
70
|
+
runId || null,
|
|
71
|
+
stackId || null,
|
|
72
|
+
userId || null
|
|
73
|
+
]
|
|
74
|
+
);
|
|
75
|
+
const row = result.rows[0];
|
|
76
|
+
if (!row) {
|
|
77
|
+
throw new Error("Failed to insert mesh event");
|
|
78
|
+
}
|
|
79
|
+
return {
|
|
80
|
+
id: row.id,
|
|
81
|
+
type,
|
|
82
|
+
payload,
|
|
83
|
+
timestamp: new Date(row.created_at).getTime(),
|
|
84
|
+
embedding,
|
|
85
|
+
metadata,
|
|
86
|
+
prevHash,
|
|
87
|
+
tracePath,
|
|
88
|
+
span,
|
|
89
|
+
interactionId,
|
|
90
|
+
runId,
|
|
91
|
+
stackId,
|
|
92
|
+
userId
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
async function queryMeshEvents(pathQuery, limit = 100) {
|
|
96
|
+
const db = await getDB();
|
|
97
|
+
const result = await db.query(
|
|
98
|
+
`SELECT * FROM mesh_events
|
|
99
|
+
WHERE trace_path ~ $1::lquery
|
|
100
|
+
ORDER BY created_at DESC
|
|
101
|
+
LIMIT $2`,
|
|
102
|
+
[pathQuery, limit]
|
|
103
|
+
);
|
|
104
|
+
return result.rows.map((row) => {
|
|
105
|
+
const eventRow = row;
|
|
106
|
+
return {
|
|
107
|
+
id: eventRow.id,
|
|
108
|
+
type: eventRow.type || eventRow.trace_path.split(".").pop() || "event",
|
|
109
|
+
payload: robustJsonParse(eventRow.payload),
|
|
110
|
+
timestamp: new Date(eventRow.created_at).getTime(),
|
|
111
|
+
embedding: eventRow.embedding ? JSON.parse(eventRow.embedding) : void 0,
|
|
112
|
+
metadata: eventRow.metadata ? robustJsonParse(eventRow.metadata) ?? void 0 : void 0,
|
|
113
|
+
prevHash: eventRow.prev_hash ?? void 0,
|
|
114
|
+
tracePath: eventRow.trace_path,
|
|
115
|
+
span: eventRow.span ? parseSpanJsonb(eventRow.span) : void 0,
|
|
116
|
+
interactionId: eventRow.interaction_id ?? void 0,
|
|
117
|
+
runId: eventRow.run_id ?? void 0,
|
|
118
|
+
stackId: eventRow.stack_id ?? void 0,
|
|
119
|
+
userId: eventRow.user_id ?? void 0
|
|
120
|
+
};
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
async function calculateTokenTrajectory(pathQuery, limit = 10) {
|
|
124
|
+
const db = await getDB();
|
|
125
|
+
const result = await db.query(
|
|
126
|
+
`SELECT embedding FROM mesh_events
|
|
127
|
+
WHERE trace_path ~ $1::lquery AND embedding IS NOT NULL
|
|
128
|
+
ORDER BY created_at DESC
|
|
129
|
+
LIMIT $2`,
|
|
130
|
+
[pathQuery, limit]
|
|
131
|
+
);
|
|
132
|
+
const vectors = result.rows.map((r) => JSON.parse(r.embedding)).reverse();
|
|
133
|
+
const distances = [];
|
|
134
|
+
for (let i = 1; i < vectors.length; i++) {
|
|
135
|
+
distances.push(cosineDistance(vectors[i - 1], vectors[i]));
|
|
136
|
+
}
|
|
137
|
+
return distances;
|
|
138
|
+
}
|
|
139
|
+
function parseSpanJsonb(spanVal) {
|
|
140
|
+
try {
|
|
141
|
+
if (typeof spanVal === "object" && Array.isArray(spanVal)) {
|
|
142
|
+
return [Number(spanVal[0]), Number(spanVal[1])];
|
|
143
|
+
}
|
|
144
|
+
const parsed = typeof spanVal === "string" ? JSON.parse(spanVal) : spanVal;
|
|
145
|
+
if (Array.isArray(parsed)) {
|
|
146
|
+
return [Number(parsed[0]), Number(parsed[1])];
|
|
147
|
+
}
|
|
148
|
+
if (typeof parsed === "string") {
|
|
149
|
+
const parts = parsed.replace(/[()]/g, "").split(",");
|
|
150
|
+
return [parseFloat(parts[0]), parseFloat(parts[1])];
|
|
151
|
+
}
|
|
152
|
+
return void 0;
|
|
153
|
+
} catch {
|
|
154
|
+
return void 0;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
function cosineDistance(a, b) {
|
|
158
|
+
if (a.length !== b.length) return 0;
|
|
159
|
+
let dot = 0, magA = 0, magB = 0;
|
|
160
|
+
for (let i = 0; i < a.length; i++) {
|
|
161
|
+
dot += a[i] * b[i];
|
|
162
|
+
magA += a[i] * a[i];
|
|
163
|
+
magB += b[i] * b[i];
|
|
164
|
+
}
|
|
165
|
+
return 1 - dot / (Math.sqrt(magA) * Math.sqrt(magB));
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
export {
|
|
169
|
+
appendMeshEvent,
|
|
170
|
+
queryMeshEvents,
|
|
171
|
+
calculateTokenTrajectory
|
|
172
|
+
};
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import {
|
|
2
|
+
isNetworkAllowed,
|
|
3
|
+
requireNetworkApproval
|
|
4
|
+
} from "./chunk-UJDUQNE2.js";
|
|
5
|
+
|
|
6
|
+
// ../../lib/terminals-tech/machines/core/runners/container-lite-shared.ts
|
|
7
|
+
import path from "path";
|
|
8
|
+
var DEFAULT_CONTAINER_LITE_TIMEOUT_MS = 12e4;
|
|
9
|
+
var DEFAULT_CONTAINER_LITE_IMAGE = "python:3.11-slim";
|
|
10
|
+
function shellQuote(value) {
|
|
11
|
+
return `'${value.replace(/'/g, `'"'"'`)}'`;
|
|
12
|
+
}
|
|
13
|
+
function asStringArray(value) {
|
|
14
|
+
if (!Array.isArray(value)) return [];
|
|
15
|
+
return value.filter((entry) => typeof entry === "string");
|
|
16
|
+
}
|
|
17
|
+
function isRecord(value) {
|
|
18
|
+
return typeof value === "object" && value !== null;
|
|
19
|
+
}
|
|
20
|
+
function extractUrls(input) {
|
|
21
|
+
const urls = [];
|
|
22
|
+
const extract = (value) => {
|
|
23
|
+
if (typeof value === "string") {
|
|
24
|
+
try {
|
|
25
|
+
const url = new URL(value);
|
|
26
|
+
if (url.protocol === "http:" || url.protocol === "https:") {
|
|
27
|
+
urls.push(value);
|
|
28
|
+
}
|
|
29
|
+
} catch {
|
|
30
|
+
return;
|
|
31
|
+
}
|
|
32
|
+
return;
|
|
33
|
+
}
|
|
34
|
+
if (Array.isArray(value)) {
|
|
35
|
+
value.forEach(extract);
|
|
36
|
+
return;
|
|
37
|
+
}
|
|
38
|
+
if (isRecord(value)) {
|
|
39
|
+
Object.values(value).forEach(extract);
|
|
40
|
+
}
|
|
41
|
+
};
|
|
42
|
+
extract(input);
|
|
43
|
+
return urls;
|
|
44
|
+
}
|
|
45
|
+
async function validateContainerLiteNetworkPolicy(urls, ctx) {
|
|
46
|
+
for (const url of urls) {
|
|
47
|
+
const allowed = isNetworkAllowed(ctx.policies, url) || await requireNetworkApproval(ctx.stackId, ctx.nodeId, ctx.policies, url);
|
|
48
|
+
if (!allowed) {
|
|
49
|
+
return { allowed: false, deniedUrl: url };
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
return { allowed: true };
|
|
53
|
+
}
|
|
54
|
+
function flattenFilesystemTree(tree, prefix = "") {
|
|
55
|
+
if (typeof tree === "string") {
|
|
56
|
+
return prefix ? [{ path: prefix, content: tree }] : [];
|
|
57
|
+
}
|
|
58
|
+
if (!isRecord(tree)) {
|
|
59
|
+
return [];
|
|
60
|
+
}
|
|
61
|
+
if ("file" in tree && isRecord(tree.file) && typeof tree.file.contents === "string" && prefix) {
|
|
62
|
+
return [{ path: prefix, content: tree.file.contents }];
|
|
63
|
+
}
|
|
64
|
+
const writes = [];
|
|
65
|
+
for (const [name, value] of Object.entries(tree)) {
|
|
66
|
+
const nextPath = prefix ? path.posix.join(prefix, name) : name;
|
|
67
|
+
writes.push(...flattenFilesystemTree(value, nextPath));
|
|
68
|
+
}
|
|
69
|
+
return writes;
|
|
70
|
+
}
|
|
71
|
+
function buildContainerLiteCommand(command, args, env) {
|
|
72
|
+
const envPrefix = Object.entries(env).map(([key, value]) => `${key}=${shellQuote(value)}`).join(" ");
|
|
73
|
+
const commandWithArgs = [shellQuote(command), ...args.map((arg) => shellQuote(arg))].join(" ");
|
|
74
|
+
return envPrefix ? `${envPrefix} ${commandWithArgs}` : commandWithArgs;
|
|
75
|
+
}
|
|
76
|
+
function resolveContainerLiteCwd(workdir, requestedCwd) {
|
|
77
|
+
if (!requestedCwd) {
|
|
78
|
+
return workdir;
|
|
79
|
+
}
|
|
80
|
+
return requestedCwd.startsWith("/") ? requestedCwd : path.posix.join(workdir, requestedCwd);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export {
|
|
84
|
+
DEFAULT_CONTAINER_LITE_TIMEOUT_MS,
|
|
85
|
+
DEFAULT_CONTAINER_LITE_IMAGE,
|
|
86
|
+
shellQuote,
|
|
87
|
+
asStringArray,
|
|
88
|
+
isRecord,
|
|
89
|
+
extractUrls,
|
|
90
|
+
validateContainerLiteNetworkPolicy,
|
|
91
|
+
flattenFilesystemTree,
|
|
92
|
+
buildContainerLiteCommand,
|
|
93
|
+
resolveContainerLiteCwd
|
|
94
|
+
};
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
// ../../lib/terminals-tech/core/ai/embeddings.ts
|
|
2
|
+
var DEFAULT_EMBEDDING_MODEL = "Xenova/all-MiniLM-L6-v2";
|
|
3
|
+
var DEFAULT_EMBEDDING_DIM = 384;
|
|
4
|
+
var embedderPromise = null;
|
|
5
|
+
var workerHandle = null;
|
|
6
|
+
var workerInitPromise = null;
|
|
7
|
+
var reqCounter = 0;
|
|
8
|
+
async function getEmbeddingsWorker() {
|
|
9
|
+
try {
|
|
10
|
+
if (typeof window === "undefined") return null;
|
|
11
|
+
if (workerHandle) return workerHandle;
|
|
12
|
+
const w = new Worker("/workers/embeddings-worker.js", {
|
|
13
|
+
type: "classic",
|
|
14
|
+
name: "embeddings-worker"
|
|
15
|
+
});
|
|
16
|
+
workerHandle = w;
|
|
17
|
+
return w;
|
|
18
|
+
} catch {
|
|
19
|
+
return null;
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
async function getEmbedder(modelName = DEFAULT_EMBEDDING_MODEL) {
|
|
23
|
+
if (typeof window === "undefined") {
|
|
24
|
+
throw new Error("embedText is only available in the browser environment");
|
|
25
|
+
}
|
|
26
|
+
const w = await getEmbeddingsWorker();
|
|
27
|
+
if (w && !workerInitPromise) {
|
|
28
|
+
workerInitPromise = (async () => {
|
|
29
|
+
try {
|
|
30
|
+
const { getDefaultOnnxExecutionProviders } = await import("./capabilities-MIPUMBLL.js");
|
|
31
|
+
const providers = await getDefaultOnnxExecutionProviders();
|
|
32
|
+
await new Promise((resolve, reject) => {
|
|
33
|
+
const onMsg = (ev) => {
|
|
34
|
+
if (ev.data && ev.data.type === "inited") {
|
|
35
|
+
w.removeEventListener("message", onMsg);
|
|
36
|
+
if (ev.data.ok) resolve();
|
|
37
|
+
else reject(new Error(ev.data.error || "worker init failed"));
|
|
38
|
+
}
|
|
39
|
+
};
|
|
40
|
+
w.addEventListener("message", onMsg);
|
|
41
|
+
w.postMessage({
|
|
42
|
+
type: "init",
|
|
43
|
+
model: modelName,
|
|
44
|
+
execProviders: providers.length ? providers : ["wasm"]
|
|
45
|
+
});
|
|
46
|
+
});
|
|
47
|
+
return true;
|
|
48
|
+
} catch {
|
|
49
|
+
return false;
|
|
50
|
+
}
|
|
51
|
+
})();
|
|
52
|
+
}
|
|
53
|
+
if (w && workerInitPromise) {
|
|
54
|
+
const ok = await workerInitPromise;
|
|
55
|
+
if (ok) return null;
|
|
56
|
+
}
|
|
57
|
+
if (!embedderPromise) {
|
|
58
|
+
const { getDefaultOnnxExecutionProviders } = await import("./capabilities-MIPUMBLL.js");
|
|
59
|
+
const { pipeline, env } = await import("@xenova/transformers");
|
|
60
|
+
env.allowRemoteModels = true;
|
|
61
|
+
env.allowLocalModels = false;
|
|
62
|
+
const providers = await getDefaultOnnxExecutionProviders();
|
|
63
|
+
env.backends.onnx.executionProviders = providers.length ? providers : ["wasm"];
|
|
64
|
+
try {
|
|
65
|
+
embedderPromise = pipeline("feature-extraction", modelName, {
|
|
66
|
+
quantized: true
|
|
67
|
+
});
|
|
68
|
+
} catch (e) {
|
|
69
|
+
try {
|
|
70
|
+
env.backends.onnx.executionProviders = ["wasm"];
|
|
71
|
+
embedderPromise = pipeline("feature-extraction", modelName, {
|
|
72
|
+
quantized: true
|
|
73
|
+
});
|
|
74
|
+
} catch (e2) {
|
|
75
|
+
throw e2;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
return embedderPromise;
|
|
80
|
+
}
|
|
81
|
+
async function embedText(text, modelName) {
|
|
82
|
+
const extractor = await getEmbedder(modelName);
|
|
83
|
+
if (!extractor) {
|
|
84
|
+
const w = await getEmbeddingsWorker();
|
|
85
|
+
if (!w) throw new Error("Embeddings worker unavailable");
|
|
86
|
+
const id = ++reqCounter;
|
|
87
|
+
return await new Promise((resolve, reject) => {
|
|
88
|
+
const onMsg = (ev) => {
|
|
89
|
+
const d = ev.data;
|
|
90
|
+
if (!d || d.id !== id) return;
|
|
91
|
+
w.removeEventListener("message", onMsg);
|
|
92
|
+
if (d.ok) {
|
|
93
|
+
const arr = d.data instanceof Float32Array ? Array.from(d.data) : Array.isArray(d.data) ? d.data : [];
|
|
94
|
+
resolve(arr);
|
|
95
|
+
} else {
|
|
96
|
+
reject(new Error(d.error || "embed failed"));
|
|
97
|
+
}
|
|
98
|
+
};
|
|
99
|
+
w.addEventListener("message", onMsg);
|
|
100
|
+
w.postMessage({ type: "embed", id, text });
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
const output = await extractor(text, { pooling: "mean", normalize: true });
|
|
104
|
+
const data = output && output.data ? output.data : output;
|
|
105
|
+
return Array.from(data);
|
|
106
|
+
}
|
|
107
|
+
async function prewarmEmbeddingsWorker(modelName = DEFAULT_EMBEDDING_MODEL, mode = "worker-only") {
|
|
108
|
+
if (typeof window === "undefined") return false;
|
|
109
|
+
const w = await getEmbeddingsWorker();
|
|
110
|
+
if (!w) return false;
|
|
111
|
+
if (mode === "worker-only") return true;
|
|
112
|
+
if (!workerInitPromise) {
|
|
113
|
+
workerInitPromise = (async () => {
|
|
114
|
+
try {
|
|
115
|
+
const { getDefaultOnnxExecutionProviders } = await import("./capabilities-MIPUMBLL.js");
|
|
116
|
+
const providers = await getDefaultOnnxExecutionProviders();
|
|
117
|
+
await new Promise((resolve, reject) => {
|
|
118
|
+
const onMsg = (ev) => {
|
|
119
|
+
if (ev.data && ev.data.type === "inited") {
|
|
120
|
+
w.removeEventListener("message", onMsg);
|
|
121
|
+
if (ev.data.ok) resolve();
|
|
122
|
+
else reject(new Error(ev.data.error || "worker init failed"));
|
|
123
|
+
}
|
|
124
|
+
};
|
|
125
|
+
w.addEventListener("message", onMsg);
|
|
126
|
+
w.postMessage({
|
|
127
|
+
type: "init",
|
|
128
|
+
model: modelName,
|
|
129
|
+
execProviders: providers.length ? providers : ["wasm"]
|
|
130
|
+
});
|
|
131
|
+
});
|
|
132
|
+
return true;
|
|
133
|
+
} catch {
|
|
134
|
+
return false;
|
|
135
|
+
}
|
|
136
|
+
})();
|
|
137
|
+
}
|
|
138
|
+
return workerInitPromise.then(Boolean).catch(() => false);
|
|
139
|
+
}
|
|
140
|
+
function getEmbeddingDimension() {
|
|
141
|
+
return DEFAULT_EMBEDDING_DIM;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
export {
|
|
145
|
+
DEFAULT_EMBEDDING_MODEL,
|
|
146
|
+
DEFAULT_EMBEDDING_DIM,
|
|
147
|
+
embedText,
|
|
148
|
+
prewarmEmbeddingsWorker,
|
|
149
|
+
getEmbeddingDimension
|
|
150
|
+
};
|