@modootoday/pmcp 0.9.2 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +167 -2
- package/dist/attachment-tU5FZrgZ.d.ts +17 -0
- package/dist/chunk-HNAVQG6R.js +1198 -0
- package/dist/chunk-JRSR4JBR.js +88 -0
- package/dist/chunk-MSTKY33C.js +147 -0
- package/dist/{chunk-A5NOHVHN.js → chunk-VRZFM2M5.js} +18 -49
- package/dist/chunk-XEH4ZGGJ.js +336 -0
- package/dist/chunk-XOH2CW6P.js +297 -0
- package/dist/{chunk-D3OZVMHN.js → chunk-ZUFFVIHQ.js} +166 -117
- package/dist/cli.js +507 -74
- package/dist/config-CJ43_oBS.d.ts +49 -0
- package/dist/embed-BBhv3y5x.d.ts +53 -0
- package/dist/encoder-DGRy9skC.d.ts +36 -0
- package/dist/http.d.ts +7 -38
- package/dist/http.js +18 -7
- package/dist/index.d.ts +25 -44
- package/dist/index.js +31 -11
- package/dist/mailbox/index.d.ts +44 -0
- package/dist/mailbox/index.js +387 -0
- package/dist/{server-C9eO6LMI.d.ts → server-CwWcRjqz.d.ts} +122 -2
- package/dist/service-qpCsOXkQ.d.ts +116 -0
- package/dist/stdio.d.ts +10 -4
- package/dist/stdio.js +9 -4
- package/package.json +5 -1
- package/dist/catalog-BU5VLLyd.d.ts +0 -121
- package/dist/chunk-CMZ7OTQD.js +0 -134
- package/dist/chunk-EIHMCKY5.js +0 -41
- package/dist/chunk-KYW3VP3L.js +0 -160
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import {
|
|
2
|
+
createSkillServer
|
|
3
|
+
} from "./chunk-VRZFM2M5.js";
|
|
4
|
+
import {
|
|
5
|
+
attachMailbox
|
|
6
|
+
} from "./chunk-HNAVQG6R.js";
|
|
7
|
+
|
|
8
|
+
// src/stdio.ts
|
|
9
|
+
import { serveStdio } from "@modelcontextprotocol/server/stdio";
|
|
10
|
+
function resolveRoots(argv, cwd) {
|
|
11
|
+
const given = argv.filter((arg) => !arg.startsWith("-"));
|
|
12
|
+
return given.length > 0 ? given : [`${cwd}/node_modules`];
|
|
13
|
+
}
|
|
14
|
+
function serve(catalog, serving = {}) {
|
|
15
|
+
const options = {
|
|
16
|
+
...catalog,
|
|
17
|
+
...serving,
|
|
18
|
+
onReject: ({ path, reason }) => process.stderr.write(`pmcp: skipped ${path}: ${reason}
|
|
19
|
+
`),
|
|
20
|
+
onWarning: ({ path, reason }) => process.stderr.write(`pmcp: warning ${path}: ${reason}
|
|
21
|
+
`)
|
|
22
|
+
};
|
|
23
|
+
return serveStdio(() => createSkillServer(options), {
|
|
24
|
+
// stdout is the protocol. A diagnostic written there corrupts the stream,
|
|
25
|
+
// so out-of-band errors go to stderr and nowhere else.
|
|
26
|
+
onerror: (error) => process.stderr.write(`pmcp: ${error.message}
|
|
27
|
+
`)
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
async function serveWithMailbox(catalog, mailbox, serving = {}) {
|
|
31
|
+
let stop;
|
|
32
|
+
const attachment = await attachMailbox({
|
|
33
|
+
...mailbox,
|
|
34
|
+
onExpired: () => {
|
|
35
|
+
process.stderr.write("pmcp: mailbox attachment expired or superseded\n");
|
|
36
|
+
void stop?.().catch(() => {
|
|
37
|
+
process.exitCode = 1;
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
});
|
|
41
|
+
let handle;
|
|
42
|
+
try {
|
|
43
|
+
handle = serve(catalog, { ...serving, mailbox: attachment.service });
|
|
44
|
+
} catch (error) {
|
|
45
|
+
await attachment.close();
|
|
46
|
+
throw error;
|
|
47
|
+
}
|
|
48
|
+
let shutdown;
|
|
49
|
+
const onExit = () => {
|
|
50
|
+
void stop().catch(() => {
|
|
51
|
+
process.exitCode = 1;
|
|
52
|
+
});
|
|
53
|
+
};
|
|
54
|
+
stop = () => {
|
|
55
|
+
if (shutdown) return shutdown;
|
|
56
|
+
process.removeListener("SIGINT", onExit);
|
|
57
|
+
process.removeListener("SIGTERM", onExit);
|
|
58
|
+
process.stdin.removeListener("end", onExit);
|
|
59
|
+
shutdown = (async () => {
|
|
60
|
+
try {
|
|
61
|
+
await handle.close();
|
|
62
|
+
} finally {
|
|
63
|
+
await attachment.close();
|
|
64
|
+
}
|
|
65
|
+
})();
|
|
66
|
+
return shutdown;
|
|
67
|
+
};
|
|
68
|
+
process.once("SIGINT", onExit);
|
|
69
|
+
process.once("SIGTERM", onExit);
|
|
70
|
+
process.stdin.once("end", onExit);
|
|
71
|
+
return stop;
|
|
72
|
+
}
|
|
73
|
+
function main(argv = process.argv.slice(2)) {
|
|
74
|
+
const scopeArg = argv.find((arg) => arg.startsWith("--scope="));
|
|
75
|
+
const scopes = scopeArg ? scopeArg.slice("--scope=".length).split(",") : [];
|
|
76
|
+
serve({
|
|
77
|
+
roots: resolveRoots(argv, process.cwd()),
|
|
78
|
+
...scopes.length > 0 ? { scopes } : {}
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export {
|
|
83
|
+
resolveRoots,
|
|
84
|
+
serve,
|
|
85
|
+
serveWithMailbox,
|
|
86
|
+
main
|
|
87
|
+
};
|
|
88
|
+
//# sourceMappingURL=chunk-JRSR4JBR.js.map
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
import {
|
|
2
|
+
resolveEncoder
|
|
3
|
+
} from "./chunk-XOH2CW6P.js";
|
|
4
|
+
|
|
5
|
+
// src/config.ts
|
|
6
|
+
import { existsSync, readFileSync } from "fs";
|
|
7
|
+
import { dirname, join, resolve } from "path";
|
|
8
|
+
import { parse } from "smol-toml";
|
|
9
|
+
var CONFIG_FILE = "pmcp.toml";
|
|
10
|
+
function embeddingSection(path, dir, value) {
|
|
11
|
+
if (value === void 0) return void 0;
|
|
12
|
+
if (value === null || typeof value !== "object" || Array.isArray(value))
|
|
13
|
+
throw new ConfigError(path, "[embedding] must be a table");
|
|
14
|
+
const section = value;
|
|
15
|
+
const allowed = /* @__PURE__ */ new Set([
|
|
16
|
+
"enabled",
|
|
17
|
+
"model",
|
|
18
|
+
"dtype",
|
|
19
|
+
"dimensions",
|
|
20
|
+
"revision",
|
|
21
|
+
"model_path",
|
|
22
|
+
"cache_dir",
|
|
23
|
+
"encoder"
|
|
24
|
+
]);
|
|
25
|
+
for (const key of Object.keys(section)) {
|
|
26
|
+
if (!allowed.has(key))
|
|
27
|
+
throw new ConfigError(path, `unknown key [embedding] ${key}`);
|
|
28
|
+
}
|
|
29
|
+
if (section.enabled !== void 0 && typeof section.enabled !== "boolean")
|
|
30
|
+
throw new ConfigError(path, "[embedding] enabled must be a boolean");
|
|
31
|
+
const string = (key) => {
|
|
32
|
+
const item = section[key];
|
|
33
|
+
if (item === void 0) return void 0;
|
|
34
|
+
if (typeof item !== "string" || !item.trim())
|
|
35
|
+
throw new ConfigError(
|
|
36
|
+
path,
|
|
37
|
+
`[embedding] ${key} must be a non-empty string`
|
|
38
|
+
);
|
|
39
|
+
return item;
|
|
40
|
+
};
|
|
41
|
+
let encoder;
|
|
42
|
+
if (section.encoder !== void 0) {
|
|
43
|
+
if (section.encoder === null || typeof section.encoder !== "object" || Array.isArray(section.encoder))
|
|
44
|
+
throw new ConfigError(path, "[embedding.encoder] must be a table");
|
|
45
|
+
const source = section.encoder;
|
|
46
|
+
const keys = /* @__PURE__ */ new Set([
|
|
47
|
+
"query_prefix",
|
|
48
|
+
"passage_prefix",
|
|
49
|
+
"pooling",
|
|
50
|
+
"projected"
|
|
51
|
+
]);
|
|
52
|
+
for (const key of Object.keys(source)) {
|
|
53
|
+
if (!keys.has(key))
|
|
54
|
+
throw new ConfigError(path, `unknown key [embedding.encoder] ${key}`);
|
|
55
|
+
}
|
|
56
|
+
encoder = {};
|
|
57
|
+
if (source.query_prefix !== void 0)
|
|
58
|
+
encoder = { ...encoder, queryPrefix: source.query_prefix };
|
|
59
|
+
if (source.passage_prefix !== void 0)
|
|
60
|
+
encoder = { ...encoder, passagePrefix: source.passage_prefix };
|
|
61
|
+
if (source.pooling !== void 0)
|
|
62
|
+
encoder = { ...encoder, pooling: source.pooling };
|
|
63
|
+
if (source.projected !== void 0)
|
|
64
|
+
encoder = { ...encoder, projected: source.projected };
|
|
65
|
+
}
|
|
66
|
+
const modelPath = string("model_path");
|
|
67
|
+
const cacheDir = string("cache_dir");
|
|
68
|
+
const result = {
|
|
69
|
+
modelId: string("model"),
|
|
70
|
+
dtype: string("dtype"),
|
|
71
|
+
dimensions: section.dimensions,
|
|
72
|
+
revision: string("revision"),
|
|
73
|
+
encoder,
|
|
74
|
+
enabled: section.enabled,
|
|
75
|
+
modelPath: modelPath === void 0 ? void 0 : resolve(dir, modelPath),
|
|
76
|
+
cacheDir: cacheDir === void 0 ? void 0 : resolve(dir, cacheDir)
|
|
77
|
+
};
|
|
78
|
+
try {
|
|
79
|
+
resolveEncoder(result.modelId, result);
|
|
80
|
+
} catch (error) {
|
|
81
|
+
throw new ConfigError(
|
|
82
|
+
path,
|
|
83
|
+
error instanceof Error ? error.message : "Invalid embedding configuration"
|
|
84
|
+
);
|
|
85
|
+
}
|
|
86
|
+
return result;
|
|
87
|
+
}
|
|
88
|
+
var ConfigError = class extends Error {
|
|
89
|
+
constructor(path, message) {
|
|
90
|
+
super(`${path}: ${message}`);
|
|
91
|
+
this.name = "ConfigError";
|
|
92
|
+
}
|
|
93
|
+
};
|
|
94
|
+
var PATH_KEYS = ["roots", "workspaces", "packages", "marketplaces"];
|
|
95
|
+
var CATALOG_KEYS = /* @__PURE__ */ new Set([...PATH_KEYS, "scopes"]);
|
|
96
|
+
function findConfig(start) {
|
|
97
|
+
let dir = resolve(start);
|
|
98
|
+
for (; ; ) {
|
|
99
|
+
const candidate = join(dir, CONFIG_FILE);
|
|
100
|
+
if (existsSync(candidate)) return candidate;
|
|
101
|
+
const parent = dirname(dir);
|
|
102
|
+
if (parent === dir) return null;
|
|
103
|
+
dir = parent;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
function stringList(path, key, value) {
|
|
107
|
+
if (!Array.isArray(value) || !value.every((v) => typeof v === "string")) {
|
|
108
|
+
throw new ConfigError(path, `[catalog] ${key} must be a list of strings`);
|
|
109
|
+
}
|
|
110
|
+
return value;
|
|
111
|
+
}
|
|
112
|
+
function readConfig(path) {
|
|
113
|
+
const absolute = resolve(path);
|
|
114
|
+
const dir = dirname(absolute);
|
|
115
|
+
let raw;
|
|
116
|
+
try {
|
|
117
|
+
raw = parse(readFileSync(absolute, "utf8"));
|
|
118
|
+
} catch (error) {
|
|
119
|
+
const reason = error instanceof Error ? error.message : "unreadable";
|
|
120
|
+
throw new ConfigError(absolute, reason.split("\n")[0] ?? reason);
|
|
121
|
+
}
|
|
122
|
+
const section = raw["catalog"];
|
|
123
|
+
const embedding = embeddingSection(absolute, dir, raw.embedding);
|
|
124
|
+
const configured = embedding === void 0 ? {} : { embedding };
|
|
125
|
+
if (section === void 0)
|
|
126
|
+
return { path: absolute, dir, catalog: {}, raw, ...configured };
|
|
127
|
+
if (section === null || typeof section !== "object" || Array.isArray(section)) {
|
|
128
|
+
throw new ConfigError(absolute, "[catalog] must be a table");
|
|
129
|
+
}
|
|
130
|
+
const catalog = {};
|
|
131
|
+
for (const [key, value] of Object.entries(section)) {
|
|
132
|
+
if (!CATALOG_KEYS.has(key)) {
|
|
133
|
+
throw new ConfigError(absolute, `unknown key [catalog] ${key}`);
|
|
134
|
+
}
|
|
135
|
+
const list = stringList(absolute, key, value);
|
|
136
|
+
catalog[key] = key === "scopes" ? list : list.map((p) => resolve(dir, p));
|
|
137
|
+
}
|
|
138
|
+
return { path: absolute, dir, catalog, raw, ...configured };
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
export {
|
|
142
|
+
CONFIG_FILE,
|
|
143
|
+
ConfigError,
|
|
144
|
+
findConfig,
|
|
145
|
+
readConfig
|
|
146
|
+
};
|
|
147
|
+
//# sourceMappingURL=chunk-MSTKY33C.js.map
|
|
@@ -1,3 +1,10 @@
|
|
|
1
|
+
import {
|
|
2
|
+
RUNTIMES,
|
|
3
|
+
detectRuntime,
|
|
4
|
+
registerMailboxTools,
|
|
5
|
+
verifiedRuntimesOf
|
|
6
|
+
} from "./chunk-XOH2CW6P.js";
|
|
7
|
+
|
|
1
8
|
// src/intent.ts
|
|
2
9
|
function normaliseIntent(intent) {
|
|
3
10
|
return intent.toLowerCase().replaceAll(/[^\p{L}\p{N}\s]/gu, " ").split(/\s+/u).filter(Boolean).join(" ");
|
|
@@ -935,6 +942,7 @@ function registerSkillResources(server, skills) {
|
|
|
935
942
|
}
|
|
936
943
|
|
|
937
944
|
// src/find.ts
|
|
945
|
+
import { createHash as createHash2 } from "crypto";
|
|
938
946
|
var DEFAULT_FIND_LIMIT = 5;
|
|
939
947
|
function tokens(text) {
|
|
940
948
|
return normaliseIntent(text).split(" ").filter(Boolean);
|
|
@@ -977,15 +985,17 @@ async function find(options) {
|
|
|
977
985
|
if (semantic) {
|
|
978
986
|
const embedder = options.embedder;
|
|
979
987
|
const vectors = options.vectors;
|
|
980
|
-
|
|
988
|
+
const namespace = createHash2("sha256").update(embedder.indexKey ?? embedder.modelId).digest("hex");
|
|
989
|
+
const cacheKey = `${namespace} ${normaliseIntent(options.intent)}`;
|
|
990
|
+
let query = options.cache?.get(cacheKey) ?? null;
|
|
981
991
|
if (!query) {
|
|
982
992
|
query = await embedder.embed(options.intent);
|
|
983
|
-
options.cache?.put(
|
|
993
|
+
options.cache?.put(cacheKey, query);
|
|
984
994
|
}
|
|
985
995
|
const scored = [];
|
|
986
996
|
for (const entry of options.entries) {
|
|
987
997
|
const vector = vectors.get(entry.name);
|
|
988
|
-
if (!vector) continue;
|
|
998
|
+
if (!vector || vector.length !== query.length) continue;
|
|
989
999
|
scored.push({
|
|
990
1000
|
name: entry.name,
|
|
991
1001
|
package: entry.package,
|
|
@@ -1035,44 +1045,6 @@ function bundleSkill(entry) {
|
|
|
1035
1045
|
};
|
|
1036
1046
|
}
|
|
1037
1047
|
|
|
1038
|
-
// src/runtime.ts
|
|
1039
|
-
import { CLIENT_INFO_META_KEY } from "@modelcontextprotocol/server";
|
|
1040
|
-
var RUNTIMES = [
|
|
1041
|
-
"claude-code",
|
|
1042
|
-
"codex-cli",
|
|
1043
|
-
"gemini-cli",
|
|
1044
|
-
"grok-cli",
|
|
1045
|
-
"antigravity"
|
|
1046
|
-
];
|
|
1047
|
-
var VERIFIED_RUNTIMES_KEY = "verified-runtimes";
|
|
1048
|
-
var isRuntime = (value) => RUNTIMES.includes(value);
|
|
1049
|
-
function verifiedRuntimesOf(metadata) {
|
|
1050
|
-
const raw = metadata?.[VERIFIED_RUNTIMES_KEY];
|
|
1051
|
-
if (raw === void 0) return [];
|
|
1052
|
-
const words = typeof raw === "string" ? raw.split(/[\s,]+/u) : Array.isArray(raw) ? raw.map((word) => String(word).trim()) : [];
|
|
1053
|
-
return [...new Set(words.filter(isRuntime))];
|
|
1054
|
-
}
|
|
1055
|
-
var CLIENT_PATTERNS = [
|
|
1056
|
-
[/claude[-_ ]?code/u, "claude-code"],
|
|
1057
|
-
[/codex/u, "codex-cli"],
|
|
1058
|
-
[/gemini/u, "gemini-cli"],
|
|
1059
|
-
[/grok/u, "grok-cli"],
|
|
1060
|
-
[/antigravity/u, "antigravity"]
|
|
1061
|
-
];
|
|
1062
|
-
function runtimeOfClient(name) {
|
|
1063
|
-
if (!name) return null;
|
|
1064
|
-
const lower = name.toLowerCase();
|
|
1065
|
-
for (const [pattern, runtime] of CLIENT_PATTERNS)
|
|
1066
|
-
if (pattern.test(lower)) return runtime;
|
|
1067
|
-
return null;
|
|
1068
|
-
}
|
|
1069
|
-
function detectRuntime(server, ctx) {
|
|
1070
|
-
const envelope = ctx?.mcpReq?.envelope;
|
|
1071
|
-
const fromEnvelope = envelope?.[CLIENT_INFO_META_KEY]?.name;
|
|
1072
|
-
if (typeof fromEnvelope === "string") return runtimeOfClient(fromEnvelope);
|
|
1073
|
-
return runtimeOfClient(server.server?.getClientVersion?.()?.name);
|
|
1074
|
-
}
|
|
1075
|
-
|
|
1076
1048
|
// src/server.ts
|
|
1077
1049
|
import { readFileSync as readFileSync6 } from "fs";
|
|
1078
1050
|
var DESCRIBE_FILE_LIMIT = FILE_LIST_LIMIT;
|
|
@@ -1292,7 +1264,7 @@ function createSkillTools(options) {
|
|
|
1292
1264
|
}
|
|
1293
1265
|
|
|
1294
1266
|
// src/extension/entry.ts
|
|
1295
|
-
import { createHash as
|
|
1267
|
+
import { createHash as createHash3 } from "crypto";
|
|
1296
1268
|
import { readFileSync as readFileSync7, readdirSync as readdirSync4, statSync as statSync6 } from "fs";
|
|
1297
1269
|
import { basename as basename5, dirname as dirname5, join as join5, relative as relative4, sep as sep3 } from "path";
|
|
1298
1270
|
var MAX_RESOURCES = 512;
|
|
@@ -1304,7 +1276,7 @@ function facts(path) {
|
|
|
1304
1276
|
const cached = digests.get(path);
|
|
1305
1277
|
if (cached?.key === key) return cached;
|
|
1306
1278
|
const bytes = servedBytes(path, readFileSync7(path));
|
|
1307
|
-
const digest = `sha256:${
|
|
1279
|
+
const digest = `sha256:${createHash3("sha256").update(bytes).digest("hex")}`;
|
|
1308
1280
|
const fresh = { key, digest, size: bytes.length };
|
|
1309
1281
|
digests.set(path, fresh);
|
|
1310
1282
|
return fresh;
|
|
@@ -1727,7 +1699,7 @@ function registerReadTool(server, tools) {
|
|
|
1727
1699
|
|
|
1728
1700
|
// src/mcp.ts
|
|
1729
1701
|
var SERVER_NAME = "pmcp";
|
|
1730
|
-
var SERVER_VERSION = "0.
|
|
1702
|
+
var SERVER_VERSION = "0.10.0";
|
|
1731
1703
|
function registerSkillTools(server, tools) {
|
|
1732
1704
|
registerCatalogTool(server, tools);
|
|
1733
1705
|
registerFindTool(server, tools);
|
|
@@ -1743,6 +1715,7 @@ function createSkillServer(options) {
|
|
|
1743
1715
|
registerSkillsExtension(server, tools.skills, options.skillsExtension);
|
|
1744
1716
|
registerSkillResources(server, tools.skills);
|
|
1745
1717
|
if (options.prompts) registerSkillPrompts(server, tools);
|
|
1718
|
+
if (options.mailbox) registerMailboxTools(server, options.mailbox);
|
|
1746
1719
|
return registerSkillTools(server, tools);
|
|
1747
1720
|
}
|
|
1748
1721
|
|
|
@@ -1782,10 +1755,6 @@ export {
|
|
|
1782
1755
|
cosine,
|
|
1783
1756
|
find,
|
|
1784
1757
|
bundleSkill,
|
|
1785
|
-
RUNTIMES,
|
|
1786
|
-
verifiedRuntimesOf,
|
|
1787
|
-
runtimeOfClient,
|
|
1788
|
-
detectRuntime,
|
|
1789
1758
|
DESCRIBE_FILE_LIMIT,
|
|
1790
1759
|
CATALOG_PAGE_SIZE,
|
|
1791
1760
|
CATALOG_PAGE_MAX,
|
|
@@ -1802,4 +1771,4 @@ export {
|
|
|
1802
1771
|
registerSkillTools,
|
|
1803
1772
|
createSkillServer
|
|
1804
1773
|
};
|
|
1805
|
-
//# sourceMappingURL=chunk-
|
|
1774
|
+
//# sourceMappingURL=chunk-VRZFM2M5.js.map
|
|
@@ -0,0 +1,336 @@
|
|
|
1
|
+
import {
|
|
2
|
+
normaliseIntent
|
|
3
|
+
} from "./chunk-VRZFM2M5.js";
|
|
4
|
+
import {
|
|
5
|
+
openSqlite,
|
|
6
|
+
resolveEncoder
|
|
7
|
+
} from "./chunk-XOH2CW6P.js";
|
|
8
|
+
|
|
9
|
+
// src/cache.ts
|
|
10
|
+
async function openDatabase(path) {
|
|
11
|
+
try {
|
|
12
|
+
return await openSqlite(path);
|
|
13
|
+
} catch {
|
|
14
|
+
return null;
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
var DEFAULT_INTENT_CACHE_LIMIT = 4096;
|
|
18
|
+
var SCHEMA = `
|
|
19
|
+
CREATE TABLE IF NOT EXISTS intent_vector (
|
|
20
|
+
intent TEXT NOT NULL PRIMARY KEY,
|
|
21
|
+
model_id TEXT NOT NULL,
|
|
22
|
+
dims INTEGER NOT NULL,
|
|
23
|
+
vector BLOB NOT NULL,
|
|
24
|
+
used_at INTEGER NOT NULL
|
|
25
|
+
);
|
|
26
|
+
CREATE INDEX IF NOT EXISTS intent_vector_used_at ON intent_vector (used_at);
|
|
27
|
+
`;
|
|
28
|
+
async function openIntentCache(options) {
|
|
29
|
+
const opened = await openDatabase(options.path);
|
|
30
|
+
if (!opened) return null;
|
|
31
|
+
const { db, provider } = opened;
|
|
32
|
+
try {
|
|
33
|
+
db.exec("PRAGMA journal_mode = WAL");
|
|
34
|
+
db.exec(SCHEMA);
|
|
35
|
+
} catch {
|
|
36
|
+
db.close();
|
|
37
|
+
return null;
|
|
38
|
+
}
|
|
39
|
+
const limit = options.limit ?? DEFAULT_INTENT_CACHE_LIMIT;
|
|
40
|
+
const now = options.now ?? (() => Date.now());
|
|
41
|
+
let hits = 0;
|
|
42
|
+
let misses = 0;
|
|
43
|
+
let mismatched = 0;
|
|
44
|
+
const selectRow = db.prepare("SELECT model_id, dims, vector FROM intent_vector WHERE intent = ?");
|
|
45
|
+
const touch = db.prepare(
|
|
46
|
+
"UPDATE intent_vector SET used_at = ? WHERE intent = ?"
|
|
47
|
+
);
|
|
48
|
+
const upsert = db.prepare(
|
|
49
|
+
`INSERT INTO intent_vector (intent, model_id, dims, vector, used_at)
|
|
50
|
+
VALUES (?, ?, ?, ?, ?)
|
|
51
|
+
ON CONFLICT(intent) DO UPDATE SET
|
|
52
|
+
model_id = excluded.model_id,
|
|
53
|
+
dims = excluded.dims,
|
|
54
|
+
vector = excluded.vector,
|
|
55
|
+
used_at = excluded.used_at`
|
|
56
|
+
);
|
|
57
|
+
const countRows = db.prepare(
|
|
58
|
+
"SELECT count(*) AS n FROM intent_vector"
|
|
59
|
+
);
|
|
60
|
+
const evict = db.prepare(
|
|
61
|
+
`DELETE FROM intent_vector WHERE intent IN (
|
|
62
|
+
SELECT intent FROM intent_vector ORDER BY used_at ASC LIMIT ?
|
|
63
|
+
)`
|
|
64
|
+
);
|
|
65
|
+
return {
|
|
66
|
+
get(intent) {
|
|
67
|
+
const key = normaliseIntent(intent);
|
|
68
|
+
const row = selectRow.get(key) ?? null;
|
|
69
|
+
if (!row) {
|
|
70
|
+
misses += 1;
|
|
71
|
+
return null;
|
|
72
|
+
}
|
|
73
|
+
if (row.model_id !== options.modelId || row.dims !== options.dims) {
|
|
74
|
+
mismatched += 1;
|
|
75
|
+
return null;
|
|
76
|
+
}
|
|
77
|
+
hits += 1;
|
|
78
|
+
touch.run(now(), key);
|
|
79
|
+
const bytes = row.vector;
|
|
80
|
+
return new Float32Array(
|
|
81
|
+
bytes.buffer.slice(
|
|
82
|
+
bytes.byteOffset,
|
|
83
|
+
bytes.byteOffset + bytes.byteLength
|
|
84
|
+
)
|
|
85
|
+
);
|
|
86
|
+
},
|
|
87
|
+
put(intent, vector) {
|
|
88
|
+
if (vector.length !== options.dims) {
|
|
89
|
+
throw new Error(
|
|
90
|
+
`intent vector is ${vector.length} wide, cache holds ${options.dims}`
|
|
91
|
+
);
|
|
92
|
+
}
|
|
93
|
+
const key = normaliseIntent(intent);
|
|
94
|
+
upsert.run(
|
|
95
|
+
key,
|
|
96
|
+
options.modelId,
|
|
97
|
+
options.dims,
|
|
98
|
+
new Uint8Array(vector.buffer, vector.byteOffset, vector.byteLength),
|
|
99
|
+
now()
|
|
100
|
+
);
|
|
101
|
+
const rows = countRows.get()?.n ?? 0;
|
|
102
|
+
if (rows > limit) evict.run(rows - limit);
|
|
103
|
+
},
|
|
104
|
+
stats() {
|
|
105
|
+
return {
|
|
106
|
+
hits,
|
|
107
|
+
misses,
|
|
108
|
+
mismatched,
|
|
109
|
+
rows: countRows.get()?.n ?? 0,
|
|
110
|
+
provider
|
|
111
|
+
};
|
|
112
|
+
},
|
|
113
|
+
close() {
|
|
114
|
+
db.close();
|
|
115
|
+
}
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
// src/embed.ts
|
|
120
|
+
import { resolve } from "path";
|
|
121
|
+
var ENTRY_SCHEMA = `
|
|
122
|
+
CREATE TABLE IF NOT EXISTS entry_vector (
|
|
123
|
+
name TEXT NOT NULL,
|
|
124
|
+
model_id TEXT NOT NULL,
|
|
125
|
+
dims INTEGER NOT NULL,
|
|
126
|
+
description_hash TEXT NOT NULL,
|
|
127
|
+
vector BLOB NOT NULL,
|
|
128
|
+
PRIMARY KEY (name, model_id)
|
|
129
|
+
);
|
|
130
|
+
`;
|
|
131
|
+
var TRANSFORMERS = "@huggingface/transformers";
|
|
132
|
+
function pretrainedFactory(value) {
|
|
133
|
+
if (!value || !["object", "function"].includes(typeof value)) return null;
|
|
134
|
+
const candidate = value;
|
|
135
|
+
return typeof candidate.from_pretrained === "function" ? candidate : null;
|
|
136
|
+
}
|
|
137
|
+
async function loadProjectedEncoder(module, selected, source, options) {
|
|
138
|
+
const tokenizers = pretrainedFactory(module.AutoTokenizer);
|
|
139
|
+
const models = pretrainedFactory(module.AutoModel);
|
|
140
|
+
if (!tokenizers || !models) return null;
|
|
141
|
+
const settings = {
|
|
142
|
+
dtype: selected.dtype,
|
|
143
|
+
revision: selected.revision,
|
|
144
|
+
local_files_only: options.localOnly ?? false
|
|
145
|
+
};
|
|
146
|
+
if (options.cacheDir !== void 0) settings.cache_dir = options.cacheDir;
|
|
147
|
+
const modelSettings = { ...settings };
|
|
148
|
+
if (selected.modelId === "onnx-community/embeddinggemma-2-ONNX") {
|
|
149
|
+
const configurations = pretrainedFactory(module.AutoConfig);
|
|
150
|
+
if (!configurations) return null;
|
|
151
|
+
const config = await configurations.from_pretrained(
|
|
152
|
+
source,
|
|
153
|
+
settings
|
|
154
|
+
);
|
|
155
|
+
config.vision_config = null;
|
|
156
|
+
config.audio_config = null;
|
|
157
|
+
modelSettings.config = config;
|
|
158
|
+
}
|
|
159
|
+
const tokenizer = await tokenizers.from_pretrained(source, settings);
|
|
160
|
+
const model = await models.from_pretrained(source, modelSettings);
|
|
161
|
+
return async (text) => {
|
|
162
|
+
const output = await model(
|
|
163
|
+
tokenizer(text, { padding: true, truncation: true })
|
|
164
|
+
);
|
|
165
|
+
const vector = output.sentence_embedding;
|
|
166
|
+
const gemma = selected.modelId.startsWith("onnx-community/embeddinggemma-");
|
|
167
|
+
if (!vector || vector.data.length === 0 || gemma && vector.data.length !== 768)
|
|
168
|
+
throw new Error(
|
|
169
|
+
"EmbeddingGemma requires its projected 768-dimensional sentence_embedding"
|
|
170
|
+
);
|
|
171
|
+
if (!Array.from(vector.data).every(Number.isFinite))
|
|
172
|
+
throw new Error("EmbeddingGemma returned a non-finite embedding");
|
|
173
|
+
return vector;
|
|
174
|
+
};
|
|
175
|
+
}
|
|
176
|
+
async function loadEmbedder(modelId, options = {}) {
|
|
177
|
+
const selected = resolveEncoder(modelId, options);
|
|
178
|
+
let module;
|
|
179
|
+
try {
|
|
180
|
+
module = options.module ?? await import(TRANSFORMERS);
|
|
181
|
+
} catch {
|
|
182
|
+
return null;
|
|
183
|
+
}
|
|
184
|
+
let source = options.modelPath ?? selected.modelId;
|
|
185
|
+
if (options.localOnly && !options.modelPath) {
|
|
186
|
+
const cache = options.cacheDir ?? module.env?.cacheDir;
|
|
187
|
+
if (typeof cache !== "string")
|
|
188
|
+
throw new Error("Provide modelPath or cacheDir for an offline encoder");
|
|
189
|
+
source = resolve(
|
|
190
|
+
cache,
|
|
191
|
+
selected.modelId,
|
|
192
|
+
selected.revision === "main" ? "." : selected.revision
|
|
193
|
+
);
|
|
194
|
+
}
|
|
195
|
+
const spec = selected.encoder;
|
|
196
|
+
let encode;
|
|
197
|
+
if (spec.projected) {
|
|
198
|
+
const projected = await loadProjectedEncoder(
|
|
199
|
+
module,
|
|
200
|
+
selected,
|
|
201
|
+
source,
|
|
202
|
+
options
|
|
203
|
+
);
|
|
204
|
+
if (!projected) return null;
|
|
205
|
+
encode = projected;
|
|
206
|
+
} else {
|
|
207
|
+
const factory = module.pipeline;
|
|
208
|
+
if (typeof factory !== "function") return null;
|
|
209
|
+
encode = await factory("feature-extraction", source, {
|
|
210
|
+
dtype: selected.dtype,
|
|
211
|
+
revision: selected.revision,
|
|
212
|
+
cache_dir: options.cacheDir,
|
|
213
|
+
local_files_only: options.localOnly ?? false
|
|
214
|
+
});
|
|
215
|
+
}
|
|
216
|
+
let dims = 0;
|
|
217
|
+
const run = async (text) => {
|
|
218
|
+
const output = await encode(text, {
|
|
219
|
+
pooling: spec.pooling,
|
|
220
|
+
normalize: true
|
|
221
|
+
});
|
|
222
|
+
let vector = Float32Array.from(output.data);
|
|
223
|
+
if (vector.length === 0 || !vector.every(Number.isFinite))
|
|
224
|
+
throw new Error("Encoder returned an invalid embedding");
|
|
225
|
+
if (selected.dimensions !== void 0) {
|
|
226
|
+
if (selected.dimensions > vector.length)
|
|
227
|
+
throw new Error(
|
|
228
|
+
"Encoder output is smaller than the requested dimensions"
|
|
229
|
+
);
|
|
230
|
+
vector = vector.slice(0, selected.dimensions);
|
|
231
|
+
const norm = Math.hypot(...vector);
|
|
232
|
+
if (norm === 0 || !Number.isFinite(norm))
|
|
233
|
+
throw new Error("Encoder returned a zero embedding");
|
|
234
|
+
vector = vector.map((value) => value / norm);
|
|
235
|
+
}
|
|
236
|
+
dims = vector.length;
|
|
237
|
+
return vector;
|
|
238
|
+
};
|
|
239
|
+
return {
|
|
240
|
+
modelId: selected.modelId,
|
|
241
|
+
indexKey: selected.indexKey,
|
|
242
|
+
get dims() {
|
|
243
|
+
return dims;
|
|
244
|
+
},
|
|
245
|
+
embed: (text) => run(spec.queryPrefix + text),
|
|
246
|
+
embedPassage: (text) => run(spec.passagePrefix + text)
|
|
247
|
+
};
|
|
248
|
+
}
|
|
249
|
+
async function encodeEntries(embedder, entries, previous = /* @__PURE__ */ new Map()) {
|
|
250
|
+
const out = /* @__PURE__ */ new Map();
|
|
251
|
+
const passage = (text) => embedder.embedPassage ? embedder.embedPassage(text) : embedder.embed(text);
|
|
252
|
+
for (const entry of entries) {
|
|
253
|
+
const hash = describeHash(entry.description);
|
|
254
|
+
const kept = previous.get(entry.name);
|
|
255
|
+
if (kept && kept.hash === hash) {
|
|
256
|
+
out.set(entry.name, kept);
|
|
257
|
+
continue;
|
|
258
|
+
}
|
|
259
|
+
out.set(entry.name, { hash, vector: await passage(entry.description) });
|
|
260
|
+
}
|
|
261
|
+
return out;
|
|
262
|
+
}
|
|
263
|
+
function describeHash(description) {
|
|
264
|
+
let hash = 2166136261;
|
|
265
|
+
for (let i = 0; i < description.length; i++) {
|
|
266
|
+
hash ^= description.charCodeAt(i);
|
|
267
|
+
hash = Math.imul(hash, 16777619) >>> 0;
|
|
268
|
+
}
|
|
269
|
+
return hash.toString(16).padStart(8, "0");
|
|
270
|
+
}
|
|
271
|
+
async function openEntryVectors(path, options = {}) {
|
|
272
|
+
const opened = options.readOnly ? await openSqlite(path, options).catch(() => null) : await openDatabase(path);
|
|
273
|
+
if (opened === null) return null;
|
|
274
|
+
const db = opened.db;
|
|
275
|
+
if (!options.readOnly) db.exec(ENTRY_SCHEMA);
|
|
276
|
+
return {
|
|
277
|
+
read(modelId) {
|
|
278
|
+
const out = /* @__PURE__ */ new Map();
|
|
279
|
+
const rows = db.prepare(
|
|
280
|
+
"SELECT name, dims, vector FROM entry_vector WHERE model_id = ?"
|
|
281
|
+
).all(modelId);
|
|
282
|
+
for (const row of rows) {
|
|
283
|
+
const bytes = row.vector;
|
|
284
|
+
const vector = new Float32Array(
|
|
285
|
+
bytes.buffer.slice(
|
|
286
|
+
bytes.byteOffset,
|
|
287
|
+
bytes.byteOffset + bytes.byteLength
|
|
288
|
+
)
|
|
289
|
+
);
|
|
290
|
+
if (vector.length === row.dims) out.set(row.name, vector);
|
|
291
|
+
}
|
|
292
|
+
return out;
|
|
293
|
+
},
|
|
294
|
+
stale(modelId, entries) {
|
|
295
|
+
const rows = db.prepare(
|
|
296
|
+
"SELECT name, description_hash FROM entry_vector WHERE model_id = ?"
|
|
297
|
+
).all(modelId);
|
|
298
|
+
const stored = new Map(
|
|
299
|
+
rows.map((row) => [row.name, row.description_hash])
|
|
300
|
+
);
|
|
301
|
+
const out = [];
|
|
302
|
+
for (const [name, description] of entries) {
|
|
303
|
+
if (stored.get(name) !== describeHash(description)) out.push(name);
|
|
304
|
+
}
|
|
305
|
+
return out;
|
|
306
|
+
},
|
|
307
|
+
write(modelId, name, description, vector) {
|
|
308
|
+
db.prepare(
|
|
309
|
+
`INSERT INTO entry_vector (name, model_id, dims, description_hash, vector)
|
|
310
|
+
VALUES (?, ?, ?, ?, ?)
|
|
311
|
+
ON CONFLICT(name, model_id) DO UPDATE SET
|
|
312
|
+
dims = excluded.dims,
|
|
313
|
+
description_hash = excluded.description_hash,
|
|
314
|
+
vector = excluded.vector`
|
|
315
|
+
).run(
|
|
316
|
+
name,
|
|
317
|
+
modelId,
|
|
318
|
+
vector.length,
|
|
319
|
+
describeHash(description),
|
|
320
|
+
new Uint8Array(vector.buffer, vector.byteOffset, vector.byteLength)
|
|
321
|
+
);
|
|
322
|
+
},
|
|
323
|
+
close() {
|
|
324
|
+
db.close();
|
|
325
|
+
}
|
|
326
|
+
};
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
export {
|
|
330
|
+
DEFAULT_INTENT_CACHE_LIMIT,
|
|
331
|
+
openIntentCache,
|
|
332
|
+
loadEmbedder,
|
|
333
|
+
encodeEntries,
|
|
334
|
+
openEntryVectors
|
|
335
|
+
};
|
|
336
|
+
//# sourceMappingURL=chunk-XEH4ZGGJ.js.map
|