@askdkc/kiokuko 0.2.7 → 0.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.ja.md +17 -22
- package/README.ko.md +16 -22
- package/README.md +23 -23
- package/README.zh-CN.md +14 -20
- package/dist/akinator/agent-task.d.ts +2 -0
- package/dist/akinator/agent-task.d.ts.map +1 -1
- package/dist/akinator/agent-task.js +62 -26
- package/dist/akinator/agent-task.js.map +1 -1
- package/dist/akinator/domain.d.ts.map +1 -1
- package/dist/akinator/domain.js +36 -19
- package/dist/akinator/domain.js.map +1 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +8 -3
- package/dist/cli.js.map +1 -1
- package/dist/commands/doctor.d.ts.map +1 -1
- package/dist/commands/doctor.js +6 -3
- package/dist/commands/doctor.js.map +1 -1
- package/dist/commands/embeddings.d.ts +9 -0
- package/dist/commands/embeddings.d.ts.map +1 -1
- package/dist/commands/embeddings.js +58 -1
- package/dist/commands/embeddings.js.map +1 -1
- package/dist/commands/use.d.ts.map +1 -1
- package/dist/commands/use.js +4 -2
- package/dist/commands/use.js.map +1 -1
- package/dist/config/paths.d.ts +5 -0
- package/dist/config/paths.d.ts.map +1 -1
- package/dist/config/paths.js +34 -0
- package/dist/config/paths.js.map +1 -1
- package/dist/embedding/backend.d.ts +1 -1
- package/dist/embedding/backend.d.ts.map +1 -1
- package/dist/embedding/backend.js +20 -4
- package/dist/embedding/backend.js.map +1 -1
- package/dist/embedding/config.d.ts +3 -0
- package/dist/embedding/config.d.ts.map +1 -1
- package/dist/embedding/config.js +17 -0
- package/dist/embedding/config.js.map +1 -1
- package/dist/embedding/diagnostics.d.ts +1 -1
- package/dist/embedding/diagnostics.d.ts.map +1 -1
- package/dist/embedding/diagnostics.js +20 -5
- package/dist/embedding/diagnostics.js.map +1 -1
- package/dist/embedding/document.d.ts +5 -0
- package/dist/embedding/document.d.ts.map +1 -1
- package/dist/embedding/document.js +20 -0
- package/dist/embedding/document.js.map +1 -1
- package/dist/embedding/local-model-loader.d.ts +24 -0
- package/dist/embedding/local-model-loader.d.ts.map +1 -0
- package/dist/embedding/local-model-loader.js +64 -0
- package/dist/embedding/local-model-loader.js.map +1 -0
- package/dist/embedding/local-transformers-provider.d.ts +17 -0
- package/dist/embedding/local-transformers-provider.d.ts.map +1 -0
- package/dist/embedding/local-transformers-provider.js +54 -0
- package/dist/embedding/local-transformers-provider.js.map +1 -0
- package/dist/embedding/model-download.d.ts +32 -0
- package/dist/embedding/model-download.d.ts.map +1 -0
- package/dist/embedding/model-download.js +41 -0
- package/dist/embedding/model-download.js.map +1 -0
- package/dist/embedding/model-installation.d.ts +17 -0
- package/dist/embedding/model-installation.d.ts.map +1 -0
- package/dist/embedding/model-installation.js +71 -0
- package/dist/embedding/model-installation.js.map +1 -0
- package/dist/embedding/model-manifest.d.ts +18 -0
- package/dist/embedding/model-manifest.d.ts.map +1 -0
- package/dist/embedding/model-manifest.js +86 -0
- package/dist/embedding/model-manifest.js.map +1 -0
- package/dist/embedding/presets/local-small.d.ts +8 -0
- package/dist/embedding/presets/local-small.d.ts.map +1 -0
- package/dist/embedding/presets/local-small.js +67 -0
- package/dist/embedding/presets/local-small.js.map +1 -0
- package/dist/embedding/presets/manifest.d.ts +27 -0
- package/dist/embedding/presets/manifest.d.ts.map +1 -0
- package/dist/embedding/presets/manifest.js +5 -0
- package/dist/embedding/presets/manifest.js.map +1 -0
- package/dist/embedding/profile.d.ts +6 -2
- package/dist/embedding/profile.d.ts.map +1 -1
- package/dist/embedding/profile.js +36 -0
- package/dist/embedding/profile.js.map +1 -1
- package/dist/embedding/query-cache.d.ts +6 -0
- package/dist/embedding/query-cache.d.ts.map +1 -1
- package/dist/embedding/query-cache.js +21 -0
- package/dist/embedding/query-cache.js.map +1 -1
- package/dist/embedding/runtime.d.ts +2 -1
- package/dist/embedding/runtime.d.ts.map +1 -1
- package/dist/embedding/runtime.js +39 -5
- package/dist/embedding/runtime.js.map +1 -1
- package/dist/embedding/settings.d.ts +6 -0
- package/dist/embedding/settings.d.ts.map +1 -0
- package/dist/embedding/settings.js +69 -0
- package/dist/embedding/settings.js.map +1 -0
- package/dist/embedding/setup-lock.d.ts +7 -0
- package/dist/embedding/setup-lock.d.ts.map +1 -0
- package/dist/embedding/setup-lock.js +35 -0
- package/dist/embedding/setup-lock.js.map +1 -0
- package/dist/embedding/setup-service.d.ts +53 -0
- package/dist/embedding/setup-service.d.ts.map +1 -0
- package/dist/embedding/setup-service.js +184 -0
- package/dist/embedding/setup-service.js.map +1 -0
- package/dist/embedding/store.d.ts +3 -1
- package/dist/embedding/store.d.ts.map +1 -1
- package/dist/embedding/store.js +124 -11
- package/dist/embedding/store.js.map +1 -1
- package/dist/embedding/types.d.ts +31 -2
- package/dist/embedding/types.d.ts.map +1 -1
- package/dist/enno-oduno/directives.js +1 -1
- package/dist/enno-oduno/directives.js.map +1 -1
- package/dist/enno-oduno/instructions.d.ts +1 -1
- package/dist/enno-oduno/instructions.d.ts.map +1 -1
- package/dist/enno-oduno/instructions.js +1 -1
- package/dist/enno-oduno/instructions.js.map +1 -1
- package/dist/enno-oduno/schemas.d.ts +9 -0
- package/dist/enno-oduno/schemas.d.ts.map +1 -1
- package/dist/enno-oduno/schemas.js +11 -0
- package/dist/enno-oduno/schemas.js.map +1 -1
- package/dist/enno-oduno/service.d.ts +7 -1
- package/dist/enno-oduno/service.d.ts.map +1 -1
- package/dist/enno-oduno/service.js +52 -1
- package/dist/enno-oduno/service.js.map +1 -1
- package/dist/enno-oduno/validation-errors.d.ts +2 -1
- package/dist/enno-oduno/validation-errors.d.ts.map +1 -1
- package/dist/enno-oduno/validation-errors.js +1 -0
- package/dist/enno-oduno/validation-errors.js.map +1 -1
- package/dist/enno-oduno/verifier.d.ts.map +1 -1
- package/dist/enno-oduno/verifier.js +27 -2
- package/dist/enno-oduno/verifier.js.map +1 -1
- package/dist/mcp/bounded-stdio-transport.d.ts +3 -10
- package/dist/mcp/bounded-stdio-transport.d.ts.map +1 -1
- package/dist/mcp/bounded-stdio-transport.js +2 -89
- package/dist/mcp/bounded-stdio-transport.js.map +1 -1
- package/dist/mcp/request-deadline.d.ts +51 -0
- package/dist/mcp/request-deadline.d.ts.map +1 -0
- package/dist/mcp/request-deadline.js +155 -0
- package/dist/mcp/request-deadline.js.map +1 -0
- package/dist/mcp/runtime-owner.d.ts.map +1 -1
- package/dist/mcp/runtime-owner.js +2 -3
- package/dist/mcp/runtime-owner.js.map +1 -1
- package/dist/mcp/server.d.ts +2 -0
- package/dist/mcp/server.d.ts.map +1 -1
- package/dist/mcp/server.js +58 -23
- package/dist/mcp/server.js.map +1 -1
- package/dist/memory/scoped-memory.d.ts +1 -1
- package/dist/memory/scoped-memory.d.ts.map +1 -1
- package/dist/memory/scoped-memory.js +5 -1
- package/dist/memory/scoped-memory.js.map +1 -1
- package/dist/server/http.d.ts +1 -1
- package/dist/server/http.d.ts.map +1 -1
- package/dist/server/http.js +6 -3
- package/dist/server/http.js.map +1 -1
- package/dist/skills/discovery-service.d.ts.map +1 -1
- package/dist/skills/discovery-service.js +10 -8
- package/dist/skills/discovery-service.js.map +1 -1
- package/dist/skills/find.d.ts +1 -0
- package/dist/skills/find.d.ts.map +1 -1
- package/dist/skills/find.js +1 -0
- package/dist/skills/find.js.map +1 -1
- package/dist/skills/materialization-authority.d.ts +1 -1
- package/dist/skills/materialization-authority.d.ts.map +1 -1
- package/dist/skills/materialization-authority.js +2 -2
- package/dist/skills/materialization-authority.js.map +1 -1
- package/dist/skills/providers/skills-sh-compat.d.ts.map +1 -1
- package/dist/skills/providers/skills-sh-compat.js +4 -2
- package/dist/skills/providers/skills-sh-compat.js.map +1 -1
- package/dist/skills/providers/skills-sh-v1.d.ts +2 -2
- package/dist/skills/providers/skills-sh-v1.d.ts.map +1 -1
- package/dist/skills/providers/skills-sh-v1.js +22 -4
- package/dist/skills/providers/skills-sh-v1.js.map +1 -1
- package/dist/skills/source/github-fetcher.d.ts.map +1 -1
- package/dist/skills/source/github-fetcher.js +8 -4
- package/dist/skills/source/github-fetcher.js.map +1 -1
- package/dist/skills/types.d.ts +3 -2
- package/dist/skills/types.d.ts.map +1 -1
- package/docs/agent-file.md +72 -0
- package/docs/agent-gateway.md +104 -0
- package/docs/architecture.md +12 -0
- package/docs/cli-contract.md +35 -0
- package/docs/client-compatibility.md +298 -0
- package/docs/database.md +11 -0
- package/docs/execution-ledger.md +89 -0
- package/docs/release-notes-0.1.18.md +103 -0
- package/docs/retrieval-evaluation.md +10 -0
- package/docs/security.md +11 -0
- package/docs/sqlite-driver-adr.md +8 -0
- package/migrations/022_embedding_setup_v2.sql +236 -0
- package/migrations/down/022_embedding_setup_v2.sql +133 -0
- package/package.json +24 -3
- package/skills/kiokuko-enno-oduno/SKILL.md +44 -0
- package/templates/AGENTS.md +1 -1
|
@@ -3,6 +3,7 @@ import { parseEmbeddingConfig, isEnabledEmbeddingConfig } from './config.js';
|
|
|
3
3
|
import { embeddingProfileId, createEmbeddingProfile } from './profile.js';
|
|
4
4
|
import { readActiveEmbeddingProfile, readEmbeddingRuntimeState } from './store.js';
|
|
5
5
|
import { decodeVector, hashVectorBytes } from './vector.js';
|
|
6
|
+
import { readPersistedEmbeddingSettings } from './settings.js';
|
|
6
7
|
const EMBEDDING_TABLES = [
|
|
7
8
|
'embedding_profiles',
|
|
8
9
|
'embedding_runtime',
|
|
@@ -89,7 +90,7 @@ function activeStatus(database, config, active, backend) {
|
|
|
89
90
|
mode: config.mode,
|
|
90
91
|
activeProfileId: active.profileId,
|
|
91
92
|
providerKind: active.identity.providerKind,
|
|
92
|
-
model: active.identity.model,
|
|
93
|
+
model: active.identity.schemaVersion === 2 ? active.identity.sourceModel : active.identity.model,
|
|
93
94
|
dimensions: active.identity.dimensions,
|
|
94
95
|
distanceMetric: active.identity.distanceMetric,
|
|
95
96
|
distanceCeiling: active.identity.distanceCeiling,
|
|
@@ -107,7 +108,7 @@ function activeStatus(database, config, active, backend) {
|
|
|
107
108
|
queryCacheRows,
|
|
108
109
|
};
|
|
109
110
|
}
|
|
110
|
-
export function readEmbeddingStatus(database, config =
|
|
111
|
+
export function readEmbeddingStatus(database, config = readPersistedEmbeddingSettings(database), backend) {
|
|
111
112
|
const runtime = readEmbeddingRuntimeState(database);
|
|
112
113
|
const active = readActiveEmbeddingProfile(database);
|
|
113
114
|
if (active !== null && runtime.activeProfileId !== active.profile.profileId) {
|
|
@@ -200,9 +201,23 @@ function safeConfig(environment) {
|
|
|
200
201
|
return { invalid: true };
|
|
201
202
|
}
|
|
202
203
|
}
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
204
|
+
function isEmbeddingConfig(value) {
|
|
205
|
+
return typeof value.mode === 'string' && typeof value.provider === 'string'
|
|
206
|
+
&& typeof value.allowRemote === 'boolean' && typeof value.vectorBackend === 'string'
|
|
207
|
+
&& typeof value.timeoutMs === 'number' && typeof value.batchSize === 'number';
|
|
208
|
+
}
|
|
209
|
+
export function inspectEmbeddingHealth(database, environment = readPersistedEmbeddingSettings(database), backend) {
|
|
210
|
+
const parsed = isEmbeddingConfig(environment)
|
|
211
|
+
? { config: environment, invalid: false }
|
|
212
|
+
: safeConfig(environment);
|
|
213
|
+
const config = (isEmbeddingConfig(environment) ? environment : parsed.config) ?? {
|
|
214
|
+
mode: 'off',
|
|
215
|
+
provider: 'openai-compatible',
|
|
216
|
+
allowRemote: false,
|
|
217
|
+
vectorBackend: 'auto',
|
|
218
|
+
timeoutMs: 30_000,
|
|
219
|
+
batchSize: 16,
|
|
220
|
+
};
|
|
206
221
|
let status = emptyStatus(config, backend);
|
|
207
222
|
let findings = parsed.invalid ? 1 : 0;
|
|
208
223
|
const tables = objectNames(database, 'table');
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"diagnostics.js","sourceRoot":"","sources":["../../src/embedding/diagnostics.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,oBAAoB,EAAE,wBAAwB,EAAE,MAAM,aAAa,CAAC;AAC7E,OAAO,EAAE,kBAAkB,EAAE,sBAAsB,EAAE,MAAM,cAAc,CAAC;AAC1E,OAAO,EAAE,0BAA0B,EAAE,yBAAyB,EAAE,MAAM,YAAY,CAAC;AACnF,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"diagnostics.js","sourceRoot":"","sources":["../../src/embedding/diagnostics.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,oBAAoB,EAAE,wBAAwB,EAAE,MAAM,aAAa,CAAC;AAC7E,OAAO,EAAE,kBAAkB,EAAE,sBAAsB,EAAE,MAAM,cAAc,CAAC;AAC1E,OAAO,EAAE,0BAA0B,EAAE,yBAAyB,EAAE,MAAM,YAAY,CAAC;AACnF,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAE5D,OAAO,EAAE,8BAA8B,EAAE,MAAM,eAAe,CAAC;AAE/D,MAAM,gBAAgB,GAAG;IACvB,oBAAoB;IACpB,mBAAmB;IACnB,kBAAkB;IAClB,gBAAgB;IAChB,kBAAkB;CACV,CAAC;AAEX,MAAM,iBAAiB,GAAG;IACxB,uCAAuC;IACvC,0BAA0B;IAC1B,0BAA0B;CAClB,CAAC;AAyCX,SAAS,SAAS,CAAC,QAAwB,EAAE,GAAW,EAAE,GAAG,UAAkC;IAC7F,OAAO,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAoB,GAAG,UAAU,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC;AACzF,CAAC;AAED,SAAS,WAAW,CAAC,QAAwB,EAAE,IAAmC;IAChF,OAAO,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC;;GAE/B,CAAC,CAAC,GAAG,CAAmB,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;AACzD,CAAC;AAED,SAAS,iBAAiB,CAAC,MAAuB,EAAE,OAAwC;IAC1F,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK;QAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC9D,IAAI,OAAO,KAAK,SAAS;QAAE,OAAO,EAAE,EAAE,EAAE,OAAO,CAAC,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,KAAK,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC7G,IAAI,MAAM,CAAC,aAAa,KAAK,YAAY;QAAE,OAAO,EAAE,EAAE,EAAE,YAAY,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;IAC1F,OAAO,EAAE,EAAE,EAAE,YAAY,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;AAC7C,CAAC;AAED,SAAS,WAAW,CAAC,MAAuB,EAAE,OAA6B;IACzE,MAAM,eAAe,GAAG,iBAAiB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC3D,OAAO;QACL,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,eAAe,EAAE,IAAI;QACrB,YAAY,EAAE,MAAM,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ;QAC5D,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,IAAI;QAC3B,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,IAAI;QACrC,cAAc,EAAE,MAAM,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ;QACvD,eAAe,EAAE,MAAM,CAAC,eAAe,IAAI,IAAI;QAC/C,OAAO,EAAE,eAAe,CAAC,EAAE;QAC3B,cAAc,EAAE,eAAe,CAAC,OAAO;QACvC,eAAe,EAAE,CAAC;QAClB,YAAY,EAAE,CAAC;QACf,YAAY,EAAE,CAAC;QACf,cAAc,EAAE,CAAC;QACjB,WAAW,EAAE,CAAC;QACd,UAAU,EAAE,CAAC;QACb,UAAU,EAAE,CAAC;QACb,WAAW,EAAE,CAAC;QACd,aAAa,EAAE,CAAC;QAChB,cAAc,EAAE,CAAC;KAClB,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CACnB,QAAwB,EACxB,MAAuB,EACvB,MAA+B,EAC/B,OAA6B;IAE7B,MAAM,eAAe,GAAG,iBAAiB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC3D,MAAM,eAAe,GAAG,SAAS,CAAC,QAAQ,EAAE,uCAAuC,CAAC,CAAC;IACrF,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;QACpB,OAAO,EAAE,GAAG,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,eAAe,EAAE,cAAc,EAAE,eAAe,EAAE,CAAC;IAC/F,CAAC;IACD,MAAM,YAAY,GAAG,SAAS,CAAC,QAAQ,EAAE;;;;;;;;GAQxC,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;IACrB,MAAM,YAAY,GAAG,SAAS,CAAC,QAAQ,EAAE,qEAAqE,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;IAClI,MAAM,IAAI,GAAG,IAAI,GAAG,CAClB,QAAQ,CAAC,OAAO,CAAC;;;;;KAKhB,CAAC,CAAC,GAAG,CAAmC,MAAM,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CACxG,CAAC;IACF,MAAM,QAAQ,GAAG;QACf,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC;QACjC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC;QAC/B,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC;QAC/B,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC;KAClC,CAAC;IACF,MAAM,cAAc,GAAG,SAAS,CAAC,QAAQ,EAAE,qEAAqE,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;IACpI,OAAO;QACL,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,eAAe,EAAE,MAAM,CAAC,SAAS;QACjC,YAAY,EAAE,MAAM,CAAC,QAAQ,CAAC,YAAY;QAC1C,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC,aAAa,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK;QAChG,UAAU,EAAE,MAAM,CAAC,QAAQ,CAAC,UAAU;QACtC,cAAc,EAAE,MAAM,CAAC,QAAQ,CAAC,cAAc;QAC9C,eAAe,EAAE,MAAM,CAAC,QAAQ,CAAC,eAAe;QAChD,OAAO,EAAE,eAAe,CAAC,EAAE;QAC3B,cAAc,EAAE,eAAe,CAAC,OAAO;QACvC,eAAe;QACf,YAAY;QACZ,YAAY,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,YAAY,GAAG,YAAY,CAAC;QACtD,cAAc,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,eAAe,GAAG,YAAY,CAAC;QAC3D,WAAW,EAAE,QAAQ,CAAC,OAAO;QAC7B,UAAU,EAAE,QAAQ,CAAC,MAAM;QAC3B,UAAU,EAAE,QAAQ,CAAC,MAAM;QAC3B,WAAW,EAAE,QAAQ,CAAC,OAAO;QAC7B,aAAa,EAAE,eAAe,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,eAAe;QACzE,cAAc;KACf,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,QAAwB,EACxB,SAA0B,8BAA8B,CAAC,QAAQ,CAAC,EAClE,OAA6B;IAE7B,MAAM,OAAO,GAAG,yBAAyB,CAAC,QAAQ,CAAC,CAAC;IACpD,MAAM,MAAM,GAAG,0BAA0B,CAAC,QAAQ,CAAC,CAAC;IACpD,IAAI,MAAM,KAAK,IAAI,IAAI,OAAO,CAAC,eAAe,KAAK,MAAM,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC;QAC5E,MAAM,IAAI,YAAY,CAAC,iBAAiB,EAAE,kDAAkD,CAAC,CAAC;IAChG,CAAC;IACD,OAAO,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,IAAI,IAAI,EAAE,OAAO,CAAC,CAAC;AAC1E,CAAC;AAED,SAAS,cAAc,CAAC,QAAwB;IAC9C,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC;;;;GAI7B,CAAC,CAAC,GAAG,EAAa,CAAC;IACpB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,CAAC;YACH,IAAI,OAAO,GAAG,CAAC,UAAU,KAAK,QAAQ,IAAI,OAAO,GAAG,CAAC,UAAU,KAAK,QAAQ,IAAI,OAAO,GAAG,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;gBACpH,QAAQ,IAAI,CAAC,CAAC;gBACd,SAAS;YACX,CAAC;YACD,MAAM,KAAK,GAAG,GAAG,CAAC,SAAS,YAAY,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC;YACzE,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,UAAU,KAAK,GAAG,CAAC,UAAU,GAAG,CAAC,IAAI,eAAe,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,WAAW,EAAE,CAAC;gBAC5G,QAAQ,IAAI,CAAC,CAAC;gBACd,SAAS;YACX,CAAC;YACD,YAAY,CAAC,KAAK,EAAE,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,WAAW,CAAC,CAAC;QACvD,CAAC;QAAC,MAAM,CAAC;YACP,QAAQ,IAAI,CAAC,CAAC;QAChB,CAAC;IACH,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,iBAAiB,CAAC,QAAwB;IACjD,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC;;;;GAI7B,CAAC,CAAC,GAAG,EAAa,CAAC;IACpB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,CAAC;YACH,IAAI,OAAO,GAAG,CAAC,UAAU,KAAK,QAAQ,IAAI,OAAO,GAAG,CAAC,WAAW,KAAK,QAAQ,IAAI,CAAC,CAAC,GAAG,CAAC,SAAS,YAAY,UAAU,CAAC,EAAE,CAAC;gBACxH,QAAQ,IAAI,CAAC,CAAC;gBACd,SAAS;YACX,CAAC;YACD,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,WAAW,CAAC,CAAC;QAC/D,CAAC;QAAC,MAAM,CAAC;YACP,QAAQ,IAAI,CAAC,CAAC;QAChB,CAAC;IACH,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,oBAAoB,CAAC,QAAwB;IACpD,OAAO,SAAS,CAAC,QAAQ,EAAE;;;;;;;;GAQ1B,CAAC,CAAC;AACL,CAAC;AAED,SAAS,oBAAoB,CAC3B,QAAwB,EACxB,MAAuB,EACvB,MAAqD,EACrD,OAAwC;IAExC,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK;QAAE,OAAO,CAAC,CAAC;IACpC,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,MAAM,KAAK,IAAI,IAAI,CAAC,wBAAwB,CAAC,MAAM,CAAC;QAAE,QAAQ,IAAI,CAAC,CAAC;SACnE,CAAC;QACJ,IAAI,CAAC;YACH,IAAI,kBAAkB,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,KAAK,MAAM,CAAC,OAAO,CAAC,SAAS;gBAAE,QAAQ,IAAI,CAAC,CAAC;QAC9G,CAAC;QAAC,MAAM,CAAC;YACP,QAAQ,IAAI,CAAC,CAAC;QAChB,CAAC;IACH,CAAC;IACD,IAAI,MAAM,CAAC,aAAa,KAAK,YAAY,IAAI,OAAO,EAAE,EAAE,KAAK,YAAY;QAAE,QAAQ,IAAI,CAAC,CAAC;IACzF,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,UAAU,CAAC,WAA8B;IAChD,IAAI,CAAC;QACH,OAAO,EAAE,MAAM,EAAE,oBAAoB,CAAC,WAAW,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IACvE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC3B,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CAAC,KAA0C;IACnE,OAAO,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,IAAI,OAAO,KAAK,CAAC,QAAQ,KAAK,QAAQ;WACtE,OAAO,KAAK,CAAC,WAAW,KAAK,SAAS,IAAI,OAAO,KAAK,CAAC,aAAa,KAAK,QAAQ;WACjF,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ,IAAI,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ,CAAC;AAClF,CAAC;AAED,MAAM,UAAU,sBAAsB,CACpC,QAAwB,EACxB,cAAmD,8BAA8B,CAAC,QAAQ,CAAC,EAC3F,OAA6B;IAE7B,MAAM,MAAM,GAAG,iBAAiB,CAAC,WAAW,CAAC;QAC3C,CAAC,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,KAAK,EAAE;QACzC,CAAC,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;IAC5B,MAAM,MAAM,GAAoB,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI;QAChG,IAAI,EAAE,KAAK;QACX,QAAQ,EAAE,mBAAmB;QAC7B,WAAW,EAAE,KAAK;QAClB,aAAa,EAAE,MAAM;QACrB,SAAS,EAAE,MAAM;QACjB,SAAS,EAAE,EAAE;KACd,CAAC;IACF,IAAI,MAAM,GAAG,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC1C,IAAI,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACtC,MAAM,MAAM,GAAG,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC9C,MAAM,OAAO,GAAG,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC/C,MAAM,QAAQ,GAAG,WAAW,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IAClD,QAAQ,IAAI,gBAAgB,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;IACxE,QAAQ,IAAI,iBAAiB,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;IAC1E,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,qCAAqC,CAAC;QAAE,QAAQ,IAAI,CAAC,CAAC;IAExE,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,yBAAyB,CAAC,QAAQ,CAAC,CAAC;QACpD,MAAM,MAAM,GAAG,0BAA0B,CAAC,QAAQ,CAAC,CAAC;QACpD,MAAM,GAAG,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,IAAI,IAAI,EAAE,OAAO,CAAC,CAAC;QAC1E,QAAQ,IAAI,oBAAoB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;QACpE,QAAQ,IAAI,cAAc,CAAC,QAAQ,CAAC,CAAC;QACrC,QAAQ,IAAI,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QACxC,QAAQ,IAAI,oBAAoB,CAAC,QAAQ,CAAC,CAAC;QAC3C,MAAM,WAAW,GAAG,SAAS,CAAC,QAAQ,EAAE,iDAAiD,CAAC,CAAC;QAC3F,IAAI,WAAW,KAAK,CAAC;YAAE,QAAQ,IAAI,CAAC,CAAC;QACrC,MAAM,KAAK,GAAG,SAAS,CAAC,QAAQ,EAAE;;;;;;KAMjC,CAAC,CAAC;QACH,IAAI,KAAK,KAAK,MAAM,CAAC,YAAY;YAAE,QAAQ,IAAI,CAAC,CAAC;IACnD,CAAC;IAAC,MAAM,CAAC;QACP,QAAQ,IAAI,CAAC,CAAC;IAChB,CAAC;IAED,OAAO;QACL,MAAM;QACN,KAAK,EAAE;YACL,EAAE,EAAE,QAAQ,KAAK,CAAC;YAClB,KAAK,EAAE,QAAQ;YACf,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY,QAAQ,yBAAyB,CAAC,CAAC,CAAC,YAAY,QAAQ,UAAU,MAAM,CAAC,IAAI,aAAa,MAAM,CAAC,OAAO,IAAI,UAAU,EAAE;SAC9J;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { type JsonObject, type EntryKind } from '../serialization/validate.js';
|
|
2
2
|
import type { EmbeddingDocument } from './types.js';
|
|
3
3
|
export declare const EMBEDDING_DOCUMENT_TEMPLATE_VERSION: 1;
|
|
4
|
+
export declare const EMBEDDING_DOCUMENT_TEMPLATE_VERSION_V2: 2;
|
|
5
|
+
export declare const EMBEDDING_INPUT_CONTRACT: "e5-query-passage-v1";
|
|
4
6
|
export declare const MAX_EMBEDDING_DOCUMENT_BYTES: number;
|
|
5
7
|
export declare const BODY_TRUNCATION_MARKER: "[body truncated]";
|
|
6
8
|
export interface EmbeddingDocumentInput {
|
|
@@ -12,4 +14,7 @@ export interface EmbeddingDocumentInput {
|
|
|
12
14
|
readonly scope: JsonObject;
|
|
13
15
|
}
|
|
14
16
|
export declare function buildEmbeddingDocument(input: EmbeddingDocumentInput): EmbeddingDocument;
|
|
17
|
+
/** Build the provider-neutral v2 memory representation for E5 models. */
|
|
18
|
+
export declare function buildCanonicalEmbeddingDocument(input: EmbeddingDocumentInput): EmbeddingDocument;
|
|
19
|
+
export declare function renderEmbeddingProviderInput(canonicalText: string, prefix?: string): string;
|
|
15
20
|
//# sourceMappingURL=document.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"document.d.ts","sourceRoot":"","sources":["../../src/embedding/document.ts"],"names":[],"mappings":"AAGA,OAAO,EAAoC,KAAK,UAAU,EAAE,KAAK,SAAS,EAAE,MAAM,8BAA8B,CAAC;AAGjH,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAEpD,eAAO,MAAM,mCAAmC,EAAG,CAAU,CAAC;AAC9D,eAAO,MAAM,4BAA4B,QAAY,CAAC;AACtD,eAAO,MAAM,sBAAsB,EAAG,kBAA2B,CAAC;AAElE,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,CAAC;IACjC,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC;CAC5B;AAmDD,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,sBAAsB,GAAG,iBAAiB,CA6CvF"}
|
|
1
|
+
{"version":3,"file":"document.d.ts","sourceRoot":"","sources":["../../src/embedding/document.ts"],"names":[],"mappings":"AAGA,OAAO,EAAoC,KAAK,UAAU,EAAE,KAAK,SAAS,EAAE,MAAM,8BAA8B,CAAC;AAGjH,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAEpD,eAAO,MAAM,mCAAmC,EAAG,CAAU,CAAC;AAC9D,eAAO,MAAM,sCAAsC,EAAG,CAAU,CAAC;AACjE,eAAO,MAAM,wBAAwB,EAAG,qBAA8B,CAAC;AACvE,eAAO,MAAM,4BAA4B,QAAY,CAAC;AACtD,eAAO,MAAM,sBAAsB,EAAG,kBAA2B,CAAC;AAElE,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,CAAC;IACjC,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC;CAC5B;AAmDD,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,sBAAsB,GAAG,iBAAiB,CA6CvF;AAED,yEAAyE;AACzE,wBAAgB,+BAA+B,CAAC,KAAK,EAAE,sBAAsB,GAAG,iBAAiB,CAUhG;AAED,wBAAgB,4BAA4B,CAAC,aAAa,EAAE,MAAM,EAAE,MAAM,SAAc,GAAG,MAAM,CAKhG"}
|
|
@@ -5,6 +5,8 @@ import { canonicalJson, canonicalTagOrder } from '../serialization/validate.js';
|
|
|
5
5
|
import { findSecret } from '../memory/secrets.js';
|
|
6
6
|
import { validateApplicability, validateSignals } from '../memory/structured-memory.js';
|
|
7
7
|
export const EMBEDDING_DOCUMENT_TEMPLATE_VERSION = 1;
|
|
8
|
+
export const EMBEDDING_DOCUMENT_TEMPLATE_VERSION_V2 = 2;
|
|
9
|
+
export const EMBEDDING_INPUT_CONTRACT = 'e5-query-passage-v1';
|
|
8
10
|
export const MAX_EMBEDDING_DOCUMENT_BYTES = 32 * 1024;
|
|
9
11
|
export const BODY_TRUNCATION_MARKER = '[body truncated]';
|
|
10
12
|
const encoder = new TextEncoder();
|
|
@@ -99,4 +101,22 @@ export function buildEmbeddingDocument(input) {
|
|
|
99
101
|
}
|
|
100
102
|
return Object.freeze({ text, bytes, documentHash: documentHash(bytes), truncated });
|
|
101
103
|
}
|
|
104
|
+
/** Build the provider-neutral v2 memory representation for E5 models. */
|
|
105
|
+
export function buildCanonicalEmbeddingDocument(input) {
|
|
106
|
+
const document = buildEmbeddingDocument(input);
|
|
107
|
+
const text = document.text.replace(/^kiokuko-memory-v1\n/u, 'kiokuko-memory-v2\n');
|
|
108
|
+
const bytes = encoder.encode(text);
|
|
109
|
+
return Object.freeze({
|
|
110
|
+
text,
|
|
111
|
+
bytes,
|
|
112
|
+
documentHash: documentHash(bytes),
|
|
113
|
+
truncated: document.truncated,
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
export function renderEmbeddingProviderInput(canonicalText, prefix = 'passage: ') {
|
|
117
|
+
if (typeof canonicalText !== 'string' || canonicalText.length === 0 || prefix !== 'passage: ') {
|
|
118
|
+
invalid('Embedding provider document input is invalid');
|
|
119
|
+
}
|
|
120
|
+
return `${prefix}${canonicalText}`;
|
|
121
|
+
}
|
|
102
122
|
//# sourceMappingURL=document.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"document.js","sourceRoot":"","sources":["../../src/embedding/document.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAmC,MAAM,8BAA8B,CAAC;AACjH,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAClD,OAAO,EAAE,qBAAqB,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AAGxF,MAAM,CAAC,MAAM,mCAAmC,GAAG,CAAU,CAAC;AAC9D,MAAM,CAAC,MAAM,4BAA4B,GAAG,EAAE,GAAG,IAAI,CAAC;AACtD,MAAM,CAAC,MAAM,sBAAsB,GAAG,kBAA2B,CAAC;AAWlE,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;AAClC,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AAE1D,SAAS,OAAO,CAAC,OAAe;IAC9B,MAAM,IAAI,YAAY,CAAC,kBAAkB,EAAE,OAAO,CAAC,CAAC;AACtD,CAAC;AAED,SAAS,aAAa,CAAC,KAAa,EAAE,KAAa;IACjD,MAAM,UAAU,GAAG,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAC3F,IAAI,iDAAiD,CAAC,IAAI,CAAC,UAAU,CAAC;QAAE,OAAO,CAAC,GAAG,KAAK,yCAAyC,CAAC,CAAC;IACnI,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAuB;IAChD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,IAAI,GAAG,CAAC,MAAM,GAAG,GAAG;YAAE,OAAO,CAAC,qCAAqC,CAAC,CAAC;IAC7H,CAAC;IACD,OAAO,iBAAiB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,aAAa,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;AACzE,CAAC;AAED,SAAS,eAAe,CAAC,KAAiB;IACxC,MAAM,SAAS,GAAG,KAAgC,CAAC;IACnD,IAAI,SAAS,CAAC,aAAa,KAAK,CAAC,IAAI,SAAS,CAAC,aAAa,KAAK,CAAC;QAAE,OAAO,EAAE,aAAa,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;IAC9G,MAAM,aAAa,GAAG,SAAS,CAAC,aAAa,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,qBAAqB,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;IAClH,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;IAC1F,OAAO;QACL,aAAa,EAAE,aAAsC;QACrD,OAAO,EAAE,OAAgC;KAC1C,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,KAAa,EAAE,QAAgB;IACnD,IAAI,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,UAAU,IAAI,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC/D,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACpC,IAAI,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;IAC/C,OAAO,GAAG,GAAG,CAAC,EAAE,CAAC;QACf,IAAI,CAAC;YACH,OAAO,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;QAChD,CAAC;QAAC,MAAM,CAAC;YACP,GAAG,IAAI,CAAC,CAAC;QACX,CAAC;IACH,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,YAAY,CAAC,KAAiB;IACrC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC1D,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,KAA6B;IAClE,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QAC/I,OAAO,CAAC,oCAAoC,CAAC,CAAC;IAChD,CAAC;IACD,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;IACzD,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC;IAC7F,MAAM,IAAI,GAAG,aAAa,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC/C,MAAM,IAAI,GAAG,iBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAChE,MAAM,YAAY,GAAG;QACnB,mBAAmB;QACnB,SAAS,KAAK,CAAC,IAAI,EAAE;QACrB,UAAU,KAAK,EAAE;QACjB,YAAY,OAAO,EAAE;QACrB,OAAO;QACP,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,KAAK,GAAG,EAAE,CAAC;QAChC,gBAAgB;QAChB,aAAa,CAAC,aAAa,CAAC;QAC5B,UAAU;QACV,aAAa,CAAC,OAAO,CAAC;QACtB,OAAO;KACR,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IACpB,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;IACnD,IAAI,aAAa,CAAC,UAAU,IAAI,4BAA4B,EAAE,CAAC;QAC7D,OAAO,CAAC,oDAAoD,CAAC,CAAC;IAChE,CAAC;IAED,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACvC,IAAI,IAAI,GAAG,YAAY,GAAG,IAAI,CAAC;IAC/B,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI,aAAa,CAAC,UAAU,GAAG,SAAS,CAAC,UAAU,GAAG,4BAA4B,EAAE,CAAC;QACnF,MAAM,MAAM,GAAG,KAAK,sBAAsB,EAAE,CAAC;QAC7C,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC3C,MAAM,SAAS,GAAG,4BAA4B,GAAG,aAAa,CAAC,UAAU,GAAG,WAAW,CAAC,UAAU,CAAC;QACnG,IAAI,SAAS,GAAG,CAAC;YAAE,OAAO,CAAC,oDAAoD,CAAC,CAAC;QACjF,IAAI,GAAG,YAAY,GAAG,YAAY,CAAC,IAAI,EAAE,SAAS,CAAC,GAAG,MAAM,CAAC;QAC7D,SAAS,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,aAAa,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IACvC,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;QAChC,MAAM,IAAI,YAAY,CAAC,oBAAoB,EAAE,wDAAwD,CAAC,CAAC;IACzG,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,YAAY,EAAE,YAAY,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC;AACtF,CAAC"}
|
|
1
|
+
{"version":3,"file":"document.js","sourceRoot":"","sources":["../../src/embedding/document.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAmC,MAAM,8BAA8B,CAAC;AACjH,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAClD,OAAO,EAAE,qBAAqB,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AAGxF,MAAM,CAAC,MAAM,mCAAmC,GAAG,CAAU,CAAC;AAC9D,MAAM,CAAC,MAAM,sCAAsC,GAAG,CAAU,CAAC;AACjE,MAAM,CAAC,MAAM,wBAAwB,GAAG,qBAA8B,CAAC;AACvE,MAAM,CAAC,MAAM,4BAA4B,GAAG,EAAE,GAAG,IAAI,CAAC;AACtD,MAAM,CAAC,MAAM,sBAAsB,GAAG,kBAA2B,CAAC;AAWlE,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;AAClC,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AAE1D,SAAS,OAAO,CAAC,OAAe;IAC9B,MAAM,IAAI,YAAY,CAAC,kBAAkB,EAAE,OAAO,CAAC,CAAC;AACtD,CAAC;AAED,SAAS,aAAa,CAAC,KAAa,EAAE,KAAa;IACjD,MAAM,UAAU,GAAG,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAC3F,IAAI,iDAAiD,CAAC,IAAI,CAAC,UAAU,CAAC;QAAE,OAAO,CAAC,GAAG,KAAK,yCAAyC,CAAC,CAAC;IACnI,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAuB;IAChD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,IAAI,GAAG,CAAC,MAAM,GAAG,GAAG;YAAE,OAAO,CAAC,qCAAqC,CAAC,CAAC;IAC7H,CAAC;IACD,OAAO,iBAAiB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,aAAa,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;AACzE,CAAC;AAED,SAAS,eAAe,CAAC,KAAiB;IACxC,MAAM,SAAS,GAAG,KAAgC,CAAC;IACnD,IAAI,SAAS,CAAC,aAAa,KAAK,CAAC,IAAI,SAAS,CAAC,aAAa,KAAK,CAAC;QAAE,OAAO,EAAE,aAAa,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;IAC9G,MAAM,aAAa,GAAG,SAAS,CAAC,aAAa,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,qBAAqB,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;IAClH,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;IAC1F,OAAO;QACL,aAAa,EAAE,aAAsC;QACrD,OAAO,EAAE,OAAgC;KAC1C,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,KAAa,EAAE,QAAgB;IACnD,IAAI,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,UAAU,IAAI,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC/D,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACpC,IAAI,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;IAC/C,OAAO,GAAG,GAAG,CAAC,EAAE,CAAC;QACf,IAAI,CAAC;YACH,OAAO,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;QAChD,CAAC;QAAC,MAAM,CAAC;YACP,GAAG,IAAI,CAAC,CAAC;QACX,CAAC;IACH,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,YAAY,CAAC,KAAiB;IACrC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC1D,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,KAA6B;IAClE,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QAC/I,OAAO,CAAC,oCAAoC,CAAC,CAAC;IAChD,CAAC;IACD,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;IACzD,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC;IAC7F,MAAM,IAAI,GAAG,aAAa,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC/C,MAAM,IAAI,GAAG,iBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAChE,MAAM,YAAY,GAAG;QACnB,mBAAmB;QACnB,SAAS,KAAK,CAAC,IAAI,EAAE;QACrB,UAAU,KAAK,EAAE;QACjB,YAAY,OAAO,EAAE;QACrB,OAAO;QACP,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,KAAK,GAAG,EAAE,CAAC;QAChC,gBAAgB;QAChB,aAAa,CAAC,aAAa,CAAC;QAC5B,UAAU;QACV,aAAa,CAAC,OAAO,CAAC;QACtB,OAAO;KACR,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IACpB,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;IACnD,IAAI,aAAa,CAAC,UAAU,IAAI,4BAA4B,EAAE,CAAC;QAC7D,OAAO,CAAC,oDAAoD,CAAC,CAAC;IAChE,CAAC;IAED,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACvC,IAAI,IAAI,GAAG,YAAY,GAAG,IAAI,CAAC;IAC/B,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI,aAAa,CAAC,UAAU,GAAG,SAAS,CAAC,UAAU,GAAG,4BAA4B,EAAE,CAAC;QACnF,MAAM,MAAM,GAAG,KAAK,sBAAsB,EAAE,CAAC;QAC7C,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC3C,MAAM,SAAS,GAAG,4BAA4B,GAAG,aAAa,CAAC,UAAU,GAAG,WAAW,CAAC,UAAU,CAAC;QACnG,IAAI,SAAS,GAAG,CAAC;YAAE,OAAO,CAAC,oDAAoD,CAAC,CAAC;QACjF,IAAI,GAAG,YAAY,GAAG,YAAY,CAAC,IAAI,EAAE,SAAS,CAAC,GAAG,MAAM,CAAC;QAC7D,SAAS,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,aAAa,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IACvC,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;QAChC,MAAM,IAAI,YAAY,CAAC,oBAAoB,EAAE,wDAAwD,CAAC,CAAC;IACzG,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,YAAY,EAAE,YAAY,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC;AACtF,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,+BAA+B,CAAC,KAA6B;IAC3E,MAAM,QAAQ,GAAG,sBAAsB,CAAC,KAAK,CAAC,CAAC;IAC/C,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,uBAAuB,EAAE,qBAAqB,CAAC,CAAC;IACnF,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACnC,OAAO,MAAM,CAAC,MAAM,CAAC;QACnB,IAAI;QACJ,KAAK;QACL,YAAY,EAAE,YAAY,CAAC,KAAK,CAAC;QACjC,SAAS,EAAE,QAAQ,CAAC,SAAS;KAC9B,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,4BAA4B,CAAC,aAAqB,EAAE,MAAM,GAAG,WAAW;IACtF,IAAI,OAAO,aAAa,KAAK,QAAQ,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,KAAK,WAAW,EAAE,CAAC;QAC9F,OAAO,CAAC,8CAA8C,CAAC,CAAC;IAC1D,CAAC;IACD,OAAO,GAAG,MAAM,GAAG,aAAa,EAAE,CAAC;AACrC,CAAC"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { LocalEmbeddingProfileIdentity } from './types.js';
|
|
2
|
+
export interface LocalModelRuntime {
|
|
3
|
+
embed(inputs: readonly string[], signal?: AbortSignal): Promise<readonly Float32Array[]>;
|
|
4
|
+
dispose?(): Promise<void> | void;
|
|
5
|
+
}
|
|
6
|
+
export interface LocalModelLoader {
|
|
7
|
+
load(profile: LocalEmbeddingProfileIdentity, modelDirectory: string): Promise<LocalModelRuntime>;
|
|
8
|
+
}
|
|
9
|
+
interface FeatureExtractionTensor {
|
|
10
|
+
readonly dims: readonly number[];
|
|
11
|
+
readonly data: ArrayLike<number>;
|
|
12
|
+
}
|
|
13
|
+
interface FeatureExtractionPipeline {
|
|
14
|
+
(inputs: readonly string[], options: {
|
|
15
|
+
pooling: 'mean';
|
|
16
|
+
normalize: true;
|
|
17
|
+
}): Promise<FeatureExtractionTensor>;
|
|
18
|
+
dispose?(): Promise<void> | void;
|
|
19
|
+
}
|
|
20
|
+
export declare function createLocalTransformersModelLoader(options?: {
|
|
21
|
+
readonly pipeline?: (task: string, model: string, options: Record<string, unknown>) => Promise<FeatureExtractionPipeline>;
|
|
22
|
+
}): LocalModelLoader;
|
|
23
|
+
export {};
|
|
24
|
+
//# sourceMappingURL=local-model-loader.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"local-model-loader.d.ts","sourceRoot":"","sources":["../../src/embedding/local-model-loader.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,YAAY,CAAC;AAEhE,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,SAAS,YAAY,EAAE,CAAC,CAAC;IACzF,OAAO,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;CAClC;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,CAAC,OAAO,EAAE,6BAA6B,EAAE,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;CAClG;AAED,UAAU,uBAAuB;IAC/B,QAAQ,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,CAAC;IACjC,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;CAClC;AAED,UAAU,yBAAyB;IACjC,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,EAAE,OAAO,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,IAAI,CAAA;KAAE,GAAG,OAAO,CAAC,uBAAuB,CAAC,CAAC;IAC7G,OAAO,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;CAClC;AAqBD,wBAAgB,kCAAkC,CAChD,OAAO,GAAE;IAAE,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,OAAO,CAAC,yBAAyB,CAAC,CAAA;CAAO,GAC1I,gBAAgB,CAqClB"}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { KiokukoError } from '../errors.js';
|
|
2
|
+
function unavailable(message, cause) {
|
|
3
|
+
const error = new KiokukoError('SERVICE_UNAVAILABLE', message);
|
|
4
|
+
if (cause !== undefined)
|
|
5
|
+
Object.defineProperty(error, 'cause', { value: cause });
|
|
6
|
+
throw error;
|
|
7
|
+
}
|
|
8
|
+
function outputRows(output, expectedCount) {
|
|
9
|
+
if (!Array.isArray(output.dims) || output.dims.length !== 2 || output.dims[0] !== expectedCount || output.dims[1] !== 384) {
|
|
10
|
+
throw new KiokukoError('VALIDATION_ERROR', 'Local model returned unexpected embedding dimensions');
|
|
11
|
+
}
|
|
12
|
+
const vectors = [];
|
|
13
|
+
for (let row = 0; row < expectedCount; row += 1) {
|
|
14
|
+
const vector = new Float32Array(384);
|
|
15
|
+
for (let column = 0; column < 384; column += 1)
|
|
16
|
+
vector[column] = output.data[row * 384 + column] ?? Number.NaN;
|
|
17
|
+
vectors.push(vector);
|
|
18
|
+
}
|
|
19
|
+
return vectors;
|
|
20
|
+
}
|
|
21
|
+
export function createLocalTransformersModelLoader(options = {}) {
|
|
22
|
+
return {
|
|
23
|
+
load: async (profile, modelDirectory) => {
|
|
24
|
+
if (profile.providerKind !== 'local-transformers' || profile.dtype !== 'q8' || profile.pooling !== 'mean' || profile.normalize !== true || profile.dimensions !== 384) {
|
|
25
|
+
throw new KiokukoError('VALIDATION_ERROR', 'Local model profile contract is invalid');
|
|
26
|
+
}
|
|
27
|
+
let pipeline = options.pipeline;
|
|
28
|
+
if (pipeline === undefined) {
|
|
29
|
+
try {
|
|
30
|
+
const transformers = await import('@huggingface/transformers');
|
|
31
|
+
transformers.env.localModelPath = `${modelDirectory.endsWith('/') ? modelDirectory : `${modelDirectory}/`}`;
|
|
32
|
+
transformers.env.allowLocalModels = true;
|
|
33
|
+
transformers.env.allowRemoteModels = false;
|
|
34
|
+
pipeline = transformers.pipeline;
|
|
35
|
+
}
|
|
36
|
+
catch (error) {
|
|
37
|
+
unavailable('The optional local Transformers.js dependency is unavailable', error);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
if (pipeline === undefined)
|
|
41
|
+
unavailable('The local Transformers.js pipeline is unavailable');
|
|
42
|
+
const pipelineFactory = pipeline;
|
|
43
|
+
let extractor;
|
|
44
|
+
try {
|
|
45
|
+
extractor = await pipelineFactory('feature-extraction', modelDirectory, { dtype: profile.dtype });
|
|
46
|
+
}
|
|
47
|
+
catch (error) {
|
|
48
|
+
unavailable('The verified local embedding model could not be loaded offline', error);
|
|
49
|
+
}
|
|
50
|
+
return {
|
|
51
|
+
embed: async (inputs, signal) => {
|
|
52
|
+
if (signal?.aborted)
|
|
53
|
+
throw new KiokukoError('SERVICE_UNAVAILABLE', 'Local embedding was interrupted');
|
|
54
|
+
const output = await extractor(inputs, { pooling: 'mean', normalize: true });
|
|
55
|
+
if (signal?.aborted)
|
|
56
|
+
throw new KiokukoError('SERVICE_UNAVAILABLE', 'Local embedding was interrupted');
|
|
57
|
+
return outputRows(output, inputs.length);
|
|
58
|
+
},
|
|
59
|
+
dispose: async () => { await extractor.dispose?.(); },
|
|
60
|
+
};
|
|
61
|
+
},
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
//# sourceMappingURL=local-model-loader.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"local-model-loader.js","sourceRoot":"","sources":["../../src/embedding/local-model-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAsB5C,SAAS,WAAW,CAAC,OAAe,EAAE,KAAe;IACnD,MAAM,KAAK,GAAG,IAAI,YAAY,CAAC,qBAAqB,EAAE,OAAO,CAAC,CAAC;IAC/D,IAAI,KAAK,KAAK,SAAS;QAAE,MAAM,CAAC,cAAc,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;IACjF,MAAM,KAAK,CAAC;AACd,CAAC;AAED,SAAS,UAAU,CAAC,MAA+B,EAAE,aAAqB;IACxE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,aAAa,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;QAC1H,MAAM,IAAI,YAAY,CAAC,kBAAkB,EAAE,sDAAsD,CAAC,CAAC;IACrG,CAAC;IACD,MAAM,OAAO,GAAmB,EAAE,CAAC;IACnC,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,aAAa,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC;QAChD,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC;QACrC,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,GAAG,EAAE,MAAM,IAAI,CAAC;YAAE,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,GAAG,GAAG,GAAG,MAAM,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC;QAC/G,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACvB,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,kCAAkC,CAChD,UAAyI,EAAE;IAE3I,OAAO;QACL,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,cAAc,EAAE,EAAE;YACtC,IAAI,OAAO,CAAC,YAAY,KAAK,oBAAoB,IAAI,OAAO,CAAC,KAAK,KAAK,IAAI,IAAI,OAAO,CAAC,OAAO,KAAK,MAAM,IAAI,OAAO,CAAC,SAAS,KAAK,IAAI,IAAI,OAAO,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;gBACtK,MAAM,IAAI,YAAY,CAAC,kBAAkB,EAAE,yCAAyC,CAAC,CAAC;YACxF,CAAC;YACD,IAAI,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;YAChC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;gBAC3B,IAAI,CAAC;oBACH,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAC;oBAC/D,YAAY,CAAC,GAAG,CAAC,cAAc,GAAG,GAAG,cAAc,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,GAAG,cAAc,GAAG,EAAE,CAAC;oBAC5G,YAAY,CAAC,GAAG,CAAC,gBAAgB,GAAG,IAAI,CAAC;oBACzC,YAAY,CAAC,GAAG,CAAC,iBAAiB,GAAG,KAAK,CAAC;oBAC3C,QAAQ,GAAG,YAAY,CAAC,QAAmD,CAAC;gBAC9E,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,WAAW,CAAC,8DAA8D,EAAE,KAAK,CAAC,CAAC;gBACrF,CAAC;YACH,CAAC;YACD,IAAI,QAAQ,KAAK,SAAS;gBAAE,WAAW,CAAC,mDAAmD,CAAC,CAAC;YAC7F,MAAM,eAAe,GAAG,QAAQ,CAAC;YACjC,IAAI,SAAoC,CAAC;YACzC,IAAI,CAAC;gBACH,SAAS,GAAG,MAAM,eAAe,CAAC,oBAAoB,EAAE,cAAc,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;YACpG,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,WAAW,CAAC,gEAAgE,EAAE,KAAK,CAAC,CAAC;YACvF,CAAC;YACD,OAAO;gBACL,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE;oBAC9B,IAAI,MAAM,EAAE,OAAO;wBAAE,MAAM,IAAI,YAAY,CAAC,qBAAqB,EAAE,iCAAiC,CAAC,CAAC;oBACtG,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;oBAC7E,IAAI,MAAM,EAAE,OAAO;wBAAE,MAAM,IAAI,YAAY,CAAC,qBAAqB,EAAE,iCAAiC,CAAC,CAAC;oBACtG,OAAO,UAAU,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;gBAC3C,CAAC;gBACD,OAAO,EAAE,KAAK,IAAI,EAAE,GAAG,MAAM,SAAS,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;aACtD,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { type LocalModelLoader } from './local-model-loader.js';
|
|
2
|
+
import type { EmbeddingProvider, LocalEmbeddingProfileIdentity } from './types.js';
|
|
3
|
+
export interface LocalTransformersEmbeddingProviderOptions {
|
|
4
|
+
readonly profile: LocalEmbeddingProfileIdentity;
|
|
5
|
+
readonly modelDirectory: string;
|
|
6
|
+
readonly loader?: LocalModelLoader;
|
|
7
|
+
}
|
|
8
|
+
export declare class LocalTransformersEmbeddingProvider implements EmbeddingProvider {
|
|
9
|
+
#private;
|
|
10
|
+
readonly profile: LocalEmbeddingProfileIdentity;
|
|
11
|
+
constructor(options: LocalTransformersEmbeddingProviderOptions);
|
|
12
|
+
embed(inputs: readonly string[], options?: {
|
|
13
|
+
signal?: AbortSignal;
|
|
14
|
+
}): Promise<readonly Float32Array[]>;
|
|
15
|
+
close(): Promise<void>;
|
|
16
|
+
}
|
|
17
|
+
//# sourceMappingURL=local-transformers-provider.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"local-transformers-provider.d.ts","sourceRoot":"","sources":["../../src/embedding/local-transformers-provider.ts"],"names":[],"mappings":"AAEA,OAAO,EAAsC,KAAK,gBAAgB,EAA0B,MAAM,yBAAyB,CAAC;AAC5H,OAAO,KAAK,EAAE,iBAAiB,EAAE,6BAA6B,EAAE,MAAM,YAAY,CAAC;AAiBnF,MAAM,WAAW,yCAAyC;IACxD,QAAQ,CAAC,OAAO,EAAE,6BAA6B,CAAC;IAChD,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,MAAM,CAAC,EAAE,gBAAgB,CAAC;CACpC;AAED,qBAAa,kCAAmC,YAAW,iBAAiB;;IAC1E,QAAQ,CAAC,OAAO,EAAE,6BAA6B,CAAC;gBAMpC,OAAO,EAAE,yCAAyC;IAaxD,KAAK,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,EAAE,OAAO,GAAE;QAAE,MAAM,CAAC,EAAE,WAAW,CAAA;KAAO,GAAG,OAAO,CAAC,SAAS,YAAY,EAAE,CAAC;IAS1G,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAM7B"}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { KiokukoError } from '../errors.js';
|
|
2
|
+
import { normalizeVector } from './vector.js';
|
|
3
|
+
import { createLocalTransformersModelLoader } from './local-model-loader.js';
|
|
4
|
+
const MAX_INPUTS = 64;
|
|
5
|
+
const MAX_INPUT_BYTES = 32 * 1024;
|
|
6
|
+
function validateInputs(inputs) {
|
|
7
|
+
if (!Array.isArray(inputs) || inputs.length === 0 || inputs.length > MAX_INPUTS) {
|
|
8
|
+
throw new KiokukoError('VALIDATION_ERROR', 'Local embedding input batch is invalid');
|
|
9
|
+
}
|
|
10
|
+
return inputs.map((input) => {
|
|
11
|
+
if (typeof input !== 'string' || input.length === 0 || Buffer.byteLength(input, 'utf8') > MAX_INPUT_BYTES || /[\u0000-\u001f\u007f]/u.test(input)) {
|
|
12
|
+
throw new KiokukoError('VALIDATION_ERROR', 'Local embedding input is invalid');
|
|
13
|
+
}
|
|
14
|
+
return input;
|
|
15
|
+
});
|
|
16
|
+
}
|
|
17
|
+
export class LocalTransformersEmbeddingProvider {
|
|
18
|
+
profile;
|
|
19
|
+
#modelDirectory;
|
|
20
|
+
#loader;
|
|
21
|
+
#runtime;
|
|
22
|
+
#closed = false;
|
|
23
|
+
constructor(options) {
|
|
24
|
+
if (options.profile.providerKind !== 'local-transformers')
|
|
25
|
+
throw new KiokukoError('VALIDATION_ERROR', 'Local provider profile is invalid');
|
|
26
|
+
this.profile = Object.freeze({ ...options.profile });
|
|
27
|
+
this.#modelDirectory = options.modelDirectory;
|
|
28
|
+
this.#loader = options.loader ?? createLocalTransformersModelLoader();
|
|
29
|
+
}
|
|
30
|
+
async #load() {
|
|
31
|
+
if (this.#closed)
|
|
32
|
+
throw new KiokukoError('SERVICE_UNAVAILABLE', 'Local embedding provider is closed');
|
|
33
|
+
this.#runtime ??= this.#loader.load(this.profile, this.#modelDirectory);
|
|
34
|
+
return this.#runtime;
|
|
35
|
+
}
|
|
36
|
+
async embed(inputs, options = {}) {
|
|
37
|
+
const validInputs = validateInputs(inputs);
|
|
38
|
+
if (options.signal?.aborted)
|
|
39
|
+
throw new KiokukoError('SERVICE_UNAVAILABLE', 'Local embedding was interrupted');
|
|
40
|
+
const runtime = await this.#load();
|
|
41
|
+
const vectors = await runtime.embed(validInputs, options.signal);
|
|
42
|
+
if (vectors.length !== validInputs.length)
|
|
43
|
+
throw new KiokukoError('VALIDATION_ERROR', 'Local model returned the wrong number of embeddings');
|
|
44
|
+
return vectors.map((vector) => normalizeVector(vector, this.profile.dimensions));
|
|
45
|
+
}
|
|
46
|
+
async close() {
|
|
47
|
+
if (this.#closed)
|
|
48
|
+
return;
|
|
49
|
+
this.#closed = true;
|
|
50
|
+
const runtime = this.#runtime === undefined ? undefined : await this.#runtime;
|
|
51
|
+
await runtime?.dispose?.();
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
//# sourceMappingURL=local-transformers-provider.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"local-transformers-provider.js","sourceRoot":"","sources":["../../src/embedding/local-transformers-provider.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,kCAAkC,EAAiD,MAAM,yBAAyB,CAAC;AAG5H,MAAM,UAAU,GAAG,EAAE,CAAC;AACtB,MAAM,eAAe,GAAG,EAAE,GAAG,IAAI,CAAC;AAElC,SAAS,cAAc,CAAC,MAAyB;IAC/C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,GAAG,UAAU,EAAE,CAAC;QAChF,MAAM,IAAI,YAAY,CAAC,kBAAkB,EAAE,wCAAwC,CAAC,CAAC;IACvF,CAAC;IACD,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QAC1B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,eAAe,IAAI,wBAAwB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YAClJ,MAAM,IAAI,YAAY,CAAC,kBAAkB,EAAE,kCAAkC,CAAC,CAAC;QACjF,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC,CAAC,CAAC;AACL,CAAC;AAQD,MAAM,OAAO,kCAAkC;IACpC,OAAO,CAAgC;IACvC,eAAe,CAAS;IACxB,OAAO,CAAmB;IACnC,QAAQ,CAAyC;IACjD,OAAO,GAAG,KAAK,CAAC;IAEhB,YAAY,OAAkD;QAC5D,IAAI,OAAO,CAAC,OAAO,CAAC,YAAY,KAAK,oBAAoB;YAAE,MAAM,IAAI,YAAY,CAAC,kBAAkB,EAAE,mCAAmC,CAAC,CAAC;QAC3I,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;QACrD,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC;QAC9C,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,MAAM,IAAI,kCAAkC,EAAE,CAAC;IACxE,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,YAAY,CAAC,qBAAqB,EAAE,oCAAoC,CAAC,CAAC;QACtG,IAAI,CAAC,QAAQ,KAAK,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC;QACxE,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,MAAyB,EAAE,UAAoC,EAAE;QAC3E,MAAM,WAAW,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;QAC3C,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;YAAE,MAAM,IAAI,YAAY,CAAC,qBAAqB,EAAE,iCAAiC,CAAC,CAAC;QAC9G,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,WAAW,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QACjE,IAAI,OAAO,CAAC,MAAM,KAAK,WAAW,CAAC,MAAM;YAAE,MAAM,IAAI,YAAY,CAAC,kBAAkB,EAAE,qDAAqD,CAAC,CAAC;QAC7I,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,eAAe,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;IACnF,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO;QACzB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC;QAC9E,MAAM,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;IAC7B,CAAC;CACF"}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { type VerifiedModelManifest } from './model-manifest.js';
|
|
2
|
+
import type { LocalEmbeddingPreset } from './presets/manifest.js';
|
|
3
|
+
export interface ModelDownloadProgress {
|
|
4
|
+
readonly file: string;
|
|
5
|
+
readonly completedBytes: number;
|
|
6
|
+
readonly totalBytes: number;
|
|
7
|
+
}
|
|
8
|
+
export interface DownloadedModel {
|
|
9
|
+
readonly directory: string;
|
|
10
|
+
readonly manifest: VerifiedModelManifest;
|
|
11
|
+
}
|
|
12
|
+
export interface ModelDownloader {
|
|
13
|
+
download(preset: LocalEmbeddingPreset, stagingDirectory: string, options?: {
|
|
14
|
+
readonly signal?: AbortSignal;
|
|
15
|
+
readonly onProgress?: (progress: ModelDownloadProgress) => void;
|
|
16
|
+
}): Promise<DownloadedModel>;
|
|
17
|
+
}
|
|
18
|
+
type HubDownloadFile = (params: {
|
|
19
|
+
repo: {
|
|
20
|
+
type: 'model';
|
|
21
|
+
name: string;
|
|
22
|
+
};
|
|
23
|
+
path: string;
|
|
24
|
+
revision: string;
|
|
25
|
+
xet: boolean;
|
|
26
|
+
fetch?: typeof fetch;
|
|
27
|
+
}) => Promise<Blob | null>;
|
|
28
|
+
export declare function createHuggingFaceModelDownloader(options?: {
|
|
29
|
+
readonly downloadFile?: HubDownloadFile;
|
|
30
|
+
}): ModelDownloader;
|
|
31
|
+
export {};
|
|
32
|
+
//# sourceMappingURL=model-download.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"model-download.d.ts","sourceRoot":"","sources":["../../src/embedding/model-download.ts"],"names":[],"mappings":"AAGA,OAAO,EAA+C,KAAK,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAC9G,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAElE,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,qBAAqB,CAAC;CAC1C;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CACN,MAAM,EAAE,oBAAoB,EAC5B,gBAAgB,EAAE,MAAM,EACxB,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC;QAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,qBAAqB,KAAK,IAAI,CAAA;KAAE,GAC3G,OAAO,CAAC,eAAe,CAAC,CAAC;CAC7B;AAED,KAAK,eAAe,GAAG,CAAC,MAAM,EAAE;IAC9B,IAAI,EAAE;QAAE,IAAI,EAAE,OAAO,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,OAAO,CAAC;IACb,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACtB,KAAK,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;AAM3B,wBAAgB,gCAAgC,CAAC,OAAO,GAAE;IAAE,QAAQ,CAAC,YAAY,CAAC,EAAE,eAAe,CAAA;CAAO,GAAG,eAAe,CA6B3H"}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { mkdir, writeFile } from 'node:fs/promises';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { KiokukoError } from '../errors.js';
|
|
4
|
+
import { createModelManifest, validatePresetManifest } from './model-manifest.js';
|
|
5
|
+
function aborted(signal) {
|
|
6
|
+
if (signal?.aborted)
|
|
7
|
+
throw new KiokukoError('SERVICE_UNAVAILABLE', 'Embedding model download was interrupted');
|
|
8
|
+
}
|
|
9
|
+
export function createHuggingFaceModelDownloader(options = {}) {
|
|
10
|
+
return {
|
|
11
|
+
download: async (preset, stagingDirectory, downloadOptions = {}) => {
|
|
12
|
+
validatePresetManifest(preset);
|
|
13
|
+
const downloadFile = options.downloadFile ?? (await import('@huggingface/hub')).downloadFile;
|
|
14
|
+
const manifest = createModelManifest(preset);
|
|
15
|
+
await mkdir(stagingDirectory, { recursive: true, mode: 0o700 });
|
|
16
|
+
let completedBytes = 0;
|
|
17
|
+
for (const file of preset.files) {
|
|
18
|
+
aborted(downloadOptions.signal);
|
|
19
|
+
const blob = await downloadFile({
|
|
20
|
+
repo: { type: 'model', name: preset.artifactRepository },
|
|
21
|
+
path: file.path,
|
|
22
|
+
revision: preset.revision,
|
|
23
|
+
xet: true,
|
|
24
|
+
});
|
|
25
|
+
if (blob === null)
|
|
26
|
+
throw new KiokukoError('NOT_FOUND', `Pinned model artifact is unavailable: ${file.path}`);
|
|
27
|
+
const bytes = new Uint8Array(await blob.arrayBuffer());
|
|
28
|
+
aborted(downloadOptions.signal);
|
|
29
|
+
if (bytes.byteLength !== file.size)
|
|
30
|
+
throw new KiokukoError('INTEGRITY_ERROR', `Downloaded model artifact size mismatch: ${file.path}`);
|
|
31
|
+
const target = path.join(stagingDirectory, file.path);
|
|
32
|
+
await mkdir(path.dirname(target), { recursive: true, mode: 0o700 });
|
|
33
|
+
await writeFile(target, bytes, { mode: 0o600, flag: 'wx' });
|
|
34
|
+
completedBytes += bytes.byteLength;
|
|
35
|
+
downloadOptions.onProgress?.({ file: file.path, completedBytes, totalBytes: manifest.totalBytes });
|
|
36
|
+
}
|
|
37
|
+
return { directory: stagingDirectory, manifest };
|
|
38
|
+
},
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
//# sourceMappingURL=model-download.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"model-download.js","sourceRoot":"","sources":["../../src/embedding/model-download.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,mBAAmB,EAAE,sBAAsB,EAA8B,MAAM,qBAAqB,CAAC;AA8B9G,SAAS,OAAO,CAAC,MAA+B;IAC9C,IAAI,MAAM,EAAE,OAAO;QAAE,MAAM,IAAI,YAAY,CAAC,qBAAqB,EAAE,0CAA0C,CAAC,CAAC;AACjH,CAAC;AAED,MAAM,UAAU,gCAAgC,CAAC,UAAuD,EAAE;IACxG,OAAO;QACL,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,gBAAgB,EAAE,eAAe,GAAG,EAAE,EAAE,EAAE;YACjE,sBAAsB,CAAC,MAAM,CAAC,CAAC;YAC/B,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,CAAC,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAC,CAAC,YAA+B,CAAC;YAChH,MAAM,QAAQ,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC;YAC7C,MAAM,KAAK,CAAC,gBAAgB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;YAChE,IAAI,cAAc,GAAG,CAAC,CAAC;YACvB,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;gBAChC,OAAO,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;gBAChC,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC;oBAC9B,IAAI,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,kBAAkB,EAAE;oBACxD,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,QAAQ,EAAE,MAAM,CAAC,QAAQ;oBACzB,GAAG,EAAE,IAAI;iBACV,CAAC,CAAC;gBACH,IAAI,IAAI,KAAK,IAAI;oBAAE,MAAM,IAAI,YAAY,CAAC,WAAW,EAAE,yCAAyC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC7G,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;gBACvD,OAAO,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;gBAChC,IAAI,KAAK,CAAC,UAAU,KAAK,IAAI,CAAC,IAAI;oBAAE,MAAM,IAAI,YAAY,CAAC,iBAAiB,EAAE,4CAA4C,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;gBACvI,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;gBACtD,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;gBACpE,MAAM,SAAS,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC5D,cAAc,IAAI,KAAK,CAAC,UAAU,CAAC;gBACnC,eAAe,CAAC,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,cAAc,EAAE,UAAU,EAAE,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;YACrG,CAAC;YACD,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,QAAQ,EAAE,CAAC;QACnD,CAAC;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { type PathEnvironment } from '../config/paths.js';
|
|
2
|
+
import { type ModelDownloader, type ModelDownloadProgress } from './model-download.js';
|
|
3
|
+
import type { LocalEmbeddingPreset } from './presets/manifest.js';
|
|
4
|
+
export interface InstalledModel {
|
|
5
|
+
readonly installation: 'installed' | 'reused';
|
|
6
|
+
readonly directory: string;
|
|
7
|
+
readonly relativePath: string;
|
|
8
|
+
readonly totalBytes: number;
|
|
9
|
+
readonly manifestHash: string;
|
|
10
|
+
}
|
|
11
|
+
export interface InstallModelOptions extends PathEnvironment {
|
|
12
|
+
readonly downloader?: ModelDownloader;
|
|
13
|
+
readonly signal?: AbortSignal;
|
|
14
|
+
readonly onProgress?: (progress: ModelDownloadProgress) => void;
|
|
15
|
+
}
|
|
16
|
+
export declare function installEmbeddingModel(preset: LocalEmbeddingPreset, options?: InstallModelOptions): Promise<InstalledModel>;
|
|
17
|
+
//# sourceMappingURL=model-installation.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"model-installation.d.ts","sourceRoot":"","sources":["../../src/embedding/model-installation.ts"],"names":[],"mappings":"AAIA,OAAO,EAA+B,KAAK,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAEvF,OAAO,EAAoC,KAAK,eAAe,EAAE,KAAK,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AACzH,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAElE,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,YAAY,EAAE,WAAW,GAAG,QAAQ,CAAC;IAC9C,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,WAAW,mBAAoB,SAAQ,eAAe;IAC1D,QAAQ,CAAC,UAAU,CAAC,EAAE,eAAe,CAAC;IACtC,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC;IAC9B,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,qBAAqB,KAAK,IAAI,CAAC;CACjE;AAkBD,wBAAsB,qBAAqB,CACzC,MAAM,EAAE,oBAAoB,EAC5B,OAAO,GAAE,mBAAwB,GAChC,OAAO,CAAC,cAAc,CAAC,CA0CzB"}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { chmod, lstat, mkdir, rename, rm, writeFile } from 'node:fs/promises';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
import { KiokukoError } from '../errors.js';
|
|
5
|
+
import { getEmbeddingPresetDirectory } from '../config/paths.js';
|
|
6
|
+
import { assertNoUnexpectedModelFiles, MODEL_MANIFEST_FILENAME, serializeModelManifest, verifyModelDirectory } from './model-manifest.js';
|
|
7
|
+
import { createHuggingFaceModelDownloader } from './model-download.js';
|
|
8
|
+
function relativeModelPath(preset) {
|
|
9
|
+
return path.posix.join('models', 'embeddings', preset.id, preset.revision);
|
|
10
|
+
}
|
|
11
|
+
async function existingDirectory(pathname) {
|
|
12
|
+
try {
|
|
13
|
+
const stat = await lstat(pathname);
|
|
14
|
+
if (!stat.isDirectory() || stat.isSymbolicLink())
|
|
15
|
+
throw new KiokukoError('SECURITY_REJECTION', 'Existing model installation is not a private directory');
|
|
16
|
+
return true;
|
|
17
|
+
}
|
|
18
|
+
catch (error) {
|
|
19
|
+
if (error instanceof KiokukoError)
|
|
20
|
+
throw error;
|
|
21
|
+
if (error instanceof Error && 'code' in error && error.code === 'ENOENT')
|
|
22
|
+
return false;
|
|
23
|
+
throw error;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
export async function installEmbeddingModel(preset, options = {}) {
|
|
27
|
+
const downloader = options.downloader ?? createHuggingFaceModelDownloader();
|
|
28
|
+
const finalDirectory = getEmbeddingPresetDirectory(preset.id, preset.revision, options);
|
|
29
|
+
if (await existingDirectory(finalDirectory)) {
|
|
30
|
+
const manifest = await verifyModelDirectory(finalDirectory, preset);
|
|
31
|
+
await assertNoUnexpectedModelFiles(finalDirectory, preset);
|
|
32
|
+
return { installation: 'reused', directory: finalDirectory, relativePath: relativeModelPath(preset), totalBytes: manifest.totalBytes, manifestHash: manifest.artifactManifestHash };
|
|
33
|
+
}
|
|
34
|
+
const stagingParent = path.dirname(path.dirname(finalDirectory));
|
|
35
|
+
const stagingDirectory = path.join(stagingParent, '.staging', randomUUID());
|
|
36
|
+
await mkdir(stagingDirectory, { recursive: true, mode: 0o700 });
|
|
37
|
+
try {
|
|
38
|
+
const downloaded = await downloader.download(preset, stagingDirectory, {
|
|
39
|
+
...(options.signal === undefined ? {} : { signal: options.signal }),
|
|
40
|
+
...(options.onProgress === undefined ? {} : { onProgress: options.onProgress }),
|
|
41
|
+
});
|
|
42
|
+
const manifest = await verifyModelDirectory(downloaded.directory, preset);
|
|
43
|
+
await assertNoUnexpectedModelFiles(downloaded.directory, preset);
|
|
44
|
+
await writeFile(path.join(downloaded.directory, MODEL_MANIFEST_FILENAME), serializeModelManifest(manifest), { mode: 0o600, flag: 'wx' });
|
|
45
|
+
await chmod(downloaded.directory, 0o700);
|
|
46
|
+
await mkdir(path.dirname(finalDirectory), { recursive: true, mode: 0o700 });
|
|
47
|
+
if (await existingDirectory(finalDirectory)) {
|
|
48
|
+
const existing = await verifyModelDirectory(finalDirectory, preset);
|
|
49
|
+
if (existing.artifactManifestHash !== manifest.artifactManifestHash)
|
|
50
|
+
throw new KiokukoError('CONFLICT', 'A different model installation already exists for this preset revision');
|
|
51
|
+
return { installation: 'reused', directory: finalDirectory, relativePath: relativeModelPath(preset), totalBytes: existing.totalBytes, manifestHash: existing.artifactManifestHash };
|
|
52
|
+
}
|
|
53
|
+
try {
|
|
54
|
+
await rename(downloaded.directory, finalDirectory);
|
|
55
|
+
return { installation: 'installed', directory: finalDirectory, relativePath: relativeModelPath(preset), totalBytes: manifest.totalBytes, manifestHash: manifest.artifactManifestHash };
|
|
56
|
+
}
|
|
57
|
+
catch (error) {
|
|
58
|
+
if (error instanceof Error && 'code' in error && error.code === 'ENOTEMPTY'
|
|
59
|
+
&& await existingDirectory(finalDirectory)) {
|
|
60
|
+
const existing = await verifyModelDirectory(finalDirectory, preset);
|
|
61
|
+
return { installation: 'reused', directory: finalDirectory, relativePath: relativeModelPath(preset), totalBytes: existing.totalBytes, manifestHash: existing.artifactManifestHash };
|
|
62
|
+
}
|
|
63
|
+
throw error;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
catch (error) {
|
|
67
|
+
await rm(stagingDirectory, { recursive: true, force: true }).catch(() => undefined);
|
|
68
|
+
throw error;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
//# sourceMappingURL=model-installation.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"model-installation.js","sourceRoot":"","sources":["../../src/embedding/model-installation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC9E,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,2BAA2B,EAAwB,MAAM,oBAAoB,CAAC;AACvF,OAAO,EAAE,4BAA4B,EAAE,uBAAuB,EAAE,sBAAsB,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAC1I,OAAO,EAAE,gCAAgC,EAAoD,MAAM,qBAAqB,CAAC;AAiBzH,SAAS,iBAAiB,CAAC,MAA4B;IACrD,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,YAAY,EAAE,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;AAC7E,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,QAAgB;IAC/C,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC;QACnC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,IAAI,CAAC,cAAc,EAAE;YAAE,MAAM,IAAI,YAAY,CAAC,oBAAoB,EAAE,wDAAwD,CAAC,CAAC;QACzJ,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,YAAY;YAAE,MAAM,KAAK,CAAC;QAC/C,IAAI,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,KAAK,CAAC;QAClH,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,MAA4B,EAC5B,UAA+B,EAAE;IAEjC,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,gCAAgC,EAAE,CAAC;IAC5E,MAAM,cAAc,GAAG,2BAA2B,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACxF,IAAI,MAAM,iBAAiB,CAAC,cAAc,CAAC,EAAE,CAAC;QAC5C,MAAM,QAAQ,GAAG,MAAM,oBAAoB,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;QACpE,MAAM,4BAA4B,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;QAC3D,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,SAAS,EAAE,cAAc,EAAE,YAAY,EAAE,iBAAiB,CAAC,MAAM,CAAC,EAAE,UAAU,EAAE,QAAQ,CAAC,UAAU,EAAE,YAAY,EAAE,QAAQ,CAAC,oBAAoB,EAAE,CAAC;IACtL,CAAC;IAED,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC;IACjE,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC,CAAC;IAC5E,MAAM,KAAK,CAAC,gBAAgB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAChE,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,MAAM,EAAE,gBAAgB,EAAE;YACrE,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;YACnE,GAAG,CAAC,OAAO,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC;SAChF,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,oBAAoB,CAAC,UAAU,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAC1E,MAAM,4BAA4B,CAAC,UAAU,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QACjE,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,uBAAuB,CAAC,EAAE,sBAAsB,CAAC,QAAQ,CAAC,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QACzI,MAAM,KAAK,CAAC,UAAU,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QACzC,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAC5E,IAAI,MAAM,iBAAiB,CAAC,cAAc,CAAC,EAAE,CAAC;YAC5C,MAAM,QAAQ,GAAG,MAAM,oBAAoB,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;YACpE,IAAI,QAAQ,CAAC,oBAAoB,KAAK,QAAQ,CAAC,oBAAoB;gBAAE,MAAM,IAAI,YAAY,CAAC,UAAU,EAAE,wEAAwE,CAAC,CAAC;YAClL,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,SAAS,EAAE,cAAc,EAAE,YAAY,EAAE,iBAAiB,CAAC,MAAM,CAAC,EAAE,UAAU,EAAE,QAAQ,CAAC,UAAU,EAAE,YAAY,EAAE,QAAQ,CAAC,oBAAoB,EAAE,CAAC;QACtL,CAAC;QACD,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,UAAU,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;YACnD,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,SAAS,EAAE,cAAc,EAAE,YAAY,EAAE,iBAAiB,CAAC,MAAM,CAAC,EAAE,UAAU,EAAE,QAAQ,CAAC,UAAU,EAAE,YAAY,EAAE,QAAQ,CAAC,oBAAoB,EAAE,CAAC;QACzL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,IAAK,KAA+B,CAAC,IAAI,KAAK,WAAW;mBACjG,MAAM,iBAAiB,CAAC,cAAc,CAAC,EAAE,CAAC;gBAC7C,MAAM,QAAQ,GAAG,MAAM,oBAAoB,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;gBACpE,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,SAAS,EAAE,cAAc,EAAE,YAAY,EAAE,iBAAiB,CAAC,MAAM,CAAC,EAAE,UAAU,EAAE,QAAQ,CAAC,UAAU,EAAE,YAAY,EAAE,QAAQ,CAAC,oBAAoB,EAAE,CAAC;YACtL,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,EAAE,CAAC,gBAAgB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QACpF,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { LocalEmbeddingPreset, LocalEmbeddingPresetFile } from './presets/manifest.js';
|
|
2
|
+
export declare const MODEL_MANIFEST_FILENAME = "kiokuko-model-manifest.json";
|
|
3
|
+
export declare const MAX_MODEL_BYTES: number;
|
|
4
|
+
export interface VerifiedModelManifest {
|
|
5
|
+
readonly schemaVersion: 1;
|
|
6
|
+
readonly presetId: 'local-small';
|
|
7
|
+
readonly repositoryId: string;
|
|
8
|
+
readonly revision: string;
|
|
9
|
+
readonly artifactManifestHash: string;
|
|
10
|
+
readonly files: readonly LocalEmbeddingPresetFile[];
|
|
11
|
+
readonly totalBytes: number;
|
|
12
|
+
}
|
|
13
|
+
export declare function validatePresetManifest(preset: LocalEmbeddingPreset): void;
|
|
14
|
+
export declare function createModelManifest(preset: LocalEmbeddingPreset): VerifiedModelManifest;
|
|
15
|
+
export declare function serializeModelManifest(manifest: VerifiedModelManifest): string;
|
|
16
|
+
export declare function verifyModelDirectory(directory: string, preset: LocalEmbeddingPreset): Promise<VerifiedModelManifest>;
|
|
17
|
+
export declare function assertNoUnexpectedModelFiles(directory: string, preset: LocalEmbeddingPreset): Promise<void>;
|
|
18
|
+
//# sourceMappingURL=model-manifest.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"model-manifest.d.ts","sourceRoot":"","sources":["../../src/embedding/model-manifest.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,oBAAoB,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AAG5F,eAAO,MAAM,uBAAuB,gCAAgC,CAAC;AACrE,eAAO,MAAM,eAAe,QAAoB,CAAC;AAEjD,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,aAAa,EAAE,CAAC,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAC;IACjC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,oBAAoB,EAAE,MAAM,CAAC;IACtC,QAAQ,CAAC,KAAK,EAAE,SAAS,wBAAwB,EAAE,CAAC;IACpD,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAUD,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,oBAAoB,GAAG,IAAI,CAuBzE;AAED,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,oBAAoB,GAAG,qBAAqB,CAYvF;AAED,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,qBAAqB,GAAG,MAAM,CAE9E;AAED,wBAAsB,oBAAoB,CACxC,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,oBAAoB,GAC3B,OAAO,CAAC,qBAAqB,CAAC,CAUhC;AAED,wBAAsB,4BAA4B,CAChD,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,oBAAoB,GAC3B,OAAO,CAAC,IAAI,CAAC,CAWf"}
|