@deployxai/dxc 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +131 -0
- package/dist/chunks/chunk-I6VZLNRZ.js +2118 -0
- package/dist/chunks/chunk-XIHX5YAF.js +16391 -0
- package/dist/chunks/knowledge-Q6MHPG6I.js +1248 -0
- package/dist/chunks/monitor-VPRVRQIS.js +694 -0
- package/dist/index.js +32367 -0
- package/docs/00-project-context.md +125 -0
- package/docs/01-north-star-architecture.md +234 -0
- package/docs/02-mvp-technical-design.md +553 -0
- package/docs/03-domain-state-api.md +599 -0
- package/docs/04-security-and-operations.md +413 -0
- package/docs/05-delivery-plan.md +407 -0
- package/docs/README.md +44 -0
- package/docs/decisions/0001-initial-architecture.md +57 -0
- package/docs/decisions/0002-mongodb-environment-boundary.md +42 -0
- package/docs/decisions/0003-staged-production-topology.md +33 -0
- package/docs/decisions/0004-local-first-agent-research-runtime.md +71 -0
- package/docs/decisions/0005-official-skill-orchestration-and-local-content-memory.md +97 -0
- package/docs/decisions/0006-separate-wechat-user-login-from-account-authorization.md +87 -0
- package/docs/decisions/0007-explicit-personal-wechat-start.md +67 -0
- package/docs/decisions/0008-end-to-end-content-workflow-continuity.md +115 -0
- package/docs/decisions/0009-privileged-multitenant-draft-scheduling.md +36 -0
- package/docs/decisions/0009-versioned-cloud-template-catalog.md +39 -0
- package/docs/eight-stage-implementation-audit.md +62 -0
- package/docs/first-user-guide.md +187 -0
- package/docs/history/content-forge-prd-v0.2-summary.md +81 -0
- package/docs/local-development.md +511 -0
- package/docs/references/aliyun-oss-production-setup.md +89 -0
- package/docs/references/legacy-content-to-wechat-contract.md +223 -0
- package/docs/references/renderer-compatibility-report.md +68 -0
- package/docs/references/source-inventory.md +179 -0
- package/docs/references/wechat-renderer-platform-validation.md +92 -0
- package/docs/references/wechat-third-party-platform-setup.md +159 -0
- package/docs/references/wechat-website-login-setup.md +137 -0
- package/docs/references/wemd-template-attribution.md +25 -0
- package/docs/research-monitoring-design.md +235 -0
- package/docs/todo-preview-local-first.md +31 -0
- package/docs/workbuddy-first-user-runbook.md +246 -0
- package/docs//345/221/230/345/267/245BCDE/347/232/204skill/employee-b-research-analyst/SKILL.md +230 -0
- package/docs//345/221/230/345/267/245BCDE/347/232/204skill/employee-c-outline-architect/SKILL.md +194 -0
- package/docs//345/221/230/345/267/245BCDE/347/232/204skill/employee-d-content-writer/SKILL.md +296 -0
- package/docs//345/221/230/345/267/245BCDE/347/232/204skill/employee-e-visual-designer/SKILL.md +268 -0
- package/package.json +25 -0
- package/skills/dxc-article-outline/SKILL.md +82 -0
- package/skills/dxc-article-outline/agents/openai.yaml +6 -0
- package/skills/dxc-article-outline/references/outline-methods.md +38 -0
- package/skills/dxc-article-write/SKILL.md +85 -0
- package/skills/dxc-article-write/agents/openai.yaml +6 -0
- package/skills/dxc-article-write/references/writing-methods.md +42 -0
- package/skills/dxc-content-brief/SKILL.md +81 -0
- package/skills/dxc-content-brief/agents/openai.yaml +6 -0
- package/skills/dxc-content-brief/references/brief-method.md +34 -0
- package/skills/dxc-content-review/SKILL.md +84 -0
- package/skills/dxc-content-review/agents/openai.yaml +6 -0
- package/skills/dxc-content-review/references/review-checklist.md +35 -0
- package/skills/dxc-content-workflow/SKILL.md +190 -0
- package/skills/dxc-content-workflow/agents/openai.yaml +6 -0
- package/skills/dxc-content-workflow/references/catalog.json +136 -0
- package/skills/dxc-content-workflow/references/onboarding-questions.md +107 -0
- package/skills/dxc-content-workflow/references/stage-contract.md +70 -0
- package/skills/dxc-research/SKILL.md +110 -0
- package/skills/dxc-research/agents/openai.yaml +6 -0
- package/skills/dxc-research/references/research-method.md +53 -0
- package/skills/dxc-title-write/SKILL.md +112 -0
- package/skills/dxc-title-write/agents/openai.yaml +6 -0
- package/skills/dxc-title-write/references/title-methods.md +26 -0
- package/skills/dxc-visual-plan/SKILL.md +119 -0
- package/skills/dxc-visual-plan/agents/openai.yaml +6 -0
- package/skills/dxc-visual-plan/references/visual-methods.md +35 -0
- package/skills/dxc-wechat-publisher/SKILL.md +157 -0
- package/skills/dxc-wechat-publisher/agents/openai.yaml +6 -0
|
@@ -0,0 +1,1248 @@
|
|
|
1
|
+
import { createRequire as __dxcCreateRequire } from "node:module"; const require = __dxcCreateRequire(import.meta.url);
|
|
2
|
+
import {
|
|
3
|
+
CONTENT_KNOWLEDGE_CHUNKER_VERSION,
|
|
4
|
+
CONTENT_KNOWLEDGE_EMBEDDING_DIMENSIONS,
|
|
5
|
+
CONTENT_KNOWLEDGE_EMBEDDING_MODEL,
|
|
6
|
+
CONTENT_KNOWLEDGE_EMBEDDING_QUANTIZATION,
|
|
7
|
+
CONTENT_KNOWLEDGE_EMBEDDING_REVISION,
|
|
8
|
+
CONTENT_KNOWLEDGE_SCHEMA_VERSION,
|
|
9
|
+
LocalStateError,
|
|
10
|
+
contentKnowledgeImportResultSchema,
|
|
11
|
+
contentKnowledgeModelSchema,
|
|
12
|
+
contentKnowledgeRemoveResultSchema,
|
|
13
|
+
contentKnowledgeSearchModeSchema,
|
|
14
|
+
contentKnowledgeSearchResultSchema,
|
|
15
|
+
contentKnowledgeStatusResultSchema,
|
|
16
|
+
ensurePrivateDirectory,
|
|
17
|
+
ensurePrivateRegularFile,
|
|
18
|
+
readPrivateBuffer,
|
|
19
|
+
readPrivateFile,
|
|
20
|
+
writePrivateBuffer,
|
|
21
|
+
writePrivateJson
|
|
22
|
+
} from "./chunk-XIHX5YAF.js";
|
|
23
|
+
|
|
24
|
+
// apps/cli/src/knowledge.ts
|
|
25
|
+
import { createHash as createHash2, randomUUID } from "node:crypto";
|
|
26
|
+
import { lstat, readFile, stat } from "node:fs/promises";
|
|
27
|
+
import path2 from "node:path";
|
|
28
|
+
import { DatabaseSync } from "node:sqlite";
|
|
29
|
+
|
|
30
|
+
// apps/cli/src/knowledge-embedding.ts
|
|
31
|
+
import { createHash } from "node:crypto";
|
|
32
|
+
import path from "node:path";
|
|
33
|
+
var MODEL_MARKER_NAME = "content-embedding-model.json";
|
|
34
|
+
var MODEL_BASE_URL = `https://huggingface.co/${CONTENT_KNOWLEDGE_EMBEDDING_MODEL}/resolve/${CONTENT_KNOWLEDGE_EMBEDDING_REVISION}`;
|
|
35
|
+
var ONNX_RUNTIME_MODULE_NAME = "onnxruntime-web";
|
|
36
|
+
var TOKENIZERS_MODULE_NAME = "@huggingface/tokenizers";
|
|
37
|
+
var QUERY_INSTRUCTION = "\u4E3A\u8FD9\u4E2A\u53E5\u5B50\u751F\u6210\u8868\u793A\u4EE5\u7528\u4E8E\u68C0\u7D22\u76F8\u5173\u6587\u7AE0\uFF1A";
|
|
38
|
+
var MAXIMUM_TOKENS = 512;
|
|
39
|
+
var EMBEDDING_BATCH_SIZE = 8;
|
|
40
|
+
var MODEL_DOWNLOAD_TIMEOUT_MS = 3e5;
|
|
41
|
+
var MODEL_ARTIFACTS = [
|
|
42
|
+
{
|
|
43
|
+
maximumBytes: 25 * 1024 * 1024,
|
|
44
|
+
name: "model_quantized.onnx",
|
|
45
|
+
remotePath: "onnx/model_quantized.onnx",
|
|
46
|
+
sha256: "15b717c382bcb518ba457b93ea6850ede7f4f1cd8937454aa06972366cd19bcc"
|
|
47
|
+
},
|
|
48
|
+
{
|
|
49
|
+
maximumBytes: 512 * 1024,
|
|
50
|
+
name: "tokenizer.json",
|
|
51
|
+
remotePath: "tokenizer.json",
|
|
52
|
+
sha256: "48cea5d44424912a6fd1ea647bf4fe50b55ab8b1e5879c3275f80e339e8fae26"
|
|
53
|
+
},
|
|
54
|
+
{
|
|
55
|
+
maximumBytes: 8 * 1024,
|
|
56
|
+
name: "tokenizer_config.json",
|
|
57
|
+
remotePath: "tokenizer_config.json",
|
|
58
|
+
sha256: "e6f3b96db926a37d4039995fbf5ad17de158dfb8f6343d607e4dbaad18d75f5a"
|
|
59
|
+
}
|
|
60
|
+
];
|
|
61
|
+
function artifactPath(cacheDirectory, artifact) {
|
|
62
|
+
return path.join(cacheDirectory, artifact.name);
|
|
63
|
+
}
|
|
64
|
+
function sha256(value) {
|
|
65
|
+
return createHash("sha256").update(value).digest("hex");
|
|
66
|
+
}
|
|
67
|
+
async function readVerifiedArtifact(cacheDirectory, artifact) {
|
|
68
|
+
try {
|
|
69
|
+
const contents = await readPrivateBuffer(
|
|
70
|
+
artifactPath(cacheDirectory, artifact),
|
|
71
|
+
artifact.maximumBytes
|
|
72
|
+
);
|
|
73
|
+
if (sha256(contents) !== artifact.sha256) {
|
|
74
|
+
throw new Error("local model artifact checksum is invalid");
|
|
75
|
+
}
|
|
76
|
+
return contents;
|
|
77
|
+
} catch (error) {
|
|
78
|
+
if (error instanceof LocalStateError && error.failure === "missing") {
|
|
79
|
+
return void 0;
|
|
80
|
+
}
|
|
81
|
+
throw error;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
async function downloadArtifact(cacheDirectory, artifact) {
|
|
85
|
+
const response = await fetch(`${MODEL_BASE_URL}/${artifact.remotePath}`, {
|
|
86
|
+
headers: {
|
|
87
|
+
"user-agent": "dxc-cli/0.1"
|
|
88
|
+
},
|
|
89
|
+
redirect: "follow",
|
|
90
|
+
signal: AbortSignal.timeout(MODEL_DOWNLOAD_TIMEOUT_MS)
|
|
91
|
+
});
|
|
92
|
+
if (!response.ok) {
|
|
93
|
+
throw new Error("local model artifact download failed");
|
|
94
|
+
}
|
|
95
|
+
const declaredSize = Number(response.headers.get("content-length"));
|
|
96
|
+
if (Number.isFinite(declaredSize) && declaredSize > artifact.maximumBytes) {
|
|
97
|
+
throw new Error("local model artifact is too large");
|
|
98
|
+
}
|
|
99
|
+
const contents = Buffer.from(await response.arrayBuffer());
|
|
100
|
+
if (contents.byteLength > artifact.maximumBytes || sha256(contents) !== artifact.sha256) {
|
|
101
|
+
throw new Error("downloaded model artifact checksum is invalid");
|
|
102
|
+
}
|
|
103
|
+
await writePrivateBuffer(artifactPath(cacheDirectory, artifact), contents);
|
|
104
|
+
return contents;
|
|
105
|
+
}
|
|
106
|
+
async function ensureModelArtifacts(cacheDirectory) {
|
|
107
|
+
await ensurePrivateDirectory(cacheDirectory);
|
|
108
|
+
const artifacts = /* @__PURE__ */ new Map();
|
|
109
|
+
for (const artifact of MODEL_ARTIFACTS) {
|
|
110
|
+
const cached = await readVerifiedArtifact(cacheDirectory, artifact);
|
|
111
|
+
artifacts.set(artifact.name, cached ?? await downloadArtifact(cacheDirectory, artifact));
|
|
112
|
+
}
|
|
113
|
+
return artifacts;
|
|
114
|
+
}
|
|
115
|
+
function modelMarkerPath(cacheDirectory) {
|
|
116
|
+
return path.join(cacheDirectory, MODEL_MARKER_NAME);
|
|
117
|
+
}
|
|
118
|
+
function parseJsonObject(value, label) {
|
|
119
|
+
const parsed = JSON.parse(value.toString("utf8"));
|
|
120
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
|
121
|
+
throw new Error(`${label} is invalid`);
|
|
122
|
+
}
|
|
123
|
+
return parsed;
|
|
124
|
+
}
|
|
125
|
+
async function loadOnnxRuntime() {
|
|
126
|
+
const loaded = await import(ONNX_RUNTIME_MODULE_NAME);
|
|
127
|
+
if (typeof loaded !== "object" || loaded === null || !("env" in loaded) || typeof loaded.env !== "object" || loaded.env === null || !("InferenceSession" in loaded) || typeof loaded.InferenceSession !== "object" && typeof loaded.InferenceSession !== "function" || loaded.InferenceSession === null || !("Tensor" in loaded) || typeof loaded.Tensor !== "function") {
|
|
128
|
+
throw new Error("ONNX runtime module is invalid");
|
|
129
|
+
}
|
|
130
|
+
return loaded;
|
|
131
|
+
}
|
|
132
|
+
async function loadTokenizerConstructor() {
|
|
133
|
+
const loaded = await import(TOKENIZERS_MODULE_NAME);
|
|
134
|
+
if (typeof loaded !== "object" || loaded === null || !("Tokenizer" in loaded) || typeof loaded.Tokenizer !== "function") {
|
|
135
|
+
throw new Error("tokenizer module is invalid");
|
|
136
|
+
}
|
|
137
|
+
return loaded.Tokenizer;
|
|
138
|
+
}
|
|
139
|
+
async function createEmbeddingRuntime(cacheDirectory) {
|
|
140
|
+
const artifacts = await ensureModelArtifacts(cacheDirectory);
|
|
141
|
+
const tokenizerJson = artifacts.get("tokenizer.json");
|
|
142
|
+
const tokenizerConfig = artifacts.get("tokenizer_config.json");
|
|
143
|
+
const model = artifacts.get("model_quantized.onnx");
|
|
144
|
+
if (tokenizerJson === void 0 || tokenizerConfig === void 0 || model === void 0) {
|
|
145
|
+
throw new Error("local model artifacts are incomplete");
|
|
146
|
+
}
|
|
147
|
+
const Tokenizer = await loadTokenizerConstructor();
|
|
148
|
+
const tokenizer = new Tokenizer(
|
|
149
|
+
parseJsonObject(tokenizerJson, "tokenizer"),
|
|
150
|
+
parseJsonObject(tokenizerConfig, "tokenizer config")
|
|
151
|
+
);
|
|
152
|
+
const onnx = await loadOnnxRuntime();
|
|
153
|
+
onnx.env.wasm.numThreads = 1;
|
|
154
|
+
const session = await onnx.InferenceSession.create(model, {
|
|
155
|
+
executionProviders: ["wasm"],
|
|
156
|
+
graphOptimizationLevel: "all"
|
|
157
|
+
});
|
|
158
|
+
if (!["input_ids", "attention_mask", "token_type_ids"].every(
|
|
159
|
+
(name) => session.inputNames.includes(name)
|
|
160
|
+
) || !session.outputNames.includes("last_hidden_state")) {
|
|
161
|
+
await session.release();
|
|
162
|
+
throw new Error("local model input or output contract is invalid");
|
|
163
|
+
}
|
|
164
|
+
return { onnx, session, tokenizer };
|
|
165
|
+
}
|
|
166
|
+
function truncateEncoding(encoding, tokenizer) {
|
|
167
|
+
const tokenTypeIds = encoding.token_type_ids ?? encoding.ids.map(() => 0);
|
|
168
|
+
if (encoding.ids.length <= MAXIMUM_TOKENS) {
|
|
169
|
+
return {
|
|
170
|
+
attentionMask: encoding.attention_mask,
|
|
171
|
+
ids: encoding.ids,
|
|
172
|
+
tokenTypeIds
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
const separatorId = tokenizer.token_to_id("[SEP]");
|
|
176
|
+
if (separatorId === void 0) {
|
|
177
|
+
throw new Error("tokenizer separator token is missing");
|
|
178
|
+
}
|
|
179
|
+
const ids = encoding.ids.slice(0, MAXIMUM_TOKENS);
|
|
180
|
+
ids[MAXIMUM_TOKENS - 1] = separatorId;
|
|
181
|
+
const attentionMask = encoding.attention_mask.slice(0, MAXIMUM_TOKENS);
|
|
182
|
+
attentionMask[MAXIMUM_TOKENS - 1] = 1;
|
|
183
|
+
const truncatedTokenTypes = tokenTypeIds.slice(0, MAXIMUM_TOKENS);
|
|
184
|
+
truncatedTokenTypes[MAXIMUM_TOKENS - 1] = 0;
|
|
185
|
+
return {
|
|
186
|
+
attentionMask,
|
|
187
|
+
ids,
|
|
188
|
+
tokenTypeIds: truncatedTokenTypes
|
|
189
|
+
};
|
|
190
|
+
}
|
|
191
|
+
function paddedTensorData(inputs, field, sequenceLength, padId) {
|
|
192
|
+
const values = new BigInt64Array(inputs.length * sequenceLength);
|
|
193
|
+
for (const [rowIndex, input] of inputs.entries()) {
|
|
194
|
+
const row = input[field];
|
|
195
|
+
for (let columnIndex = 0; columnIndex < sequenceLength; columnIndex += 1) {
|
|
196
|
+
const fallback = field === "ids" ? padId : 0;
|
|
197
|
+
values[rowIndex * sequenceLength + columnIndex] = BigInt(row[columnIndex] ?? fallback);
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
return values;
|
|
201
|
+
}
|
|
202
|
+
async function embedBatch(runtime, texts) {
|
|
203
|
+
if (texts.length === 0) {
|
|
204
|
+
return [];
|
|
205
|
+
}
|
|
206
|
+
const inputs = texts.map(
|
|
207
|
+
(text) => truncateEncoding(
|
|
208
|
+
runtime.tokenizer.encode(text, { return_token_type_ids: true }),
|
|
209
|
+
runtime.tokenizer
|
|
210
|
+
)
|
|
211
|
+
);
|
|
212
|
+
const sequenceLength = Math.max(...inputs.map((input) => input.ids.length));
|
|
213
|
+
const dimensions = [inputs.length, sequenceLength];
|
|
214
|
+
const padId = runtime.tokenizer.token_to_id("[PAD]");
|
|
215
|
+
if (padId === void 0) {
|
|
216
|
+
throw new Error("tokenizer padding token is missing");
|
|
217
|
+
}
|
|
218
|
+
const output = await runtime.session.run({
|
|
219
|
+
attention_mask: new runtime.onnx.Tensor(
|
|
220
|
+
"int64",
|
|
221
|
+
paddedTensorData(inputs, "attentionMask", sequenceLength, padId),
|
|
222
|
+
dimensions
|
|
223
|
+
),
|
|
224
|
+
input_ids: new runtime.onnx.Tensor(
|
|
225
|
+
"int64",
|
|
226
|
+
paddedTensorData(inputs, "ids", sequenceLength, padId),
|
|
227
|
+
dimensions
|
|
228
|
+
),
|
|
229
|
+
token_type_ids: new runtime.onnx.Tensor(
|
|
230
|
+
"int64",
|
|
231
|
+
paddedTensorData(inputs, "tokenTypeIds", sequenceLength, padId),
|
|
232
|
+
dimensions
|
|
233
|
+
)
|
|
234
|
+
});
|
|
235
|
+
const tensor = output["last_hidden_state"];
|
|
236
|
+
if (tensor === void 0 || tensor.type !== "float32" || tensor.dims.length !== 3 || tensor.dims[0] !== inputs.length || tensor.dims[1] !== sequenceLength || tensor.dims[2] !== CONTENT_KNOWLEDGE_EMBEDDING_DIMENSIONS) {
|
|
237
|
+
throw new Error("embedding output contract is invalid");
|
|
238
|
+
}
|
|
239
|
+
const vectors = [];
|
|
240
|
+
const rowStride = sequenceLength * CONTENT_KNOWLEDGE_EMBEDDING_DIMENSIONS;
|
|
241
|
+
for (let rowIndex = 0; rowIndex < inputs.length; rowIndex += 1) {
|
|
242
|
+
const vector = new Float32Array(CONTENT_KNOWLEDGE_EMBEDDING_DIMENSIONS);
|
|
243
|
+
let squaredNorm = 0;
|
|
244
|
+
const offset = rowIndex * rowStride;
|
|
245
|
+
for (let index = 0; index < vector.length; index += 1) {
|
|
246
|
+
const value = tensor.data[offset + index];
|
|
247
|
+
if (typeof value !== "number" || !Number.isFinite(value)) {
|
|
248
|
+
throw new Error("embedding output contains an invalid value");
|
|
249
|
+
}
|
|
250
|
+
vector[index] = value;
|
|
251
|
+
squaredNorm += value * value;
|
|
252
|
+
}
|
|
253
|
+
const norm = Math.sqrt(squaredNorm);
|
|
254
|
+
if (!Number.isFinite(norm) || norm === 0) {
|
|
255
|
+
throw new Error("embedding output norm is invalid");
|
|
256
|
+
}
|
|
257
|
+
for (let index = 0; index < vector.length; index += 1) {
|
|
258
|
+
vector[index] = (vector[index] ?? 0) / norm;
|
|
259
|
+
}
|
|
260
|
+
vectors.push(vector);
|
|
261
|
+
}
|
|
262
|
+
return vectors;
|
|
263
|
+
}
|
|
264
|
+
async function isKnowledgeModelCached(cacheDirectory) {
|
|
265
|
+
try {
|
|
266
|
+
const raw = await readPrivateFile(modelMarkerPath(cacheDirectory), 16 * 1024);
|
|
267
|
+
const value = JSON.parse(raw);
|
|
268
|
+
if (!contentKnowledgeModelSchema.safeParse(value.model).success || value.runtime !== "onnxruntime-web@1.22.0" || !Array.isArray(value.artifacts) || value.artifacts.join(",") !== MODEL_ARTIFACTS.map((artifact) => artifact.sha256).join(",")) {
|
|
269
|
+
return false;
|
|
270
|
+
}
|
|
271
|
+
return (await Promise.all(
|
|
272
|
+
MODEL_ARTIFACTS.map((artifact) => readVerifiedArtifact(cacheDirectory, artifact))
|
|
273
|
+
)).every((artifact) => artifact !== void 0);
|
|
274
|
+
} catch {
|
|
275
|
+
return false;
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
var LocalBgeKnowledgeEmbedder = class {
|
|
279
|
+
model = contentKnowledgeModelSchema.parse({
|
|
280
|
+
dimensions: CONTENT_KNOWLEDGE_EMBEDDING_DIMENSIONS,
|
|
281
|
+
id: CONTENT_KNOWLEDGE_EMBEDDING_MODEL,
|
|
282
|
+
quantization: CONTENT_KNOWLEDGE_EMBEDDING_QUANTIZATION,
|
|
283
|
+
revision: CONTENT_KNOWLEDGE_EMBEDDING_REVISION
|
|
284
|
+
});
|
|
285
|
+
#cacheDirectory;
|
|
286
|
+
#now;
|
|
287
|
+
#runtimePromise;
|
|
288
|
+
constructor(cacheDirectory, now = () => /* @__PURE__ */ new Date()) {
|
|
289
|
+
this.#cacheDirectory = cacheDirectory;
|
|
290
|
+
this.#now = now;
|
|
291
|
+
}
|
|
292
|
+
async #runtime() {
|
|
293
|
+
if (this.#runtimePromise === void 0) {
|
|
294
|
+
this.#runtimePromise = (async () => {
|
|
295
|
+
const runtime = await createEmbeddingRuntime(this.#cacheDirectory);
|
|
296
|
+
await writePrivateJson(modelMarkerPath(this.#cacheDirectory), {
|
|
297
|
+
artifacts: MODEL_ARTIFACTS.map((artifact) => artifact.sha256),
|
|
298
|
+
model: this.model,
|
|
299
|
+
readyAt: this.#now().toISOString(),
|
|
300
|
+
runtime: "onnxruntime-web@1.22.0"
|
|
301
|
+
});
|
|
302
|
+
return runtime;
|
|
303
|
+
})();
|
|
304
|
+
}
|
|
305
|
+
return this.#runtimePromise;
|
|
306
|
+
}
|
|
307
|
+
async embedDocuments(texts) {
|
|
308
|
+
const runtime = await this.#runtime();
|
|
309
|
+
const results = [];
|
|
310
|
+
for (let index = 0; index < texts.length; index += EMBEDDING_BATCH_SIZE) {
|
|
311
|
+
results.push(
|
|
312
|
+
...await embedBatch(runtime, texts.slice(index, index + EMBEDDING_BATCH_SIZE))
|
|
313
|
+
);
|
|
314
|
+
}
|
|
315
|
+
return results;
|
|
316
|
+
}
|
|
317
|
+
async embedQuery(query) {
|
|
318
|
+
const runtime = await this.#runtime();
|
|
319
|
+
const [vector] = await embedBatch(runtime, [`${QUERY_INSTRUCTION}${query}`]);
|
|
320
|
+
if (vector === void 0) {
|
|
321
|
+
throw new Error("embedding output is empty");
|
|
322
|
+
}
|
|
323
|
+
return vector;
|
|
324
|
+
}
|
|
325
|
+
};
|
|
326
|
+
|
|
327
|
+
// apps/cli/src/knowledge.ts
|
|
328
|
+
var DATABASE_DISPLAY_PATH = "~/.dxc/content-memory.sqlite";
|
|
329
|
+
var MAXIMUM_ARTICLE_BYTES = 5 * 1024 * 1024;
|
|
330
|
+
var MAXIMUM_CHUNK_CHARACTERS = 420;
|
|
331
|
+
var TARGET_CHUNK_CHARACTERS = 280;
|
|
332
|
+
var MAXIMUM_RESULTS = 20;
|
|
333
|
+
var SEARCH_CANDIDATE_LIMIT = 80;
|
|
334
|
+
var RRF_CONSTANT = 60;
|
|
335
|
+
var SEMANTIC_MINIMUM_SIMILARITY = 0.28;
|
|
336
|
+
var SEMANTIC_MAXIMUM_DISTANCE_FROM_BEST = 0.12;
|
|
337
|
+
var SUPPORTED_EXTENSIONS = /* @__PURE__ */ new Set([".md", ".markdown", ".txt"]);
|
|
338
|
+
var UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu;
|
|
339
|
+
var KnowledgeCliError = class extends Error {
|
|
340
|
+
constructor(code, message) {
|
|
341
|
+
super(message);
|
|
342
|
+
this.code = code;
|
|
343
|
+
this.name = "KnowledgeCliError";
|
|
344
|
+
}
|
|
345
|
+
code;
|
|
346
|
+
};
|
|
347
|
+
function errorCode(error) {
|
|
348
|
+
return typeof error === "object" && error !== null && "code" in error ? String(error.code) : void 0;
|
|
349
|
+
}
|
|
350
|
+
function sha2562(value) {
|
|
351
|
+
return createHash2("sha256").update(value).digest("hex");
|
|
352
|
+
}
|
|
353
|
+
function normalizedMarkdown(value) {
|
|
354
|
+
return value.replaceAll("\r\n", "\n").replaceAll("\r", "\n").trim();
|
|
355
|
+
}
|
|
356
|
+
function removeFrontmatter(value) {
|
|
357
|
+
return value.replace(/^---\n[\s\S]*?\n---(?:\n|$)/u, "");
|
|
358
|
+
}
|
|
359
|
+
function plainMarkdown(value) {
|
|
360
|
+
return value.replace(/<!--[\s\S]*?-->/gu, " ").replace(/!\[([^\]]*)\]\([^)]+\)/gu, "$1").replace(/\[([^\]]+)\]\([^)]+\)/gu, "$1").replace(/<[^>]+>/gu, " ").replace(/```[^\n]*\n?/gu, "").replace(/`([^`]+)`/gu, "$1").replace(/^\s{0,3}(?:[-*+]|\d+[.)])\s+/gmu, "").replace(/^\s{0,3}>\s?/gmu, "").replace(/[*_~]{1,3}/gu, "").replace(/[ \t]+/gu, " ").replace(/\n+/gu, " ").trim();
|
|
361
|
+
}
|
|
362
|
+
function extractTitle(value, sourceLabel) {
|
|
363
|
+
const withoutFrontmatter = removeFrontmatter(value);
|
|
364
|
+
const heading = withoutFrontmatter.match(/^\s{0,3}#\s+(.+)$/mu)?.[1];
|
|
365
|
+
const fallback = path2.basename(sourceLabel, path2.extname(sourceLabel));
|
|
366
|
+
const title = plainMarkdown(heading ?? fallback).slice(0, 300).trim();
|
|
367
|
+
if (title.length === 0) {
|
|
368
|
+
throw new KnowledgeCliError("DXC_KNOWLEDGE_ARTICLE_INVALID", "Article title is empty");
|
|
369
|
+
}
|
|
370
|
+
return title;
|
|
371
|
+
}
|
|
372
|
+
function splitOversizedText(value) {
|
|
373
|
+
if ([...value].length <= MAXIMUM_CHUNK_CHARACTERS) {
|
|
374
|
+
return [value];
|
|
375
|
+
}
|
|
376
|
+
const sentences = value.match(/[^。!?!?;;\n]+(?:[。!?!?;;]+|$)/gu)?.map((item) => item.trim()) ?? [value];
|
|
377
|
+
const parts = [];
|
|
378
|
+
let current = "";
|
|
379
|
+
const flush = () => {
|
|
380
|
+
if (current.length > 0) {
|
|
381
|
+
parts.push(current);
|
|
382
|
+
current = "";
|
|
383
|
+
}
|
|
384
|
+
};
|
|
385
|
+
for (const sentence of sentences) {
|
|
386
|
+
if ([...sentence].length > MAXIMUM_CHUNK_CHARACTERS) {
|
|
387
|
+
flush();
|
|
388
|
+
const characters = [...sentence];
|
|
389
|
+
for (let index = 0; index < characters.length; index += MAXIMUM_CHUNK_CHARACTERS) {
|
|
390
|
+
parts.push(characters.slice(index, index + MAXIMUM_CHUNK_CHARACTERS).join(""));
|
|
391
|
+
}
|
|
392
|
+
continue;
|
|
393
|
+
}
|
|
394
|
+
const combined = current.length === 0 ? sentence : `${current} ${sentence}`;
|
|
395
|
+
if ([...combined].length > MAXIMUM_CHUNK_CHARACTERS) {
|
|
396
|
+
flush();
|
|
397
|
+
current = sentence;
|
|
398
|
+
} else {
|
|
399
|
+
current = combined;
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
flush();
|
|
403
|
+
return parts;
|
|
404
|
+
}
|
|
405
|
+
function chunkHistoricalArticle(markdown) {
|
|
406
|
+
const value = removeFrontmatter(markdown);
|
|
407
|
+
const lines = value.split("\n");
|
|
408
|
+
const paragraphs = [];
|
|
409
|
+
let heading = null;
|
|
410
|
+
let paragraphLines = [];
|
|
411
|
+
const flushParagraph = () => {
|
|
412
|
+
const text = plainMarkdown(paragraphLines.join("\n"));
|
|
413
|
+
paragraphLines = [];
|
|
414
|
+
if (text.length === 0) {
|
|
415
|
+
return;
|
|
416
|
+
}
|
|
417
|
+
for (const part of splitOversizedText(text)) {
|
|
418
|
+
paragraphs.push({ heading, text: part });
|
|
419
|
+
}
|
|
420
|
+
};
|
|
421
|
+
for (const line of lines) {
|
|
422
|
+
const headingMatch = line.match(/^\s{0,3}#{1,6}\s+(.+)$/u);
|
|
423
|
+
if (headingMatch !== null) {
|
|
424
|
+
flushParagraph();
|
|
425
|
+
heading = plainMarkdown(headingMatch[1] ?? "").slice(0, 300).trim() || null;
|
|
426
|
+
continue;
|
|
427
|
+
}
|
|
428
|
+
if (line.trim().length === 0) {
|
|
429
|
+
flushParagraph();
|
|
430
|
+
} else {
|
|
431
|
+
paragraphLines.push(line);
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
flushParagraph();
|
|
435
|
+
const merged = [];
|
|
436
|
+
for (const paragraph of paragraphs) {
|
|
437
|
+
const previous = merged.at(-1);
|
|
438
|
+
const combined = previous === void 0 ? paragraph.text : `${previous.body}
|
|
439
|
+
|
|
440
|
+
${paragraph.text}`;
|
|
441
|
+
if (previous !== void 0 && previous.heading === paragraph.heading && [...previous.body].length < TARGET_CHUNK_CHARACTERS && [...combined].length <= MAXIMUM_CHUNK_CHARACTERS) {
|
|
442
|
+
merged[merged.length - 1] = {
|
|
443
|
+
body: combined,
|
|
444
|
+
heading: previous.heading
|
|
445
|
+
};
|
|
446
|
+
} else {
|
|
447
|
+
merged.push({
|
|
448
|
+
body: paragraph.text,
|
|
449
|
+
heading: paragraph.heading
|
|
450
|
+
});
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
return merged.map((chunk, ordinal) => ({
|
|
454
|
+
...chunk,
|
|
455
|
+
ordinal,
|
|
456
|
+
sha256: sha2562(chunk.body)
|
|
457
|
+
}));
|
|
458
|
+
}
|
|
459
|
+
async function prepareArticle(currentDirectory, inputPath) {
|
|
460
|
+
const absolutePath = path2.resolve(currentDirectory, inputPath);
|
|
461
|
+
const extension = path2.extname(absolutePath).toLowerCase();
|
|
462
|
+
if (!SUPPORTED_EXTENSIONS.has(extension)) {
|
|
463
|
+
throw new KnowledgeCliError(
|
|
464
|
+
"DXC_KNOWLEDGE_ARTICLE_INVALID",
|
|
465
|
+
"Only Markdown and text articles can be imported"
|
|
466
|
+
);
|
|
467
|
+
}
|
|
468
|
+
let metadata;
|
|
469
|
+
try {
|
|
470
|
+
metadata = await stat(absolutePath);
|
|
471
|
+
} catch (error) {
|
|
472
|
+
if (errorCode(error) === "ENOENT") {
|
|
473
|
+
throw new KnowledgeCliError("DXC_KNOWLEDGE_INPUT_NOT_FOUND", "Article input was not found");
|
|
474
|
+
}
|
|
475
|
+
throw new KnowledgeCliError("DXC_KNOWLEDGE_IO_FAILED", "Unable to read article input");
|
|
476
|
+
}
|
|
477
|
+
if (!metadata.isFile()) {
|
|
478
|
+
throw new KnowledgeCliError("DXC_KNOWLEDGE_ARTICLE_INVALID", "Article input is invalid");
|
|
479
|
+
}
|
|
480
|
+
if (metadata.size > MAXIMUM_ARTICLE_BYTES) {
|
|
481
|
+
throw new KnowledgeCliError(
|
|
482
|
+
"DXC_KNOWLEDGE_FILE_TOO_LARGE",
|
|
483
|
+
"Article input exceeds the 5 MB limit"
|
|
484
|
+
);
|
|
485
|
+
}
|
|
486
|
+
let content;
|
|
487
|
+
try {
|
|
488
|
+
content = normalizedMarkdown(await readFile(absolutePath, "utf8"));
|
|
489
|
+
} catch {
|
|
490
|
+
throw new KnowledgeCliError("DXC_KNOWLEDGE_IO_FAILED", "Unable to read article input");
|
|
491
|
+
}
|
|
492
|
+
if (content.length === 0 || content.includes("\0")) {
|
|
493
|
+
throw new KnowledgeCliError("DXC_KNOWLEDGE_ARTICLE_INVALID", "Article content is invalid");
|
|
494
|
+
}
|
|
495
|
+
const sourceLabel = path2.basename(absolutePath).slice(0, 255);
|
|
496
|
+
const chunks = chunkHistoricalArticle(content);
|
|
497
|
+
if (chunks.length === 0) {
|
|
498
|
+
throw new KnowledgeCliError(
|
|
499
|
+
"DXC_KNOWLEDGE_ARTICLE_INVALID",
|
|
500
|
+
"Article has no indexable content"
|
|
501
|
+
);
|
|
502
|
+
}
|
|
503
|
+
return {
|
|
504
|
+
chunks,
|
|
505
|
+
contentSha256: sha2562(content),
|
|
506
|
+
sourceKey: sha2562(absolutePath),
|
|
507
|
+
sourceLabel,
|
|
508
|
+
title: extractTitle(content, sourceLabel)
|
|
509
|
+
};
|
|
510
|
+
}
|
|
511
|
+
function sqlRow(value) {
|
|
512
|
+
if (typeof value !== "object" || value === null) {
|
|
513
|
+
throw new KnowledgeCliError("DXC_KNOWLEDGE_STATE_INVALID", "Knowledge state is invalid");
|
|
514
|
+
}
|
|
515
|
+
return value;
|
|
516
|
+
}
|
|
517
|
+
function stringColumn(row, name) {
|
|
518
|
+
const value = row[name];
|
|
519
|
+
if (typeof value !== "string") {
|
|
520
|
+
throw new KnowledgeCliError("DXC_KNOWLEDGE_STATE_INVALID", "Knowledge state is invalid");
|
|
521
|
+
}
|
|
522
|
+
return value;
|
|
523
|
+
}
|
|
524
|
+
function numberColumn(row, name) {
|
|
525
|
+
const value = row[name];
|
|
526
|
+
if (typeof value !== "number" || !Number.isFinite(value)) {
|
|
527
|
+
throw new KnowledgeCliError("DXC_KNOWLEDGE_STATE_INVALID", "Knowledge state is invalid");
|
|
528
|
+
}
|
|
529
|
+
return value;
|
|
530
|
+
}
|
|
531
|
+
function articleRow(value) {
|
|
532
|
+
const row = sqlRow(value);
|
|
533
|
+
return {
|
|
534
|
+
chunkCount: numberColumn(row, "chunk_count"),
|
|
535
|
+
chunkerVersion: stringColumn(row, "chunker_version"),
|
|
536
|
+
contentSha256: stringColumn(row, "content_sha256"),
|
|
537
|
+
id: stringColumn(row, "id"),
|
|
538
|
+
importedAt: stringColumn(row, "imported_at"),
|
|
539
|
+
sourceLabel: stringColumn(row, "source_label"),
|
|
540
|
+
title: stringColumn(row, "title"),
|
|
541
|
+
updatedAt: stringColumn(row, "updated_at")
|
|
542
|
+
};
|
|
543
|
+
}
|
|
544
|
+
function articleSummary(row) {
|
|
545
|
+
return {
|
|
546
|
+
articleId: row.id,
|
|
547
|
+
chunkCount: row.chunkCount,
|
|
548
|
+
contentSha256: row.contentSha256,
|
|
549
|
+
importedAt: row.importedAt,
|
|
550
|
+
sourceLabel: row.sourceLabel,
|
|
551
|
+
title: row.title,
|
|
552
|
+
updatedAt: row.updatedAt
|
|
553
|
+
};
|
|
554
|
+
}
|
|
555
|
+
function vectorBuffer(vector) {
|
|
556
|
+
return Buffer.from(vector.buffer, vector.byteOffset, vector.byteLength);
|
|
557
|
+
}
|
|
558
|
+
function vectorFromColumn(value) {
|
|
559
|
+
if (value === null) {
|
|
560
|
+
return null;
|
|
561
|
+
}
|
|
562
|
+
if (!(value instanceof Uint8Array)) {
|
|
563
|
+
throw new KnowledgeCliError("DXC_KNOWLEDGE_STATE_INVALID", "Knowledge vector is invalid");
|
|
564
|
+
}
|
|
565
|
+
const expectedBytes = CONTENT_KNOWLEDGE_EMBEDDING_DIMENSIONS * Float32Array.BYTES_PER_ELEMENT;
|
|
566
|
+
if (value.byteLength !== expectedBytes) {
|
|
567
|
+
throw new KnowledgeCliError("DXC_KNOWLEDGE_STATE_INVALID", "Knowledge vector is invalid");
|
|
568
|
+
}
|
|
569
|
+
return new Float32Array(Uint8Array.from(value).buffer);
|
|
570
|
+
}
|
|
571
|
+
function chunkRow(value) {
|
|
572
|
+
const row = sqlRow(value);
|
|
573
|
+
const headingValue = row["heading"];
|
|
574
|
+
if (headingValue !== null && typeof headingValue !== "string") {
|
|
575
|
+
throw new KnowledgeCliError("DXC_KNOWLEDGE_STATE_INVALID", "Knowledge state is invalid");
|
|
576
|
+
}
|
|
577
|
+
return {
|
|
578
|
+
articleId: stringColumn(row, "article_id"),
|
|
579
|
+
body: stringColumn(row, "body"),
|
|
580
|
+
chunkSha256: stringColumn(row, "chunk_sha256"),
|
|
581
|
+
embedding: vectorFromColumn(row["embedding"]),
|
|
582
|
+
heading: headingValue,
|
|
583
|
+
id: stringColumn(row, "id"),
|
|
584
|
+
ordinal: numberColumn(row, "ordinal"),
|
|
585
|
+
sourceLabel: stringColumn(row, "source_label"),
|
|
586
|
+
title: stringColumn(row, "title")
|
|
587
|
+
};
|
|
588
|
+
}
|
|
589
|
+
function databaseSchema(database) {
|
|
590
|
+
const versionRow = sqlRow(database.prepare("PRAGMA user_version").get());
|
|
591
|
+
const version = numberColumn(versionRow, "user_version");
|
|
592
|
+
if (version !== 0 && version !== Number(CONTENT_KNOWLEDGE_SCHEMA_VERSION)) {
|
|
593
|
+
throw new KnowledgeCliError(
|
|
594
|
+
"DXC_KNOWLEDGE_SCHEMA_UNSUPPORTED",
|
|
595
|
+
"Knowledge database version is unsupported"
|
|
596
|
+
);
|
|
597
|
+
}
|
|
598
|
+
database.exec(`
|
|
599
|
+
PRAGMA foreign_keys = ON;
|
|
600
|
+
PRAGMA busy_timeout = 5000;
|
|
601
|
+
PRAGMA journal_mode = WAL;
|
|
602
|
+
PRAGMA synchronous = NORMAL;
|
|
603
|
+
|
|
604
|
+
CREATE TABLE IF NOT EXISTS articles (
|
|
605
|
+
id TEXT PRIMARY KEY,
|
|
606
|
+
source_key TEXT NOT NULL UNIQUE,
|
|
607
|
+
source_label TEXT NOT NULL,
|
|
608
|
+
title TEXT NOT NULL,
|
|
609
|
+
content_sha256 TEXT NOT NULL,
|
|
610
|
+
chunker_version TEXT NOT NULL,
|
|
611
|
+
imported_at TEXT NOT NULL,
|
|
612
|
+
updated_at TEXT NOT NULL,
|
|
613
|
+
chunk_count INTEGER NOT NULL CHECK (chunk_count > 0)
|
|
614
|
+
);
|
|
615
|
+
|
|
616
|
+
CREATE INDEX IF NOT EXISTS articles_content_sha256_idx
|
|
617
|
+
ON articles(content_sha256);
|
|
618
|
+
|
|
619
|
+
CREATE TABLE IF NOT EXISTS chunks (
|
|
620
|
+
id TEXT PRIMARY KEY,
|
|
621
|
+
article_id TEXT NOT NULL REFERENCES articles(id) ON DELETE CASCADE,
|
|
622
|
+
ordinal INTEGER NOT NULL CHECK (ordinal >= 0),
|
|
623
|
+
heading TEXT,
|
|
624
|
+
body TEXT NOT NULL,
|
|
625
|
+
chunk_sha256 TEXT NOT NULL,
|
|
626
|
+
embedding BLOB,
|
|
627
|
+
embedding_model TEXT,
|
|
628
|
+
embedding_revision TEXT,
|
|
629
|
+
embedding_quantization TEXT,
|
|
630
|
+
embedding_dimensions INTEGER,
|
|
631
|
+
CHECK (
|
|
632
|
+
(
|
|
633
|
+
embedding IS NULL
|
|
634
|
+
AND embedding_model IS NULL
|
|
635
|
+
AND embedding_revision IS NULL
|
|
636
|
+
AND embedding_quantization IS NULL
|
|
637
|
+
AND embedding_dimensions IS NULL
|
|
638
|
+
)
|
|
639
|
+
OR
|
|
640
|
+
(
|
|
641
|
+
embedding IS NOT NULL
|
|
642
|
+
AND embedding_model IS NOT NULL
|
|
643
|
+
AND embedding_revision IS NOT NULL
|
|
644
|
+
AND embedding_quantization IS NOT NULL
|
|
645
|
+
AND embedding_dimensions IS NOT NULL
|
|
646
|
+
)
|
|
647
|
+
),
|
|
648
|
+
UNIQUE(article_id, ordinal)
|
|
649
|
+
);
|
|
650
|
+
|
|
651
|
+
CREATE INDEX IF NOT EXISTS chunks_article_id_idx
|
|
652
|
+
ON chunks(article_id);
|
|
653
|
+
|
|
654
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS chunk_search USING fts5(
|
|
655
|
+
chunk_id UNINDEXED,
|
|
656
|
+
article_id UNINDEXED,
|
|
657
|
+
title,
|
|
658
|
+
heading,
|
|
659
|
+
body,
|
|
660
|
+
tokenize = 'trigram'
|
|
661
|
+
);
|
|
662
|
+
|
|
663
|
+
PRAGMA user_version = 1;
|
|
664
|
+
`);
|
|
665
|
+
}
|
|
666
|
+
async function openDatabase(target) {
|
|
667
|
+
try {
|
|
668
|
+
await ensurePrivateRegularFile(target);
|
|
669
|
+
const database = new DatabaseSync(target);
|
|
670
|
+
try {
|
|
671
|
+
databaseSchema(database);
|
|
672
|
+
return database;
|
|
673
|
+
} catch (error) {
|
|
674
|
+
database.close();
|
|
675
|
+
throw error;
|
|
676
|
+
}
|
|
677
|
+
} catch (error) {
|
|
678
|
+
if (error instanceof KnowledgeCliError) {
|
|
679
|
+
throw error;
|
|
680
|
+
}
|
|
681
|
+
throw new KnowledgeCliError("DXC_KNOWLEDGE_IO_FAILED", "Unable to open knowledge database");
|
|
682
|
+
}
|
|
683
|
+
}
|
|
684
|
+
async function databaseExists(target) {
|
|
685
|
+
try {
|
|
686
|
+
const metadata = await lstat(target);
|
|
687
|
+
if (!metadata.isFile() || metadata.isSymbolicLink()) {
|
|
688
|
+
throw new KnowledgeCliError("DXC_KNOWLEDGE_STATE_INVALID", "Knowledge database is invalid");
|
|
689
|
+
}
|
|
690
|
+
return true;
|
|
691
|
+
} catch (error) {
|
|
692
|
+
if (error instanceof KnowledgeCliError) {
|
|
693
|
+
throw error;
|
|
694
|
+
}
|
|
695
|
+
if (errorCode(error) === "ENOENT") {
|
|
696
|
+
return false;
|
|
697
|
+
}
|
|
698
|
+
throw new KnowledgeCliError("DXC_KNOWLEDGE_IO_FAILED", "Unable to inspect knowledge database");
|
|
699
|
+
}
|
|
700
|
+
}
|
|
701
|
+
function embeddingText(article, chunk) {
|
|
702
|
+
return [article.title.slice(0, 120), chunk.heading?.slice(0, 120) ?? "", chunk.body].filter((value) => value.length > 0).join("\n");
|
|
703
|
+
}
|
|
704
|
+
async function embedArticle(embedder, article) {
|
|
705
|
+
let vectors;
|
|
706
|
+
try {
|
|
707
|
+
vectors = await embedder.embedDocuments(
|
|
708
|
+
article.chunks.map((chunk) => embeddingText(article, chunk))
|
|
709
|
+
);
|
|
710
|
+
} catch {
|
|
711
|
+
throw new KnowledgeCliError(
|
|
712
|
+
"DXC_KNOWLEDGE_MODEL_FAILED",
|
|
713
|
+
"Local semantic model could not index the article"
|
|
714
|
+
);
|
|
715
|
+
}
|
|
716
|
+
if (vectors.length !== article.chunks.length || vectors.some((vector) => vector.length !== CONTENT_KNOWLEDGE_EMBEDDING_DIMENSIONS)) {
|
|
717
|
+
throw new KnowledgeCliError(
|
|
718
|
+
"DXC_KNOWLEDGE_MODEL_FAILED",
|
|
719
|
+
"Local semantic model returned invalid vectors"
|
|
720
|
+
);
|
|
721
|
+
}
|
|
722
|
+
return vectors;
|
|
723
|
+
}
|
|
724
|
+
function findExistingArticle(database, article) {
|
|
725
|
+
const statement = database.prepare(`
|
|
726
|
+
SELECT
|
|
727
|
+
id, source_label, title, content_sha256, chunker_version,
|
|
728
|
+
imported_at, updated_at, chunk_count
|
|
729
|
+
FROM articles
|
|
730
|
+
WHERE source_key = ?
|
|
731
|
+
LIMIT 1
|
|
732
|
+
`);
|
|
733
|
+
const bySource = statement.get(article.sourceKey);
|
|
734
|
+
if (bySource !== void 0) {
|
|
735
|
+
return articleRow(bySource);
|
|
736
|
+
}
|
|
737
|
+
const byContent = database.prepare(
|
|
738
|
+
`
|
|
739
|
+
SELECT
|
|
740
|
+
id, source_label, title, content_sha256, chunker_version,
|
|
741
|
+
imported_at, updated_at, chunk_count
|
|
742
|
+
FROM articles
|
|
743
|
+
WHERE content_sha256 = ?
|
|
744
|
+
ORDER BY imported_at ASC
|
|
745
|
+
LIMIT 1
|
|
746
|
+
`
|
|
747
|
+
).get(article.contentSha256);
|
|
748
|
+
return byContent === void 0 ? void 0 : articleRow(byContent);
|
|
749
|
+
}
|
|
750
|
+
function semanticIndexComplete(database, articleId) {
|
|
751
|
+
const row = sqlRow(
|
|
752
|
+
database.prepare(
|
|
753
|
+
`
|
|
754
|
+
SELECT
|
|
755
|
+
COUNT(*) AS total,
|
|
756
|
+
COALESCE(SUM(
|
|
757
|
+
CASE
|
|
758
|
+
WHEN embedding IS NOT NULL
|
|
759
|
+
AND embedding_model = ?
|
|
760
|
+
AND embedding_revision = ?
|
|
761
|
+
AND embedding_quantization = ?
|
|
762
|
+
AND embedding_dimensions = ?
|
|
763
|
+
THEN 1
|
|
764
|
+
ELSE 0
|
|
765
|
+
END
|
|
766
|
+
), 0) AS indexed
|
|
767
|
+
FROM chunks
|
|
768
|
+
WHERE article_id = ?
|
|
769
|
+
`
|
|
770
|
+
).get(
|
|
771
|
+
CONTENT_KNOWLEDGE_EMBEDDING_MODEL,
|
|
772
|
+
CONTENT_KNOWLEDGE_EMBEDDING_REVISION,
|
|
773
|
+
CONTENT_KNOWLEDGE_EMBEDDING_QUANTIZATION,
|
|
774
|
+
CONTENT_KNOWLEDGE_EMBEDDING_DIMENSIONS,
|
|
775
|
+
articleId
|
|
776
|
+
)
|
|
777
|
+
);
|
|
778
|
+
return numberColumn(row, "total") > 0 && numberColumn(row, "total") === numberColumn(row, "indexed");
|
|
779
|
+
}
|
|
780
|
+
function saveArticle(database, article, existing, vectors, timestamp, createId) {
|
|
781
|
+
const articleId = existing?.id ?? createId();
|
|
782
|
+
database.exec("BEGIN IMMEDIATE");
|
|
783
|
+
try {
|
|
784
|
+
if (existing === void 0) {
|
|
785
|
+
database.prepare(
|
|
786
|
+
`
|
|
787
|
+
INSERT INTO articles (
|
|
788
|
+
id, source_key, source_label, title, content_sha256, chunker_version,
|
|
789
|
+
imported_at, updated_at, chunk_count
|
|
790
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
791
|
+
`
|
|
792
|
+
).run(
|
|
793
|
+
articleId,
|
|
794
|
+
article.sourceKey,
|
|
795
|
+
article.sourceLabel,
|
|
796
|
+
article.title,
|
|
797
|
+
article.contentSha256,
|
|
798
|
+
CONTENT_KNOWLEDGE_CHUNKER_VERSION,
|
|
799
|
+
timestamp,
|
|
800
|
+
timestamp,
|
|
801
|
+
article.chunks.length
|
|
802
|
+
);
|
|
803
|
+
} else {
|
|
804
|
+
database.prepare("DELETE FROM chunk_search WHERE article_id = ?").run(articleId);
|
|
805
|
+
database.prepare("DELETE FROM chunks WHERE article_id = ?").run(articleId);
|
|
806
|
+
database.prepare(
|
|
807
|
+
`
|
|
808
|
+
UPDATE articles
|
|
809
|
+
SET
|
|
810
|
+
source_key = ?,
|
|
811
|
+
source_label = ?,
|
|
812
|
+
title = ?,
|
|
813
|
+
content_sha256 = ?,
|
|
814
|
+
chunker_version = ?,
|
|
815
|
+
updated_at = ?,
|
|
816
|
+
chunk_count = ?
|
|
817
|
+
WHERE id = ?
|
|
818
|
+
`
|
|
819
|
+
).run(
|
|
820
|
+
article.sourceKey,
|
|
821
|
+
article.sourceLabel,
|
|
822
|
+
article.title,
|
|
823
|
+
article.contentSha256,
|
|
824
|
+
CONTENT_KNOWLEDGE_CHUNKER_VERSION,
|
|
825
|
+
timestamp,
|
|
826
|
+
article.chunks.length,
|
|
827
|
+
articleId
|
|
828
|
+
);
|
|
829
|
+
}
|
|
830
|
+
const insertChunk = database.prepare(`
|
|
831
|
+
INSERT INTO chunks (
|
|
832
|
+
id, article_id, ordinal, heading, body, chunk_sha256,
|
|
833
|
+
embedding, embedding_model, embedding_revision, embedding_quantization,
|
|
834
|
+
embedding_dimensions
|
|
835
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
836
|
+
`);
|
|
837
|
+
const insertSearch = database.prepare(`
|
|
838
|
+
INSERT INTO chunk_search (chunk_id, article_id, title, heading, body)
|
|
839
|
+
VALUES (?, ?, ?, ?, ?)
|
|
840
|
+
`);
|
|
841
|
+
for (const [index, chunk] of article.chunks.entries()) {
|
|
842
|
+
const chunkId = createId();
|
|
843
|
+
const vector = vectors?.[index];
|
|
844
|
+
insertChunk.run(
|
|
845
|
+
chunkId,
|
|
846
|
+
articleId,
|
|
847
|
+
chunk.ordinal,
|
|
848
|
+
chunk.heading,
|
|
849
|
+
chunk.body,
|
|
850
|
+
chunk.sha256,
|
|
851
|
+
vector === void 0 ? null : vectorBuffer(vector),
|
|
852
|
+
vector === void 0 ? null : CONTENT_KNOWLEDGE_EMBEDDING_MODEL,
|
|
853
|
+
vector === void 0 ? null : CONTENT_KNOWLEDGE_EMBEDDING_REVISION,
|
|
854
|
+
vector === void 0 ? null : CONTENT_KNOWLEDGE_EMBEDDING_QUANTIZATION,
|
|
855
|
+
vector === void 0 ? null : CONTENT_KNOWLEDGE_EMBEDDING_DIMENSIONS
|
|
856
|
+
);
|
|
857
|
+
insertSearch.run(chunkId, articleId, article.title, chunk.heading ?? "", chunk.body);
|
|
858
|
+
}
|
|
859
|
+
database.exec("COMMIT");
|
|
860
|
+
} catch (error) {
|
|
861
|
+
try {
|
|
862
|
+
database.exec("ROLLBACK");
|
|
863
|
+
} catch {
|
|
864
|
+
}
|
|
865
|
+
if (error instanceof KnowledgeCliError) {
|
|
866
|
+
throw error;
|
|
867
|
+
}
|
|
868
|
+
throw new KnowledgeCliError("DXC_KNOWLEDGE_IO_FAILED", "Unable to save article index");
|
|
869
|
+
}
|
|
870
|
+
return {
|
|
871
|
+
chunkCount: article.chunks.length,
|
|
872
|
+
chunkerVersion: CONTENT_KNOWLEDGE_CHUNKER_VERSION,
|
|
873
|
+
contentSha256: article.contentSha256,
|
|
874
|
+
id: articleId,
|
|
875
|
+
importedAt: existing?.importedAt ?? timestamp,
|
|
876
|
+
sourceLabel: article.sourceLabel,
|
|
877
|
+
title: article.title,
|
|
878
|
+
updatedAt: timestamp
|
|
879
|
+
};
|
|
880
|
+
}
|
|
881
|
+
function loadChunks(database) {
|
|
882
|
+
return database.prepare(
|
|
883
|
+
`
|
|
884
|
+
SELECT
|
|
885
|
+
c.id,
|
|
886
|
+
c.article_id,
|
|
887
|
+
c.ordinal,
|
|
888
|
+
c.heading,
|
|
889
|
+
c.body,
|
|
890
|
+
c.chunk_sha256,
|
|
891
|
+
CASE
|
|
892
|
+
WHEN c.embedding_model = ?
|
|
893
|
+
AND c.embedding_revision = ?
|
|
894
|
+
AND c.embedding_quantization = ?
|
|
895
|
+
AND c.embedding_dimensions = ?
|
|
896
|
+
THEN c.embedding
|
|
897
|
+
ELSE NULL
|
|
898
|
+
END AS embedding,
|
|
899
|
+
a.title,
|
|
900
|
+
a.source_label
|
|
901
|
+
FROM chunks c
|
|
902
|
+
INNER JOIN articles a ON a.id = c.article_id
|
|
903
|
+
ORDER BY a.imported_at ASC, c.ordinal ASC
|
|
904
|
+
`
|
|
905
|
+
).all(
|
|
906
|
+
CONTENT_KNOWLEDGE_EMBEDDING_MODEL,
|
|
907
|
+
CONTENT_KNOWLEDGE_EMBEDDING_REVISION,
|
|
908
|
+
CONTENT_KNOWLEDGE_EMBEDDING_QUANTIZATION,
|
|
909
|
+
CONTENT_KNOWLEDGE_EMBEDDING_DIMENSIONS
|
|
910
|
+
).map(chunkRow);
|
|
911
|
+
}
|
|
912
|
+
function ftsExpression(query) {
|
|
913
|
+
const terms = query.normalize("NFKC").split(/[\s,,。!?!?;;::、/\\|()[\]{}]+/u).map((term) => term.trim()).filter((term) => [...term].length >= 3).slice(0, 12);
|
|
914
|
+
if (terms.length === 0) {
|
|
915
|
+
return null;
|
|
916
|
+
}
|
|
917
|
+
return terms.map((term) => `"${term.replaceAll('"', '""')}"`).join(" OR ");
|
|
918
|
+
}
|
|
919
|
+
function lexicalRanking(database, query) {
|
|
920
|
+
const expression = ftsExpression(query);
|
|
921
|
+
if (expression === null) {
|
|
922
|
+
return [];
|
|
923
|
+
}
|
|
924
|
+
try {
|
|
925
|
+
return database.prepare(
|
|
926
|
+
`
|
|
927
|
+
SELECT chunk_id
|
|
928
|
+
FROM chunk_search
|
|
929
|
+
WHERE chunk_search MATCH ?
|
|
930
|
+
ORDER BY bm25(chunk_search, 0.0, 0.0, 4.0, 2.0, 1.0)
|
|
931
|
+
LIMIT ?
|
|
932
|
+
`
|
|
933
|
+
).all(expression, SEARCH_CANDIDATE_LIMIT).map((value, rank) => ({
|
|
934
|
+
id: stringColumn(sqlRow(value), "chunk_id"),
|
|
935
|
+
rank
|
|
936
|
+
}));
|
|
937
|
+
} catch {
|
|
938
|
+
throw new KnowledgeCliError("DXC_KNOWLEDGE_QUERY_INVALID", "Knowledge query is invalid");
|
|
939
|
+
}
|
|
940
|
+
}
|
|
941
|
+
function dotProduct(left, right) {
|
|
942
|
+
let result = 0;
|
|
943
|
+
for (let index = 0; index < left.length; index += 1) {
|
|
944
|
+
result += (left[index] ?? 0) * (right[index] ?? 0);
|
|
945
|
+
}
|
|
946
|
+
return result;
|
|
947
|
+
}
|
|
948
|
+
function semanticRanking(chunks, queryVector) {
|
|
949
|
+
const candidates = chunks.filter(
|
|
950
|
+
(chunk) => chunk.embedding !== null
|
|
951
|
+
).map((chunk) => ({
|
|
952
|
+
id: chunk.id,
|
|
953
|
+
similarity: dotProduct(queryVector, chunk.embedding)
|
|
954
|
+
})).sort((left, right) => right.similarity - left.similarity || left.id.localeCompare(right.id)).slice(0, SEARCH_CANDIDATE_LIMIT);
|
|
955
|
+
const bestSimilarity = candidates[0]?.similarity;
|
|
956
|
+
if (bestSimilarity === void 0 || bestSimilarity < SEMANTIC_MINIMUM_SIMILARITY) {
|
|
957
|
+
return [];
|
|
958
|
+
}
|
|
959
|
+
return candidates.filter(
|
|
960
|
+
(candidate) => candidate.similarity >= SEMANTIC_MINIMUM_SIMILARITY && bestSimilarity - candidate.similarity <= SEMANTIC_MAXIMUM_DISTANCE_FROM_BEST
|
|
961
|
+
).slice(0, SEARCH_CANDIDATE_LIMIT).map((candidate, rank) => ({
|
|
962
|
+
id: candidate.id,
|
|
963
|
+
rank
|
|
964
|
+
}));
|
|
965
|
+
}
|
|
966
|
+
function snippet(body) {
|
|
967
|
+
const characters = [...body];
|
|
968
|
+
return characters.length <= 360 ? body : `${characters.slice(0, 359).join("")}\u2026`;
|
|
969
|
+
}
|
|
970
|
+
function fusedResults(chunks, mode, lexical, semantic, limit) {
|
|
971
|
+
const scores = /* @__PURE__ */ new Map();
|
|
972
|
+
const addRanking = (ranking, lane, weight) => {
|
|
973
|
+
for (const candidate of ranking) {
|
|
974
|
+
const current = scores.get(candidate.id) ?? { lanes: /* @__PURE__ */ new Set(), score: 0 };
|
|
975
|
+
current.lanes.add(lane);
|
|
976
|
+
current.score += weight / (RRF_CONSTANT + candidate.rank + 1);
|
|
977
|
+
scores.set(candidate.id, current);
|
|
978
|
+
}
|
|
979
|
+
};
|
|
980
|
+
const semanticWeight = mode === "hybrid" ? 0.7 : mode === "semantic" ? 1 : 0;
|
|
981
|
+
const lexicalWeight = mode === "hybrid" ? 0.3 : mode === "lexical" ? 1 : 0;
|
|
982
|
+
addRanking(semantic, "semantic", semanticWeight);
|
|
983
|
+
addRanking(lexical, "lexical", lexicalWeight);
|
|
984
|
+
const byId = new Map(chunks.map((chunk) => [chunk.id, chunk]));
|
|
985
|
+
const articleCounts = /* @__PURE__ */ new Map();
|
|
986
|
+
const results = [];
|
|
987
|
+
for (const [chunkId, score] of [...scores.entries()].sort(
|
|
988
|
+
(left, right) => right[1].score - left[1].score || left[0].localeCompare(right[0])
|
|
989
|
+
)) {
|
|
990
|
+
const chunk = byId.get(chunkId);
|
|
991
|
+
if (chunk === void 0 || (articleCounts.get(chunk.articleId) ?? 0) >= 2) {
|
|
992
|
+
continue;
|
|
993
|
+
}
|
|
994
|
+
articleCounts.set(chunk.articleId, (articleCounts.get(chunk.articleId) ?? 0) + 1);
|
|
995
|
+
results.push({
|
|
996
|
+
articleId: chunk.articleId,
|
|
997
|
+
chunkId: chunk.id,
|
|
998
|
+
chunkSha256: chunk.chunkSha256,
|
|
999
|
+
heading: chunk.heading,
|
|
1000
|
+
lanes: [...score.lanes].sort(),
|
|
1001
|
+
ordinal: chunk.ordinal,
|
|
1002
|
+
rank: results.length + 1,
|
|
1003
|
+
snippet: snippet(chunk.body),
|
|
1004
|
+
sourceLabel: chunk.sourceLabel,
|
|
1005
|
+
title: chunk.title
|
|
1006
|
+
});
|
|
1007
|
+
if (results.length >= limit) {
|
|
1008
|
+
break;
|
|
1009
|
+
}
|
|
1010
|
+
}
|
|
1011
|
+
return results;
|
|
1012
|
+
}
|
|
1013
|
+
var KnowledgeCli = class {
|
|
1014
|
+
#currentDirectory;
|
|
1015
|
+
#databasePath;
|
|
1016
|
+
#embedder;
|
|
1017
|
+
#isModelCached;
|
|
1018
|
+
#now;
|
|
1019
|
+
#randomUUID;
|
|
1020
|
+
#writeStderr;
|
|
1021
|
+
constructor(dependencies) {
|
|
1022
|
+
const now = dependencies.now ?? (() => /* @__PURE__ */ new Date());
|
|
1023
|
+
const modelCacheDirectory = dependencies.modelCacheDirectory ?? path2.join(dependencies.homeDirectory, ".dxc", "models", "bge-small-zh-v1.5");
|
|
1024
|
+
this.#currentDirectory = dependencies.currentDirectory;
|
|
1025
|
+
this.#databasePath = dependencies.databasePath ?? path2.join(dependencies.homeDirectory, ".dxc", "content-memory.sqlite");
|
|
1026
|
+
this.#embedder = dependencies.embedder ?? new LocalBgeKnowledgeEmbedder(modelCacheDirectory, now);
|
|
1027
|
+
this.#isModelCached = dependencies.isModelCached ?? (() => isKnowledgeModelCached(modelCacheDirectory));
|
|
1028
|
+
this.#now = now;
|
|
1029
|
+
this.#randomUUID = dependencies.randomUUID ?? randomUUID;
|
|
1030
|
+
this.#writeStderr = dependencies.writeStderr ?? (() => void 0);
|
|
1031
|
+
}
|
|
1032
|
+
async importArticles(options) {
|
|
1033
|
+
if (options.files.length === 0) {
|
|
1034
|
+
throw new KnowledgeCliError(
|
|
1035
|
+
"DXC_KNOWLEDGE_ARTICLE_INVALID",
|
|
1036
|
+
"At least one article file is required"
|
|
1037
|
+
);
|
|
1038
|
+
}
|
|
1039
|
+
if (!options.lexicalOnly) {
|
|
1040
|
+
this.#writeStderr(
|
|
1041
|
+
"Preparing the local semantic index; first use downloads a quantized model of about 24 MB.\n"
|
|
1042
|
+
);
|
|
1043
|
+
}
|
|
1044
|
+
const database = await openDatabase(this.#databasePath);
|
|
1045
|
+
try {
|
|
1046
|
+
const imported = [];
|
|
1047
|
+
for (const file of options.files) {
|
|
1048
|
+
const article = await prepareArticle(this.#currentDirectory, file);
|
|
1049
|
+
const existing = findExistingArticle(database, article);
|
|
1050
|
+
const unchanged = existing?.contentSha256 === article.contentSha256 && existing.chunkerVersion === CONTENT_KNOWLEDGE_CHUNKER_VERSION;
|
|
1051
|
+
const semanticComplete = existing === void 0 ? false : semanticIndexComplete(database, existing.id);
|
|
1052
|
+
if (unchanged && (options.lexicalOnly || semanticComplete)) {
|
|
1053
|
+
imported.push({
|
|
1054
|
+
article: articleSummary(existing),
|
|
1055
|
+
semanticIndexed: semanticComplete,
|
|
1056
|
+
status: "unchanged"
|
|
1057
|
+
});
|
|
1058
|
+
continue;
|
|
1059
|
+
}
|
|
1060
|
+
const vectors = options.lexicalOnly ? void 0 : await embedArticle(this.#embedder, article);
|
|
1061
|
+
const saved = saveArticle(
|
|
1062
|
+
database,
|
|
1063
|
+
article,
|
|
1064
|
+
existing,
|
|
1065
|
+
vectors,
|
|
1066
|
+
this.#now().toISOString(),
|
|
1067
|
+
this.#randomUUID
|
|
1068
|
+
);
|
|
1069
|
+
imported.push({
|
|
1070
|
+
article: articleSummary(saved),
|
|
1071
|
+
semanticIndexed: vectors !== void 0,
|
|
1072
|
+
status: existing === void 0 ? "created" : "updated"
|
|
1073
|
+
});
|
|
1074
|
+
}
|
|
1075
|
+
return contentKnowledgeImportResultSchema.parse({
|
|
1076
|
+
command: "knowledge.import",
|
|
1077
|
+
data: {
|
|
1078
|
+
imported,
|
|
1079
|
+
model: options.lexicalOnly ? null : this.#embedder.model
|
|
1080
|
+
},
|
|
1081
|
+
ok: true
|
|
1082
|
+
});
|
|
1083
|
+
} finally {
|
|
1084
|
+
database.close();
|
|
1085
|
+
}
|
|
1086
|
+
}
|
|
1087
|
+
async remove(articleId) {
|
|
1088
|
+
if (!UUID_PATTERN.test(articleId)) {
|
|
1089
|
+
throw new KnowledgeCliError(
|
|
1090
|
+
"DXC_KNOWLEDGE_ARTICLE_INVALID",
|
|
1091
|
+
"Knowledge article ID is invalid"
|
|
1092
|
+
);
|
|
1093
|
+
}
|
|
1094
|
+
if (!await databaseExists(this.#databasePath)) {
|
|
1095
|
+
return contentKnowledgeRemoveResultSchema.parse({
|
|
1096
|
+
command: "knowledge.remove",
|
|
1097
|
+
data: { articleId, removed: false },
|
|
1098
|
+
ok: true
|
|
1099
|
+
});
|
|
1100
|
+
}
|
|
1101
|
+
const database = await openDatabase(this.#databasePath);
|
|
1102
|
+
try {
|
|
1103
|
+
database.exec("BEGIN IMMEDIATE");
|
|
1104
|
+
try {
|
|
1105
|
+
database.prepare("DELETE FROM chunk_search WHERE article_id = ?").run(articleId);
|
|
1106
|
+
const result = database.prepare("DELETE FROM articles WHERE id = ?").run(articleId);
|
|
1107
|
+
database.exec("COMMIT");
|
|
1108
|
+
return contentKnowledgeRemoveResultSchema.parse({
|
|
1109
|
+
command: "knowledge.remove",
|
|
1110
|
+
data: {
|
|
1111
|
+
articleId,
|
|
1112
|
+
removed: result.changes > 0
|
|
1113
|
+
},
|
|
1114
|
+
ok: true
|
|
1115
|
+
});
|
|
1116
|
+
} catch {
|
|
1117
|
+
database.exec("ROLLBACK");
|
|
1118
|
+
throw new KnowledgeCliError(
|
|
1119
|
+
"DXC_KNOWLEDGE_IO_FAILED",
|
|
1120
|
+
"Unable to remove knowledge article"
|
|
1121
|
+
);
|
|
1122
|
+
}
|
|
1123
|
+
} finally {
|
|
1124
|
+
database.close();
|
|
1125
|
+
}
|
|
1126
|
+
}
|
|
1127
|
+
async search(queryInput, modeInput, limitInput) {
|
|
1128
|
+
const query = queryInput.trim();
|
|
1129
|
+
const mode = contentKnowledgeSearchModeSchema.safeParse(modeInput);
|
|
1130
|
+
if (query.length === 0 || query.length > 500 || !mode.success || !Number.isSafeInteger(limitInput) || limitInput < 1 || limitInput > MAXIMUM_RESULTS) {
|
|
1131
|
+
throw new KnowledgeCliError("DXC_KNOWLEDGE_QUERY_INVALID", "Knowledge query is invalid");
|
|
1132
|
+
}
|
|
1133
|
+
if (!await databaseExists(this.#databasePath)) {
|
|
1134
|
+
return contentKnowledgeSearchResultSchema.parse({
|
|
1135
|
+
command: "knowledge.search",
|
|
1136
|
+
data: {
|
|
1137
|
+
mode: mode.data,
|
|
1138
|
+
model: mode.data === "lexical" ? null : this.#embedder.model,
|
|
1139
|
+
query,
|
|
1140
|
+
results: []
|
|
1141
|
+
},
|
|
1142
|
+
ok: true
|
|
1143
|
+
});
|
|
1144
|
+
}
|
|
1145
|
+
const database = await openDatabase(this.#databasePath);
|
|
1146
|
+
try {
|
|
1147
|
+
const chunks = loadChunks(database);
|
|
1148
|
+
const usesSemantic = mode.data !== "lexical";
|
|
1149
|
+
if (usesSemantic && chunks.some((chunk) => chunk.embedding === null)) {
|
|
1150
|
+
throw new KnowledgeCliError(
|
|
1151
|
+
"DXC_KNOWLEDGE_SEMANTIC_NOT_READY",
|
|
1152
|
+
"All articles must have a semantic index before hybrid search"
|
|
1153
|
+
);
|
|
1154
|
+
}
|
|
1155
|
+
const lexical = mode.data === "semantic" ? [] : lexicalRanking(database, query);
|
|
1156
|
+
let semantic = [];
|
|
1157
|
+
if (usesSemantic && chunks.length > 0) {
|
|
1158
|
+
let queryVector;
|
|
1159
|
+
try {
|
|
1160
|
+
queryVector = await this.#embedder.embedQuery(query);
|
|
1161
|
+
} catch {
|
|
1162
|
+
throw new KnowledgeCliError(
|
|
1163
|
+
"DXC_KNOWLEDGE_MODEL_FAILED",
|
|
1164
|
+
"Local semantic model could not process the query"
|
|
1165
|
+
);
|
|
1166
|
+
}
|
|
1167
|
+
semantic = semanticRanking(chunks, queryVector);
|
|
1168
|
+
}
|
|
1169
|
+
return contentKnowledgeSearchResultSchema.parse({
|
|
1170
|
+
command: "knowledge.search",
|
|
1171
|
+
data: {
|
|
1172
|
+
mode: mode.data,
|
|
1173
|
+
model: usesSemantic ? this.#embedder.model : null,
|
|
1174
|
+
query,
|
|
1175
|
+
results: fusedResults(chunks, mode.data, lexical, semantic, limitInput)
|
|
1176
|
+
},
|
|
1177
|
+
ok: true
|
|
1178
|
+
});
|
|
1179
|
+
} finally {
|
|
1180
|
+
database.close();
|
|
1181
|
+
}
|
|
1182
|
+
}
|
|
1183
|
+
async status() {
|
|
1184
|
+
let articleCount = 0;
|
|
1185
|
+
let chunkCount = 0;
|
|
1186
|
+
let semanticIndexedChunkCount = 0;
|
|
1187
|
+
if (await databaseExists(this.#databasePath)) {
|
|
1188
|
+
const database = await openDatabase(this.#databasePath);
|
|
1189
|
+
try {
|
|
1190
|
+
const articleRowValue = sqlRow(
|
|
1191
|
+
database.prepare("SELECT COUNT(*) AS count FROM articles").get()
|
|
1192
|
+
);
|
|
1193
|
+
const chunkRowValue = sqlRow(
|
|
1194
|
+
database.prepare(
|
|
1195
|
+
`
|
|
1196
|
+
SELECT
|
|
1197
|
+
COUNT(*) AS count,
|
|
1198
|
+
COALESCE(SUM(
|
|
1199
|
+
CASE
|
|
1200
|
+
WHEN embedding IS NOT NULL
|
|
1201
|
+
AND embedding_model = ?
|
|
1202
|
+
AND embedding_revision = ?
|
|
1203
|
+
AND embedding_quantization = ?
|
|
1204
|
+
AND embedding_dimensions = ?
|
|
1205
|
+
THEN 1
|
|
1206
|
+
ELSE 0
|
|
1207
|
+
END
|
|
1208
|
+
), 0) AS semantic_count
|
|
1209
|
+
FROM chunks
|
|
1210
|
+
`
|
|
1211
|
+
).get(
|
|
1212
|
+
CONTENT_KNOWLEDGE_EMBEDDING_MODEL,
|
|
1213
|
+
CONTENT_KNOWLEDGE_EMBEDDING_REVISION,
|
|
1214
|
+
CONTENT_KNOWLEDGE_EMBEDDING_QUANTIZATION,
|
|
1215
|
+
CONTENT_KNOWLEDGE_EMBEDDING_DIMENSIONS
|
|
1216
|
+
)
|
|
1217
|
+
);
|
|
1218
|
+
articleCount = numberColumn(articleRowValue, "count");
|
|
1219
|
+
chunkCount = numberColumn(chunkRowValue, "count");
|
|
1220
|
+
semanticIndexedChunkCount = numberColumn(chunkRowValue, "semantic_count");
|
|
1221
|
+
} finally {
|
|
1222
|
+
database.close();
|
|
1223
|
+
}
|
|
1224
|
+
}
|
|
1225
|
+
return contentKnowledgeStatusResultSchema.parse({
|
|
1226
|
+
command: "knowledge.status",
|
|
1227
|
+
data: {
|
|
1228
|
+
articleCount,
|
|
1229
|
+
chunkCount,
|
|
1230
|
+
databasePath: DATABASE_DISPLAY_PATH,
|
|
1231
|
+
model: contentKnowledgeModelSchema.parse({
|
|
1232
|
+
dimensions: CONTENT_KNOWLEDGE_EMBEDDING_DIMENSIONS,
|
|
1233
|
+
id: CONTENT_KNOWLEDGE_EMBEDDING_MODEL,
|
|
1234
|
+
quantization: CONTENT_KNOWLEDGE_EMBEDDING_QUANTIZATION,
|
|
1235
|
+
revision: CONTENT_KNOWLEDGE_EMBEDDING_REVISION
|
|
1236
|
+
}),
|
|
1237
|
+
modelCached: await this.#isModelCached(),
|
|
1238
|
+
semanticIndexedChunkCount
|
|
1239
|
+
},
|
|
1240
|
+
ok: true
|
|
1241
|
+
});
|
|
1242
|
+
}
|
|
1243
|
+
};
|
|
1244
|
+
export {
|
|
1245
|
+
KnowledgeCli,
|
|
1246
|
+
KnowledgeCliError,
|
|
1247
|
+
chunkHistoricalArticle
|
|
1248
|
+
};
|