@deployxai/dxc 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/README.md +131 -0
  2. package/dist/chunks/chunk-I6VZLNRZ.js +2118 -0
  3. package/dist/chunks/chunk-XIHX5YAF.js +16391 -0
  4. package/dist/chunks/knowledge-Q6MHPG6I.js +1248 -0
  5. package/dist/chunks/monitor-VPRVRQIS.js +694 -0
  6. package/dist/index.js +32367 -0
  7. package/docs/00-project-context.md +125 -0
  8. package/docs/01-north-star-architecture.md +234 -0
  9. package/docs/02-mvp-technical-design.md +553 -0
  10. package/docs/03-domain-state-api.md +599 -0
  11. package/docs/04-security-and-operations.md +413 -0
  12. package/docs/05-delivery-plan.md +407 -0
  13. package/docs/README.md +44 -0
  14. package/docs/decisions/0001-initial-architecture.md +57 -0
  15. package/docs/decisions/0002-mongodb-environment-boundary.md +42 -0
  16. package/docs/decisions/0003-staged-production-topology.md +33 -0
  17. package/docs/decisions/0004-local-first-agent-research-runtime.md +71 -0
  18. package/docs/decisions/0005-official-skill-orchestration-and-local-content-memory.md +97 -0
  19. package/docs/decisions/0006-separate-wechat-user-login-from-account-authorization.md +87 -0
  20. package/docs/decisions/0007-explicit-personal-wechat-start.md +67 -0
  21. package/docs/decisions/0008-end-to-end-content-workflow-continuity.md +115 -0
  22. package/docs/decisions/0009-privileged-multitenant-draft-scheduling.md +36 -0
  23. package/docs/decisions/0009-versioned-cloud-template-catalog.md +39 -0
  24. package/docs/eight-stage-implementation-audit.md +62 -0
  25. package/docs/first-user-guide.md +187 -0
  26. package/docs/history/content-forge-prd-v0.2-summary.md +81 -0
  27. package/docs/local-development.md +511 -0
  28. package/docs/references/aliyun-oss-production-setup.md +89 -0
  29. package/docs/references/legacy-content-to-wechat-contract.md +223 -0
  30. package/docs/references/renderer-compatibility-report.md +68 -0
  31. package/docs/references/source-inventory.md +179 -0
  32. package/docs/references/wechat-renderer-platform-validation.md +92 -0
  33. package/docs/references/wechat-third-party-platform-setup.md +159 -0
  34. package/docs/references/wechat-website-login-setup.md +137 -0
  35. package/docs/references/wemd-template-attribution.md +25 -0
  36. package/docs/research-monitoring-design.md +235 -0
  37. package/docs/todo-preview-local-first.md +31 -0
  38. package/docs/workbuddy-first-user-runbook.md +246 -0
  39. package/docs//345/221/230/345/267/245BCDE/347/232/204skill/employee-b-research-analyst/SKILL.md +230 -0
  40. package/docs//345/221/230/345/267/245BCDE/347/232/204skill/employee-c-outline-architect/SKILL.md +194 -0
  41. package/docs//345/221/230/345/267/245BCDE/347/232/204skill/employee-d-content-writer/SKILL.md +296 -0
  42. package/docs//345/221/230/345/267/245BCDE/347/232/204skill/employee-e-visual-designer/SKILL.md +268 -0
  43. package/package.json +25 -0
  44. package/skills/dxc-article-outline/SKILL.md +82 -0
  45. package/skills/dxc-article-outline/agents/openai.yaml +6 -0
  46. package/skills/dxc-article-outline/references/outline-methods.md +38 -0
  47. package/skills/dxc-article-write/SKILL.md +85 -0
  48. package/skills/dxc-article-write/agents/openai.yaml +6 -0
  49. package/skills/dxc-article-write/references/writing-methods.md +42 -0
  50. package/skills/dxc-content-brief/SKILL.md +81 -0
  51. package/skills/dxc-content-brief/agents/openai.yaml +6 -0
  52. package/skills/dxc-content-brief/references/brief-method.md +34 -0
  53. package/skills/dxc-content-review/SKILL.md +84 -0
  54. package/skills/dxc-content-review/agents/openai.yaml +6 -0
  55. package/skills/dxc-content-review/references/review-checklist.md +35 -0
  56. package/skills/dxc-content-workflow/SKILL.md +190 -0
  57. package/skills/dxc-content-workflow/agents/openai.yaml +6 -0
  58. package/skills/dxc-content-workflow/references/catalog.json +136 -0
  59. package/skills/dxc-content-workflow/references/onboarding-questions.md +107 -0
  60. package/skills/dxc-content-workflow/references/stage-contract.md +70 -0
  61. package/skills/dxc-research/SKILL.md +110 -0
  62. package/skills/dxc-research/agents/openai.yaml +6 -0
  63. package/skills/dxc-research/references/research-method.md +53 -0
  64. package/skills/dxc-title-write/SKILL.md +112 -0
  65. package/skills/dxc-title-write/agents/openai.yaml +6 -0
  66. package/skills/dxc-title-write/references/title-methods.md +26 -0
  67. package/skills/dxc-visual-plan/SKILL.md +119 -0
  68. package/skills/dxc-visual-plan/agents/openai.yaml +6 -0
  69. package/skills/dxc-visual-plan/references/visual-methods.md +35 -0
  70. package/skills/dxc-wechat-publisher/SKILL.md +157 -0
  71. package/skills/dxc-wechat-publisher/agents/openai.yaml +6 -0
@@ -0,0 +1,1248 @@
1
+ import { createRequire as __dxcCreateRequire } from "node:module"; const require = __dxcCreateRequire(import.meta.url);
2
+ import {
3
+ CONTENT_KNOWLEDGE_CHUNKER_VERSION,
4
+ CONTENT_KNOWLEDGE_EMBEDDING_DIMENSIONS,
5
+ CONTENT_KNOWLEDGE_EMBEDDING_MODEL,
6
+ CONTENT_KNOWLEDGE_EMBEDDING_QUANTIZATION,
7
+ CONTENT_KNOWLEDGE_EMBEDDING_REVISION,
8
+ CONTENT_KNOWLEDGE_SCHEMA_VERSION,
9
+ LocalStateError,
10
+ contentKnowledgeImportResultSchema,
11
+ contentKnowledgeModelSchema,
12
+ contentKnowledgeRemoveResultSchema,
13
+ contentKnowledgeSearchModeSchema,
14
+ contentKnowledgeSearchResultSchema,
15
+ contentKnowledgeStatusResultSchema,
16
+ ensurePrivateDirectory,
17
+ ensurePrivateRegularFile,
18
+ readPrivateBuffer,
19
+ readPrivateFile,
20
+ writePrivateBuffer,
21
+ writePrivateJson
22
+ } from "./chunk-XIHX5YAF.js";
23
+
24
+ // apps/cli/src/knowledge.ts
25
+ import { createHash as createHash2, randomUUID } from "node:crypto";
26
+ import { lstat, readFile, stat } from "node:fs/promises";
27
+ import path2 from "node:path";
28
+ import { DatabaseSync } from "node:sqlite";
29
+
30
+ // apps/cli/src/knowledge-embedding.ts
31
+ import { createHash } from "node:crypto";
32
+ import path from "node:path";
33
+ var MODEL_MARKER_NAME = "content-embedding-model.json";
34
+ var MODEL_BASE_URL = `https://huggingface.co/${CONTENT_KNOWLEDGE_EMBEDDING_MODEL}/resolve/${CONTENT_KNOWLEDGE_EMBEDDING_REVISION}`;
35
+ var ONNX_RUNTIME_MODULE_NAME = "onnxruntime-web";
36
+ var TOKENIZERS_MODULE_NAME = "@huggingface/tokenizers";
37
+ var QUERY_INSTRUCTION = "\u4E3A\u8FD9\u4E2A\u53E5\u5B50\u751F\u6210\u8868\u793A\u4EE5\u7528\u4E8E\u68C0\u7D22\u76F8\u5173\u6587\u7AE0\uFF1A";
38
+ var MAXIMUM_TOKENS = 512;
39
+ var EMBEDDING_BATCH_SIZE = 8;
40
+ var MODEL_DOWNLOAD_TIMEOUT_MS = 3e5;
41
+ var MODEL_ARTIFACTS = [
42
+ {
43
+ maximumBytes: 25 * 1024 * 1024,
44
+ name: "model_quantized.onnx",
45
+ remotePath: "onnx/model_quantized.onnx",
46
+ sha256: "15b717c382bcb518ba457b93ea6850ede7f4f1cd8937454aa06972366cd19bcc"
47
+ },
48
+ {
49
+ maximumBytes: 512 * 1024,
50
+ name: "tokenizer.json",
51
+ remotePath: "tokenizer.json",
52
+ sha256: "48cea5d44424912a6fd1ea647bf4fe50b55ab8b1e5879c3275f80e339e8fae26"
53
+ },
54
+ {
55
+ maximumBytes: 8 * 1024,
56
+ name: "tokenizer_config.json",
57
+ remotePath: "tokenizer_config.json",
58
+ sha256: "e6f3b96db926a37d4039995fbf5ad17de158dfb8f6343d607e4dbaad18d75f5a"
59
+ }
60
+ ];
61
+ function artifactPath(cacheDirectory, artifact) {
62
+ return path.join(cacheDirectory, artifact.name);
63
+ }
64
+ function sha256(value) {
65
+ return createHash("sha256").update(value).digest("hex");
66
+ }
67
+ async function readVerifiedArtifact(cacheDirectory, artifact) {
68
+ try {
69
+ const contents = await readPrivateBuffer(
70
+ artifactPath(cacheDirectory, artifact),
71
+ artifact.maximumBytes
72
+ );
73
+ if (sha256(contents) !== artifact.sha256) {
74
+ throw new Error("local model artifact checksum is invalid");
75
+ }
76
+ return contents;
77
+ } catch (error) {
78
+ if (error instanceof LocalStateError && error.failure === "missing") {
79
+ return void 0;
80
+ }
81
+ throw error;
82
+ }
83
+ }
84
+ async function downloadArtifact(cacheDirectory, artifact) {
85
+ const response = await fetch(`${MODEL_BASE_URL}/${artifact.remotePath}`, {
86
+ headers: {
87
+ "user-agent": "dxc-cli/0.1"
88
+ },
89
+ redirect: "follow",
90
+ signal: AbortSignal.timeout(MODEL_DOWNLOAD_TIMEOUT_MS)
91
+ });
92
+ if (!response.ok) {
93
+ throw new Error("local model artifact download failed");
94
+ }
95
+ const declaredSize = Number(response.headers.get("content-length"));
96
+ if (Number.isFinite(declaredSize) && declaredSize > artifact.maximumBytes) {
97
+ throw new Error("local model artifact is too large");
98
+ }
99
+ const contents = Buffer.from(await response.arrayBuffer());
100
+ if (contents.byteLength > artifact.maximumBytes || sha256(contents) !== artifact.sha256) {
101
+ throw new Error("downloaded model artifact checksum is invalid");
102
+ }
103
+ await writePrivateBuffer(artifactPath(cacheDirectory, artifact), contents);
104
+ return contents;
105
+ }
106
+ async function ensureModelArtifacts(cacheDirectory) {
107
+ await ensurePrivateDirectory(cacheDirectory);
108
+ const artifacts = /* @__PURE__ */ new Map();
109
+ for (const artifact of MODEL_ARTIFACTS) {
110
+ const cached = await readVerifiedArtifact(cacheDirectory, artifact);
111
+ artifacts.set(artifact.name, cached ?? await downloadArtifact(cacheDirectory, artifact));
112
+ }
113
+ return artifacts;
114
+ }
115
+ function modelMarkerPath(cacheDirectory) {
116
+ return path.join(cacheDirectory, MODEL_MARKER_NAME);
117
+ }
118
+ function parseJsonObject(value, label) {
119
+ const parsed = JSON.parse(value.toString("utf8"));
120
+ if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
121
+ throw new Error(`${label} is invalid`);
122
+ }
123
+ return parsed;
124
+ }
125
+ async function loadOnnxRuntime() {
126
+ const loaded = await import(ONNX_RUNTIME_MODULE_NAME);
127
+ if (typeof loaded !== "object" || loaded === null || !("env" in loaded) || typeof loaded.env !== "object" || loaded.env === null || !("InferenceSession" in loaded) || typeof loaded.InferenceSession !== "object" && typeof loaded.InferenceSession !== "function" || loaded.InferenceSession === null || !("Tensor" in loaded) || typeof loaded.Tensor !== "function") {
128
+ throw new Error("ONNX runtime module is invalid");
129
+ }
130
+ return loaded;
131
+ }
132
+ async function loadTokenizerConstructor() {
133
+ const loaded = await import(TOKENIZERS_MODULE_NAME);
134
+ if (typeof loaded !== "object" || loaded === null || !("Tokenizer" in loaded) || typeof loaded.Tokenizer !== "function") {
135
+ throw new Error("tokenizer module is invalid");
136
+ }
137
+ return loaded.Tokenizer;
138
+ }
139
+ async function createEmbeddingRuntime(cacheDirectory) {
140
+ const artifacts = await ensureModelArtifacts(cacheDirectory);
141
+ const tokenizerJson = artifacts.get("tokenizer.json");
142
+ const tokenizerConfig = artifacts.get("tokenizer_config.json");
143
+ const model = artifacts.get("model_quantized.onnx");
144
+ if (tokenizerJson === void 0 || tokenizerConfig === void 0 || model === void 0) {
145
+ throw new Error("local model artifacts are incomplete");
146
+ }
147
+ const Tokenizer = await loadTokenizerConstructor();
148
+ const tokenizer = new Tokenizer(
149
+ parseJsonObject(tokenizerJson, "tokenizer"),
150
+ parseJsonObject(tokenizerConfig, "tokenizer config")
151
+ );
152
+ const onnx = await loadOnnxRuntime();
153
+ onnx.env.wasm.numThreads = 1;
154
+ const session = await onnx.InferenceSession.create(model, {
155
+ executionProviders: ["wasm"],
156
+ graphOptimizationLevel: "all"
157
+ });
158
+ if (!["input_ids", "attention_mask", "token_type_ids"].every(
159
+ (name) => session.inputNames.includes(name)
160
+ ) || !session.outputNames.includes("last_hidden_state")) {
161
+ await session.release();
162
+ throw new Error("local model input or output contract is invalid");
163
+ }
164
+ return { onnx, session, tokenizer };
165
+ }
166
+ function truncateEncoding(encoding, tokenizer) {
167
+ const tokenTypeIds = encoding.token_type_ids ?? encoding.ids.map(() => 0);
168
+ if (encoding.ids.length <= MAXIMUM_TOKENS) {
169
+ return {
170
+ attentionMask: encoding.attention_mask,
171
+ ids: encoding.ids,
172
+ tokenTypeIds
173
+ };
174
+ }
175
+ const separatorId = tokenizer.token_to_id("[SEP]");
176
+ if (separatorId === void 0) {
177
+ throw new Error("tokenizer separator token is missing");
178
+ }
179
+ const ids = encoding.ids.slice(0, MAXIMUM_TOKENS);
180
+ ids[MAXIMUM_TOKENS - 1] = separatorId;
181
+ const attentionMask = encoding.attention_mask.slice(0, MAXIMUM_TOKENS);
182
+ attentionMask[MAXIMUM_TOKENS - 1] = 1;
183
+ const truncatedTokenTypes = tokenTypeIds.slice(0, MAXIMUM_TOKENS);
184
+ truncatedTokenTypes[MAXIMUM_TOKENS - 1] = 0;
185
+ return {
186
+ attentionMask,
187
+ ids,
188
+ tokenTypeIds: truncatedTokenTypes
189
+ };
190
+ }
191
+ function paddedTensorData(inputs, field, sequenceLength, padId) {
192
+ const values = new BigInt64Array(inputs.length * sequenceLength);
193
+ for (const [rowIndex, input] of inputs.entries()) {
194
+ const row = input[field];
195
+ for (let columnIndex = 0; columnIndex < sequenceLength; columnIndex += 1) {
196
+ const fallback = field === "ids" ? padId : 0;
197
+ values[rowIndex * sequenceLength + columnIndex] = BigInt(row[columnIndex] ?? fallback);
198
+ }
199
+ }
200
+ return values;
201
+ }
202
+ async function embedBatch(runtime, texts) {
203
+ if (texts.length === 0) {
204
+ return [];
205
+ }
206
+ const inputs = texts.map(
207
+ (text) => truncateEncoding(
208
+ runtime.tokenizer.encode(text, { return_token_type_ids: true }),
209
+ runtime.tokenizer
210
+ )
211
+ );
212
+ const sequenceLength = Math.max(...inputs.map((input) => input.ids.length));
213
+ const dimensions = [inputs.length, sequenceLength];
214
+ const padId = runtime.tokenizer.token_to_id("[PAD]");
215
+ if (padId === void 0) {
216
+ throw new Error("tokenizer padding token is missing");
217
+ }
218
+ const output = await runtime.session.run({
219
+ attention_mask: new runtime.onnx.Tensor(
220
+ "int64",
221
+ paddedTensorData(inputs, "attentionMask", sequenceLength, padId),
222
+ dimensions
223
+ ),
224
+ input_ids: new runtime.onnx.Tensor(
225
+ "int64",
226
+ paddedTensorData(inputs, "ids", sequenceLength, padId),
227
+ dimensions
228
+ ),
229
+ token_type_ids: new runtime.onnx.Tensor(
230
+ "int64",
231
+ paddedTensorData(inputs, "tokenTypeIds", sequenceLength, padId),
232
+ dimensions
233
+ )
234
+ });
235
+ const tensor = output["last_hidden_state"];
236
+ if (tensor === void 0 || tensor.type !== "float32" || tensor.dims.length !== 3 || tensor.dims[0] !== inputs.length || tensor.dims[1] !== sequenceLength || tensor.dims[2] !== CONTENT_KNOWLEDGE_EMBEDDING_DIMENSIONS) {
237
+ throw new Error("embedding output contract is invalid");
238
+ }
239
+ const vectors = [];
240
+ const rowStride = sequenceLength * CONTENT_KNOWLEDGE_EMBEDDING_DIMENSIONS;
241
+ for (let rowIndex = 0; rowIndex < inputs.length; rowIndex += 1) {
242
+ const vector = new Float32Array(CONTENT_KNOWLEDGE_EMBEDDING_DIMENSIONS);
243
+ let squaredNorm = 0;
244
+ const offset = rowIndex * rowStride;
245
+ for (let index = 0; index < vector.length; index += 1) {
246
+ const value = tensor.data[offset + index];
247
+ if (typeof value !== "number" || !Number.isFinite(value)) {
248
+ throw new Error("embedding output contains an invalid value");
249
+ }
250
+ vector[index] = value;
251
+ squaredNorm += value * value;
252
+ }
253
+ const norm = Math.sqrt(squaredNorm);
254
+ if (!Number.isFinite(norm) || norm === 0) {
255
+ throw new Error("embedding output norm is invalid");
256
+ }
257
+ for (let index = 0; index < vector.length; index += 1) {
258
+ vector[index] = (vector[index] ?? 0) / norm;
259
+ }
260
+ vectors.push(vector);
261
+ }
262
+ return vectors;
263
+ }
264
+ async function isKnowledgeModelCached(cacheDirectory) {
265
+ try {
266
+ const raw = await readPrivateFile(modelMarkerPath(cacheDirectory), 16 * 1024);
267
+ const value = JSON.parse(raw);
268
+ if (!contentKnowledgeModelSchema.safeParse(value.model).success || value.runtime !== "onnxruntime-web@1.22.0" || !Array.isArray(value.artifacts) || value.artifacts.join(",") !== MODEL_ARTIFACTS.map((artifact) => artifact.sha256).join(",")) {
269
+ return false;
270
+ }
271
+ return (await Promise.all(
272
+ MODEL_ARTIFACTS.map((artifact) => readVerifiedArtifact(cacheDirectory, artifact))
273
+ )).every((artifact) => artifact !== void 0);
274
+ } catch {
275
+ return false;
276
+ }
277
+ }
278
+ var LocalBgeKnowledgeEmbedder = class {
279
+ model = contentKnowledgeModelSchema.parse({
280
+ dimensions: CONTENT_KNOWLEDGE_EMBEDDING_DIMENSIONS,
281
+ id: CONTENT_KNOWLEDGE_EMBEDDING_MODEL,
282
+ quantization: CONTENT_KNOWLEDGE_EMBEDDING_QUANTIZATION,
283
+ revision: CONTENT_KNOWLEDGE_EMBEDDING_REVISION
284
+ });
285
+ #cacheDirectory;
286
+ #now;
287
+ #runtimePromise;
288
+ constructor(cacheDirectory, now = () => /* @__PURE__ */ new Date()) {
289
+ this.#cacheDirectory = cacheDirectory;
290
+ this.#now = now;
291
+ }
292
+ async #runtime() {
293
+ if (this.#runtimePromise === void 0) {
294
+ this.#runtimePromise = (async () => {
295
+ const runtime = await createEmbeddingRuntime(this.#cacheDirectory);
296
+ await writePrivateJson(modelMarkerPath(this.#cacheDirectory), {
297
+ artifacts: MODEL_ARTIFACTS.map((artifact) => artifact.sha256),
298
+ model: this.model,
299
+ readyAt: this.#now().toISOString(),
300
+ runtime: "onnxruntime-web@1.22.0"
301
+ });
302
+ return runtime;
303
+ })();
304
+ }
305
+ return this.#runtimePromise;
306
+ }
307
+ async embedDocuments(texts) {
308
+ const runtime = await this.#runtime();
309
+ const results = [];
310
+ for (let index = 0; index < texts.length; index += EMBEDDING_BATCH_SIZE) {
311
+ results.push(
312
+ ...await embedBatch(runtime, texts.slice(index, index + EMBEDDING_BATCH_SIZE))
313
+ );
314
+ }
315
+ return results;
316
+ }
317
+ async embedQuery(query) {
318
+ const runtime = await this.#runtime();
319
+ const [vector] = await embedBatch(runtime, [`${QUERY_INSTRUCTION}${query}`]);
320
+ if (vector === void 0) {
321
+ throw new Error("embedding output is empty");
322
+ }
323
+ return vector;
324
+ }
325
+ };
326
+
327
+ // apps/cli/src/knowledge.ts
328
+ var DATABASE_DISPLAY_PATH = "~/.dxc/content-memory.sqlite";
329
+ var MAXIMUM_ARTICLE_BYTES = 5 * 1024 * 1024;
330
+ var MAXIMUM_CHUNK_CHARACTERS = 420;
331
+ var TARGET_CHUNK_CHARACTERS = 280;
332
+ var MAXIMUM_RESULTS = 20;
333
+ var SEARCH_CANDIDATE_LIMIT = 80;
334
+ var RRF_CONSTANT = 60;
335
+ var SEMANTIC_MINIMUM_SIMILARITY = 0.28;
336
+ var SEMANTIC_MAXIMUM_DISTANCE_FROM_BEST = 0.12;
337
+ var SUPPORTED_EXTENSIONS = /* @__PURE__ */ new Set([".md", ".markdown", ".txt"]);
338
+ var UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu;
339
+ var KnowledgeCliError = class extends Error {
340
+ constructor(code, message) {
341
+ super(message);
342
+ this.code = code;
343
+ this.name = "KnowledgeCliError";
344
+ }
345
+ code;
346
+ };
347
+ function errorCode(error) {
348
+ return typeof error === "object" && error !== null && "code" in error ? String(error.code) : void 0;
349
+ }
350
+ function sha2562(value) {
351
+ return createHash2("sha256").update(value).digest("hex");
352
+ }
353
+ function normalizedMarkdown(value) {
354
+ return value.replaceAll("\r\n", "\n").replaceAll("\r", "\n").trim();
355
+ }
356
+ function removeFrontmatter(value) {
357
+ return value.replace(/^---\n[\s\S]*?\n---(?:\n|$)/u, "");
358
+ }
359
+ function plainMarkdown(value) {
360
+ return value.replace(/<!--[\s\S]*?-->/gu, " ").replace(/!\[([^\]]*)\]\([^)]+\)/gu, "$1").replace(/\[([^\]]+)\]\([^)]+\)/gu, "$1").replace(/<[^>]+>/gu, " ").replace(/```[^\n]*\n?/gu, "").replace(/`([^`]+)`/gu, "$1").replace(/^\s{0,3}(?:[-*+]|\d+[.)])\s+/gmu, "").replace(/^\s{0,3}>\s?/gmu, "").replace(/[*_~]{1,3}/gu, "").replace(/[ \t]+/gu, " ").replace(/\n+/gu, " ").trim();
361
+ }
362
+ function extractTitle(value, sourceLabel) {
363
+ const withoutFrontmatter = removeFrontmatter(value);
364
+ const heading = withoutFrontmatter.match(/^\s{0,3}#\s+(.+)$/mu)?.[1];
365
+ const fallback = path2.basename(sourceLabel, path2.extname(sourceLabel));
366
+ const title = plainMarkdown(heading ?? fallback).slice(0, 300).trim();
367
+ if (title.length === 0) {
368
+ throw new KnowledgeCliError("DXC_KNOWLEDGE_ARTICLE_INVALID", "Article title is empty");
369
+ }
370
+ return title;
371
+ }
372
+ function splitOversizedText(value) {
373
+ if ([...value].length <= MAXIMUM_CHUNK_CHARACTERS) {
374
+ return [value];
375
+ }
376
+ const sentences = value.match(/[^。!?!?;;\n]+(?:[。!?!?;;]+|$)/gu)?.map((item) => item.trim()) ?? [value];
377
+ const parts = [];
378
+ let current = "";
379
+ const flush = () => {
380
+ if (current.length > 0) {
381
+ parts.push(current);
382
+ current = "";
383
+ }
384
+ };
385
+ for (const sentence of sentences) {
386
+ if ([...sentence].length > MAXIMUM_CHUNK_CHARACTERS) {
387
+ flush();
388
+ const characters = [...sentence];
389
+ for (let index = 0; index < characters.length; index += MAXIMUM_CHUNK_CHARACTERS) {
390
+ parts.push(characters.slice(index, index + MAXIMUM_CHUNK_CHARACTERS).join(""));
391
+ }
392
+ continue;
393
+ }
394
+ const combined = current.length === 0 ? sentence : `${current} ${sentence}`;
395
+ if ([...combined].length > MAXIMUM_CHUNK_CHARACTERS) {
396
+ flush();
397
+ current = sentence;
398
+ } else {
399
+ current = combined;
400
+ }
401
+ }
402
+ flush();
403
+ return parts;
404
+ }
405
+ function chunkHistoricalArticle(markdown) {
406
+ const value = removeFrontmatter(markdown);
407
+ const lines = value.split("\n");
408
+ const paragraphs = [];
409
+ let heading = null;
410
+ let paragraphLines = [];
411
+ const flushParagraph = () => {
412
+ const text = plainMarkdown(paragraphLines.join("\n"));
413
+ paragraphLines = [];
414
+ if (text.length === 0) {
415
+ return;
416
+ }
417
+ for (const part of splitOversizedText(text)) {
418
+ paragraphs.push({ heading, text: part });
419
+ }
420
+ };
421
+ for (const line of lines) {
422
+ const headingMatch = line.match(/^\s{0,3}#{1,6}\s+(.+)$/u);
423
+ if (headingMatch !== null) {
424
+ flushParagraph();
425
+ heading = plainMarkdown(headingMatch[1] ?? "").slice(0, 300).trim() || null;
426
+ continue;
427
+ }
428
+ if (line.trim().length === 0) {
429
+ flushParagraph();
430
+ } else {
431
+ paragraphLines.push(line);
432
+ }
433
+ }
434
+ flushParagraph();
435
+ const merged = [];
436
+ for (const paragraph of paragraphs) {
437
+ const previous = merged.at(-1);
438
+ const combined = previous === void 0 ? paragraph.text : `${previous.body}
439
+
440
+ ${paragraph.text}`;
441
+ if (previous !== void 0 && previous.heading === paragraph.heading && [...previous.body].length < TARGET_CHUNK_CHARACTERS && [...combined].length <= MAXIMUM_CHUNK_CHARACTERS) {
442
+ merged[merged.length - 1] = {
443
+ body: combined,
444
+ heading: previous.heading
445
+ };
446
+ } else {
447
+ merged.push({
448
+ body: paragraph.text,
449
+ heading: paragraph.heading
450
+ });
451
+ }
452
+ }
453
+ return merged.map((chunk, ordinal) => ({
454
+ ...chunk,
455
+ ordinal,
456
+ sha256: sha2562(chunk.body)
457
+ }));
458
+ }
459
+ async function prepareArticle(currentDirectory, inputPath) {
460
+ const absolutePath = path2.resolve(currentDirectory, inputPath);
461
+ const extension = path2.extname(absolutePath).toLowerCase();
462
+ if (!SUPPORTED_EXTENSIONS.has(extension)) {
463
+ throw new KnowledgeCliError(
464
+ "DXC_KNOWLEDGE_ARTICLE_INVALID",
465
+ "Only Markdown and text articles can be imported"
466
+ );
467
+ }
468
+ let metadata;
469
+ try {
470
+ metadata = await stat(absolutePath);
471
+ } catch (error) {
472
+ if (errorCode(error) === "ENOENT") {
473
+ throw new KnowledgeCliError("DXC_KNOWLEDGE_INPUT_NOT_FOUND", "Article input was not found");
474
+ }
475
+ throw new KnowledgeCliError("DXC_KNOWLEDGE_IO_FAILED", "Unable to read article input");
476
+ }
477
+ if (!metadata.isFile()) {
478
+ throw new KnowledgeCliError("DXC_KNOWLEDGE_ARTICLE_INVALID", "Article input is invalid");
479
+ }
480
+ if (metadata.size > MAXIMUM_ARTICLE_BYTES) {
481
+ throw new KnowledgeCliError(
482
+ "DXC_KNOWLEDGE_FILE_TOO_LARGE",
483
+ "Article input exceeds the 5 MB limit"
484
+ );
485
+ }
486
+ let content;
487
+ try {
488
+ content = normalizedMarkdown(await readFile(absolutePath, "utf8"));
489
+ } catch {
490
+ throw new KnowledgeCliError("DXC_KNOWLEDGE_IO_FAILED", "Unable to read article input");
491
+ }
492
+ if (content.length === 0 || content.includes("\0")) {
493
+ throw new KnowledgeCliError("DXC_KNOWLEDGE_ARTICLE_INVALID", "Article content is invalid");
494
+ }
495
+ const sourceLabel = path2.basename(absolutePath).slice(0, 255);
496
+ const chunks = chunkHistoricalArticle(content);
497
+ if (chunks.length === 0) {
498
+ throw new KnowledgeCliError(
499
+ "DXC_KNOWLEDGE_ARTICLE_INVALID",
500
+ "Article has no indexable content"
501
+ );
502
+ }
503
+ return {
504
+ chunks,
505
+ contentSha256: sha2562(content),
506
+ sourceKey: sha2562(absolutePath),
507
+ sourceLabel,
508
+ title: extractTitle(content, sourceLabel)
509
+ };
510
+ }
511
+ function sqlRow(value) {
512
+ if (typeof value !== "object" || value === null) {
513
+ throw new KnowledgeCliError("DXC_KNOWLEDGE_STATE_INVALID", "Knowledge state is invalid");
514
+ }
515
+ return value;
516
+ }
517
+ function stringColumn(row, name) {
518
+ const value = row[name];
519
+ if (typeof value !== "string") {
520
+ throw new KnowledgeCliError("DXC_KNOWLEDGE_STATE_INVALID", "Knowledge state is invalid");
521
+ }
522
+ return value;
523
+ }
524
+ function numberColumn(row, name) {
525
+ const value = row[name];
526
+ if (typeof value !== "number" || !Number.isFinite(value)) {
527
+ throw new KnowledgeCliError("DXC_KNOWLEDGE_STATE_INVALID", "Knowledge state is invalid");
528
+ }
529
+ return value;
530
+ }
531
+ function articleRow(value) {
532
+ const row = sqlRow(value);
533
+ return {
534
+ chunkCount: numberColumn(row, "chunk_count"),
535
+ chunkerVersion: stringColumn(row, "chunker_version"),
536
+ contentSha256: stringColumn(row, "content_sha256"),
537
+ id: stringColumn(row, "id"),
538
+ importedAt: stringColumn(row, "imported_at"),
539
+ sourceLabel: stringColumn(row, "source_label"),
540
+ title: stringColumn(row, "title"),
541
+ updatedAt: stringColumn(row, "updated_at")
542
+ };
543
+ }
544
+ function articleSummary(row) {
545
+ return {
546
+ articleId: row.id,
547
+ chunkCount: row.chunkCount,
548
+ contentSha256: row.contentSha256,
549
+ importedAt: row.importedAt,
550
+ sourceLabel: row.sourceLabel,
551
+ title: row.title,
552
+ updatedAt: row.updatedAt
553
+ };
554
+ }
555
+ function vectorBuffer(vector) {
556
+ return Buffer.from(vector.buffer, vector.byteOffset, vector.byteLength);
557
+ }
558
+ function vectorFromColumn(value) {
559
+ if (value === null) {
560
+ return null;
561
+ }
562
+ if (!(value instanceof Uint8Array)) {
563
+ throw new KnowledgeCliError("DXC_KNOWLEDGE_STATE_INVALID", "Knowledge vector is invalid");
564
+ }
565
+ const expectedBytes = CONTENT_KNOWLEDGE_EMBEDDING_DIMENSIONS * Float32Array.BYTES_PER_ELEMENT;
566
+ if (value.byteLength !== expectedBytes) {
567
+ throw new KnowledgeCliError("DXC_KNOWLEDGE_STATE_INVALID", "Knowledge vector is invalid");
568
+ }
569
+ return new Float32Array(Uint8Array.from(value).buffer);
570
+ }
571
+ function chunkRow(value) {
572
+ const row = sqlRow(value);
573
+ const headingValue = row["heading"];
574
+ if (headingValue !== null && typeof headingValue !== "string") {
575
+ throw new KnowledgeCliError("DXC_KNOWLEDGE_STATE_INVALID", "Knowledge state is invalid");
576
+ }
577
+ return {
578
+ articleId: stringColumn(row, "article_id"),
579
+ body: stringColumn(row, "body"),
580
+ chunkSha256: stringColumn(row, "chunk_sha256"),
581
+ embedding: vectorFromColumn(row["embedding"]),
582
+ heading: headingValue,
583
+ id: stringColumn(row, "id"),
584
+ ordinal: numberColumn(row, "ordinal"),
585
+ sourceLabel: stringColumn(row, "source_label"),
586
+ title: stringColumn(row, "title")
587
+ };
588
+ }
589
+ function databaseSchema(database) {
590
+ const versionRow = sqlRow(database.prepare("PRAGMA user_version").get());
591
+ const version = numberColumn(versionRow, "user_version");
592
+ if (version !== 0 && version !== Number(CONTENT_KNOWLEDGE_SCHEMA_VERSION)) {
593
+ throw new KnowledgeCliError(
594
+ "DXC_KNOWLEDGE_SCHEMA_UNSUPPORTED",
595
+ "Knowledge database version is unsupported"
596
+ );
597
+ }
598
+ database.exec(`
599
+ PRAGMA foreign_keys = ON;
600
+ PRAGMA busy_timeout = 5000;
601
+ PRAGMA journal_mode = WAL;
602
+ PRAGMA synchronous = NORMAL;
603
+
604
+ CREATE TABLE IF NOT EXISTS articles (
605
+ id TEXT PRIMARY KEY,
606
+ source_key TEXT NOT NULL UNIQUE,
607
+ source_label TEXT NOT NULL,
608
+ title TEXT NOT NULL,
609
+ content_sha256 TEXT NOT NULL,
610
+ chunker_version TEXT NOT NULL,
611
+ imported_at TEXT NOT NULL,
612
+ updated_at TEXT NOT NULL,
613
+ chunk_count INTEGER NOT NULL CHECK (chunk_count > 0)
614
+ );
615
+
616
+ CREATE INDEX IF NOT EXISTS articles_content_sha256_idx
617
+ ON articles(content_sha256);
618
+
619
+ CREATE TABLE IF NOT EXISTS chunks (
620
+ id TEXT PRIMARY KEY,
621
+ article_id TEXT NOT NULL REFERENCES articles(id) ON DELETE CASCADE,
622
+ ordinal INTEGER NOT NULL CHECK (ordinal >= 0),
623
+ heading TEXT,
624
+ body TEXT NOT NULL,
625
+ chunk_sha256 TEXT NOT NULL,
626
+ embedding BLOB,
627
+ embedding_model TEXT,
628
+ embedding_revision TEXT,
629
+ embedding_quantization TEXT,
630
+ embedding_dimensions INTEGER,
631
+ CHECK (
632
+ (
633
+ embedding IS NULL
634
+ AND embedding_model IS NULL
635
+ AND embedding_revision IS NULL
636
+ AND embedding_quantization IS NULL
637
+ AND embedding_dimensions IS NULL
638
+ )
639
+ OR
640
+ (
641
+ embedding IS NOT NULL
642
+ AND embedding_model IS NOT NULL
643
+ AND embedding_revision IS NOT NULL
644
+ AND embedding_quantization IS NOT NULL
645
+ AND embedding_dimensions IS NOT NULL
646
+ )
647
+ ),
648
+ UNIQUE(article_id, ordinal)
649
+ );
650
+
651
+ CREATE INDEX IF NOT EXISTS chunks_article_id_idx
652
+ ON chunks(article_id);
653
+
654
+ CREATE VIRTUAL TABLE IF NOT EXISTS chunk_search USING fts5(
655
+ chunk_id UNINDEXED,
656
+ article_id UNINDEXED,
657
+ title,
658
+ heading,
659
+ body,
660
+ tokenize = 'trigram'
661
+ );
662
+
663
+ PRAGMA user_version = 1;
664
+ `);
665
+ }
666
+ async function openDatabase(target) {
667
+ try {
668
+ await ensurePrivateRegularFile(target);
669
+ const database = new DatabaseSync(target);
670
+ try {
671
+ databaseSchema(database);
672
+ return database;
673
+ } catch (error) {
674
+ database.close();
675
+ throw error;
676
+ }
677
+ } catch (error) {
678
+ if (error instanceof KnowledgeCliError) {
679
+ throw error;
680
+ }
681
+ throw new KnowledgeCliError("DXC_KNOWLEDGE_IO_FAILED", "Unable to open knowledge database");
682
+ }
683
+ }
684
+ async function databaseExists(target) {
685
+ try {
686
+ const metadata = await lstat(target);
687
+ if (!metadata.isFile() || metadata.isSymbolicLink()) {
688
+ throw new KnowledgeCliError("DXC_KNOWLEDGE_STATE_INVALID", "Knowledge database is invalid");
689
+ }
690
+ return true;
691
+ } catch (error) {
692
+ if (error instanceof KnowledgeCliError) {
693
+ throw error;
694
+ }
695
+ if (errorCode(error) === "ENOENT") {
696
+ return false;
697
+ }
698
+ throw new KnowledgeCliError("DXC_KNOWLEDGE_IO_FAILED", "Unable to inspect knowledge database");
699
+ }
700
+ }
701
+ function embeddingText(article, chunk) {
702
+ return [article.title.slice(0, 120), chunk.heading?.slice(0, 120) ?? "", chunk.body].filter((value) => value.length > 0).join("\n");
703
+ }
704
+ async function embedArticle(embedder, article) {
705
+ let vectors;
706
+ try {
707
+ vectors = await embedder.embedDocuments(
708
+ article.chunks.map((chunk) => embeddingText(article, chunk))
709
+ );
710
+ } catch {
711
+ throw new KnowledgeCliError(
712
+ "DXC_KNOWLEDGE_MODEL_FAILED",
713
+ "Local semantic model could not index the article"
714
+ );
715
+ }
716
+ if (vectors.length !== article.chunks.length || vectors.some((vector) => vector.length !== CONTENT_KNOWLEDGE_EMBEDDING_DIMENSIONS)) {
717
+ throw new KnowledgeCliError(
718
+ "DXC_KNOWLEDGE_MODEL_FAILED",
719
+ "Local semantic model returned invalid vectors"
720
+ );
721
+ }
722
+ return vectors;
723
+ }
724
+ function findExistingArticle(database, article) {
725
+ const statement = database.prepare(`
726
+ SELECT
727
+ id, source_label, title, content_sha256, chunker_version,
728
+ imported_at, updated_at, chunk_count
729
+ FROM articles
730
+ WHERE source_key = ?
731
+ LIMIT 1
732
+ `);
733
+ const bySource = statement.get(article.sourceKey);
734
+ if (bySource !== void 0) {
735
+ return articleRow(bySource);
736
+ }
737
+ const byContent = database.prepare(
738
+ `
739
+ SELECT
740
+ id, source_label, title, content_sha256, chunker_version,
741
+ imported_at, updated_at, chunk_count
742
+ FROM articles
743
+ WHERE content_sha256 = ?
744
+ ORDER BY imported_at ASC
745
+ LIMIT 1
746
+ `
747
+ ).get(article.contentSha256);
748
+ return byContent === void 0 ? void 0 : articleRow(byContent);
749
+ }
750
+ function semanticIndexComplete(database, articleId) {
751
+ const row = sqlRow(
752
+ database.prepare(
753
+ `
754
+ SELECT
755
+ COUNT(*) AS total,
756
+ COALESCE(SUM(
757
+ CASE
758
+ WHEN embedding IS NOT NULL
759
+ AND embedding_model = ?
760
+ AND embedding_revision = ?
761
+ AND embedding_quantization = ?
762
+ AND embedding_dimensions = ?
763
+ THEN 1
764
+ ELSE 0
765
+ END
766
+ ), 0) AS indexed
767
+ FROM chunks
768
+ WHERE article_id = ?
769
+ `
770
+ ).get(
771
+ CONTENT_KNOWLEDGE_EMBEDDING_MODEL,
772
+ CONTENT_KNOWLEDGE_EMBEDDING_REVISION,
773
+ CONTENT_KNOWLEDGE_EMBEDDING_QUANTIZATION,
774
+ CONTENT_KNOWLEDGE_EMBEDDING_DIMENSIONS,
775
+ articleId
776
+ )
777
+ );
778
+ return numberColumn(row, "total") > 0 && numberColumn(row, "total") === numberColumn(row, "indexed");
779
+ }
780
+ function saveArticle(database, article, existing, vectors, timestamp, createId) {
781
+ const articleId = existing?.id ?? createId();
782
+ database.exec("BEGIN IMMEDIATE");
783
+ try {
784
+ if (existing === void 0) {
785
+ database.prepare(
786
+ `
787
+ INSERT INTO articles (
788
+ id, source_key, source_label, title, content_sha256, chunker_version,
789
+ imported_at, updated_at, chunk_count
790
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
791
+ `
792
+ ).run(
793
+ articleId,
794
+ article.sourceKey,
795
+ article.sourceLabel,
796
+ article.title,
797
+ article.contentSha256,
798
+ CONTENT_KNOWLEDGE_CHUNKER_VERSION,
799
+ timestamp,
800
+ timestamp,
801
+ article.chunks.length
802
+ );
803
+ } else {
804
+ database.prepare("DELETE FROM chunk_search WHERE article_id = ?").run(articleId);
805
+ database.prepare("DELETE FROM chunks WHERE article_id = ?").run(articleId);
806
+ database.prepare(
807
+ `
808
+ UPDATE articles
809
+ SET
810
+ source_key = ?,
811
+ source_label = ?,
812
+ title = ?,
813
+ content_sha256 = ?,
814
+ chunker_version = ?,
815
+ updated_at = ?,
816
+ chunk_count = ?
817
+ WHERE id = ?
818
+ `
819
+ ).run(
820
+ article.sourceKey,
821
+ article.sourceLabel,
822
+ article.title,
823
+ article.contentSha256,
824
+ CONTENT_KNOWLEDGE_CHUNKER_VERSION,
825
+ timestamp,
826
+ article.chunks.length,
827
+ articleId
828
+ );
829
+ }
830
+ const insertChunk = database.prepare(`
831
+ INSERT INTO chunks (
832
+ id, article_id, ordinal, heading, body, chunk_sha256,
833
+ embedding, embedding_model, embedding_revision, embedding_quantization,
834
+ embedding_dimensions
835
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
836
+ `);
837
+ const insertSearch = database.prepare(`
838
+ INSERT INTO chunk_search (chunk_id, article_id, title, heading, body)
839
+ VALUES (?, ?, ?, ?, ?)
840
+ `);
841
+ for (const [index, chunk] of article.chunks.entries()) {
842
+ const chunkId = createId();
843
+ const vector = vectors?.[index];
844
+ insertChunk.run(
845
+ chunkId,
846
+ articleId,
847
+ chunk.ordinal,
848
+ chunk.heading,
849
+ chunk.body,
850
+ chunk.sha256,
851
+ vector === void 0 ? null : vectorBuffer(vector),
852
+ vector === void 0 ? null : CONTENT_KNOWLEDGE_EMBEDDING_MODEL,
853
+ vector === void 0 ? null : CONTENT_KNOWLEDGE_EMBEDDING_REVISION,
854
+ vector === void 0 ? null : CONTENT_KNOWLEDGE_EMBEDDING_QUANTIZATION,
855
+ vector === void 0 ? null : CONTENT_KNOWLEDGE_EMBEDDING_DIMENSIONS
856
+ );
857
+ insertSearch.run(chunkId, articleId, article.title, chunk.heading ?? "", chunk.body);
858
+ }
859
+ database.exec("COMMIT");
860
+ } catch (error) {
861
+ try {
862
+ database.exec("ROLLBACK");
863
+ } catch {
864
+ }
865
+ if (error instanceof KnowledgeCliError) {
866
+ throw error;
867
+ }
868
+ throw new KnowledgeCliError("DXC_KNOWLEDGE_IO_FAILED", "Unable to save article index");
869
+ }
870
+ return {
871
+ chunkCount: article.chunks.length,
872
+ chunkerVersion: CONTENT_KNOWLEDGE_CHUNKER_VERSION,
873
+ contentSha256: article.contentSha256,
874
+ id: articleId,
875
+ importedAt: existing?.importedAt ?? timestamp,
876
+ sourceLabel: article.sourceLabel,
877
+ title: article.title,
878
+ updatedAt: timestamp
879
+ };
880
+ }
881
+ function loadChunks(database) {
882
+ return database.prepare(
883
+ `
884
+ SELECT
885
+ c.id,
886
+ c.article_id,
887
+ c.ordinal,
888
+ c.heading,
889
+ c.body,
890
+ c.chunk_sha256,
891
+ CASE
892
+ WHEN c.embedding_model = ?
893
+ AND c.embedding_revision = ?
894
+ AND c.embedding_quantization = ?
895
+ AND c.embedding_dimensions = ?
896
+ THEN c.embedding
897
+ ELSE NULL
898
+ END AS embedding,
899
+ a.title,
900
+ a.source_label
901
+ FROM chunks c
902
+ INNER JOIN articles a ON a.id = c.article_id
903
+ ORDER BY a.imported_at ASC, c.ordinal ASC
904
+ `
905
+ ).all(
906
+ CONTENT_KNOWLEDGE_EMBEDDING_MODEL,
907
+ CONTENT_KNOWLEDGE_EMBEDDING_REVISION,
908
+ CONTENT_KNOWLEDGE_EMBEDDING_QUANTIZATION,
909
+ CONTENT_KNOWLEDGE_EMBEDDING_DIMENSIONS
910
+ ).map(chunkRow);
911
+ }
912
+ function ftsExpression(query) {
913
+ const terms = query.normalize("NFKC").split(/[\s,,。!?!?;;::、/\\|()[\]{}]+/u).map((term) => term.trim()).filter((term) => [...term].length >= 3).slice(0, 12);
914
+ if (terms.length === 0) {
915
+ return null;
916
+ }
917
+ return terms.map((term) => `"${term.replaceAll('"', '""')}"`).join(" OR ");
918
+ }
919
+ function lexicalRanking(database, query) {
920
+ const expression = ftsExpression(query);
921
+ if (expression === null) {
922
+ return [];
923
+ }
924
+ try {
925
+ return database.prepare(
926
+ `
927
+ SELECT chunk_id
928
+ FROM chunk_search
929
+ WHERE chunk_search MATCH ?
930
+ ORDER BY bm25(chunk_search, 0.0, 0.0, 4.0, 2.0, 1.0)
931
+ LIMIT ?
932
+ `
933
+ ).all(expression, SEARCH_CANDIDATE_LIMIT).map((value, rank) => ({
934
+ id: stringColumn(sqlRow(value), "chunk_id"),
935
+ rank
936
+ }));
937
+ } catch {
938
+ throw new KnowledgeCliError("DXC_KNOWLEDGE_QUERY_INVALID", "Knowledge query is invalid");
939
+ }
940
+ }
941
+ function dotProduct(left, right) {
942
+ let result = 0;
943
+ for (let index = 0; index < left.length; index += 1) {
944
+ result += (left[index] ?? 0) * (right[index] ?? 0);
945
+ }
946
+ return result;
947
+ }
948
+ function semanticRanking(chunks, queryVector) {
949
+ const candidates = chunks.filter(
950
+ (chunk) => chunk.embedding !== null
951
+ ).map((chunk) => ({
952
+ id: chunk.id,
953
+ similarity: dotProduct(queryVector, chunk.embedding)
954
+ })).sort((left, right) => right.similarity - left.similarity || left.id.localeCompare(right.id)).slice(0, SEARCH_CANDIDATE_LIMIT);
955
+ const bestSimilarity = candidates[0]?.similarity;
956
+ if (bestSimilarity === void 0 || bestSimilarity < SEMANTIC_MINIMUM_SIMILARITY) {
957
+ return [];
958
+ }
959
+ return candidates.filter(
960
+ (candidate) => candidate.similarity >= SEMANTIC_MINIMUM_SIMILARITY && bestSimilarity - candidate.similarity <= SEMANTIC_MAXIMUM_DISTANCE_FROM_BEST
961
+ ).slice(0, SEARCH_CANDIDATE_LIMIT).map((candidate, rank) => ({
962
+ id: candidate.id,
963
+ rank
964
+ }));
965
+ }
966
+ function snippet(body) {
967
+ const characters = [...body];
968
+ return characters.length <= 360 ? body : `${characters.slice(0, 359).join("")}\u2026`;
969
+ }
970
+ function fusedResults(chunks, mode, lexical, semantic, limit) {
971
+ const scores = /* @__PURE__ */ new Map();
972
+ const addRanking = (ranking, lane, weight) => {
973
+ for (const candidate of ranking) {
974
+ const current = scores.get(candidate.id) ?? { lanes: /* @__PURE__ */ new Set(), score: 0 };
975
+ current.lanes.add(lane);
976
+ current.score += weight / (RRF_CONSTANT + candidate.rank + 1);
977
+ scores.set(candidate.id, current);
978
+ }
979
+ };
980
+ const semanticWeight = mode === "hybrid" ? 0.7 : mode === "semantic" ? 1 : 0;
981
+ const lexicalWeight = mode === "hybrid" ? 0.3 : mode === "lexical" ? 1 : 0;
982
+ addRanking(semantic, "semantic", semanticWeight);
983
+ addRanking(lexical, "lexical", lexicalWeight);
984
+ const byId = new Map(chunks.map((chunk) => [chunk.id, chunk]));
985
+ const articleCounts = /* @__PURE__ */ new Map();
986
+ const results = [];
987
+ for (const [chunkId, score] of [...scores.entries()].sort(
988
+ (left, right) => right[1].score - left[1].score || left[0].localeCompare(right[0])
989
+ )) {
990
+ const chunk = byId.get(chunkId);
991
+ if (chunk === void 0 || (articleCounts.get(chunk.articleId) ?? 0) >= 2) {
992
+ continue;
993
+ }
994
+ articleCounts.set(chunk.articleId, (articleCounts.get(chunk.articleId) ?? 0) + 1);
995
+ results.push({
996
+ articleId: chunk.articleId,
997
+ chunkId: chunk.id,
998
+ chunkSha256: chunk.chunkSha256,
999
+ heading: chunk.heading,
1000
+ lanes: [...score.lanes].sort(),
1001
+ ordinal: chunk.ordinal,
1002
+ rank: results.length + 1,
1003
+ snippet: snippet(chunk.body),
1004
+ sourceLabel: chunk.sourceLabel,
1005
+ title: chunk.title
1006
+ });
1007
+ if (results.length >= limit) {
1008
+ break;
1009
+ }
1010
+ }
1011
+ return results;
1012
+ }
1013
+ var KnowledgeCli = class {
1014
+ #currentDirectory;
1015
+ #databasePath;
1016
+ #embedder;
1017
+ #isModelCached;
1018
+ #now;
1019
+ #randomUUID;
1020
+ #writeStderr;
1021
+ constructor(dependencies) {
1022
+ const now = dependencies.now ?? (() => /* @__PURE__ */ new Date());
1023
+ const modelCacheDirectory = dependencies.modelCacheDirectory ?? path2.join(dependencies.homeDirectory, ".dxc", "models", "bge-small-zh-v1.5");
1024
+ this.#currentDirectory = dependencies.currentDirectory;
1025
+ this.#databasePath = dependencies.databasePath ?? path2.join(dependencies.homeDirectory, ".dxc", "content-memory.sqlite");
1026
+ this.#embedder = dependencies.embedder ?? new LocalBgeKnowledgeEmbedder(modelCacheDirectory, now);
1027
+ this.#isModelCached = dependencies.isModelCached ?? (() => isKnowledgeModelCached(modelCacheDirectory));
1028
+ this.#now = now;
1029
+ this.#randomUUID = dependencies.randomUUID ?? randomUUID;
1030
+ this.#writeStderr = dependencies.writeStderr ?? (() => void 0);
1031
+ }
1032
+ async importArticles(options) {
1033
+ if (options.files.length === 0) {
1034
+ throw new KnowledgeCliError(
1035
+ "DXC_KNOWLEDGE_ARTICLE_INVALID",
1036
+ "At least one article file is required"
1037
+ );
1038
+ }
1039
+ if (!options.lexicalOnly) {
1040
+ this.#writeStderr(
1041
+ "Preparing the local semantic index; first use downloads a quantized model of about 24 MB.\n"
1042
+ );
1043
+ }
1044
+ const database = await openDatabase(this.#databasePath);
1045
+ try {
1046
+ const imported = [];
1047
+ for (const file of options.files) {
1048
+ const article = await prepareArticle(this.#currentDirectory, file);
1049
+ const existing = findExistingArticle(database, article);
1050
+ const unchanged = existing?.contentSha256 === article.contentSha256 && existing.chunkerVersion === CONTENT_KNOWLEDGE_CHUNKER_VERSION;
1051
+ const semanticComplete = existing === void 0 ? false : semanticIndexComplete(database, existing.id);
1052
+ if (unchanged && (options.lexicalOnly || semanticComplete)) {
1053
+ imported.push({
1054
+ article: articleSummary(existing),
1055
+ semanticIndexed: semanticComplete,
1056
+ status: "unchanged"
1057
+ });
1058
+ continue;
1059
+ }
1060
+ const vectors = options.lexicalOnly ? void 0 : await embedArticle(this.#embedder, article);
1061
+ const saved = saveArticle(
1062
+ database,
1063
+ article,
1064
+ existing,
1065
+ vectors,
1066
+ this.#now().toISOString(),
1067
+ this.#randomUUID
1068
+ );
1069
+ imported.push({
1070
+ article: articleSummary(saved),
1071
+ semanticIndexed: vectors !== void 0,
1072
+ status: existing === void 0 ? "created" : "updated"
1073
+ });
1074
+ }
1075
+ return contentKnowledgeImportResultSchema.parse({
1076
+ command: "knowledge.import",
1077
+ data: {
1078
+ imported,
1079
+ model: options.lexicalOnly ? null : this.#embedder.model
1080
+ },
1081
+ ok: true
1082
+ });
1083
+ } finally {
1084
+ database.close();
1085
+ }
1086
+ }
1087
+ async remove(articleId) {
1088
+ if (!UUID_PATTERN.test(articleId)) {
1089
+ throw new KnowledgeCliError(
1090
+ "DXC_KNOWLEDGE_ARTICLE_INVALID",
1091
+ "Knowledge article ID is invalid"
1092
+ );
1093
+ }
1094
+ if (!await databaseExists(this.#databasePath)) {
1095
+ return contentKnowledgeRemoveResultSchema.parse({
1096
+ command: "knowledge.remove",
1097
+ data: { articleId, removed: false },
1098
+ ok: true
1099
+ });
1100
+ }
1101
+ const database = await openDatabase(this.#databasePath);
1102
+ try {
1103
+ database.exec("BEGIN IMMEDIATE");
1104
+ try {
1105
+ database.prepare("DELETE FROM chunk_search WHERE article_id = ?").run(articleId);
1106
+ const result = database.prepare("DELETE FROM articles WHERE id = ?").run(articleId);
1107
+ database.exec("COMMIT");
1108
+ return contentKnowledgeRemoveResultSchema.parse({
1109
+ command: "knowledge.remove",
1110
+ data: {
1111
+ articleId,
1112
+ removed: result.changes > 0
1113
+ },
1114
+ ok: true
1115
+ });
1116
+ } catch {
1117
+ database.exec("ROLLBACK");
1118
+ throw new KnowledgeCliError(
1119
+ "DXC_KNOWLEDGE_IO_FAILED",
1120
+ "Unable to remove knowledge article"
1121
+ );
1122
+ }
1123
+ } finally {
1124
+ database.close();
1125
+ }
1126
+ }
1127
+ async search(queryInput, modeInput, limitInput) {
1128
+ const query = queryInput.trim();
1129
+ const mode = contentKnowledgeSearchModeSchema.safeParse(modeInput);
1130
+ if (query.length === 0 || query.length > 500 || !mode.success || !Number.isSafeInteger(limitInput) || limitInput < 1 || limitInput > MAXIMUM_RESULTS) {
1131
+ throw new KnowledgeCliError("DXC_KNOWLEDGE_QUERY_INVALID", "Knowledge query is invalid");
1132
+ }
1133
+ if (!await databaseExists(this.#databasePath)) {
1134
+ return contentKnowledgeSearchResultSchema.parse({
1135
+ command: "knowledge.search",
1136
+ data: {
1137
+ mode: mode.data,
1138
+ model: mode.data === "lexical" ? null : this.#embedder.model,
1139
+ query,
1140
+ results: []
1141
+ },
1142
+ ok: true
1143
+ });
1144
+ }
1145
+ const database = await openDatabase(this.#databasePath);
1146
+ try {
1147
+ const chunks = loadChunks(database);
1148
+ const usesSemantic = mode.data !== "lexical";
1149
+ if (usesSemantic && chunks.some((chunk) => chunk.embedding === null)) {
1150
+ throw new KnowledgeCliError(
1151
+ "DXC_KNOWLEDGE_SEMANTIC_NOT_READY",
1152
+ "All articles must have a semantic index before hybrid search"
1153
+ );
1154
+ }
1155
+ const lexical = mode.data === "semantic" ? [] : lexicalRanking(database, query);
1156
+ let semantic = [];
1157
+ if (usesSemantic && chunks.length > 0) {
1158
+ let queryVector;
1159
+ try {
1160
+ queryVector = await this.#embedder.embedQuery(query);
1161
+ } catch {
1162
+ throw new KnowledgeCliError(
1163
+ "DXC_KNOWLEDGE_MODEL_FAILED",
1164
+ "Local semantic model could not process the query"
1165
+ );
1166
+ }
1167
+ semantic = semanticRanking(chunks, queryVector);
1168
+ }
1169
+ return contentKnowledgeSearchResultSchema.parse({
1170
+ command: "knowledge.search",
1171
+ data: {
1172
+ mode: mode.data,
1173
+ model: usesSemantic ? this.#embedder.model : null,
1174
+ query,
1175
+ results: fusedResults(chunks, mode.data, lexical, semantic, limitInput)
1176
+ },
1177
+ ok: true
1178
+ });
1179
+ } finally {
1180
+ database.close();
1181
+ }
1182
+ }
1183
+ async status() {
1184
+ let articleCount = 0;
1185
+ let chunkCount = 0;
1186
+ let semanticIndexedChunkCount = 0;
1187
+ if (await databaseExists(this.#databasePath)) {
1188
+ const database = await openDatabase(this.#databasePath);
1189
+ try {
1190
+ const articleRowValue = sqlRow(
1191
+ database.prepare("SELECT COUNT(*) AS count FROM articles").get()
1192
+ );
1193
+ const chunkRowValue = sqlRow(
1194
+ database.prepare(
1195
+ `
1196
+ SELECT
1197
+ COUNT(*) AS count,
1198
+ COALESCE(SUM(
1199
+ CASE
1200
+ WHEN embedding IS NOT NULL
1201
+ AND embedding_model = ?
1202
+ AND embedding_revision = ?
1203
+ AND embedding_quantization = ?
1204
+ AND embedding_dimensions = ?
1205
+ THEN 1
1206
+ ELSE 0
1207
+ END
1208
+ ), 0) AS semantic_count
1209
+ FROM chunks
1210
+ `
1211
+ ).get(
1212
+ CONTENT_KNOWLEDGE_EMBEDDING_MODEL,
1213
+ CONTENT_KNOWLEDGE_EMBEDDING_REVISION,
1214
+ CONTENT_KNOWLEDGE_EMBEDDING_QUANTIZATION,
1215
+ CONTENT_KNOWLEDGE_EMBEDDING_DIMENSIONS
1216
+ )
1217
+ );
1218
+ articleCount = numberColumn(articleRowValue, "count");
1219
+ chunkCount = numberColumn(chunkRowValue, "count");
1220
+ semanticIndexedChunkCount = numberColumn(chunkRowValue, "semantic_count");
1221
+ } finally {
1222
+ database.close();
1223
+ }
1224
+ }
1225
+ return contentKnowledgeStatusResultSchema.parse({
1226
+ command: "knowledge.status",
1227
+ data: {
1228
+ articleCount,
1229
+ chunkCount,
1230
+ databasePath: DATABASE_DISPLAY_PATH,
1231
+ model: contentKnowledgeModelSchema.parse({
1232
+ dimensions: CONTENT_KNOWLEDGE_EMBEDDING_DIMENSIONS,
1233
+ id: CONTENT_KNOWLEDGE_EMBEDDING_MODEL,
1234
+ quantization: CONTENT_KNOWLEDGE_EMBEDDING_QUANTIZATION,
1235
+ revision: CONTENT_KNOWLEDGE_EMBEDDING_REVISION
1236
+ }),
1237
+ modelCached: await this.#isModelCached(),
1238
+ semanticIndexedChunkCount
1239
+ },
1240
+ ok: true
1241
+ });
1242
+ }
1243
+ };
1244
+ export {
1245
+ KnowledgeCli,
1246
+ KnowledgeCliError,
1247
+ chunkHistoricalArticle
1248
+ };