@llangtop/pwiki-core 0.3.4 → 0.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/WikiEngine.d.ts +25 -37
- package/dist/WikiEngine.d.ts.map +1 -1
- package/dist/WikiEngine.js +157 -298
- package/dist/WikiEngine.js.map +1 -1
- package/dist/ast-chunker.d.ts +23 -0
- package/dist/ast-chunker.d.ts.map +1 -0
- package/dist/ast-chunker.js +434 -0
- package/dist/ast-chunker.js.map +1 -0
- package/dist/content-cache.d.ts +13 -0
- package/dist/content-cache.d.ts.map +1 -0
- package/dist/content-cache.js +33 -0
- package/dist/content-cache.js.map +1 -0
- package/dist/embedder.d.ts +38 -0
- package/dist/embedder.d.ts.map +1 -0
- package/dist/embedder.js +267 -0
- package/dist/embedder.js.map +1 -0
- package/dist/file-manifest.d.ts +46 -0
- package/dist/file-manifest.d.ts.map +1 -0
- package/dist/file-manifest.js +121 -0
- package/dist/file-manifest.js.map +1 -0
- package/dist/index.d.ts +18 -8
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +25 -7
- package/dist/index.js.map +1 -1
- package/dist/indexer-compile.d.ts +20 -0
- package/dist/indexer-compile.d.ts.map +1 -0
- package/dist/indexer-compile.js +198 -0
- package/dist/indexer-compile.js.map +1 -0
- package/dist/indexer-embed.d.ts +21 -0
- package/dist/indexer-embed.d.ts.map +1 -0
- package/dist/indexer-embed.js +248 -0
- package/dist/indexer-embed.js.map +1 -0
- package/dist/indexer-scan.d.ts +4 -0
- package/dist/indexer-scan.d.ts.map +1 -0
- package/dist/indexer-scan.js +51 -0
- package/dist/indexer-scan.js.map +1 -0
- package/dist/indexer.d.ts +4 -0
- package/dist/indexer.d.ts.map +1 -0
- package/dist/indexer.js +7 -0
- package/dist/indexer.js.map +1 -0
- package/dist/model-registry.d.ts +32 -0
- package/dist/model-registry.d.ts.map +1 -0
- package/dist/model-registry.js +82 -0
- package/dist/model-registry.js.map +1 -0
- package/dist/parser.d.ts +9 -0
- package/dist/parser.d.ts.map +1 -0
- package/dist/parser.js +54 -0
- package/dist/parser.js.map +1 -0
- package/dist/preprocessor.d.ts +36 -0
- package/dist/preprocessor.d.ts.map +1 -0
- package/dist/preprocessor.js +209 -0
- package/dist/preprocessor.js.map +1 -0
- package/dist/search.d.ts +6 -0
- package/dist/search.d.ts.map +1 -0
- package/dist/search.js +91 -0
- package/dist/search.js.map +1 -0
- package/dist/semantic-compiler.d.ts +44 -0
- package/dist/semantic-compiler.d.ts.map +1 -0
- package/dist/semantic-compiler.js +376 -0
- package/dist/semantic-compiler.js.map +1 -0
- package/dist/semantic-search.d.ts +11 -0
- package/dist/semantic-search.d.ts.map +1 -0
- package/dist/semantic-search.js +217 -0
- package/dist/semantic-search.js.map +1 -0
- package/dist/store-settings.d.ts +32 -0
- package/dist/store-settings.d.ts.map +1 -0
- package/dist/store-settings.js +138 -0
- package/dist/store-settings.js.map +1 -0
- package/dist/store-vectors.d.ts +13 -0
- package/dist/store-vectors.d.ts.map +1 -0
- package/dist/store-vectors.js +101 -0
- package/dist/store-vectors.js.map +1 -0
- package/dist/store.d.ts +11 -0
- package/dist/store.d.ts.map +1 -0
- package/dist/store.js +28 -0
- package/dist/store.js.map +1 -0
- package/dist/types.d.ts +75 -92
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +1 -1
- package/dist/types.js.map +1 -1
- package/dist/wiki-paths.d.ts +3 -0
- package/dist/wiki-paths.d.ts.map +1 -0
- package/dist/wiki-paths.js +13 -0
- package/dist/wiki-paths.js.map +1 -0
- package/package.json +38 -38
- package/dist/compile/compiler.d.ts +0 -39
- package/dist/compile/compiler.d.ts.map +0 -1
- package/dist/compile/compiler.js +0 -227
- package/dist/compile/compiler.js.map +0 -1
- package/dist/compile/index.d.ts +0 -3
- package/dist/compile/index.d.ts.map +0 -1
- package/dist/compile/index.js +0 -2
- package/dist/compile/index.js.map +0 -1
- package/dist/embed/WikiEmbedder.d.ts +0 -28
- package/dist/embed/WikiEmbedder.d.ts.map +0 -1
- package/dist/embed/WikiEmbedder.js +0 -147
- package/dist/embed/WikiEmbedder.js.map +0 -1
- package/dist/embed/index.d.ts +0 -2
- package/dist/embed/index.d.ts.map +0 -1
- package/dist/embed/index.js +0 -2
- package/dist/embed/index.js.map +0 -1
- package/dist/llm/WikiLLM.d.ts +0 -24
- package/dist/llm/WikiLLM.d.ts.map +0 -1
- package/dist/llm/WikiLLM.js +0 -46
- package/dist/llm/WikiLLM.js.map +0 -1
- package/dist/llm/index.d.ts +0 -3
- package/dist/llm/index.d.ts.map +0 -1
- package/dist/llm/index.js +0 -2
- package/dist/llm/index.js.map +0 -1
- package/dist/models.d.ts +0 -5
- package/dist/models.d.ts.map +0 -1
- package/dist/models.js +0 -54
- package/dist/models.js.map +0 -1
- package/dist/search/WikiSearch.d.ts +0 -14
- package/dist/search/WikiSearch.d.ts.map +0 -1
- package/dist/search/WikiSearch.js +0 -223
- package/dist/search/WikiSearch.js.map +0 -1
- package/dist/search/index.d.ts +0 -2
- package/dist/search/index.d.ts.map +0 -1
- package/dist/search/index.js +0 -2
- package/dist/search/index.js.map +0 -1
- package/dist/store/WikiStore.d.ts +0 -47
- package/dist/store/WikiStore.d.ts.map +0 -1
- package/dist/store/WikiStore.js +0 -301
- package/dist/store/WikiStore.js.map +0 -1
- package/dist/store/index.d.ts +0 -2
- package/dist/store/index.d.ts.map +0 -1
- package/dist/store/index.js +0 -2
- package/dist/store/index.js.map +0 -1
- package/dist/util/fs.d.ts +0 -7
- package/dist/util/fs.d.ts.map +0 -1
- package/dist/util/fs.js +0 -36
- package/dist/util/fs.js.map +0 -1
- package/dist/util/index.d.ts +0 -3
- package/dist/util/index.d.ts.map +0 -1
- package/dist/util/index.js +0 -3
- package/dist/util/index.js.map +0 -1
- package/dist/util/paths.d.ts +0 -17
- package/dist/util/paths.d.ts.map +0 -1
- package/dist/util/paths.js +0 -31
- package/dist/util/paths.js.map +0 -1
package/dist/WikiEngine.d.ts
CHANGED
|
@@ -1,34 +1,25 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
|
|
1
|
+
import type { ModelInfo } from "./model-registry.js";
|
|
2
|
+
import type { SearchMode, SearchHit, FileEntry } from "./types.js";
|
|
3
|
+
export interface EngineConfig {
|
|
4
|
+
basePath?: string;
|
|
5
|
+
modelId?: string;
|
|
6
|
+
}
|
|
4
7
|
export declare class WikiEngine {
|
|
5
|
-
readonly store: WikiStore;
|
|
6
|
-
readonly search: WikiSearch;
|
|
7
|
-
private embedder;
|
|
8
|
-
private llm;
|
|
9
8
|
constructor(config?: EngineConfig);
|
|
10
9
|
get sources(): string[];
|
|
11
10
|
addSource(absPath: string): boolean;
|
|
12
11
|
removeSource(target: string): string | null;
|
|
12
|
+
loadSource(absPath: string): Promise<number>;
|
|
13
13
|
load(): Promise<{
|
|
14
14
|
files: number;
|
|
15
15
|
sources: number;
|
|
16
16
|
}>;
|
|
17
|
-
loadSource(absPath: string): Promise<number>;
|
|
18
17
|
search_(query: string, mode?: SearchMode): Promise<SearchHit[]>;
|
|
19
|
-
/** 创建 .md 条目 */
|
|
20
18
|
createEntry(sourceDir: string, relPath: string, title: string, tags?: string[], content?: string): string;
|
|
21
|
-
/** 读取条目全文 */
|
|
22
19
|
readEntry(relPath: string): {
|
|
23
20
|
entry: FileEntry;
|
|
24
21
|
content: string;
|
|
25
22
|
} | null;
|
|
26
|
-
/** 重命名(修改 frontmatter title) */
|
|
27
|
-
renameEntry(relPath: string, newTitle: string): boolean;
|
|
28
|
-
/** 移动文件 */
|
|
29
|
-
moveEntry(relPath: string, newRelPath: string): boolean;
|
|
30
|
-
/** 修改内容 */
|
|
31
|
-
modifyEntry(relPath: string, content: string): boolean;
|
|
32
23
|
get semanticEnabled(): boolean;
|
|
33
24
|
enableSemantic(modelId?: string): Promise<{
|
|
34
25
|
ok: boolean;
|
|
@@ -39,43 +30,40 @@ export declare class WikiEngine {
|
|
|
39
30
|
embedded: number;
|
|
40
31
|
skipped: number;
|
|
41
32
|
}>;
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
}): void;
|
|
56
|
-
/** 编译状态查询 */
|
|
57
|
-
compileStatus(sourceDir?: string): {
|
|
58
|
-
total: number;
|
|
33
|
+
downloadModel(): Promise<{
|
|
34
|
+
ok: boolean;
|
|
35
|
+
msg: string;
|
|
36
|
+
}>;
|
|
37
|
+
status(): {
|
|
38
|
+
configPath: string;
|
|
39
|
+
sources: string[];
|
|
40
|
+
files: number;
|
|
41
|
+
lastScan: string;
|
|
42
|
+
semantic: boolean;
|
|
43
|
+
embeddings: number;
|
|
44
|
+
model: string;
|
|
45
|
+
modelDim: number;
|
|
59
46
|
compiled: number;
|
|
60
|
-
uncompiled: string[];
|
|
61
47
|
};
|
|
62
48
|
get llmInfo(): {
|
|
63
49
|
apiBase: string;
|
|
64
50
|
model: string;
|
|
65
51
|
hasKey: boolean;
|
|
66
52
|
};
|
|
67
|
-
/** 编译单个文件(调 LLM + 存储结果) */
|
|
68
53
|
compileFile(relPath: string): Promise<{
|
|
69
54
|
ok: boolean;
|
|
70
55
|
msg: string;
|
|
71
56
|
}>;
|
|
72
|
-
/** 批量编译 */
|
|
73
57
|
compileAll(sourceDir?: string, limit?: number): Promise<{
|
|
74
58
|
compiled: number;
|
|
75
59
|
failed: number;
|
|
76
60
|
msgs: string[];
|
|
77
61
|
}>;
|
|
78
|
-
|
|
62
|
+
compileStatus(_sourceDir?: string): {
|
|
63
|
+
total: number;
|
|
64
|
+
compiled: number;
|
|
65
|
+
uncompiled: string[];
|
|
66
|
+
};
|
|
79
67
|
listModels(): ModelInfo[];
|
|
80
68
|
}
|
|
81
69
|
//# sourceMappingURL=WikiEngine.d.ts.map
|
package/dist/WikiEngine.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"WikiEngine.d.ts","sourceRoot":"","sources":["../src/WikiEngine.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"WikiEngine.d.ts","sourceRoot":"","sources":["../src/WikiEngine.ts"],"names":[],"mappings":"AAeA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAInE,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,qBAAa,UAAU;gBACT,MAAM,GAAE,YAAiB;IAMrC,IAAI,OAAO,IAAI,MAAM,EAAE,CAA+B;IAEtD,SAAS,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO;IAInC,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI;IAIrC,UAAU,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAY5C,IAAI,IAAI,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;IAUnD,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,GAAE,UAAsB,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IAShF,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,GAAE,MAAM,EAAO,EAAE,OAAO,SAAK,GAAG,MAAM;IA0BzG,SAAS,CAAC,OAAO,EAAE,MAAM,GAAG;QAAE,KAAK,EAAE,SAAS,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IASxE,IAAI,eAAe,IAAI,OAAO,CAAuC;IAE/D,cAAc,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,EAAE,EAAE,OAAO,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC;IAY7E,eAAe,IAAI,IAAI;IAKjB,kBAAkB,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;IAUtF,aAAa,IAAI,OAAO,CAAC;QAAE,EAAE,EAAE,OAAO,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC;IAK5D,MAAM;;;;;;;;;;;IAiBN,IAAI,OAAO;;;;MASV;IAEK,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,EAAE,EAAE,OAAO,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC;IA2CnE,UAAU,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,KAAK,SAAK,GAAG,OAAO,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;IAe/G,aAAa,CAAC,UAAU,CAAC,EAAE,MAAM;;;;;IAYjC,UAAU,IAAI,SAAS,EAAE;CAG1B"}
|
package/dist/WikiEngine.js
CHANGED
|
@@ -1,371 +1,230 @@
|
|
|
1
|
-
// WikiEngine.ts —
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import
|
|
7
|
-
import
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
}
|
|
1
|
+
// WikiEngine.ts — Adapter: extensions/lib API → WikiEngine class (CLI-compatible)
|
|
2
|
+
import { resolve } from "node:path";
|
|
3
|
+
import { existsSync, mkdirSync, writeFileSync, readFileSync } from "node:fs";
|
|
4
|
+
import { dirname } from "node:path";
|
|
5
|
+
import { setWikiHome, getWikiHome } from "./wiki-paths.js";
|
|
6
|
+
import * as store from "./store-settings.js";
|
|
7
|
+
import * as embedder from "./embedder.js";
|
|
8
|
+
import { keywordSearch } from "./search.js";
|
|
9
|
+
import { semanticSearch, hybridSearch } from "./semantic-search.js";
|
|
10
|
+
import { scanDir } from "./indexer-scan.js";
|
|
11
|
+
import { generateEmbeddings as doGenerateEmbeddings } from "./indexer-embed.js";
|
|
12
|
+
import { getRawChunks, storeCompiledChunks } from "./indexer-compile.js";
|
|
13
|
+
import { setContent, getContent } from "./content-cache.js";
|
|
14
|
+
import { findModel, getCurrentModel, selectModel, getBuiltinModels } from "./model-registry.js";
|
|
15
|
+
import { stats } from "./store.js";
|
|
16
|
+
import { getFileState } from "./file-manifest.js";
|
|
15
17
|
export class WikiEngine {
|
|
16
|
-
store;
|
|
17
|
-
search;
|
|
18
|
-
embedder;
|
|
19
|
-
llm;
|
|
20
18
|
constructor(config = {}) {
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
this.embedder = new WikiEmbedder(model, this.store.paths.modelsDir);
|
|
26
|
-
this.search = new WikiSearch(this.store, (text) => this.embedder.embed(text));
|
|
27
|
-
this.llm = new WikiLLM({ apiBase: this.store.config.llmApiBase, apiKey: this.store.config.llmApiKey, model: this.store.config.llmModel });
|
|
28
|
-
// 如果 config.json 已有 semantic=true,自动准备 embedder
|
|
29
|
-
if (this.store.config.semantic) {
|
|
30
|
-
this.embedder.init().catch(() => { });
|
|
31
|
-
}
|
|
19
|
+
if (config.basePath)
|
|
20
|
+
setWikiHome(config.basePath);
|
|
21
|
+
if (config.modelId)
|
|
22
|
+
selectModel(config.modelId);
|
|
32
23
|
}
|
|
33
|
-
//
|
|
34
|
-
|
|
35
|
-
// ==========================================================================
|
|
36
|
-
get sources() { return this.store.config.sources; }
|
|
24
|
+
// ---- Sources ----
|
|
25
|
+
get sources() { return store.getSources(); }
|
|
37
26
|
addSource(absPath) {
|
|
38
|
-
return
|
|
27
|
+
return store.addSource(absPath);
|
|
39
28
|
}
|
|
40
29
|
removeSource(target) {
|
|
41
|
-
return
|
|
30
|
+
return store.removeSource(target);
|
|
31
|
+
}
|
|
32
|
+
async loadSource(absPath) {
|
|
33
|
+
const entries = await scanDir(absPath);
|
|
34
|
+
store.mergeIndex(entries);
|
|
35
|
+
for (const e of entries) {
|
|
36
|
+
try {
|
|
37
|
+
const content = readFileSync(resolve(e.sourceDir, e.relPath), "utf-8");
|
|
38
|
+
setContent(e.relPath, content);
|
|
39
|
+
}
|
|
40
|
+
catch { }
|
|
41
|
+
}
|
|
42
|
+
return entries.length;
|
|
42
43
|
}
|
|
43
|
-
// ==========================================================================
|
|
44
|
-
// 扫描 + 索引
|
|
45
|
-
// ==========================================================================
|
|
46
44
|
async load() {
|
|
47
|
-
const sources =
|
|
45
|
+
const sources = store.getSources();
|
|
48
46
|
let totalFiles = 0;
|
|
49
47
|
for (const src of sources) {
|
|
50
|
-
|
|
51
|
-
this.store.setEntries(entries);
|
|
52
|
-
totalFiles += entries.length;
|
|
48
|
+
totalFiles += await this.loadSource(src);
|
|
53
49
|
}
|
|
54
50
|
return { files: totalFiles, sources: sources.length };
|
|
55
51
|
}
|
|
56
|
-
|
|
57
|
-
const entries = await this.store.scanDir(absPath);
|
|
58
|
-
this.store.setEntries(entries);
|
|
59
|
-
return entries.length;
|
|
60
|
-
}
|
|
61
|
-
// ==========================================================================
|
|
62
|
-
// 搜索
|
|
63
|
-
// ==========================================================================
|
|
52
|
+
// ---- Search ----
|
|
64
53
|
async search_(query, mode = "keyword") {
|
|
65
|
-
|
|
54
|
+
switch (mode) {
|
|
55
|
+
case "keyword": return keywordSearch(query);
|
|
56
|
+
case "semantic": return semanticSearch(query);
|
|
57
|
+
case "hybrid": return hybridSearch(query);
|
|
58
|
+
}
|
|
66
59
|
}
|
|
67
|
-
//
|
|
68
|
-
// 条目 CRUD
|
|
69
|
-
// ==========================================================================
|
|
70
|
-
/** 创建 .md 条目 */
|
|
60
|
+
// ---- CRUD ----
|
|
71
61
|
createEntry(sourceDir, relPath, title, tags = [], content = "") {
|
|
72
|
-
const
|
|
73
|
-
if (existsSync(absPath))
|
|
74
|
-
return `已存在: ${absPath}`;
|
|
75
|
-
// 确保 .md 后缀
|
|
76
|
-
const finalPath = relPath.endsWith(".md") ? relPath : `${relPath}.md`;
|
|
62
|
+
const finalPath = relPath.endsWith(".md") ? relPath : relPath + ".md";
|
|
77
63
|
const finalAbs = resolve(sourceDir, finalPath);
|
|
64
|
+
if (existsSync(finalAbs))
|
|
65
|
+
return "exists: " + finalAbs;
|
|
78
66
|
const fm = [
|
|
79
67
|
"---",
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
68
|
+
"title: " + title,
|
|
69
|
+
"tags: [" + tags.join(", ") + "]",
|
|
70
|
+
"created: " + new Date().toISOString(),
|
|
83
71
|
"---", "",
|
|
84
72
|
].join("\n");
|
|
85
73
|
mkdirSync(dirname(finalAbs), { recursive: true });
|
|
86
74
|
writeFileSync(finalAbs, fm + content, "utf-8");
|
|
87
|
-
// 更新索引
|
|
88
75
|
const entry = {
|
|
89
76
|
title, tags, sourceDir,
|
|
90
77
|
relPath: finalPath.replace(/\\/g, "/"),
|
|
91
78
|
mtime: new Date().toISOString(),
|
|
92
79
|
};
|
|
93
|
-
|
|
94
|
-
|
|
80
|
+
store.mergeIndex([entry]);
|
|
81
|
+
setContent(entry.relPath, fm + content);
|
|
95
82
|
return entry.relPath;
|
|
96
83
|
}
|
|
97
|
-
/** 读取条目全文 */
|
|
98
84
|
readEntry(relPath) {
|
|
99
|
-
const entry =
|
|
85
|
+
const entry = store.getEntry(relPath);
|
|
100
86
|
if (!entry)
|
|
101
87
|
return null;
|
|
102
|
-
const content =
|
|
88
|
+
const content = getContent(relPath);
|
|
103
89
|
if (!content)
|
|
104
90
|
return null;
|
|
105
91
|
return { entry, content };
|
|
106
92
|
}
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
const entry = this.store.getEntry(relPath);
|
|
110
|
-
const content = this.store.getContent(relPath);
|
|
111
|
-
if (!entry || !content)
|
|
112
|
-
return false;
|
|
113
|
-
const updated = content.replace(/^(title:\s*).+$/m, `$1${newTitle}`);
|
|
114
|
-
// 写文件
|
|
115
|
-
const absPath = resolve(entry.sourceDir, relPath);
|
|
116
|
-
writeFileSync(absPath, updated, "utf-8");
|
|
117
|
-
// 更新索引
|
|
118
|
-
entry.title = newTitle;
|
|
119
|
-
entry.mtime = new Date().toISOString();
|
|
120
|
-
this.store.setEntries([entry]);
|
|
121
|
-
this.store.setContent(relPath, updated);
|
|
122
|
-
return true;
|
|
123
|
-
}
|
|
124
|
-
/** 移动文件 */
|
|
125
|
-
moveEntry(relPath, newRelPath) {
|
|
126
|
-
const entry = this.store.getEntry(relPath);
|
|
127
|
-
if (!entry)
|
|
128
|
-
return false;
|
|
129
|
-
const oldAbs = resolve(entry.sourceDir, relPath);
|
|
130
|
-
const newAbs = resolve(entry.sourceDir, newRelPath);
|
|
131
|
-
if (existsSync(newAbs))
|
|
132
|
-
return false;
|
|
133
|
-
mkdirSync(dirname(newAbs), { recursive: true });
|
|
134
|
-
renameSync(oldAbs, newAbs);
|
|
135
|
-
const newEntry = { ...entry, relPath: newRelPath, mtime: new Date().toISOString() };
|
|
136
|
-
this.store.removeEntry(relPath);
|
|
137
|
-
this.store.setEntries([newEntry]);
|
|
138
|
-
// 迁移向量
|
|
139
|
-
const vec = this.store.getEmbedding(relPath);
|
|
140
|
-
if (vec) {
|
|
141
|
-
this.store.setSingleEmbedding(newRelPath, vec);
|
|
142
|
-
this.store.removeEmbeddingsByPath(relPath);
|
|
143
|
-
this.store.flushVectors();
|
|
144
|
-
}
|
|
145
|
-
return true;
|
|
146
|
-
}
|
|
147
|
-
/** 修改内容 */
|
|
148
|
-
modifyEntry(relPath, content) {
|
|
149
|
-
const entry = this.store.getEntry(relPath);
|
|
150
|
-
if (!entry)
|
|
151
|
-
return false;
|
|
152
|
-
const absPath = resolve(entry.sourceDir, relPath);
|
|
153
|
-
writeFileSync(absPath, content, "utf-8");
|
|
154
|
-
entry.mtime = new Date().toISOString();
|
|
155
|
-
this.store.setEntries([entry]);
|
|
156
|
-
this.store.setContent(relPath, content);
|
|
157
|
-
return true;
|
|
158
|
-
}
|
|
159
|
-
// ==========================================================================
|
|
160
|
-
// 语义搜索控制
|
|
161
|
-
// ==========================================================================
|
|
162
|
-
get semanticEnabled() { return this.store.config.semantic; }
|
|
93
|
+
// ---- Semantic ----
|
|
94
|
+
get semanticEnabled() { return store.getSemanticEnabled(); }
|
|
163
95
|
async enableSemantic(modelId) {
|
|
164
96
|
if (modelId) {
|
|
165
97
|
const m = findModel(modelId);
|
|
166
98
|
if (!m)
|
|
167
|
-
return { ok: false, msg:
|
|
168
|
-
|
|
169
|
-
this.store.saveConfig({ model: m.id });
|
|
99
|
+
return { ok: false, msg: "Unknown model: " + modelId };
|
|
100
|
+
selectModel(modelId);
|
|
170
101
|
}
|
|
171
|
-
const ok = await
|
|
102
|
+
const ok = await embedder.initialize();
|
|
172
103
|
if (!ok)
|
|
173
|
-
return { ok: false, msg:
|
|
174
|
-
|
|
175
|
-
return { ok: true, msg:
|
|
104
|
+
return { ok: false, msg: "Init failed: " + (embedder.getInitError() || "unknown") };
|
|
105
|
+
store.setSemanticEnabled(true);
|
|
106
|
+
return { ok: true, msg: "Semantic search ON. Model: " + embedder.getModelName() + ". Source: " + embedder.getModelSource() };
|
|
176
107
|
}
|
|
177
108
|
disableSemantic() {
|
|
178
|
-
|
|
109
|
+
store.setSemanticEnabled(false);
|
|
179
110
|
}
|
|
180
|
-
//
|
|
181
|
-
// 向量生成
|
|
182
|
-
// ==========================================================================
|
|
111
|
+
// ---- Embeddings ----
|
|
183
112
|
async generateEmbeddings(sourceDir) {
|
|
184
|
-
if (!
|
|
185
|
-
await
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
let embedded = 0, skipped = 0;
|
|
190
|
-
for (const entry of entries) {
|
|
191
|
-
const content = this.store.getContent(entry.relPath);
|
|
192
|
-
if (!content) {
|
|
193
|
-
skipped++;
|
|
194
|
-
continue;
|
|
195
|
-
}
|
|
196
|
-
const mft = this.store.getManifestEntry(entry.relPath);
|
|
197
|
-
const md5 = md5Hash(content);
|
|
198
|
-
if (mft?.embedded && mft.md5 === md5) {
|
|
199
|
-
skipped++;
|
|
200
|
-
continue;
|
|
201
|
-
}
|
|
202
|
-
const chunks = chunkByHeadings(content, entry.relPath);
|
|
203
|
-
if (chunks.length <= 1) {
|
|
204
|
-
// 单块文件:直接嵌入全文
|
|
205
|
-
const vec = await this.embedder.embed(content.slice(0, 800));
|
|
206
|
-
this.store.setSingleEmbedding(entry.relPath, vec);
|
|
207
|
-
}
|
|
208
|
-
else {
|
|
209
|
-
// 多块文件:逐块嵌入
|
|
210
|
-
for (const chunk of chunks) {
|
|
211
|
-
const pre = preprocess(chunk.text, chunk.heading, chunk.level);
|
|
212
|
-
const embedText = buildEmbeddingText(chunk.text, pre);
|
|
213
|
-
const vec = await this.embedder.embed(embedText);
|
|
214
|
-
this.store.setSingleEmbedding(chunk.key, vec);
|
|
215
|
-
this.store.setChunkMeta(chunk.key, {
|
|
216
|
-
heading: chunk.heading, level: chunk.level,
|
|
217
|
-
...pre,
|
|
218
|
-
});
|
|
219
|
-
}
|
|
220
|
-
}
|
|
221
|
-
// 更新 manifest
|
|
222
|
-
this.store.setManifestEntry(entry.relPath, {
|
|
223
|
-
md5, size: content.length, chunks: chunks.length,
|
|
224
|
-
compiled: mft?.compiled ?? false,
|
|
225
|
-
embedded: true,
|
|
226
|
-
updatedAt: new Date().toISOString(),
|
|
227
|
-
});
|
|
228
|
-
embedded++;
|
|
229
|
-
}
|
|
230
|
-
this.store.flushVectors();
|
|
231
|
-
this.store.vectors.model = this.embedder.modelInfo.hfRepo;
|
|
232
|
-
this.store.vectors.dim = this.embedder.modelInfo.dim;
|
|
233
|
-
this.store.flushVectors();
|
|
234
|
-
return { embedded, skipped };
|
|
113
|
+
if (!embedder.isAvailable())
|
|
114
|
+
await embedder.initialize();
|
|
115
|
+
const entries = Object.values(store.getIndex()).filter(e => !sourceDir || e.sourceDir === sourceDir);
|
|
116
|
+
const count = await doGenerateEmbeddings(sourceDir || "", entries);
|
|
117
|
+
return { embedded: count, skipped: 0 };
|
|
235
118
|
}
|
|
236
|
-
//
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
/** 获取待编译的原始块 */
|
|
240
|
-
getRawChunks(sourceDir, uncompiledOnly = true) {
|
|
241
|
-
const entries = Object.values(this.store.entries).filter(e => !sourceDir || e.sourceDir === sourceDir);
|
|
242
|
-
const result = [];
|
|
243
|
-
for (const entry of entries) {
|
|
244
|
-
if (uncompiledOnly) {
|
|
245
|
-
const mft = this.store.getManifestEntry(entry.relPath);
|
|
246
|
-
if (mft?.compiled)
|
|
247
|
-
continue;
|
|
248
|
-
}
|
|
249
|
-
const content = this.store.getContent(entry.relPath);
|
|
250
|
-
if (!content)
|
|
251
|
-
continue;
|
|
252
|
-
const chunks = chunkByHeadings(content, entry.relPath);
|
|
253
|
-
for (const chunk of chunks) {
|
|
254
|
-
const key = chunk.key;
|
|
255
|
-
result.push({
|
|
256
|
-
key, relPath: entry.relPath,
|
|
257
|
-
heading: chunk.heading,
|
|
258
|
-
rawText: chunk.text,
|
|
259
|
-
compiled: !!this.store.getChunkMeta(key)?.topic,
|
|
260
|
-
});
|
|
261
|
-
}
|
|
262
|
-
}
|
|
263
|
-
return result;
|
|
119
|
+
// ---- Model download ----
|
|
120
|
+
async downloadModel() {
|
|
121
|
+
return embedder.downloadModel();
|
|
264
122
|
}
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
const
|
|
268
|
-
|
|
269
|
-
return null;
|
|
270
|
-
const chunks = chunkByHeadings(content, relPath);
|
|
271
|
-
if (chunks.length === 0)
|
|
272
|
-
return null;
|
|
273
|
-
// 合并为单文件 prompt
|
|
274
|
-
const texts = chunks.map(c => `### ${c.heading}\n${c.text}`).join("\n\n");
|
|
123
|
+
// ---- Status ----
|
|
124
|
+
status() {
|
|
125
|
+
const s = stats();
|
|
126
|
+
const model = getCurrentModel();
|
|
275
127
|
return {
|
|
276
|
-
|
|
277
|
-
|
|
128
|
+
configPath: getWikiHome(),
|
|
129
|
+
sources: store.getSources(),
|
|
130
|
+
files: s.files,
|
|
131
|
+
lastScan: s.lastScan || "",
|
|
132
|
+
semantic: store.getSemanticEnabled(),
|
|
133
|
+
embeddings: s.embeddings || 0,
|
|
134
|
+
model: model?.id || "unknown",
|
|
135
|
+
modelDim: model?.dim || 0,
|
|
136
|
+
compiled: Object.keys(store.getIndex()).filter(p => getFileState(p)?.llmCompiled).length,
|
|
278
137
|
};
|
|
279
138
|
}
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
const
|
|
283
|
-
const
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
...result,
|
|
291
|
-
embeddingText: buildEmbeddingText(result.normalizedText, { topic: result.topic, concepts: result.concepts, aliases: result.aliases }),
|
|
292
|
-
};
|
|
293
|
-
this.store.setCompiled(relPath, record);
|
|
294
|
-
// 存储 LLM 块元数据
|
|
295
|
-
this.store.setChunkMeta(`${relPath}###llm`, {
|
|
296
|
-
heading: "(file-level)",
|
|
297
|
-
level: 0,
|
|
298
|
-
topic: result.topic,
|
|
299
|
-
summary: result.normalizedText.slice(0, 200),
|
|
300
|
-
concepts: result.concepts,
|
|
301
|
-
aliases: result.aliases,
|
|
302
|
-
});
|
|
303
|
-
markCompiled(this.store, relPath);
|
|
139
|
+
// ---- LLM ----
|
|
140
|
+
get llmInfo() {
|
|
141
|
+
const apiKey = process.env.DEEPSEEK_API_KEY || process.env.OPENAI_API_KEY || "";
|
|
142
|
+
const apiBase = process.env.DEEPSEEK_API_KEY
|
|
143
|
+
? "https://api.deepseek.com/v1"
|
|
144
|
+
: process.env.OPENAI_API_KEY
|
|
145
|
+
? "https://api.openai.com/v1"
|
|
146
|
+
: "";
|
|
147
|
+
const model = process.env.LLM_MODEL || "deepseek-chat";
|
|
148
|
+
return { apiBase, model, hasKey: !!apiKey };
|
|
304
149
|
}
|
|
305
|
-
/** 编译状态查询 */
|
|
306
|
-
compileStatus(sourceDir) {
|
|
307
|
-
return getCompileStats(this.store);
|
|
308
|
-
}
|
|
309
|
-
// ==========================================================================
|
|
310
|
-
// 状态
|
|
311
|
-
// ==========================================================================
|
|
312
|
-
// ==========================================================================
|
|
313
|
-
// LLM 编译(内置调用)
|
|
314
|
-
// ==========================================================================
|
|
315
|
-
get llmInfo() { return this.llm.info; }
|
|
316
|
-
/** 编译单个文件(调 LLM + 存储结果) */
|
|
317
150
|
async compileFile(relPath) {
|
|
318
|
-
const content =
|
|
151
|
+
const content = getContent(relPath);
|
|
319
152
|
if (!content)
|
|
320
|
-
return { ok: false, msg:
|
|
321
|
-
const chunks =
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
const
|
|
153
|
+
return { ok: false, msg: "Not found: " + relPath };
|
|
154
|
+
const chunks = await getRawChunks(undefined, false);
|
|
155
|
+
const fileChunks = chunks.filter(c => c.relPath === relPath);
|
|
156
|
+
if (!fileChunks.length)
|
|
157
|
+
return { ok: false, msg: "No chunks: " + relPath };
|
|
158
|
+
const { COMPILE_SYSTEM_PROMPT, buildCompilePrompt, parseCompiledResult } = await import("./semantic-compiler.js");
|
|
159
|
+
const prompt = buildCompilePrompt(fileChunks);
|
|
326
160
|
try {
|
|
327
|
-
const
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
161
|
+
const apiKey = process.env.DEEPSEEK_API_KEY || process.env.OPENAI_API_KEY;
|
|
162
|
+
if (!apiKey)
|
|
163
|
+
return { ok: false, msg: "No API key. Set DEEPSEEK_API_KEY or OPENAI_API_KEY" };
|
|
164
|
+
const apiBase = process.env.DEEPSEEK_API_KEY
|
|
165
|
+
? "https://api.deepseek.com/v1"
|
|
166
|
+
: "https://api.openai.com/v1";
|
|
167
|
+
const model = process.env.LLM_MODEL || "deepseek-chat";
|
|
168
|
+
const res = await fetch(apiBase + "/chat/completions", {
|
|
169
|
+
method: "POST",
|
|
170
|
+
headers: { "Content-Type": "application/json", "Authorization": "Bearer " + apiKey },
|
|
171
|
+
body: JSON.stringify({
|
|
172
|
+
model,
|
|
173
|
+
messages: [
|
|
174
|
+
{ role: "system", content: COMPILE_SYSTEM_PROMPT },
|
|
175
|
+
{ role: "user", content: prompt },
|
|
176
|
+
],
|
|
177
|
+
response_format: { type: "json_object" },
|
|
178
|
+
temperature: 0.1,
|
|
179
|
+
}),
|
|
180
|
+
});
|
|
181
|
+
const data = await res.json();
|
|
182
|
+
const raw = data?.choices?.[0]?.message?.content || "";
|
|
183
|
+
const compiled = parseCompiledResult(raw);
|
|
184
|
+
if (!compiled)
|
|
185
|
+
return { ok: false, msg: "Invalid JSON: " + raw.slice(0, 200) };
|
|
186
|
+
await storeCompiledChunks(compiled);
|
|
187
|
+
const topic = compiled[0]?.topic || "unknown";
|
|
188
|
+
return { ok: true, msg: "Compiled: " + relPath + ' -> "' + topic + '"' };
|
|
333
189
|
}
|
|
334
190
|
catch (e) {
|
|
335
|
-
return { ok: false, msg:
|
|
191
|
+
return { ok: false, msg: "LLM error: " + e.message };
|
|
336
192
|
}
|
|
337
193
|
}
|
|
338
|
-
/** 批量编译 */
|
|
339
194
|
async compileAll(sourceDir, limit = 10) {
|
|
340
|
-
|
|
341
|
-
const targets = sourceDir
|
|
342
|
-
? stats.uncompiled.filter(p => this.store.getEntry(p)?.sourceDir === sourceDir)
|
|
343
|
-
: stats.uncompiled;
|
|
344
|
-
const batch = targets.slice(0, limit);
|
|
345
|
-
let compiled = 0, failed = 0;
|
|
195
|
+
let count = 0, failed = 0;
|
|
346
196
|
const msgs = [];
|
|
347
|
-
|
|
348
|
-
|
|
197
|
+
const entries = Object.values(store.getIndex()).filter(e => !sourceDir || e.sourceDir === sourceDir);
|
|
198
|
+
for (const entry of entries) {
|
|
199
|
+
if (count >= limit)
|
|
200
|
+
break;
|
|
201
|
+
const r = await this.compileFile(entry.relPath);
|
|
349
202
|
if (r.ok) {
|
|
350
|
-
|
|
351
|
-
msgs.push(
|
|
203
|
+
count++;
|
|
204
|
+
msgs.push(" " + r.msg);
|
|
352
205
|
}
|
|
353
206
|
else {
|
|
354
207
|
failed++;
|
|
355
|
-
msgs.push(
|
|
208
|
+
msgs.push(" FAIL: " + r.msg);
|
|
356
209
|
}
|
|
357
210
|
}
|
|
358
|
-
return { compiled, failed, msgs };
|
|
359
|
-
}
|
|
360
|
-
|
|
361
|
-
|
|
211
|
+
return { compiled: count, failed, msgs };
|
|
212
|
+
}
|
|
213
|
+
compileStatus(_sourceDir) {
|
|
214
|
+
const all = Object.keys(store.getIndex());
|
|
215
|
+
const compiled = [];
|
|
216
|
+
const uncompiled = [];
|
|
217
|
+
for (const relPath of all) {
|
|
218
|
+
const state = getFileState(relPath);
|
|
219
|
+
if (state?.llmCompiled)
|
|
220
|
+
compiled.push(relPath);
|
|
221
|
+
else
|
|
222
|
+
uncompiled.push(relPath);
|
|
223
|
+
}
|
|
224
|
+
return { total: all.length, compiled: compiled.length, uncompiled };
|
|
362
225
|
}
|
|
363
226
|
listModels() {
|
|
364
|
-
|
|
365
|
-
return MODELS;
|
|
227
|
+
return getBuiltinModels();
|
|
366
228
|
}
|
|
367
229
|
}
|
|
368
|
-
function md5Hash(content) {
|
|
369
|
-
return createHash("md5").update(content).digest("hex");
|
|
370
|
-
}
|
|
371
230
|
//# sourceMappingURL=WikiEngine.js.map
|