crewly 1.20.20 → 1.20.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config/roles/generalist/prompt.md +8 -2
- package/config/roles/team-leader/prompt.md +2 -1
- package/config/skills/agent/core/wiki-query/SKILL.md +11 -0
- package/config/skills/agent/core/wiki-query/execute.sh +22 -3
- package/config/skills/orchestrator/wiki-ingest/SKILL.md +19 -0
- package/config/skills/orchestrator/wiki-ingest/execute.sh +36 -1
- package/config/skills/team-leader/wiki-review-proposals/SKILL.md +35 -0
- package/config/skills/team-leader/wiki-review-proposals/execute.sh +40 -0
- package/config/skills/team-leader/wiki-supersede/SKILL.md +38 -0
- package/config/skills/team-leader/wiki-supersede/execute.sh +44 -0
- package/dist/backend/backend/src/constants.d.ts +39 -0
- package/dist/backend/backend/src/constants.d.ts.map +1 -1
- package/dist/backend/backend/src/constants.js +37 -0
- package/dist/backend/backend/src/constants.js.map +1 -1
- package/dist/backend/backend/src/controllers/wiki/wiki.controller.d.ts +29 -0
- package/dist/backend/backend/src/controllers/wiki/wiki.controller.d.ts.map +1 -1
- package/dist/backend/backend/src/controllers/wiki/wiki.controller.js +196 -4
- package/dist/backend/backend/src/controllers/wiki/wiki.controller.js.map +1 -1
- package/dist/backend/backend/src/controllers/wiki/wiki.routes.d.ts.map +1 -1
- package/dist/backend/backend/src/controllers/wiki/wiki.routes.js +9 -1
- package/dist/backend/backend/src/controllers/wiki/wiki.routes.js.map +1 -1
- package/dist/backend/backend/src/index.d.ts.map +1 -1
- package/dist/backend/backend/src/index.js +3 -0
- package/dist/backend/backend/src/index.js.map +1 -1
- package/dist/backend/backend/src/services/memory/memory.service.d.ts +2 -0
- package/dist/backend/backend/src/services/memory/memory.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/memory/memory.service.js +16 -4
- package/dist/backend/backend/src/services/memory/memory.service.js.map +1 -1
- package/dist/backend/backend/src/services/slack/slack-agent-dm.service.d.ts +3 -1
- package/dist/backend/backend/src/services/slack/slack-agent-dm.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/slack/slack-agent-dm.service.js +14 -3
- package/dist/backend/backend/src/services/slack/slack-agent-dm.service.js.map +1 -1
- package/dist/backend/backend/src/services/slack/slack-initializer.d.ts +0 -10
- package/dist/backend/backend/src/services/slack/slack-initializer.d.ts.map +1 -1
- package/dist/backend/backend/src/services/slack/slack-initializer.js +21 -0
- package/dist/backend/backend/src/services/slack/slack-initializer.js.map +1 -1
- package/dist/backend/backend/src/services/slack/slack-team-channel.service.d.ts +3 -1
- package/dist/backend/backend/src/services/slack/slack-team-channel.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/slack/slack-team-channel.service.js +26 -9
- package/dist/backend/backend/src/services/slack/slack-team-channel.service.js.map +1 -1
- package/dist/backend/backend/src/services/slack/slack-typing-placeholder.service.d.ts +22 -1
- package/dist/backend/backend/src/services/slack/slack-typing-placeholder.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/slack/slack-typing-placeholder.service.js +69 -7
- package/dist/backend/backend/src/services/slack/slack-typing-placeholder.service.js.map +1 -1
- package/dist/backend/backend/src/services/wiki/schema-loader.service.d.ts +9 -0
- package/dist/backend/backend/src/services/wiki/schema-loader.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/wiki/schema-loader.service.js +40 -1
- package/dist/backend/backend/src/services/wiki/schema-loader.service.js.map +1 -1
- package/dist/backend/backend/src/services/wiki/wiki-bookkeep.service.d.ts +32 -0
- package/dist/backend/backend/src/services/wiki/wiki-bookkeep.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/wiki/wiki-bookkeep.service.js +63 -1
- package/dist/backend/backend/src/services/wiki/wiki-bookkeep.service.js.map +1 -1
- package/dist/backend/backend/src/services/wiki/wiki-cleanup.service.d.ts +2 -0
- package/dist/backend/backend/src/services/wiki/wiki-cleanup.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/wiki/wiki-cleanup.service.js +9 -2
- package/dist/backend/backend/src/services/wiki/wiki-cleanup.service.js.map +1 -1
- package/dist/backend/backend/src/services/wiki/wiki-curation.service.d.ts +106 -0
- package/dist/backend/backend/src/services/wiki/wiki-curation.service.d.ts.map +1 -0
- package/dist/backend/backend/src/services/wiki/wiki-curation.service.js +211 -0
- package/dist/backend/backend/src/services/wiki/wiki-curation.service.js.map +1 -0
- package/dist/backend/backend/src/services/wiki/wiki-history.service.d.ts +63 -0
- package/dist/backend/backend/src/services/wiki/wiki-history.service.d.ts.map +1 -0
- package/dist/backend/backend/src/services/wiki/wiki-history.service.js +124 -0
- package/dist/backend/backend/src/services/wiki/wiki-history.service.js.map +1 -0
- package/dist/backend/backend/src/services/wiki/wiki-index.service.d.ts +119 -0
- package/dist/backend/backend/src/services/wiki/wiki-index.service.d.ts.map +1 -0
- package/dist/backend/backend/src/services/wiki/wiki-index.service.js +263 -0
- package/dist/backend/backend/src/services/wiki/wiki-index.service.js.map +1 -0
- package/dist/backend/backend/src/services/wiki/wiki-ingest.service.d.ts +66 -11
- package/dist/backend/backend/src/services/wiki/wiki-ingest.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/wiki/wiki-ingest.service.js +162 -44
- package/dist/backend/backend/src/services/wiki/wiki-ingest.service.js.map +1 -1
- package/dist/backend/backend/src/services/wiki/wiki-lint.service.d.ts +31 -0
- package/dist/backend/backend/src/services/wiki/wiki-lint.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/wiki/wiki-lint.service.js +69 -0
- package/dist/backend/backend/src/services/wiki/wiki-lint.service.js.map +1 -1
- package/dist/backend/backend/src/services/wiki/wiki-owner.resolver.d.ts +29 -0
- package/dist/backend/backend/src/services/wiki/wiki-owner.resolver.d.ts.map +1 -0
- package/dist/backend/backend/src/services/wiki/wiki-owner.resolver.js +55 -0
- package/dist/backend/backend/src/services/wiki/wiki-owner.resolver.js.map +1 -0
- package/dist/backend/backend/src/services/wiki/wiki-page.d.ts +101 -0
- package/dist/backend/backend/src/services/wiki/wiki-page.d.ts.map +1 -0
- package/dist/backend/backend/src/services/wiki/wiki-page.js +130 -0
- package/dist/backend/backend/src/services/wiki/wiki-page.js.map +1 -0
- package/dist/backend/backend/src/services/wiki/wiki-policy.d.ts +35 -0
- package/dist/backend/backend/src/services/wiki/wiki-policy.d.ts.map +1 -0
- package/dist/backend/backend/src/services/wiki/wiki-policy.js +44 -0
- package/dist/backend/backend/src/services/wiki/wiki-policy.js.map +1 -0
- package/dist/backend/backend/src/services/wiki/wiki-process.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/wiki/wiki-process.service.js +5 -2
- package/dist/backend/backend/src/services/wiki/wiki-process.service.js.map +1 -1
- package/dist/backend/backend/src/services/wiki/wiki-query.service.d.ts +57 -13
- package/dist/backend/backend/src/services/wiki/wiki-query.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/wiki/wiki-query.service.js +91 -54
- package/dist/backend/backend/src/services/wiki/wiki-query.service.js.map +1 -1
- package/dist/backend/backend/src/services/wiki/wiki-redaction.d.ts +74 -0
- package/dist/backend/backend/src/services/wiki/wiki-redaction.d.ts.map +1 -0
- package/dist/backend/backend/src/services/wiki/wiki-redaction.js +94 -0
- package/dist/backend/backend/src/services/wiki/wiki-redaction.js.map +1 -0
- package/dist/backend/backend/src/services/wiki/wiki-search.service.d.ts +12 -3
- package/dist/backend/backend/src/services/wiki/wiki-search.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/wiki/wiki-search.service.js +19 -9
- package/dist/backend/backend/src/services/wiki/wiki-search.service.js.map +1 -1
- package/dist/backend/backend/src/services/wiki/wiki-usage.service.d.ts +96 -0
- package/dist/backend/backend/src/services/wiki/wiki-usage.service.d.ts.map +1 -0
- package/dist/backend/backend/src/services/wiki/wiki-usage.service.js +147 -0
- package/dist/backend/backend/src/services/wiki/wiki-usage.service.js.map +1 -0
- package/dist/backend/backend/src/services/wiki/wiki-vault-walk.d.ts +25 -0
- package/dist/backend/backend/src/services/wiki/wiki-vault-walk.d.ts.map +1 -0
- package/dist/backend/backend/src/services/wiki/wiki-vault-walk.js +52 -0
- package/dist/backend/backend/src/services/wiki/wiki-vault-walk.js.map +1 -0
- package/dist/backend/backend/src/services/wiki/wiki-workitem-bridge.service.d.ts +19 -1
- package/dist/backend/backend/src/services/wiki/wiki-workitem-bridge.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/wiki/wiki-workitem-bridge.service.js +47 -6
- package/dist/backend/backend/src/services/wiki/wiki-workitem-bridge.service.js.map +1 -1
- package/dist/backend/backend/src/services/wiki/wiki.types.d.ts +24 -0
- package/dist/backend/backend/src/services/wiki/wiki.types.d.ts.map +1 -1
- package/dist/backend/build-info.json +2 -2
- package/dist/cli/backend/src/constants.d.ts +39 -0
- package/dist/cli/backend/src/constants.d.ts.map +1 -1
- package/dist/cli/backend/src/constants.js +37 -0
- package/dist/cli/backend/src/constants.js.map +1 -1
- package/dist/cli/backend/src/services/memory/memory.service.d.ts +2 -0
- package/dist/cli/backend/src/services/memory/memory.service.d.ts.map +1 -1
- package/dist/cli/backend/src/services/memory/memory.service.js +16 -4
- package/dist/cli/backend/src/services/memory/memory.service.js.map +1 -1
- package/dist/cli/backend/src/services/slack/slack-agent-dm.service.d.ts +3 -1
- package/dist/cli/backend/src/services/slack/slack-agent-dm.service.d.ts.map +1 -1
- package/dist/cli/backend/src/services/slack/slack-agent-dm.service.js +14 -3
- package/dist/cli/backend/src/services/slack/slack-agent-dm.service.js.map +1 -1
- package/dist/cli/backend/src/services/slack/slack-team-channel.service.d.ts +3 -1
- package/dist/cli/backend/src/services/slack/slack-team-channel.service.d.ts.map +1 -1
- package/dist/cli/backend/src/services/slack/slack-team-channel.service.js +26 -9
- package/dist/cli/backend/src/services/slack/slack-team-channel.service.js.map +1 -1
- package/dist/cli/backend/src/services/slack/slack-typing-placeholder.service.d.ts +22 -1
- package/dist/cli/backend/src/services/slack/slack-typing-placeholder.service.d.ts.map +1 -1
- package/dist/cli/backend/src/services/slack/slack-typing-placeholder.service.js +69 -7
- package/dist/cli/backend/src/services/slack/slack-typing-placeholder.service.js.map +1 -1
- package/dist/cli/backend/src/services/wiki/schema-loader.service.d.ts +9 -0
- package/dist/cli/backend/src/services/wiki/schema-loader.service.d.ts.map +1 -1
- package/dist/cli/backend/src/services/wiki/schema-loader.service.js +40 -1
- package/dist/cli/backend/src/services/wiki/schema-loader.service.js.map +1 -1
- package/dist/cli/backend/src/services/wiki/wiki-history.service.d.ts +63 -0
- package/dist/cli/backend/src/services/wiki/wiki-history.service.d.ts.map +1 -0
- package/dist/cli/backend/src/services/wiki/wiki-history.service.js +124 -0
- package/dist/cli/backend/src/services/wiki/wiki-history.service.js.map +1 -0
- package/dist/cli/backend/src/services/wiki/wiki-index.service.d.ts +119 -0
- package/dist/cli/backend/src/services/wiki/wiki-index.service.d.ts.map +1 -0
- package/dist/cli/backend/src/services/wiki/wiki-index.service.js +263 -0
- package/dist/cli/backend/src/services/wiki/wiki-index.service.js.map +1 -0
- package/dist/cli/backend/src/services/wiki/wiki-ingest.service.d.ts +66 -11
- package/dist/cli/backend/src/services/wiki/wiki-ingest.service.d.ts.map +1 -1
- package/dist/cli/backend/src/services/wiki/wiki-ingest.service.js +162 -44
- package/dist/cli/backend/src/services/wiki/wiki-ingest.service.js.map +1 -1
- package/dist/cli/backend/src/services/wiki/wiki-page.d.ts +101 -0
- package/dist/cli/backend/src/services/wiki/wiki-page.d.ts.map +1 -0
- package/dist/cli/backend/src/services/wiki/wiki-page.js +130 -0
- package/dist/cli/backend/src/services/wiki/wiki-page.js.map +1 -0
- package/dist/cli/backend/src/services/wiki/wiki-policy.d.ts +35 -0
- package/dist/cli/backend/src/services/wiki/wiki-policy.d.ts.map +1 -0
- package/dist/cli/backend/src/services/wiki/wiki-policy.js +44 -0
- package/dist/cli/backend/src/services/wiki/wiki-policy.js.map +1 -0
- package/dist/cli/backend/src/services/wiki/wiki-redaction.d.ts +74 -0
- package/dist/cli/backend/src/services/wiki/wiki-redaction.d.ts.map +1 -0
- package/dist/cli/backend/src/services/wiki/wiki-redaction.js +94 -0
- package/dist/cli/backend/src/services/wiki/wiki-redaction.js.map +1 -0
- package/dist/cli/backend/src/services/wiki/wiki-search.service.d.ts +12 -3
- package/dist/cli/backend/src/services/wiki/wiki-search.service.d.ts.map +1 -1
- package/dist/cli/backend/src/services/wiki/wiki-search.service.js +19 -9
- package/dist/cli/backend/src/services/wiki/wiki-search.service.js.map +1 -1
- package/dist/cli/backend/src/services/wiki/wiki-usage.service.d.ts +96 -0
- package/dist/cli/backend/src/services/wiki/wiki-usage.service.d.ts.map +1 -0
- package/dist/cli/backend/src/services/wiki/wiki-usage.service.js +147 -0
- package/dist/cli/backend/src/services/wiki/wiki-usage.service.js.map +1 -0
- package/dist/cli/backend/src/services/wiki/wiki-vault-walk.d.ts +25 -0
- package/dist/cli/backend/src/services/wiki/wiki-vault-walk.d.ts.map +1 -0
- package/dist/cli/backend/src/services/wiki/wiki-vault-walk.js +52 -0
- package/dist/cli/backend/src/services/wiki/wiki-vault-walk.js.map +1 -0
- package/dist/cli/backend/src/services/wiki/wiki.types.d.ts +24 -0
- package/dist/cli/backend/src/services/wiki/wiki.types.d.ts.map +1 -1
- package/package.json +1 -1
|
@@ -1,10 +1,17 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* WikiIngestService — write
|
|
2
|
+
* WikiIngestService — write into a vault's `llm-curated/`.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
4
|
+
* Two kinds of write:
|
|
5
|
+
* - **log entry** (default): append to `llm-curated/log.md`. Cheap,
|
|
6
|
+
* append-only, no gate beyond confidentiality. This is where "maybe
|
|
7
|
+
* useful" goes — the default is NOT to keep.
|
|
8
|
+
* - **page** (`targetRelativePath` under llm-curated/): a piece of
|
|
9
|
+
* knowledge. Passes the retention gate (title + one-line summary +
|
|
10
|
+
* keep_because), the vault's write policy (a `proposed_only` role's
|
|
11
|
+
* page lands in `_proposed/` until a canonical role accepts), the
|
|
12
|
+
* confidentiality gate (secrets never; PII per vault), gets YAML
|
|
13
|
+
* frontmatter, an index line, and a history snapshot of what it
|
|
14
|
+
* replaced.
|
|
8
15
|
*
|
|
9
16
|
* Frozen-path contract (§2): refuses ANY write whose target lives under a
|
|
10
17
|
* `hardcoded:` folder. SchemaLoaderService.isFrozenPath() is the gate.
|
|
@@ -12,6 +19,9 @@
|
|
|
12
19
|
* @module services/wiki/wiki-ingest.service
|
|
13
20
|
*/
|
|
14
21
|
import { SchemaLoaderService } from './schema-loader.service.js';
|
|
22
|
+
import { type WikiKeepBecause } from '../../constants.js';
|
|
23
|
+
import { WikiIndexService } from './wiki-index.service.js';
|
|
24
|
+
import { WikiHistoryService } from './wiki-history.service.js';
|
|
15
25
|
/**
|
|
16
26
|
* Categories of sources that trigger ingest. Mirrors §4 (ingest trigger
|
|
17
27
|
* taxonomy) plus `user_chat` for the demo path.
|
|
@@ -28,14 +38,39 @@ export interface WikiIngestInput {
|
|
|
28
38
|
sourceBody: string;
|
|
29
39
|
/** Session/user that authored the source — appears in the log header. */
|
|
30
40
|
callerSession?: string;
|
|
41
|
+
/** Writer's role as resolved by the server (undefined = owner/UI). Drives write_policy. */
|
|
42
|
+
callerRole?: string;
|
|
31
43
|
/** Optional override of the relative target path; defaults to `llm-curated/log.md`. */
|
|
32
44
|
targetRelativePath?: string;
|
|
45
|
+
/** Page writes only: the page's name. */
|
|
46
|
+
title?: string;
|
|
47
|
+
/** Page writes only: the one-line conclusion — what this means for us. */
|
|
48
|
+
summary?: string;
|
|
49
|
+
/** Page writes only: why it earns a page. */
|
|
50
|
+
keepBecause?: WikiKeepBecause | string;
|
|
51
|
+
tags?: string[];
|
|
52
|
+
/** Page writes only: roles that may read it (default: the vault's default_visibility). */
|
|
53
|
+
visibility?: string[];
|
|
54
|
+
/** Page writes only: replace the page instead of appending to it (default: append). */
|
|
55
|
+
replace?: boolean;
|
|
33
56
|
}
|
|
34
57
|
export interface WikiIngestResult {
|
|
35
58
|
ok: true;
|
|
36
59
|
pagesWritten: string[];
|
|
37
60
|
logEntry: string;
|
|
38
61
|
frozenPathsTouched: string[];
|
|
62
|
+
/** True when the page landed in `_proposed/` (writer is a proposed_only role). */
|
|
63
|
+
proposed?: boolean;
|
|
64
|
+
/** PII pattern names that were masked (vault privacy `mask`). */
|
|
65
|
+
masked?: string[];
|
|
66
|
+
indexUpdated?: boolean;
|
|
67
|
+
}
|
|
68
|
+
export interface WikiIngestRefusedGate {
|
|
69
|
+
ok: false;
|
|
70
|
+
reason: 'retention_gate' | 'secret_detected' | 'pii_refused';
|
|
71
|
+
message: string;
|
|
72
|
+
/** retention_gate: missing fields; privacy: matched pattern names. */
|
|
73
|
+
details: string[];
|
|
39
74
|
}
|
|
40
75
|
export interface WikiIngestRefusedFrozen {
|
|
41
76
|
ok: false;
|
|
@@ -48,7 +83,7 @@ export interface WikiIngestRefusedInvalid {
|
|
|
48
83
|
reason: 'invalid_input' | 'schema_missing' | 'empty_body';
|
|
49
84
|
message: string;
|
|
50
85
|
}
|
|
51
|
-
export type WikiIngestOutcome = WikiIngestResult | WikiIngestRefusedFrozen | WikiIngestRefusedInvalid;
|
|
86
|
+
export type WikiIngestOutcome = WikiIngestResult | WikiIngestRefusedFrozen | WikiIngestRefusedInvalid | WikiIngestRefusedGate;
|
|
52
87
|
/**
|
|
53
88
|
* Writes ingest pages to a vault. Construct one per process; stateless.
|
|
54
89
|
*/
|
|
@@ -56,7 +91,9 @@ export declare class WikiIngestService {
|
|
|
56
91
|
private static instance;
|
|
57
92
|
private readonly logger;
|
|
58
93
|
private readonly schemaLoader;
|
|
59
|
-
|
|
94
|
+
private readonly index;
|
|
95
|
+
private readonly history;
|
|
96
|
+
constructor(schemaLoader?: SchemaLoaderService, index?: WikiIndexService, history?: WikiHistoryService);
|
|
60
97
|
static getInstance(): WikiIngestService;
|
|
61
98
|
/** Test-only reset. */
|
|
62
99
|
static _resetForTesting(): void;
|
|
@@ -68,6 +105,25 @@ export declare class WikiIngestService {
|
|
|
68
105
|
* callers can fold the result into their normal control flow.
|
|
69
106
|
*/
|
|
70
107
|
ingest(input: WikiIngestInput): Promise<WikiIngestOutcome>;
|
|
108
|
+
/**
|
|
109
|
+
* Write (create, append to, or replace) a knowledge page. See the module
|
|
110
|
+
* header for the gates it passes.
|
|
111
|
+
*
|
|
112
|
+
* @param schema - Vault schema
|
|
113
|
+
* @param target - Vault-relative page path (validated, not frozen)
|
|
114
|
+
* @param input - Ingest input with a body that already passed the privacy gate
|
|
115
|
+
* @param masked - PII pattern names masked in the body
|
|
116
|
+
* @returns Outcome
|
|
117
|
+
*/
|
|
118
|
+
private writePage;
|
|
119
|
+
/**
|
|
120
|
+
* The provenance block appended to a page for each ingest.
|
|
121
|
+
*
|
|
122
|
+
* @param input - Ingest input
|
|
123
|
+
* @returns Markdown block
|
|
124
|
+
*/
|
|
125
|
+
private formatPageEntry;
|
|
126
|
+
private titleFromBody;
|
|
71
127
|
private validateInput;
|
|
72
128
|
/**
|
|
73
129
|
* Build the markdown entry. Format is chosen to be safely append-only
|
|
@@ -82,10 +138,9 @@ export declare class WikiIngestService {
|
|
|
82
138
|
private formatLogEntry;
|
|
83
139
|
private appendOrCreate;
|
|
84
140
|
/**
|
|
85
|
-
*
|
|
86
|
-
*
|
|
87
|
-
*
|
|
88
|
-
* - anything else → minimal title block from the source ref
|
|
141
|
+
* Header for a freshly-created `log.md` (pages are built by {@link writePage}).
|
|
142
|
+
*
|
|
143
|
+
* @returns The activity-log preamble
|
|
89
144
|
*/
|
|
90
145
|
private buildPageHeader;
|
|
91
146
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"wiki-ingest.service.d.ts","sourceRoot":"","sources":["../../../../../../backend/src/services/wiki/wiki-ingest.service.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"wiki-ingest.service.d.ts","sourceRoot":"","sources":["../../../../../../backend/src/services/wiki/wiki-ingest.service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAMH,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAEjE,OAAO,EAAqB,KAAK,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAI7E,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAE/D;;;GAGG;AACH,MAAM,MAAM,cAAc,GACtB,WAAW,GACX,eAAe,GACf,WAAW,GACX,UAAU,GACV,iBAAiB,GACjB,eAAe,CAAC;AAEpB,MAAM,WAAW,eAAe;IAC9B,8DAA8D;IAC9D,SAAS,EAAE,MAAM,CAAC;IAClB,mCAAmC;IACnC,UAAU,EAAE,cAAc,CAAC;IAC3B,uEAAuE;IACvE,SAAS,EAAE,MAAM,CAAC;IAClB,yDAAyD;IACzD,UAAU,EAAE,MAAM,CAAC;IACnB,yEAAyE;IACzE,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,2FAA2F;IAC3F,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,uFAAuF;IACvF,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,yCAAyC;IACzC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,0EAA0E;IAC1E,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,6CAA6C;IAC7C,WAAW,CAAC,EAAE,eAAe,GAAG,MAAM,CAAC;IACvC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,0FAA0F;IAC1F,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,uFAAuF;IACvF,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,IAAI,CAAC;IACT,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,kBAAkB,EAAE,MAAM,EAAE,CAAC;IAC7B,kFAAkF;IAClF,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,iEAAiE;IACjE,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,qBAAqB;IACpC,EAAE,EAAE,KAAK,CAAC;IACV,MAAM,EAAE,gBAAgB,GAAG,iBAAiB,GAAG,aAAa,CAAC;IAC7D,OAAO,EAAE,MAAM,CAAC;IAChB,sEAAsE;IACtE,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB;AAED,MAAM,WAAW,uBAAuB;IACtC,EAAE,EAAE,KAAK,CAAC;IACV,MAAM,EAAE,aAAa,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,EAAE,CAAC;CACzB;AAED,MAAM,WAAW,wBAAwB;IACvC,EAAE,EAAE,KAAK,CAAC;IACV,MAAM,EAAE,eAAe,GAAG,gBAAgB,GAAG,YAAY,CAAC;IAC1D,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,MAAM,iBAAiB,GACzB,gBAAgB,GAChB,uBAAuB,GACvB,wBAAwB,GACxB,qBAAqB,CAAC;AAc1B;;GAEG;AACH,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAkC;IACzD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAkB;IACzC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAsB;IACnD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAmB;IACzC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqB;gBAEjC,YAAY,CAAC,EAAE,mBAAmB,EAAE,KAAK,CAAC,EAAE,gBAAgB,EAAE,OAAO,CAAC,EAAE,kBAAkB;IAOtG,MAAM,CAAC,WAAW,IAAI,iBAAiB;IAOvC,uBAAuB;IACvB,MAAM,CAAC,gBAAgB,IAAI,IAAI;IAI/B;;;;;;OAMG;IACG,MAAM,CAAC,KAAK,EAAE,eAAe,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAyDhE;;;;;;;;;OASG;YACW,SAAS;IA4FvB;;;;;OAKG;IACH,OAAO,CAAC,eAAe;IAYvB,OAAO,CAAC,aAAa;IASrB,OAAO,CAAC,aAAa;IAgCrB;;;;;;;;;OASG;IACH,OAAO,CAAC,cAAc;YAgBR,cAAc;IAS5B;;;;OAIG;IACH,OAAO,CAAC,eAAe;IAIvB;;;;;OAKG;IACH,OAAO,CAAC,YAAY;IAQpB,OAAO,CAAC,eAAe;CAKxB"}
|
|
@@ -1,10 +1,17 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* WikiIngestService — write
|
|
2
|
+
* WikiIngestService — write into a vault's `llm-curated/`.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
4
|
+
* Two kinds of write:
|
|
5
|
+
* - **log entry** (default): append to `llm-curated/log.md`. Cheap,
|
|
6
|
+
* append-only, no gate beyond confidentiality. This is where "maybe
|
|
7
|
+
* useful" goes — the default is NOT to keep.
|
|
8
|
+
* - **page** (`targetRelativePath` under llm-curated/): a piece of
|
|
9
|
+
* knowledge. Passes the retention gate (title + one-line summary +
|
|
10
|
+
* keep_because), the vault's write policy (a `proposed_only` role's
|
|
11
|
+
* page lands in `_proposed/` until a canonical role accepts), the
|
|
12
|
+
* confidentiality gate (secrets never; PII per vault), gets YAML
|
|
13
|
+
* frontmatter, an index line, and a history snapshot of what it
|
|
14
|
+
* replaced.
|
|
8
15
|
*
|
|
9
16
|
* Frozen-path contract (§2): refuses ANY write whose target lives under a
|
|
10
17
|
* `hardcoded:` folder. SchemaLoaderService.isFrozenPath() is the gate.
|
|
@@ -16,6 +23,12 @@ import * as fs from 'fs/promises';
|
|
|
16
23
|
import { existsSync } from 'fs';
|
|
17
24
|
import { LoggerService } from '../core/logger.service.js';
|
|
18
25
|
import { SchemaLoaderService } from './schema-loader.service.js';
|
|
26
|
+
import { WIKI_KB_CONSTANTS } from '../../constants.js';
|
|
27
|
+
import { checkRetention, parsePage, serializePage, oneLine } from './wiki-page.js';
|
|
28
|
+
import { applyPrivacyGate } from './wiki-redaction.js';
|
|
29
|
+
import { decideWrite } from './wiki-policy.js';
|
|
30
|
+
import { WikiIndexService } from './wiki-index.service.js';
|
|
31
|
+
import { WikiHistoryService } from './wiki-history.service.js';
|
|
19
32
|
const DEFAULT_LOG_RELATIVE_PATH = 'llm-curated/log.md';
|
|
20
33
|
const MAX_BODY_BYTES = 64 * 1024;
|
|
21
34
|
// Spec note (Steve, 2026-05-22): the earlier keyword-based
|
|
@@ -33,9 +46,13 @@ export class WikiIngestService {
|
|
|
33
46
|
static instance = null;
|
|
34
47
|
logger;
|
|
35
48
|
schemaLoader;
|
|
36
|
-
|
|
49
|
+
index;
|
|
50
|
+
history;
|
|
51
|
+
constructor(schemaLoader, index, history) {
|
|
37
52
|
this.logger = LoggerService.getInstance().createComponentLogger('WikiIngest');
|
|
38
53
|
this.schemaLoader = schemaLoader ?? new SchemaLoaderService();
|
|
54
|
+
this.index = index ?? WikiIndexService.getInstance();
|
|
55
|
+
this.history = history ?? WikiHistoryService.getInstance();
|
|
39
56
|
}
|
|
40
57
|
static getInstance() {
|
|
41
58
|
if (!this.instance) {
|
|
@@ -70,7 +87,7 @@ export class WikiIngestService {
|
|
|
70
87
|
message: err.message,
|
|
71
88
|
};
|
|
72
89
|
}
|
|
73
|
-
const target = input.targetRelativePath ?? DEFAULT_LOG_RELATIVE_PATH;
|
|
90
|
+
const target = (input.targetRelativePath ?? DEFAULT_LOG_RELATIVE_PATH).replace(/\\/g, '/').replace(/^\/+/, '');
|
|
74
91
|
if (this.schemaLoader.isFrozenPath(schema, target)) {
|
|
75
92
|
return {
|
|
76
93
|
ok: false,
|
|
@@ -79,24 +96,145 @@ export class WikiIngestService {
|
|
|
79
96
|
frozenFolders: this.schemaLoader.getFrozenPaths(schema),
|
|
80
97
|
};
|
|
81
98
|
}
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
99
|
+
if (target.includes('..')) {
|
|
100
|
+
return { ok: false, reason: 'invalid_input', message: 'targetRelativePath may not contain ".."' };
|
|
101
|
+
}
|
|
102
|
+
// Confidentiality gate — applies to log entries and pages alike.
|
|
103
|
+
const gate = applyPrivacyGate(input.sourceBody, schema.privacy);
|
|
104
|
+
if (!gate.ok) {
|
|
105
|
+
this.logger.warn('WikiIngest refused by the confidentiality gate', { vault: input.vaultPath, target, reason: gate.reason, patterns: gate.patterns });
|
|
106
|
+
return { ok: false, reason: gate.reason, message: gate.message, details: gate.patterns };
|
|
107
|
+
}
|
|
108
|
+
const safeInput = { ...input, sourceBody: gate.body };
|
|
109
|
+
const isLogTarget = path.basename(target) === 'log.md';
|
|
110
|
+
if (isLogTarget) {
|
|
111
|
+
const absoluteTarget = path.join(input.vaultPath, target);
|
|
112
|
+
await fs.mkdir(path.dirname(absoluteTarget), { recursive: true });
|
|
113
|
+
const logEntry = this.formatLogEntry(safeInput);
|
|
114
|
+
await this.appendOrCreate(absoluteTarget, logEntry);
|
|
115
|
+
this.logger.info('WikiIngest wrote log entry', {
|
|
116
|
+
vault: input.vaultPath,
|
|
117
|
+
target,
|
|
118
|
+
sourceType: input.sourceType,
|
|
119
|
+
sourceRef: input.sourceRef,
|
|
120
|
+
bodyBytes: safeInput.sourceBody.length,
|
|
121
|
+
});
|
|
122
|
+
return { ok: true, pagesWritten: [target], logEntry, frozenPathsTouched: [], masked: gate.masked };
|
|
123
|
+
}
|
|
124
|
+
return this.writePage(schema, target, safeInput, gate.masked);
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* Write (create, append to, or replace) a knowledge page. See the module
|
|
128
|
+
* header for the gates it passes.
|
|
129
|
+
*
|
|
130
|
+
* @param schema - Vault schema
|
|
131
|
+
* @param target - Vault-relative page path (validated, not frozen)
|
|
132
|
+
* @param input - Ingest input with a body that already passed the privacy gate
|
|
133
|
+
* @param masked - PII pattern names masked in the body
|
|
134
|
+
* @returns Outcome
|
|
135
|
+
*/
|
|
136
|
+
async writePage(schema, target, input, masked) {
|
|
137
|
+
if (!target.startsWith('llm-curated/')) {
|
|
138
|
+
return { ok: false, reason: 'invalid_input', message: 'pages live under llm-curated/ (log entries go to llm-curated/log.md)' };
|
|
139
|
+
}
|
|
140
|
+
const existingAbs = path.join(input.vaultPath, target);
|
|
141
|
+
const existing = existsSync(existingAbs) ? parsePage(await fs.readFile(existingAbs, 'utf8')) : null;
|
|
142
|
+
// Retention gate: a page needs title + summary + keep_because. An
|
|
143
|
+
// append to an existing page that already has them inherits them.
|
|
144
|
+
const supplied = {
|
|
145
|
+
...(existing?.frontmatter ?? {}),
|
|
146
|
+
...(input.title ? { title: oneLine(input.title, WIKI_KB_CONSTANTS.TITLE_MAX_CHARS) } : {}),
|
|
147
|
+
...(input.summary ? { summary: oneLine(input.summary, WIKI_KB_CONSTANTS.SUMMARY_MAX_CHARS + 1) } : {}),
|
|
148
|
+
...(input.keepBecause ? { keep_because: input.keepBecause } : {}),
|
|
149
|
+
...(input.tags ? { tags: input.tags } : {}),
|
|
150
|
+
...(input.visibility ? { visibility: input.visibility } : {}),
|
|
151
|
+
};
|
|
152
|
+
if (!supplied.title && existing === null)
|
|
153
|
+
supplied.title = this.titleFromBody(input.sourceBody);
|
|
154
|
+
const refusal = checkRetention(supplied);
|
|
155
|
+
if (refusal) {
|
|
156
|
+
return { ok: false, reason: 'retention_gate', message: refusal.message, details: refusal.missing };
|
|
157
|
+
}
|
|
158
|
+
if (!schema.retention.keep_because.includes(String(supplied.keep_because))) {
|
|
159
|
+
return {
|
|
160
|
+
ok: false,
|
|
161
|
+
reason: 'retention_gate',
|
|
162
|
+
message: `This vault keeps pages only for: ${schema.retention.keep_because.join(' | ')}.`,
|
|
163
|
+
details: ['keep_because'],
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
// Write policy: proposed_only roles land in _proposed/.
|
|
167
|
+
const decision = decideWrite(schema, input.callerRole);
|
|
168
|
+
const proposed = decision === 'proposed';
|
|
169
|
+
const finalTarget = proposed && !target.startsWith(`${WIKI_KB_CONSTANTS.PROPOSED_DIR}/`)
|
|
170
|
+
? `${WIKI_KB_CONSTANTS.PROPOSED_DIR}/${target.replace(/^llm-curated\//, '')}`
|
|
171
|
+
: target;
|
|
172
|
+
const abs = path.join(input.vaultPath, finalTarget);
|
|
173
|
+
const prior = existsSync(abs) ? parsePage(await fs.readFile(abs, 'utf8')) : null;
|
|
174
|
+
const now = new Date().toISOString();
|
|
175
|
+
const author = input.callerSession ?? input.sourceRef;
|
|
176
|
+
const fm = {
|
|
177
|
+
...(prior?.frontmatter ?? {}),
|
|
178
|
+
...supplied,
|
|
179
|
+
source: oneLine(input.sourceRef, 200),
|
|
180
|
+
caller: oneLine(author, 80),
|
|
181
|
+
recorded: prior?.frontmatter.recorded ?? now,
|
|
182
|
+
updated: now,
|
|
183
|
+
...(proposed ? { proposed_by: oneLine(author, 80) } : {}),
|
|
184
|
+
};
|
|
185
|
+
if (!fm.visibility && schema.privacy.default_visibility.length > 0)
|
|
186
|
+
fm.visibility = schema.privacy.default_visibility;
|
|
187
|
+
const entry = this.formatPageEntry(input);
|
|
188
|
+
const body = prior && !input.replace
|
|
189
|
+
? `${prior.body.replace(/\s+$/, '')}\n${entry}`
|
|
190
|
+
: `# ${fm.title}\n${entry}`;
|
|
191
|
+
await this.history.snapshot(input.vaultPath, finalTarget, author, 'write');
|
|
192
|
+
await fs.mkdir(path.dirname(abs), { recursive: true });
|
|
193
|
+
await fs.writeFile(abs, serializePage(fm, body), 'utf8');
|
|
194
|
+
let indexUpdated = false;
|
|
195
|
+
if (!proposed) {
|
|
196
|
+
await this.index.upsert(input.vaultPath, finalTarget, fm);
|
|
197
|
+
indexUpdated = true;
|
|
198
|
+
}
|
|
199
|
+
this.logger.info(proposed ? 'WikiIngest wrote a proposed page' : 'WikiIngest wrote page', {
|
|
87
200
|
vault: input.vaultPath,
|
|
88
|
-
target,
|
|
201
|
+
target: finalTarget,
|
|
89
202
|
sourceType: input.sourceType,
|
|
90
|
-
|
|
91
|
-
|
|
203
|
+
keepBecause: fm.keep_because,
|
|
204
|
+
callerRole: input.callerRole ?? 'owner',
|
|
205
|
+
appended: !!prior && !input.replace,
|
|
92
206
|
});
|
|
93
207
|
return {
|
|
94
208
|
ok: true,
|
|
95
|
-
pagesWritten: [
|
|
96
|
-
logEntry,
|
|
209
|
+
pagesWritten: [finalTarget],
|
|
210
|
+
logEntry: entry,
|
|
97
211
|
frozenPathsTouched: [],
|
|
212
|
+
proposed,
|
|
213
|
+
masked,
|
|
214
|
+
indexUpdated,
|
|
98
215
|
};
|
|
99
216
|
}
|
|
217
|
+
/**
|
|
218
|
+
* The provenance block appended to a page for each ingest.
|
|
219
|
+
*
|
|
220
|
+
* @param input - Ingest input
|
|
221
|
+
* @returns Markdown block
|
|
222
|
+
*/
|
|
223
|
+
formatPageEntry(input) {
|
|
224
|
+
const ts = new Date().toISOString();
|
|
225
|
+
const caller = this.sanitizeOneLine(input.callerSession ?? input.sourceRef, 80);
|
|
226
|
+
return [
|
|
227
|
+
'',
|
|
228
|
+
this.sanitizeBody(input.sourceBody),
|
|
229
|
+
'',
|
|
230
|
+
`<sub>${input.sourceType} · ${this.sanitizeOneLine(input.sourceRef, 200)} · ${caller} · ${ts}</sub>`,
|
|
231
|
+
'',
|
|
232
|
+
].join('\n');
|
|
233
|
+
}
|
|
234
|
+
titleFromBody(body) {
|
|
235
|
+
const heading = body.match(/^#\s+(.+)$/m);
|
|
236
|
+
return oneLine(heading ? heading[1] : body, WIKI_KB_CONSTANTS.TITLE_MAX_CHARS);
|
|
237
|
+
}
|
|
100
238
|
// ---------------------------------------------------------------------------
|
|
101
239
|
// Internals
|
|
102
240
|
// ---------------------------------------------------------------------------
|
|
@@ -156,41 +294,21 @@ export class WikiIngestService {
|
|
|
156
294
|
'',
|
|
157
295
|
].join('\n');
|
|
158
296
|
}
|
|
159
|
-
async appendOrCreate(absolutePath, entry
|
|
297
|
+
async appendOrCreate(absolutePath, entry) {
|
|
160
298
|
if (!existsSync(absolutePath)) {
|
|
161
|
-
const header = this.buildPageHeader(
|
|
299
|
+
const header = this.buildPageHeader();
|
|
162
300
|
await fs.writeFile(absolutePath, header + entry, 'utf8');
|
|
163
301
|
return;
|
|
164
302
|
}
|
|
165
303
|
await fs.appendFile(absolutePath, entry, 'utf8');
|
|
166
304
|
}
|
|
167
305
|
/**
|
|
168
|
-
*
|
|
169
|
-
*
|
|
170
|
-
*
|
|
171
|
-
* - anything else → minimal title block from the source ref
|
|
306
|
+
* Header for a freshly-created `log.md` (pages are built by {@link writePage}).
|
|
307
|
+
*
|
|
308
|
+
* @returns The activity-log preamble
|
|
172
309
|
*/
|
|
173
|
-
buildPageHeader(
|
|
174
|
-
|
|
175
|
-
if (basename === 'log.md') {
|
|
176
|
-
return '# Activity log\n\nAppend-only log of ingested sources. Each entry: `## [<ISO>] <sourceType> | <caller>`.\n';
|
|
177
|
-
}
|
|
178
|
-
if (absolutePath.includes('/decisions/')) {
|
|
179
|
-
const title = this.sanitizeOneLine(input.sourceBody, 80);
|
|
180
|
-
const caller = input.callerSession ?? input.sourceRef;
|
|
181
|
-
return [
|
|
182
|
-
`# ${title}`,
|
|
183
|
-
'',
|
|
184
|
-
`> **source:** \`${this.sanitizeOneLine(input.sourceRef, 200)}\` `,
|
|
185
|
-
`> **caller:** \`${this.sanitizeOneLine(caller, 80)}\` `,
|
|
186
|
-
`> **recorded:** ${new Date().toISOString()}`,
|
|
187
|
-
'',
|
|
188
|
-
'---',
|
|
189
|
-
'',
|
|
190
|
-
].join('\n');
|
|
191
|
-
}
|
|
192
|
-
// Generic non-log target: minimal title block.
|
|
193
|
-
return `# ${this.sanitizeOneLine(input.sourceRef, 80)}\n\n`;
|
|
310
|
+
buildPageHeader() {
|
|
311
|
+
return '# Activity log\n\nAppend-only log of ingested sources. Each entry: `## [<ISO>] <sourceType> | <caller>`.\n';
|
|
194
312
|
}
|
|
195
313
|
/**
|
|
196
314
|
* Defuse markers a future skill might mis-route on: `[CHAT]`, `[NOTIFY]`,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"wiki-ingest.service.js","sourceRoot":"","sources":["../../../../../../backend/src/services/wiki/wiki-ingest.service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,EAAE,MAAM,aAAa,CAAC;AAClC,OAAO,EAAE,UAAU,EAAE,MAAM,IAAI,CAAC;AAChC,OAAO,EAAE,aAAa,EAAmB,MAAM,2BAA2B,CAAC;AAC3E,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAuDjE,MAAM,yBAAyB,GAAG,oBAAoB,CAAC;AACvD,MAAM,cAAc,GAAG,EAAE,GAAG,IAAI,CAAC;AAEjC,2DAA2D;AAC3D,0EAA0E;AAC1E,yEAAyE;AACzE,0EAA0E;AAC1E,wEAAwE;AACxE,0EAA0E;AAC1E,sEAAsE;AACtE,iEAAiE;AAEjE;;GAEG;AACH,MAAM,OAAO,iBAAiB;IACpB,MAAM,CAAC,QAAQ,GAA6B,IAAI,CAAC;IACxC,MAAM,CAAkB;IACxB,YAAY,CAAsB;IAEnD,YAAY,YAAkC;QAC5C,IAAI,CAAC,MAAM,GAAG,aAAa,CAAC,WAAW,EAAE,CAAC,qBAAqB,CAAC,YAAY,CAAC,CAAC;QAC9E,IAAI,CAAC,YAAY,GAAG,YAAY,IAAI,IAAI,mBAAmB,EAAE,CAAC;IAChE,CAAC;IAED,MAAM,CAAC,WAAW;QAChB,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnB,IAAI,CAAC,QAAQ,GAAG,IAAI,iBAAiB,EAAE,CAAC;QAC1C,CAAC;QACD,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED,uBAAuB;IACvB,MAAM,CAAC,gBAAgB;QACrB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;IACvB,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,MAAM,CAAC,KAAsB;QACjC,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;QAC7C,IAAI,UAAU,EAAE,CAAC;YACf,OAAO,UAAU,CAAC;QACpB,CAAC;QAED,IAAI,MAAmB,CAAC;QACxB,IAAI,CAAC;YACH,MAAM,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QACzD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO;gBACL,EAAE,EAAE,KAAK;gBACT,MAAM,EAAE,gBAAgB;gBACxB,OAAO,EAAG,GAAa,CAAC,OAAO;aAChC,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAAG,KAAK,CAAC,kBAAkB,IAAI,yBAAyB,CAAC;QACrE,IAAI,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;YACnD,OAAO;gBACL,EAAE,EAAE,KAAK;gBACT,MAAM,EAAE,aAAa;gBACrB,aAAa,EAAE,MAAM;gBACrB,aAAa,EAAE,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,MAAM,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAC1D,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAElE,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;QAE3D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,4BAA4B,EAAE;YAC7C,KAAK,EAAE,KAAK,CAAC,SAAS;YACtB,MAAM;YACN,UAAU,EAAE,KAAK,CAAC,UAAU;YAC5B,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,SAAS,EAAE,KAAK,CAAC,UAAU,CAAC,MAAM;SACnC,CAAC,CAAC;QAEH,OAAO;YACL,EAAE,EAAE,IAAI;YACR,YAAY,EAAE,CAAC,MAAM,CAAC;YACtB,QAAQ;YACR,kBAAkB,EAAE,EAAE;SACvB,CAAC;IACJ,CAAC;IAED,8EAA8E;IAC9E,YAAY;IACZ,8EAA8E;IAEtE,aAAa,CAAC,KAAsB;QAC1C,IAAI,CAAC,KAAK,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC;YAC1D,OAAO;gBACL,EAAE,EAAE,KAAK;gBACT,MAAM,EAAE,eAAe;gBACvB,OAAO,EAAE,4CAA4C,KAAK,CAAC,SAAS,IAAI,EAAE,GAAG;aAC9E,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC;YACtB,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,eAAe,EAAE,OAAO,EAAE,wBAAwB,EAAE,CAAC;QACnF,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;YACrB,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,eAAe,EAAE,OAAO,EAAE,uBAAuB,EAAE,CAAC;QAClF,CAAC;QACD,MAAM,IAAI,GAAG,KAAK,CAAC,UAAU,IAAI,EAAE,CAAC;QACpC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7B,OAAO;gBACL,EAAE,EAAE,KAAK;gBACT,MAAM,EAAE,YAAY;gBACpB,OAAO,EAAE,+CAA+C;aACzD,CAAC;QACJ,CAAC;QACD,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,cAAc,EAAE,CAAC;YACrD,OAAO;gBACL,EAAE,EAAE,KAAK;gBACT,MAAM,EAAE,eAAe;gBACvB,OAAO,EAAE,sBAAsB,cAAc,QAAQ;aACtD,CAAC;QACJ,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;;;;;;OASG;IACK,cAAc,CAAC,KAAsB;QAC3C,MAAM,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,aAAa,IAAI,KAAK,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QAChF,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QACjD,MAAM,GAAG,GAAG,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QACvD,OAAO;YACL,EAAE;YACF,OAAO,EAAE,KAAK,KAAK,CAAC,UAAU,MAAM,MAAM,EAAE;YAC5C,EAAE;YACF,QAAQ,GAAG,EAAE;YACb,EAAE;YACF,IAAI;YACJ,EAAE;SACH,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACf,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,YAAoB,EACpB,KAAa,EACb,KAAsB;QAEtB,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;YAC9B,MAAM,MAAM,GAAG,IAAI,CAAC,eAAe,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;YACzD,MAAM,EAAE,CAAC,SAAS,CAAC,YAAY,EAAE,MAAM,GAAG,KAAK,EAAE,MAAM,CAAC,CAAC;YACzD,OAAO;QACT,CAAC;QACD,MAAM,EAAE,CAAC,UAAU,CAAC,YAAY,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IACnD,CAAC;IAED;;;;;OAKG;IACK,eAAe,CAAC,YAAoB,EAAE,KAAsB;QAClE,MAAM,QAAQ,GAAG,YAAY,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;QACzD,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAC1B,OAAO,4GAA4G,CAAC;QACtH,CAAC;QACD,IAAI,YAAY,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;YACzC,MAAM,KAAK,GAAG,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;YACzD,MAAM,MAAM,GAAG,KAAK,CAAC,aAAa,IAAI,KAAK,CAAC,SAAS,CAAC;YACtD,OAAO;gBACL,KAAK,KAAK,EAAE;gBACZ,EAAE;gBACF,mBAAmB,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,SAAS,EAAE,GAAG,CAAC,MAAM;gBACnE,mBAAmB,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM;gBACzD,mBAAmB,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE;gBAC7C,EAAE;gBACF,KAAK;gBACL,EAAE;aACH,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACf,CAAC;QACD,+CAA+C;QAC/C,OAAO,KAAK,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,CAAC,MAAM,CAAC;IAC9D,CAAC;IAED;;;;;OAKG;IACK,YAAY,CAAC,IAAY;QAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;QACnD,OAAO,OAAO,CAAC,OAAO,CACpB,qCAAqC,EACrC,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC,KAAK,GAAG,GAAG,CAC7B,CAAC;IACJ,CAAC;IAEO,eAAe,CAAC,KAAa,EAAE,MAAc;QACnD,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QACxE,IAAI,IAAI,CAAC,MAAM,IAAI,MAAM;YAAE,OAAO,IAAI,CAAC;QACvC,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC;IACzC,CAAC"}
|
|
1
|
+
{"version":3,"file":"wiki-ingest.service.js","sourceRoot":"","sources":["../../../../../../backend/src/services/wiki/wiki-ingest.service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,EAAE,MAAM,aAAa,CAAC;AAClC,OAAO,EAAE,UAAU,EAAE,MAAM,IAAI,CAAC;AAChC,OAAO,EAAE,aAAa,EAAmB,MAAM,2BAA2B,CAAC;AAC3E,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAEjE,OAAO,EAAE,iBAAiB,EAAwB,MAAM,oBAAoB,CAAC;AAC7E,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,aAAa,EAAE,OAAO,EAA4B,MAAM,gBAAgB,CAAC;AAC7G,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAiF/D,MAAM,yBAAyB,GAAG,oBAAoB,CAAC;AACvD,MAAM,cAAc,GAAG,EAAE,GAAG,IAAI,CAAC;AAEjC,2DAA2D;AAC3D,0EAA0E;AAC1E,yEAAyE;AACzE,0EAA0E;AAC1E,wEAAwE;AACxE,0EAA0E;AAC1E,sEAAsE;AACtE,iEAAiE;AAEjE;;GAEG;AACH,MAAM,OAAO,iBAAiB;IACpB,MAAM,CAAC,QAAQ,GAA6B,IAAI,CAAC;IACxC,MAAM,CAAkB;IACxB,YAAY,CAAsB;IAClC,KAAK,CAAmB;IACxB,OAAO,CAAqB;IAE7C,YAAY,YAAkC,EAAE,KAAwB,EAAE,OAA4B;QACpG,IAAI,CAAC,MAAM,GAAG,aAAa,CAAC,WAAW,EAAE,CAAC,qBAAqB,CAAC,YAAY,CAAC,CAAC;QAC9E,IAAI,CAAC,YAAY,GAAG,YAAY,IAAI,IAAI,mBAAmB,EAAE,CAAC;QAC9D,IAAI,CAAC,KAAK,GAAG,KAAK,IAAI,gBAAgB,CAAC,WAAW,EAAE,CAAC;QACrD,IAAI,CAAC,OAAO,GAAG,OAAO,IAAI,kBAAkB,CAAC,WAAW,EAAE,CAAC;IAC7D,CAAC;IAED,MAAM,CAAC,WAAW;QAChB,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnB,IAAI,CAAC,QAAQ,GAAG,IAAI,iBAAiB,EAAE,CAAC;QAC1C,CAAC;QACD,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED,uBAAuB;IACvB,MAAM,CAAC,gBAAgB;QACrB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;IACvB,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,MAAM,CAAC,KAAsB;QACjC,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;QAC7C,IAAI,UAAU,EAAE,CAAC;YACf,OAAO,UAAU,CAAC;QACpB,CAAC;QAED,IAAI,MAAmB,CAAC;QACxB,IAAI,CAAC;YACH,MAAM,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QACzD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO;gBACL,EAAE,EAAE,KAAK;gBACT,MAAM,EAAE,gBAAgB;gBACxB,OAAO,EAAG,GAAa,CAAC,OAAO;aAChC,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAAG,CAAC,KAAK,CAAC,kBAAkB,IAAI,yBAAyB,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC/G,IAAI,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;YACnD,OAAO;gBACL,EAAE,EAAE,KAAK;gBACT,MAAM,EAAE,aAAa;gBACrB,aAAa,EAAE,MAAM;gBACrB,aAAa,EAAE,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,MAAM,CAAC;aACxD,CAAC;QACJ,CAAC;QACD,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1B,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,eAAe,EAAE,OAAO,EAAE,yCAAyC,EAAE,CAAC;QACpG,CAAC;QAED,iEAAiE;QACjE,MAAM,IAAI,GAAG,gBAAgB,CAAC,KAAK,CAAC,UAAU,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;QAChE,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;YACb,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gDAAgD,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;YACrJ,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC3F,CAAC;QACD,MAAM,SAAS,GAAoB,EAAE,GAAG,KAAK,EAAE,UAAU,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC;QAEvE,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,QAAQ,CAAC;QACvD,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;YAC1D,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAClE,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;YAChD,MAAM,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC;YACpD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,4BAA4B,EAAE;gBAC7C,KAAK,EAAE,KAAK,CAAC,SAAS;gBACtB,MAAM;gBACN,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,SAAS,EAAE,KAAK,CAAC,SAAS;gBAC1B,SAAS,EAAE,SAAS,CAAC,UAAU,CAAC,MAAM;aACvC,CAAC,CAAC;YACH,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,kBAAkB,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;QACrG,CAAC;QAED,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IAChE,CAAC;IAED;;;;;;;;;OASG;IACK,KAAK,CAAC,SAAS,CACrB,MAAmB,EACnB,MAAc,EACd,KAAsB,EACtB,MAAgB;QAEhB,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,cAAc,CAAC,EAAE,CAAC;YACvC,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,eAAe,EAAE,OAAO,EAAE,sEAAsE,EAAE,CAAC;QACjI,CAAC;QACD,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QACvD,MAAM,QAAQ,GAAG,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAEpG,kEAAkE;QAClE,kEAAkE;QAClE,MAAM,QAAQ,GAAiC;YAC7C,GAAG,CAAC,QAAQ,EAAE,WAAW,IAAI,EAAE,CAAC;YAChC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,iBAAiB,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1F,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,iBAAiB,CAAC,iBAAiB,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACtG,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,KAAK,CAAC,WAA8B,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACpF,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3C,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC9D,CAAC;QACF,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,QAAQ,KAAK,IAAI;YAAE,QAAQ,CAAC,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QAChG,MAAM,OAAO,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;QACzC,IAAI,OAAO,EAAE,CAAC;YACZ,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,gBAAgB,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC;QACrG,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,EAAE,CAAC;YAC3E,OAAO;gBACL,EAAE,EAAE,KAAK;gBACT,MAAM,EAAE,gBAAgB;gBACxB,OAAO,EAAE,oCAAoC,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG;gBACzF,OAAO,EAAE,CAAC,cAAc,CAAC;aAC1B,CAAC;QACJ,CAAC;QAED,wDAAwD;QACxD,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;QACvD,MAAM,QAAQ,GAAG,QAAQ,KAAK,UAAU,CAAC;QACzC,MAAM,WAAW,GAAG,QAAQ,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,iBAAiB,CAAC,YAAY,GAAG,CAAC;YACtF,CAAC,CAAC,GAAG,iBAAiB,CAAC,YAAY,IAAI,MAAM,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,EAAE;YAC7E,CAAC,CAAC,MAAM,CAAC;QACX,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;QACpD,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAEjF,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACrC,MAAM,MAAM,GAAG,KAAK,CAAC,aAAa,IAAI,KAAK,CAAC,SAAS,CAAC;QACtD,MAAM,EAAE,GAAiC;YACvC,GAAG,CAAC,KAAK,EAAE,WAAW,IAAI,EAAE,CAAC;YAC7B,GAAG,QAAQ;YACX,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE,GAAG,CAAC;YACrC,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YAC3B,QAAQ,EAAE,KAAK,EAAE,WAAW,CAAC,QAAQ,IAAI,GAAG;YAC5C,OAAO,EAAE,GAAG;YACZ,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC1D,CAAC;QACF,IAAI,CAAC,EAAE,CAAC,UAAU,IAAI,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC;YAAE,EAAE,CAAC,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC;QAEtH,MAAM,KAAK,GAAG,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QAC1C,MAAM,IAAI,GAAG,KAAK,IAAI,CAAC,KAAK,CAAC,OAAO;YAClC,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,KAAK,KAAK,EAAE;YAC/C,CAAC,CAAC,KAAK,EAAE,CAAC,KAAK,KAAK,KAAK,EAAE,CAAC;QAE9B,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAS,EAAE,WAAW,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;QAC3E,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACvD,MAAM,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,aAAa,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC;QAEzD,IAAI,YAAY,GAAG,KAAK,CAAC;QACzB,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,EAAE,WAAW,EAAE,EAAE,CAAC,CAAC;YAC1D,YAAY,GAAG,IAAI,CAAC;QACtB,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,kCAAkC,CAAC,CAAC,CAAC,uBAAuB,EAAE;YACxF,KAAK,EAAE,KAAK,CAAC,SAAS;YACtB,MAAM,EAAE,WAAW;YACnB,UAAU,EAAE,KAAK,CAAC,UAAU;YAC5B,WAAW,EAAE,EAAE,CAAC,YAAY;YAC5B,UAAU,EAAE,KAAK,CAAC,UAAU,IAAI,OAAO;YACvC,QAAQ,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,OAAO;SACpC,CAAC,CAAC;QACH,OAAO;YACL,EAAE,EAAE,IAAI;YACR,YAAY,EAAE,CAAC,WAAW,CAAC;YAC3B,QAAQ,EAAE,KAAK;YACf,kBAAkB,EAAE,EAAE;YACtB,QAAQ;YACR,MAAM;YACN,YAAY;SACb,CAAC;IACJ,CAAC;IAED;;;;;OAKG;IACK,eAAe,CAAC,KAAsB;QAC5C,MAAM,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,aAAa,IAAI,KAAK,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QAChF,OAAO;YACL,EAAE;YACF,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,UAAU,CAAC;YACnC,EAAE;YACF,QAAQ,KAAK,CAAC,UAAU,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,SAAS,EAAE,GAAG,CAAC,MAAM,MAAM,MAAM,EAAE,QAAQ;YACpG,EAAE;SACH,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACf,CAAC;IAEO,aAAa,CAAC,IAAY;QAChC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QAC1C,OAAO,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,iBAAiB,CAAC,eAAe,CAAC,CAAC;IACjF,CAAC;IAED,8EAA8E;IAC9E,YAAY;IACZ,8EAA8E;IAEtE,aAAa,CAAC,KAAsB;QAC1C,IAAI,CAAC,KAAK,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC;YAC1D,OAAO;gBACL,EAAE,EAAE,KAAK;gBACT,MAAM,EAAE,eAAe;gBACvB,OAAO,EAAE,4CAA4C,KAAK,CAAC,SAAS,IAAI,EAAE,GAAG;aAC9E,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC;YACtB,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,eAAe,EAAE,OAAO,EAAE,wBAAwB,EAAE,CAAC;QACnF,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;YACrB,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,eAAe,EAAE,OAAO,EAAE,uBAAuB,EAAE,CAAC;QAClF,CAAC;QACD,MAAM,IAAI,GAAG,KAAK,CAAC,UAAU,IAAI,EAAE,CAAC;QACpC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7B,OAAO;gBACL,EAAE,EAAE,KAAK;gBACT,MAAM,EAAE,YAAY;gBACpB,OAAO,EAAE,+CAA+C;aACzD,CAAC;QACJ,CAAC;QACD,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,cAAc,EAAE,CAAC;YACrD,OAAO;gBACL,EAAE,EAAE,KAAK;gBACT,MAAM,EAAE,eAAe;gBACvB,OAAO,EAAE,sBAAsB,cAAc,QAAQ;aACtD,CAAC;QACJ,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;;;;;;OASG;IACK,cAAc,CAAC,KAAsB;QAC3C,MAAM,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,aAAa,IAAI,KAAK,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QAChF,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QACjD,MAAM,GAAG,GAAG,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QACvD,OAAO;YACL,EAAE;YACF,OAAO,EAAE,KAAK,KAAK,CAAC,UAAU,MAAM,MAAM,EAAE;YAC5C,EAAE;YACF,QAAQ,GAAG,EAAE;YACb,EAAE;YACF,IAAI;YACJ,EAAE;SACH,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACf,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,YAAoB,EAAE,KAAa;QAC9D,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;YAC9B,MAAM,MAAM,GAAG,IAAI,CAAC,eAAe,EAAE,CAAC;YACtC,MAAM,EAAE,CAAC,SAAS,CAAC,YAAY,EAAE,MAAM,GAAG,KAAK,EAAE,MAAM,CAAC,CAAC;YACzD,OAAO;QACT,CAAC;QACD,MAAM,EAAE,CAAC,UAAU,CAAC,YAAY,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IACnD,CAAC;IAED;;;;OAIG;IACK,eAAe;QACrB,OAAO,4GAA4G,CAAC;IACtH,CAAC;IAED;;;;;OAKG;IACK,YAAY,CAAC,IAAY;QAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;QACnD,OAAO,OAAO,CAAC,OAAO,CACpB,qCAAqC,EACrC,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC,KAAK,GAAG,GAAG,CAC7B,CAAC;IACJ,CAAC;IAEO,eAAe,CAAC,KAAa,EAAE,MAAc;QACnD,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QACxE,IAAI,IAAI,CAAC,MAAM,IAAI,MAAM;YAAE,OAAO,IAAI,CAAC;QACvC,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC;IACzC,CAAC"}
|
|
@@ -96,8 +96,30 @@ export interface WikiLintReport {
|
|
|
96
96
|
orphanPages: string[];
|
|
97
97
|
staleClaims: string[];
|
|
98
98
|
restructureProposals: WikiLintRestructureProposal[];
|
|
99
|
+
/**
|
|
100
|
+
* Pairs of pages in the same folder whose title+summary overlap enough
|
|
101
|
+
* to be the same claim written twice — or two different conclusions on
|
|
102
|
+
* one topic. The agent decides: merge, or supersede one with the other.
|
|
103
|
+
*/
|
|
104
|
+
contradictionCandidates: WikiLintContradictionCandidate[];
|
|
105
|
+
/** Pages under `_proposed/` awaiting review. */
|
|
106
|
+
proposalsPending: number;
|
|
107
|
+
/** Index vs. disk. */
|
|
108
|
+
index: {
|
|
109
|
+
pages: number;
|
|
110
|
+
indexed: number;
|
|
111
|
+
missingFromIndex: number;
|
|
112
|
+
indexedButMissing: number;
|
|
113
|
+
};
|
|
99
114
|
truncated: boolean;
|
|
100
115
|
}
|
|
116
|
+
/** Two pages that look like one claim (or one topic, two answers). */
|
|
117
|
+
export interface WikiLintContradictionCandidate {
|
|
118
|
+
pages: [string, string];
|
|
119
|
+
/** Jaccard similarity of title+summary token sets. */
|
|
120
|
+
similarity: number;
|
|
121
|
+
claim: string;
|
|
122
|
+
}
|
|
101
123
|
export type WikiLintOutcome = {
|
|
102
124
|
ok: true;
|
|
103
125
|
report: WikiLintReport;
|
|
@@ -157,6 +179,15 @@ export declare class WikiLintService {
|
|
|
157
179
|
*
|
|
158
180
|
* Results sort by referenceCount desc — highest-leverage concepts first.
|
|
159
181
|
*/
|
|
182
|
+
/**
|
|
183
|
+
* Pages in the same folder whose title+summary token sets overlap above
|
|
184
|
+
* {@link WIKI_KB_CONSTANTS.DUPLICATE_JACCARD}. Superseded pages are
|
|
185
|
+
* excluded (that pair has already been resolved).
|
|
186
|
+
*
|
|
187
|
+
* @param vaultPath - Vault root
|
|
188
|
+
* @returns Candidate pairs, most similar first
|
|
189
|
+
*/
|
|
190
|
+
private detectContradictionCandidates;
|
|
160
191
|
private detectMissingConcepts;
|
|
161
192
|
/**
|
|
162
193
|
* Files whose content is older than `staleDays`.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"wiki-lint.service.d.ts","sourceRoot":"","sources":["../../../../../../backend/src/services/wiki/wiki-lint.service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;
|
|
1
|
+
{"version":3,"file":"wiki-lint.service.d.ts","sourceRoot":"","sources":["../../../../../../backend/src/services/wiki/wiki-lint.service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAeH,6DAA6D;AAC7D,eAAO,MAAM,4BAA4B,KAAK,CAAC;AAC/C,4EAA4E;AAC5E,eAAO,MAAM,0BAA0B,KAAK,CAAC;AAC7C,gDAAgD;AAChD,eAAO,MAAM,mBAAmB,OAAO,CAAC;AACxC,oDAAoD;AACpD,eAAO,MAAM,8BAA8B,KAAK,CAAC;AACjD,2DAA2D;AAC3D,eAAO,MAAM,wBAAwB,aAKnC,CAAC;AAEH;;;;;GAKG;AACH,eAAO,MAAM,mCAAmC,IAAI,CAAC;AAKrD,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,uBAAuB;IACtC,2CAA2C;IAC3C,IAAI,EAAE,MAAM,CAAC;IACb,qCAAqC;IACrC,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,qBAAqB;IACpC,gDAAgD;IAChD,UAAU,EAAE,MAAM,CAAC;IACnB,sCAAsC;IACtC,MAAM,EAAE,MAAM,CAAC;IACf,gDAAgD;IAChD,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,2BAA2B;IAC1C,oIAAoI;IACpI,WAAW,EAAE,MAAM,CAAC;IACpB,8CAA8C;IAC9C,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,sBAAsB;IACrC,6DAA6D;IAC7D,MAAM,EAAE,MAAM,CAAC;IACf,kEAAkE;IAClE,cAAc,EAAE,MAAM,CAAC;IACvB,2DAA2D;IAC3D,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB;AAED,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IACnD,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,mBAAmB,EAAE,OAAO,CAAC;IAC7B,gBAAgB,EAAE,uBAAuB,EAAE,CAAC;IAC5C,eAAe,EAAE,qBAAqB,EAAE,CAAC;IACzC,oEAAoE;IACpE,eAAe,EAAE,sBAAsB,EAAE,CAAC;IAC1C,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,oBAAoB,EAAE,2BAA2B,EAAE,CAAC;IACpD;;;;OAIG;IACH,uBAAuB,EAAE,8BAA8B,EAAE,CAAC;IAC1D,gDAAgD;IAChD,gBAAgB,EAAE,MAAM,CAAC;IACzB,sBAAsB;IACtB,KAAK,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,gBAAgB,EAAE,MAAM,CAAC;QAAC,iBAAiB,EAAE,MAAM,CAAA;KAAE,CAAC;IAC/F,SAAS,EAAE,OAAO,CAAC;CACpB;AAED,sEAAsE;AACtE,MAAM,WAAW,8BAA8B;IAC7C,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACxB,sDAAsD;IACtD,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,MAAM,eAAe,GACvB;IAAE,EAAE,EAAE,IAAI,CAAC;IAAC,MAAM,EAAE,cAAc,CAAA;CAAE,GACpC;IACE,EAAE,EAAE,KAAK,CAAC;IACV,MAAM,EAAE,eAAe,GAAG,gBAAgB,GAAG,eAAe,CAAC;IAC7D,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEN;;;GAGG;AACH,qBAAa,eAAe;IAC1B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAgC;IACvD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAkB;IACzC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAsB;IAEnD,OAAO;IAKP,MAAM,CAAC,WAAW,IAAI,eAAe;IAKrC,MAAM,CAAC,aAAa,IAAI,IAAI;IAI5B;;OAEG;IACG,QAAQ,CAAC,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,eAAe,CAAC;IAgG9D;;OAEG;YACW,YAAY;IAoC1B;;;;;;;OAOG;IACH,OAAO,CAAC,sBAAsB;IA2B9B;;;;OAIG;YACW,aAAa;IAoE3B;;;OAGG;IACH,OAAO,CAAC,aAAa;IAUrB;;;;;;;;;;;OAWG;IACH;;;;;;;OAOG;YACW,6BAA6B;IAuC3C,OAAO,CAAC,qBAAqB;IAkB7B;;;;;;;;OAQG;IACH,OAAO,CAAC,WAAW;IAUnB;;;;;;;;OAQG;IACH,OAAO,CAAC,mBAAmB;CA6D5B"}
|
|
@@ -28,6 +28,12 @@ import * as fs from 'fs/promises';
|
|
|
28
28
|
import { existsSync } from 'fs';
|
|
29
29
|
import { LoggerService } from '../core/logger.service.js';
|
|
30
30
|
import { SchemaLoaderService } from './schema-loader.service.js';
|
|
31
|
+
import { WikiCurationService } from './wiki-curation.service.js';
|
|
32
|
+
import { WikiIndexService } from './wiki-index.service.js';
|
|
33
|
+
import { walkCuratedPages, isSeedFile } from './wiki-vault-walk.js';
|
|
34
|
+
import { parsePage } from './wiki-page.js';
|
|
35
|
+
import { tokenize } from './wiki-search.service.js';
|
|
36
|
+
import { WIKI_KB_CONSTANTS } from '../../constants.js';
|
|
31
37
|
/** Default age threshold (days) for marking a page stale. */
|
|
32
38
|
export const WIKI_LINT_DEFAULT_STALE_DAYS = 90;
|
|
33
39
|
/** Folder size threshold above which lint proposes an `index.md` rollup. */
|
|
@@ -130,6 +136,9 @@ export class WikiLintService {
|
|
|
130
136
|
const staleClaims = this.detectStale(files, staleDays);
|
|
131
137
|
const restructureProposals = this.proposeRestructures(files);
|
|
132
138
|
const missingConcepts = this.detectMissingConcepts(unresolvedFrequency);
|
|
139
|
+
const contradictionCandidates = await this.detectContradictionCandidates(vaultPath);
|
|
140
|
+
const proposalsPending = (await WikiCurationService.getInstance().listProposals(vaultPath)).length;
|
|
141
|
+
const coverage = await WikiIndexService.getInstance().coverage(vaultPath);
|
|
133
142
|
const report = {
|
|
134
143
|
vault: {
|
|
135
144
|
scope: schema.vault_scope,
|
|
@@ -145,6 +154,14 @@ export class WikiLintService {
|
|
|
145
154
|
orphanPages: orphanPages.slice(0, WIKI_LINT_MAX_ROWS_PER_SECTION),
|
|
146
155
|
staleClaims: staleClaims.slice(0, WIKI_LINT_MAX_ROWS_PER_SECTION),
|
|
147
156
|
restructureProposals: restructureProposals.slice(0, WIKI_LINT_MAX_ROWS_PER_SECTION),
|
|
157
|
+
contradictionCandidates: contradictionCandidates.slice(0, WIKI_LINT_MAX_ROWS_PER_SECTION),
|
|
158
|
+
proposalsPending,
|
|
159
|
+
index: {
|
|
160
|
+
pages: coverage.pages,
|
|
161
|
+
indexed: coverage.indexed,
|
|
162
|
+
missingFromIndex: coverage.missingFromIndex.length,
|
|
163
|
+
indexedButMissing: coverage.indexedButMissing.length,
|
|
164
|
+
},
|
|
148
165
|
truncated,
|
|
149
166
|
};
|
|
150
167
|
return { ok: true, report };
|
|
@@ -314,6 +331,58 @@ export class WikiLintService {
|
|
|
314
331
|
*
|
|
315
332
|
* Results sort by referenceCount desc — highest-leverage concepts first.
|
|
316
333
|
*/
|
|
334
|
+
/**
|
|
335
|
+
* Pages in the same folder whose title+summary token sets overlap above
|
|
336
|
+
* {@link WIKI_KB_CONSTANTS.DUPLICATE_JACCARD}. Superseded pages are
|
|
337
|
+
* excluded (that pair has already been resolved).
|
|
338
|
+
*
|
|
339
|
+
* @param vaultPath - Vault root
|
|
340
|
+
* @returns Candidate pairs, most similar first
|
|
341
|
+
*/
|
|
342
|
+
async detectContradictionCandidates(vaultPath) {
|
|
343
|
+
const byFolder = new Map();
|
|
344
|
+
for (const page of await walkCuratedPages(vaultPath)) {
|
|
345
|
+
if (isSeedFile(page.relativePath))
|
|
346
|
+
continue;
|
|
347
|
+
let raw;
|
|
348
|
+
try {
|
|
349
|
+
raw = await fs.readFile(page.absPath, 'utf8');
|
|
350
|
+
}
|
|
351
|
+
catch {
|
|
352
|
+
continue;
|
|
353
|
+
}
|
|
354
|
+
const { frontmatter, body } = parsePage(raw);
|
|
355
|
+
if (frontmatter.superseded_by)
|
|
356
|
+
continue;
|
|
357
|
+
const title = String(frontmatter.title ?? body.match(/^#\s+(.+)$/m)?.[1] ?? path.basename(page.relativePath, '.md'));
|
|
358
|
+
const text = `${title} ${String(frontmatter.summary ?? '')}`;
|
|
359
|
+
const tokens = new Set(tokenize(text).filter((t) => t.length > 1));
|
|
360
|
+
if (tokens.size < 3)
|
|
361
|
+
continue;
|
|
362
|
+
const folder = path.posix.dirname(page.relativePath);
|
|
363
|
+
const list = byFolder.get(folder) ?? [];
|
|
364
|
+
list.push({ rel: page.relativePath, tokens, title });
|
|
365
|
+
byFolder.set(folder, list);
|
|
366
|
+
}
|
|
367
|
+
const out = [];
|
|
368
|
+
for (const list of byFolder.values()) {
|
|
369
|
+
for (let i = 0; i < list.length; i++) {
|
|
370
|
+
for (let j = i + 1; j < list.length; j++) {
|
|
371
|
+
const a = list[i], b = list[j];
|
|
372
|
+
let inter = 0;
|
|
373
|
+
for (const t of a.tokens)
|
|
374
|
+
if (b.tokens.has(t))
|
|
375
|
+
inter++;
|
|
376
|
+
const union = a.tokens.size + b.tokens.size - inter;
|
|
377
|
+
const sim = union === 0 ? 0 : inter / union;
|
|
378
|
+
if (sim >= WIKI_KB_CONSTANTS.DUPLICATE_JACCARD) {
|
|
379
|
+
out.push({ pages: [a.rel, b.rel], similarity: Math.round(sim * 100) / 100, claim: a.title });
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
return out.sort((x, y) => y.similarity - x.similarity);
|
|
385
|
+
}
|
|
317
386
|
detectMissingConcepts(unresolvedFrequency) {
|
|
318
387
|
const out = [];
|
|
319
388
|
for (const [target, { count, sources }] of unresolvedFrequency) {
|