dsh-data-domain-knowledge 0.1.0-alpha.11 → 0.1.0-alpha.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +64 -13
- package/eval/README.md +19 -8
- package/eval/evaluate-retrieval.mjs +17 -8
- package/eval/queries.example.jsonl +1 -0
- package/lib/client.js +2294 -2096
- package/lib/doc-page.js +1 -1
- package/lib/pages.js +2 -2
- package/lib/types/axis-registry.d.ts +9 -4
- package/lib/types/axis-registry.d.ts.map +1 -1
- package/lib/types/axis-registry.js +62 -23
- package/lib/types/axis-registry.js.map +1 -1
- package/lib/types/card-organize.d.ts +5 -0
- package/lib/types/card-organize.d.ts.map +1 -1
- package/lib/types/card-organize.js +15 -0
- package/lib/types/card-organize.js.map +1 -1
- package/lib/types/compiler/embedding-identity.d.ts +3 -0
- package/lib/types/compiler/embedding-identity.d.ts.map +1 -0
- package/lib/types/compiler/embedding-identity.js +24 -0
- package/lib/types/compiler/embedding-identity.js.map +1 -0
- package/lib/types/compiler/snapshot.d.ts +16 -0
- package/lib/types/compiler/snapshot.d.ts.map +1 -0
- package/lib/types/compiler/snapshot.js +2 -0
- package/lib/types/compiler/snapshot.js.map +1 -0
- package/lib/types/diagnosis/store.d.ts.map +1 -1
- package/lib/types/diagnosis/store.js +2 -0
- package/lib/types/diagnosis/store.js.map +1 -1
- package/lib/types/environment/install.d.ts +7 -0
- package/lib/types/environment/install.d.ts.map +1 -0
- package/lib/types/environment/install.js +172 -0
- package/lib/types/environment/install.js.map +1 -0
- package/lib/types/environment/manifest.d.ts +208 -0
- package/lib/types/environment/manifest.d.ts.map +1 -0
- package/lib/types/environment/manifest.js +26 -0
- package/lib/types/environment/manifest.js.map +1 -0
- package/lib/types/environment/model-catalog.d.ts +191 -0
- package/lib/types/environment/model-catalog.d.ts.map +1 -0
- package/lib/types/environment/model-catalog.js +91 -0
- package/lib/types/environment/model-catalog.js.map +1 -0
- package/lib/types/environment/process.d.ts +10 -0
- package/lib/types/environment/process.d.ts.map +1 -0
- package/lib/types/environment/process.js +54 -0
- package/lib/types/environment/process.js.map +1 -0
- package/lib/types/environment/selection.d.ts +9 -0
- package/lib/types/environment/selection.d.ts.map +1 -0
- package/lib/types/environment/selection.js +29 -0
- package/lib/types/environment/selection.js.map +1 -0
- package/lib/types/environment/service.d.ts +47 -0
- package/lib/types/environment/service.d.ts.map +1 -0
- package/lib/types/environment/service.js +253 -0
- package/lib/types/environment/service.js.map +1 -0
- package/lib/types/environment/types.d.ts +37 -0
- package/lib/types/environment/types.d.ts.map +1 -0
- package/lib/types/environment/types.js +2 -0
- package/lib/types/environment/types.js.map +1 -0
- package/lib/types/events.d.ts +10 -0
- package/lib/types/events.d.ts.map +1 -1
- package/lib/types/events.js +38 -1
- package/lib/types/events.js.map +1 -1
- package/lib/types/images.d.ts +26 -0
- package/lib/types/images.d.ts.map +1 -0
- package/lib/types/images.js +196 -0
- package/lib/types/images.js.map +1 -0
- package/lib/types/index.d.ts.map +1 -1
- package/lib/types/index.js +59 -3
- package/lib/types/index.js.map +1 -1
- package/lib/types/knowledge-bases.d.ts +3 -0
- package/lib/types/knowledge-bases.d.ts.map +1 -1
- package/lib/types/knowledge-bases.js +9 -0
- package/lib/types/knowledge-bases.js.map +1 -1
- package/lib/types/knowledge.d.ts +14 -6
- package/lib/types/knowledge.d.ts.map +1 -1
- package/lib/types/knowledge.js +56 -39
- package/lib/types/knowledge.js.map +1 -1
- package/lib/types/layout.d.ts +2 -0
- package/lib/types/layout.d.ts.map +1 -1
- package/lib/types/layout.js +2 -0
- package/lib/types/layout.js.map +1 -1
- package/lib/types/retrieval/vector-index.d.ts +31 -0
- package/lib/types/retrieval/vector-index.d.ts.map +1 -0
- package/lib/types/retrieval/vector-index.js +15 -0
- package/lib/types/retrieval/vector-index.js.map +1 -0
- package/lib/types/retrieval/visibility.d.ts +6 -0
- package/lib/types/retrieval/visibility.d.ts.map +1 -0
- package/lib/types/retrieval/visibility.js +11 -0
- package/lib/types/retrieval/visibility.js.map +1 -0
- package/lib/types/runtime.d.ts +36 -6
- package/lib/types/runtime.d.ts.map +1 -1
- package/lib/types/runtime.js +520 -190
- package/lib/types/runtime.js.map +1 -1
- package/lib/types/source-evidence.d.ts +6 -0
- package/lib/types/source-evidence.d.ts.map +1 -0
- package/lib/types/source-evidence.js +28 -0
- package/lib/types/source-evidence.js.map +1 -0
- package/lib/types/store/axis-references.d.ts +5 -0
- package/lib/types/store/axis-references.d.ts.map +1 -0
- package/lib/types/store/axis-references.js +106 -0
- package/lib/types/store/axis-references.js.map +1 -0
- package/lib/types/store/derived.d.ts +11 -0
- package/lib/types/store/derived.d.ts.map +1 -1
- package/lib/types/store/derived.js +31 -6
- package/lib/types/store/derived.js.map +1 -1
- package/lib/types/store/knowledge-db.d.ts +4 -1
- package/lib/types/store/knowledge-db.d.ts.map +1 -1
- package/lib/types/store/knowledge-db.js +15 -9
- package/lib/types/store/knowledge-db.js.map +1 -1
- package/lib/types/store/migrate.d.ts.map +1 -1
- package/lib/types/store/migrate.js +16 -5
- package/lib/types/store/migrate.js.map +1 -1
- package/lib/types/types.d.ts +7 -1
- package/lib/types/types.d.ts.map +1 -1
- package/lib/types/types.js +3 -4
- package/lib/types/types.js.map +1 -1
- package/package.json +3 -1
- package/python/prepare-models.py +19 -0
- package/python/requirements.txt +8 -6
- package/scripts/doctor.mjs +14 -2
- package/scripts/environment.mjs +29 -0
- package/scripts/setup-embedding.sh +11 -2
- package/src/axis-registry.ts +59 -23
- package/src/card-organize.ts +15 -0
- package/src/client/EnvironmentSetup.tsx +145 -0
- package/src/client/KnowledgePanel.tsx +9 -4
- package/src/compiler/embedding-identity.ts +21 -0
- package/src/compiler/snapshot.ts +17 -0
- package/src/diagnosis/store.ts +1 -0
- package/src/environment/install.ts +129 -0
- package/src/environment/manifest.ts +28 -0
- package/src/environment/model-catalog.ts +105 -0
- package/src/environment/process.ts +54 -0
- package/src/environment/selection.ts +29 -0
- package/src/environment/service.ts +211 -0
- package/src/environment/types.ts +34 -0
- package/src/events.ts +46 -0
- package/src/images.ts +203 -0
- package/src/index.ts +54 -3
- package/src/knowledge-bases.ts +9 -0
- package/src/knowledge.ts +56 -40
- package/src/layout.ts +2 -0
- package/src/retrieval/vector-index.ts +44 -0
- package/src/retrieval/visibility.ts +14 -0
- package/src/runtime.ts +473 -233
- package/src/source-evidence.ts +30 -0
- package/src/store/axis-references.ts +96 -0
- package/src/store/derived.ts +31 -6
- package/src/store/knowledge-db.ts +16 -9
- package/src/store/migrate.ts +15 -5
- package/src/types.ts +10 -5
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { Knowledge, KnowledgeSource } from './knowledge.js'
|
|
2
|
+
import type { KnowledgeDb } from './store/knowledge-db.js'
|
|
3
|
+
|
|
4
|
+
/** Checking an evidence anchor does not establish whether a claim is true. */
|
|
5
|
+
export function assessKnowledgeEvidence(knowledge: Knowledge, db: KnowledgeDb): Knowledge {
|
|
6
|
+
return {
|
|
7
|
+
...knowledge,
|
|
8
|
+
sources: knowledge.sources.map(source => assessSource(source, db)),
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
function assessSource(source: KnowledgeSource, db: KnowledgeDb): KnowledgeSource {
|
|
13
|
+
if (!source.documentId) return { ...source, evidenceStatus: 'human' }
|
|
14
|
+
const head = db.source(source.documentId)
|
|
15
|
+
if (!head) return { ...source, evidenceStatus: 'missing', quoteInCurrentVersion: false }
|
|
16
|
+
const flatten = (text: string): string => text.replace(/\s+/g, '')
|
|
17
|
+
const quote = flatten(source.quote)
|
|
18
|
+
return {
|
|
19
|
+
...source,
|
|
20
|
+
currentVersion: head.version,
|
|
21
|
+
evidenceStatus: source.toVersion === undefined ? 'unversioned'
|
|
22
|
+
: source.toVersion === head.version ? 'current' : 'historical',
|
|
23
|
+
quoteInCurrentVersion: quote !== '' && flatten(head.record.content).includes(quote),
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function hasCurrentSupport(knowledge: Knowledge): boolean {
|
|
28
|
+
return knowledge.sources.some(source => source.evidenceStatus === 'human'
|
|
29
|
+
|| (source.evidenceStatus === 'current' && source.quoteInCurrentVersion === true))
|
|
30
|
+
}
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto'
|
|
2
|
+
import type { AxisKind } from '../axis-registry.js'
|
|
3
|
+
import { compositeEventIdentity, eventEntityFromRecords, type EventEntity } from '../events.js'
|
|
4
|
+
import { cardCoordinateKey, type CardCoordinate } from '../types.js'
|
|
5
|
+
import type { KnowledgeDb, CardKnowledgeRow } from './knowledge-db.js'
|
|
6
|
+
|
|
7
|
+
/** Runs inside the axis merge transaction on the same validated connection. */
|
|
8
|
+
export function redirectAxisReferences(db: KnowledgeDb, kind: AxisKind, fromId: string, intoId: string): void {
|
|
9
|
+
const column = { object: 'object_id', action: 'action_id', scene: 'scene_id' }[kind]
|
|
10
|
+
const field = { object: 'objectId', action: 'actionId', scene: 'sceneId' }[kind] as keyof CardCoordinate
|
|
11
|
+
const redirect = (coordinate: CardCoordinate): CardCoordinate => ({
|
|
12
|
+
...coordinate, [field]: coordinate[field] === fromId ? intoId : coordinate[field],
|
|
13
|
+
})
|
|
14
|
+
const hasKnowledge = db.raw("SELECT 1 FROM sqlite_master WHERE name = 'knowledge'").get()
|
|
15
|
+
if (hasKnowledge) db.raw(`UPDATE knowledge SET ${column} = ?, updated_at = ? WHERE ${column} = ?`).run(intoId, new Date().toISOString(), fromId)
|
|
16
|
+
const placements = db.placements()
|
|
17
|
+
const cards = db.cards()
|
|
18
|
+
const byCoordinate = new Map(cards.filter(row => row.card.coordinate).map(row => [cardCoordinateKey(row.card.coordinate!), row]))
|
|
19
|
+
db.transact({ opId: `axis-merge:${fromId}:${intoId}:${Date.now()}`, actor: 'user', summary: '合并轴实体并迁移知识、卡片与事件引用' }, tx => {
|
|
20
|
+
for (const row of cards) {
|
|
21
|
+
const coordinate = row.card.coordinate
|
|
22
|
+
if (!coordinate || coordinate[field] !== fromId) continue
|
|
23
|
+
const next = redirect(coordinate)
|
|
24
|
+
const key = cardCoordinateKey(next)
|
|
25
|
+
const target = byCoordinate.get(key)
|
|
26
|
+
const base = target && target.card.entityId !== row.card.entityId ? target : row
|
|
27
|
+
const merged = base !== row
|
|
28
|
+
const eventRefs = [...new Set([...db.cardEventIds(base.card.entityId), ...db.cardEventIds(row.card.entityId)])]
|
|
29
|
+
const card = {
|
|
30
|
+
...base.card, coordinate: next,
|
|
31
|
+
content: merged ? `${base.card.content}\n\n${row.card.content}` : row.card.content,
|
|
32
|
+
sourceRefs: [...new Set([...base.card.sourceRefs, ...row.card.sourceRefs])],
|
|
33
|
+
annotations: [...new Map([...(base.card.annotations ?? []), ...(row.card.annotations ?? [])].map(annotation => [annotation.id, annotation])).values()],
|
|
34
|
+
metadata: { ...base.card.metadata, layout: undefined, citations: undefined,
|
|
35
|
+
aliases: [...new Set([...(Array.isArray(base.card.metadata.aliases) ? base.card.metadata.aliases : []), ...(Array.isArray(row.card.metadata.aliases) ? row.card.metadata.aliases : []), base.card.title, row.card.title])],
|
|
36
|
+
...(eventRefs.length ? { event_refs: JSON.stringify(eventRefs) } : {}) },
|
|
37
|
+
}
|
|
38
|
+
const rows = [...(placements.get(base.card.entityId) ?? []), ...(merged ? placements.get(row.card.entityId) ?? [] : [])]
|
|
39
|
+
const combined: CardKnowledgeRow[] = [...new Map(rows.map(placement => [placement.knowledgeId, placement])).values()]
|
|
40
|
+
if (merged) tx.removeEntity('card', row.card.entityId)
|
|
41
|
+
const frontmatter = { ...base.frontmatter, coordinate: { object: next.objectId, action: next.actionId, scene: next.sceneId }, source_refs: card.sourceRefs, metadata: card.metadata, annotations: card.annotations }
|
|
42
|
+
tx.putCard(card, frontmatter, { actor: 'user', kind: 'named', summary: '轴合并后的卡片引用' }, combined, true)
|
|
43
|
+
db.setCardEvents(card.entityId, eventRefs)
|
|
44
|
+
byCoordinate.delete(cardCoordinateKey(coordinate))
|
|
45
|
+
byCoordinate.set(key, { ...base, card, frontmatter })
|
|
46
|
+
placements.set(card.entityId, combined)
|
|
47
|
+
}
|
|
48
|
+
const changed = new Set<string>()
|
|
49
|
+
const events = db.events().map(event => {
|
|
50
|
+
if (event.coordinateIds[field] !== fromId && !event.records.some(record => record.coordinateIds[field] === fromId)) return event
|
|
51
|
+
changed.add(event.id)
|
|
52
|
+
return { ...event, coordinateIds: redirect(event.coordinateIds), records: event.records.map(record => ({ ...record, coordinateIds: redirect(record.coordinateIds) })), updatedAt: new Date().toISOString() }
|
|
53
|
+
})
|
|
54
|
+
const externalBases = new Map(events.filter(event => event.identityType === 'external_id' && !event.identityKey.includes(':coordinate:')).map(event => [event.identityKey, event]))
|
|
55
|
+
const groups = new Map<string, EventEntity[]>()
|
|
56
|
+
for (const event of events) {
|
|
57
|
+
let key = event.identityKey
|
|
58
|
+
if (changed.has(event.id) && event.identityType === 'composite') key = compositeEventIdentity(event.occurredAt, event.coordinateIds, event.symptom)
|
|
59
|
+
if (event.identityType === 'external_id' && key.includes(':coordinate:')) {
|
|
60
|
+
const baseKey = key.slice(0, key.indexOf(':coordinate:'))
|
|
61
|
+
const base = externalBases.get(baseKey)
|
|
62
|
+
if (base && cardCoordinateKey(base.coordinateIds) === cardCoordinateKey(event.coordinateIds)) key = baseKey
|
|
63
|
+
else if (changed.has(event.id)) key = baseKey + ':coordinate:' + createHash('sha256').update(JSON.stringify(event.coordinateIds)).digest('hex').slice(0, 24)
|
|
64
|
+
}
|
|
65
|
+
const group = groups.get(key) ?? []
|
|
66
|
+
group.push(event)
|
|
67
|
+
groups.set(key, group)
|
|
68
|
+
}
|
|
69
|
+
const redirects = new Map<string, string>()
|
|
70
|
+
const affectedCards = new Set<string>()
|
|
71
|
+
for (const [key, group] of groups) {
|
|
72
|
+
if (group.length === 1 && !changed.has(group[0]!.id) && key === group[0]!.identityKey) continue
|
|
73
|
+
const target = group.find(event => event.identityKey === key) ?? group[0]!
|
|
74
|
+
const records = [...new Map(group.flatMap(event => event.records).map(record => [`${record.sourceId}\u0001${record.localId}`, record])).values()]
|
|
75
|
+
for (const event of group) {
|
|
76
|
+
if (event.id === target.id) continue
|
|
77
|
+
redirects.set(event.id, target.id)
|
|
78
|
+
for (const row of db.raw('SELECT card_id FROM card_event WHERE event_id = ?').all(event.id) as Array<{ card_id: string }>) affectedCards.add(row.card_id)
|
|
79
|
+
db.raw(`INSERT OR IGNORE INTO card_event(card_id, event_id, ordinal)
|
|
80
|
+
SELECT card_id, ?, ordinal FROM card_event WHERE event_id = ?`).run(target.id, event.id)
|
|
81
|
+
db.removeEvent(event.id)
|
|
82
|
+
}
|
|
83
|
+
db.writeEvent(group.length === 1 ? { ...target, identityKey: key } : eventEntityFromRecords({ ...target, identityKey: key, records }))
|
|
84
|
+
}
|
|
85
|
+
if (redirects.size) {
|
|
86
|
+
for (const row of db.cards()) {
|
|
87
|
+
let content = row.card.content
|
|
88
|
+
for (const [from, into] of redirects) content = content.replaceAll(from, into)
|
|
89
|
+
if (content === row.card.content && !affectedCards.has(row.card.entityId)) continue
|
|
90
|
+
const metadata = { ...row.card.metadata, event_refs: JSON.stringify(db.cardEventIds(row.card.entityId)) }
|
|
91
|
+
const frontmatter = { ...row.frontmatter, metadata }
|
|
92
|
+
tx.putCard({ ...row.card, content, metadata }, frontmatter, { actor: 'user', kind: 'named', summary: '轴合并后的 Event 引用' })
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
})
|
|
96
|
+
}
|
package/src/store/derived.ts
CHANGED
|
@@ -76,6 +76,12 @@ export interface FtsDocument {
|
|
|
76
76
|
content: string
|
|
77
77
|
}
|
|
78
78
|
|
|
79
|
+
export interface DerivedPublication {
|
|
80
|
+
artifacts: Map<string, string>
|
|
81
|
+
fts?: FtsDocument[]
|
|
82
|
+
vectors?: { manifest: unknown; binary: Buffer; int8: Buffer } | null
|
|
83
|
+
}
|
|
84
|
+
|
|
79
85
|
export type DerivedOpenResult =
|
|
80
86
|
| { ok: true; db: DerivedDb; /** 这次开库是否把旧结构整体重建了。 */ rebuilt: boolean }
|
|
81
87
|
| { ok: false; code: 'NOT_A_DERIVED_DB' | 'OPEN_FAILED'; message: string }
|
|
@@ -150,10 +156,10 @@ export class DerivedDb {
|
|
|
150
156
|
return { ok: false, code: 'OPEN_FAILED', message: `打开失败:${cause instanceof Error ? cause.message : String(cause)}` }
|
|
151
157
|
}
|
|
152
158
|
try {
|
|
153
|
-
db.exec('PRAGMA journal_mode = WAL')
|
|
154
159
|
const owner = readPragma(db, 'application_id')
|
|
155
160
|
const version = readPragma(db, 'user_version')
|
|
156
|
-
if (userTableCount(db) === 0) {
|
|
161
|
+
if (userTableCount(db) === 0 && owner === 0) {
|
|
162
|
+
db.exec('PRAGMA journal_mode = WAL')
|
|
157
163
|
db.exec(schemaSql())
|
|
158
164
|
db.exec(`PRAGMA application_id = ${DERIVED_DB_APPLICATION_ID}`)
|
|
159
165
|
db.exec(`PRAGMA user_version = ${DERIVED_DB_SCHEMA}`)
|
|
@@ -163,6 +169,7 @@ export class DerivedDb {
|
|
|
163
169
|
db.close()
|
|
164
170
|
return { ok: false, code: 'NOT_A_DERIVED_DB', message: `这个文件不是知识库的派生库(属主标记 ${owner}):${file}` }
|
|
165
171
|
}
|
|
172
|
+
db.exec('PRAGMA journal_mode = WAL')
|
|
166
173
|
if (version !== DERIVED_DB_SCHEMA) {
|
|
167
174
|
// 整体重建。**没有迁移分支是刻意的**(见模块头)。
|
|
168
175
|
db.exec('DROP TABLE IF EXISTS fts')
|
|
@@ -207,6 +214,24 @@ export class DerivedDb {
|
|
|
207
214
|
.run(name, body, hashOf(body), Buffer.byteLength(body, 'utf8'), new Date().toISOString())
|
|
208
215
|
}
|
|
209
216
|
|
|
217
|
+
/** Model calls finish before this short transaction starts. */
|
|
218
|
+
publish(publication: DerivedPublication): void {
|
|
219
|
+
this.db.exec('BEGIN IMMEDIATE')
|
|
220
|
+
try {
|
|
221
|
+
for (const [name, body] of publication.artifacts) this.putArtifact(name, body)
|
|
222
|
+
if (publication.fts !== undefined) this.rebuildFts(publication.fts)
|
|
223
|
+
if (publication.vectors === null) this.raw('DELETE FROM vector_index').run()
|
|
224
|
+
else if (publication.vectors !== undefined) {
|
|
225
|
+
const { manifest, binary, int8 } = publication.vectors
|
|
226
|
+
this.putVectorIndex(manifest, binary, int8)
|
|
227
|
+
}
|
|
228
|
+
this.db.exec('COMMIT')
|
|
229
|
+
} catch (cause) {
|
|
230
|
+
try { this.db.exec('ROLLBACK') } catch { /* preserve the original failure */ }
|
|
231
|
+
throw cause
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
|
|
210
235
|
removeArtifact(name: string): void {
|
|
211
236
|
this.raw('DELETE FROM artifact WHERE name = ?').run(name)
|
|
212
237
|
}
|
|
@@ -239,14 +264,14 @@ export class DerivedDb {
|
|
|
239
264
|
*/
|
|
240
265
|
rebuildFts(documents: FtsDocument[]): void {
|
|
241
266
|
/*
|
|
242
|
-
* `
|
|
267
|
+
* `SAVEPOINT` 在 DDL **之前**:SQLite 的 DDL 是事务性的(实测 `CREATE VIRTUAL TABLE` 与它
|
|
243
268
|
* 的 shadow 表一起回滚),而放在事务外就变成自动提交 —— 于是构建中途失败会留下一张
|
|
244
269
|
* **空的** `fts` 表,`hasFts()` 还说它在,查询却一条也回不来。
|
|
245
270
|
*
|
|
246
271
|
* 放进同一个事务之后的实际行为:**上一版索引原样留着**(还能查),或者(本来就还没有
|
|
247
272
|
* 索引时)仍然没有索引。两种都比「空表」好 —— 空表是一种既不报错也不可用的状态。
|
|
248
273
|
*/
|
|
249
|
-
this.db.exec('
|
|
274
|
+
this.db.exec('SAVEPOINT rebuild_fts')
|
|
250
275
|
try {
|
|
251
276
|
this.db.exec('DROP TABLE IF EXISTS fts')
|
|
252
277
|
this.db.exec(ftsSql())
|
|
@@ -256,9 +281,9 @@ export class DerivedDb {
|
|
|
256
281
|
const tokens = [...segment(document.title), ...segment(document.title), ...segment(document.content)]
|
|
257
282
|
insert.run(document.unitId, tokens.join(' '))
|
|
258
283
|
}
|
|
259
|
-
this.db.exec('
|
|
284
|
+
this.db.exec('RELEASE rebuild_fts')
|
|
260
285
|
} catch (cause) {
|
|
261
|
-
try { this.db.exec('ROLLBACK') } catch { /* rollback of a dead tx */ }
|
|
286
|
+
try { this.db.exec('ROLLBACK TO rebuild_fts; RELEASE rebuild_fts') } catch { /* rollback of a dead tx */ }
|
|
262
287
|
throw cause
|
|
263
288
|
}
|
|
264
289
|
// `optimize` 在事务外:它是一次纯内部整理,失败不该把已经建好的索引回滚掉。
|
|
@@ -713,7 +713,7 @@ export class KnowledgeTx {
|
|
|
713
713
|
* 会把「只改备注」「只补图」那几条路径变成一次静默的关联丢失 —— 它们本来就不该
|
|
714
714
|
* 碰落位(那件事只有编译知道)。
|
|
715
715
|
*/
|
|
716
|
-
putCard(card: KnowledgeCard, frontmatter: Record<string, unknown>, journal: Journal, placements?: CardKnowledgeRow[]): WriteOutcome {
|
|
716
|
+
putCard(card: KnowledgeCard, frontmatter: Record<string, unknown>, journal: Journal, placements?: CardKnowledgeRow[], allowCoordinateMove = false): WriteOutcome {
|
|
717
717
|
const outcome = this.putEntity('card', card.entityId, card.content, frontmatter, journal, () => {
|
|
718
718
|
const existing = this.raw('SELECT coordinate_key, object_id, action_id, scene_id FROM card WHERE entity_id = ?').get(card.entityId) as Record<string, unknown> | undefined
|
|
719
719
|
/*
|
|
@@ -726,7 +726,7 @@ export class KnowledgeTx {
|
|
|
726
726
|
: cardCoordinateKey(storedCoordinate)
|
|
727
727
|
const oldKey = typeof existing?.coordinate_key === 'string' ? existing.coordinate_key : undefined
|
|
728
728
|
if (existing !== undefined) {
|
|
729
|
-
if (oldKey !== undefined && oldKey !== coordinateKey && !oldKey.startsWith('legacy:')) {
|
|
729
|
+
if (!allowCoordinateMove && oldKey !== undefined && oldKey !== coordinateKey && !oldKey.startsWith('legacy:')) {
|
|
730
730
|
throw new Error(`CARD_COORDINATE_IMMUTABLE: ${card.entityId} 已绑定坐标,不能直接改主键`)
|
|
731
731
|
}
|
|
732
732
|
this.raw(`UPDATE card SET coordinate_key = ?, entity_type = ?, title = ?, status = ?, owner = ?,
|
|
@@ -850,6 +850,7 @@ export class KnowledgeTx {
|
|
|
850
850
|
* 短事务。异步化只会把调用方的顺序打散,换不到任何东西。
|
|
851
851
|
*/
|
|
852
852
|
export class KnowledgeDb {
|
|
853
|
+
private transactionSequence = 0
|
|
853
854
|
private constructor(private readonly db: SqliteDatabase, readonly file: string) {}
|
|
854
855
|
|
|
855
856
|
/**
|
|
@@ -878,7 +879,6 @@ export class KnowledgeDb {
|
|
|
878
879
|
return { ok: false, code: 'OPEN_FAILED', message: `打开失败:${cause instanceof Error ? cause.message : String(cause)}` }
|
|
879
880
|
}
|
|
880
881
|
try {
|
|
881
|
-
db.exec('PRAGMA journal_mode = WAL')
|
|
882
882
|
const owner = readPragma(db, 'application_id')
|
|
883
883
|
const version = readPragma(db, 'user_version')
|
|
884
884
|
/*
|
|
@@ -904,9 +904,10 @@ export class KnowledgeDb {
|
|
|
904
904
|
*/
|
|
905
905
|
const ownTables = ['knowledge', 'knowledge_sources', 'axis_entities', 'axis_aliases', 'event', 'card_event']
|
|
906
906
|
const hasUserTables = userTableCount(db) > 0
|
|
907
|
-
const fresh = version === 0 && !hasUserTables
|
|
908
|
-
const unmarkedKnowledgeDb = version === 0 && ownTables.some(name => tableExists(db, name))
|
|
907
|
+
const fresh = owner === 0 && version === 0 && !hasUserTables
|
|
908
|
+
const unmarkedKnowledgeDb = owner === 0 && version === 0 && ownTables.some(name => tableExists(db, name))
|
|
909
909
|
if (fresh) {
|
|
910
|
+
db.exec('PRAGMA journal_mode = WAL')
|
|
910
911
|
db.exec(schemaSql())
|
|
911
912
|
db.exec(`PRAGMA application_id = ${KNOWLEDGE_DB_APPLICATION_ID}`)
|
|
912
913
|
db.exec(`PRAGMA user_version = ${KNOWLEDGE_DB_SCHEMA}`)
|
|
@@ -920,6 +921,7 @@ export class KnowledgeDb {
|
|
|
920
921
|
db.close()
|
|
921
922
|
return { ok: false, code: 'SCHEMA_TOO_NEW', message: `知识库的 schema 是 ${version},本版本只认到 ${KNOWLEDGE_DB_SCHEMA}:${file}` }
|
|
922
923
|
}
|
|
924
|
+
db.exec('PRAGMA journal_mode = WAL')
|
|
923
925
|
if (version < KNOWLEDGE_DB_SCHEMA) {
|
|
924
926
|
migrateSchema(db, version)
|
|
925
927
|
}
|
|
@@ -938,6 +940,9 @@ export class KnowledgeDb {
|
|
|
938
940
|
|
|
939
941
|
raw(sql: string): SqliteStatement { return this.db.prepare(sql) }
|
|
940
942
|
|
|
943
|
+
/** Repositories share this validated connection; only its owner closes it. */
|
|
944
|
+
connection(): SqliteDatabase { return this.db }
|
|
945
|
+
|
|
941
946
|
/**
|
|
942
947
|
* 一次操作 = 一个事务 = 一行 `tree` + 若干行 `revision`。
|
|
943
948
|
*
|
|
@@ -951,7 +956,8 @@ export class KnowledgeDb {
|
|
|
951
956
|
transact<T>(entry: TreeEntry, fn: (tx: KnowledgeTx) => T): T {
|
|
952
957
|
const tx = new KnowledgeTx(this)
|
|
953
958
|
const at = entry.at ?? new Date().toISOString()
|
|
954
|
-
this.
|
|
959
|
+
const savepoint = `knowledge_op_${++this.transactionSequence}`
|
|
960
|
+
this.db.exec(`SAVEPOINT ${savepoint}`)
|
|
955
961
|
try {
|
|
956
962
|
/*
|
|
957
963
|
* `INSERT OR REPLACE`:`op_id` 是**幂等键**。
|
|
@@ -964,10 +970,10 @@ export class KnowledgeDb {
|
|
|
964
970
|
.run(entry.opId, at, entry.actor, entry.summary, '{}')
|
|
965
971
|
const result = fn(tx)
|
|
966
972
|
this.raw('UPDATE tree SET manifest = ? WHERE op_id = ?').run(tx.manifestJson(), entry.opId)
|
|
967
|
-
this.db.exec(
|
|
973
|
+
this.db.exec(`RELEASE ${savepoint}`)
|
|
968
974
|
return result
|
|
969
975
|
} catch (cause) {
|
|
970
|
-
try { this.db.exec(
|
|
976
|
+
try { this.db.exec(`ROLLBACK TO ${savepoint}; RELEASE ${savepoint}`) } catch { /* rollback of a dead tx */ }
|
|
971
977
|
throw cause
|
|
972
978
|
}
|
|
973
979
|
}
|
|
@@ -1360,7 +1366,8 @@ export class KnowledgeDb {
|
|
|
1360
1366
|
const mainNames = new Set(mainColumns.map(column => column.name))
|
|
1361
1367
|
const legacyNames = new Set(legacyColumns.map(column => column.name))
|
|
1362
1368
|
const shared = legacyColumns.filter(column => mainNames.has(column.name)).map(column => column.name)
|
|
1363
|
-
const dropped = legacyColumns.filter(column => !mainNames.has(column.name)
|
|
1369
|
+
const dropped = legacyColumns.filter(column => !mainNames.has(column.name)
|
|
1370
|
+
&& !((name === 'knowledge_sources' || name === 'source') && column.name === 'source_type')).map(column => column.name)
|
|
1364
1371
|
const required = mainColumns.filter(column => !legacyNames.has(column.name) && column.notNull && !column.hasDefault)
|
|
1365
1372
|
if (dropped.length > 0) problems.push(`${name} 的这几列在目标表里没有,内容搬不过来:${dropped.join('、')}`)
|
|
1366
1373
|
if (required.length > 0) {
|
package/src/store/migrate.ts
CHANGED
|
@@ -35,7 +35,7 @@
|
|
|
35
35
|
import { existsSync } from 'node:fs'
|
|
36
36
|
import { cp, mkdir, readFile, readdir, rename, rm, writeFile } from 'node:fs/promises'
|
|
37
37
|
import { basename, dirname, join, relative } from 'node:path'
|
|
38
|
-
import { ASSETS_DIR, SOURCE_RAW_DIR } from '../layout.js'
|
|
38
|
+
import { ASSETS_DIR, PUBLISHED_IMAGE_MARKER, SOURCE_RAW_DIR } from '../layout.js'
|
|
39
39
|
import type { KnowledgeCard, SourceRaw } from '../types.js'
|
|
40
40
|
import type { KnowledgeDb } from './knowledge-db.js'
|
|
41
41
|
|
|
@@ -240,7 +240,17 @@ export async function migrateKnowledgeTree(input: MigrationInput): Promise<Migra
|
|
|
240
240
|
})
|
|
241
241
|
}
|
|
242
242
|
db.transact({ opId: sourceOp(opId, 'source', sourceId), actor: 'migration', summary: '迁移来源', at: record.updatedAt }, tx => {
|
|
243
|
-
|
|
243
|
+
const frontmatter = { ...parsed.frontmatter }
|
|
244
|
+
const origin = typeof record.metadata.source_url === 'string' ? record.metadata.source_url.trim() : ''
|
|
245
|
+
if (origin) {
|
|
246
|
+
frontmatter.resource ??= origin
|
|
247
|
+
frontmatter.sources ??= [{ id: 'origin', resource: origin, title: record.title }]
|
|
248
|
+
}
|
|
249
|
+
const parser = typeof record.metadata.parser === 'string' ? record.metadata.parser.trim() : ''
|
|
250
|
+
if (parser && typeof parsed.frontmatter.created_at === 'string') {
|
|
251
|
+
frontmatter.generated ??= { by: parser.startsWith('convert/') ? parser : `convert/${parser}`, at: parsed.frontmatter.created_at }
|
|
252
|
+
}
|
|
253
|
+
tx.putSource(record, frontmatter, { actor: 'migration', kind: 'named', at: record.createdAt })
|
|
244
254
|
})
|
|
245
255
|
report.sources += 1
|
|
246
256
|
}
|
|
@@ -290,7 +300,7 @@ export async function migrateKnowledgeTree(input: MigrationInput): Promise<Migra
|
|
|
290
300
|
const to = join(kbRoot, ASSETS_DIR, bucket)
|
|
291
301
|
report.assetBuckets += 1
|
|
292
302
|
if (dryRun) {
|
|
293
|
-
report.assetFiles += (await listFiles(from, '')).length
|
|
303
|
+
report.assetFiles += (await listFiles(from, '')).filter(path => basename(path) !== PUBLISHED_IMAGE_MARKER).length
|
|
294
304
|
continue
|
|
295
305
|
}
|
|
296
306
|
try {
|
|
@@ -306,7 +316,7 @@ export async function migrateKnowledgeTree(input: MigrationInput): Promise<Migra
|
|
|
306
316
|
*/
|
|
307
317
|
if (!existsSync(to)) await rename(from, to)
|
|
308
318
|
else await mergeAssetBucket(from, to)
|
|
309
|
-
report.assetFiles += (await listFiles(to, '')).length
|
|
319
|
+
report.assetFiles += (await listFiles(to, '')).filter(path => basename(path) !== PUBLISHED_IMAGE_MARKER).length
|
|
310
320
|
} catch (cause) {
|
|
311
321
|
problems.push(`搬不动图片桶 ${bucket}:${cause instanceof Error ? cause.message : String(cause)}`)
|
|
312
322
|
}
|
|
@@ -555,7 +565,7 @@ async function copyContentTree(from: string, to: string, overwrite: boolean): Pr
|
|
|
555
565
|
if (!overwrite && existsSync(target)) continue
|
|
556
566
|
await mkdir(dirname(target), { recursive: true })
|
|
557
567
|
await cp(source, target, { force: true })
|
|
558
|
-
copied += 1
|
|
568
|
+
if (basename(source) !== PUBLISHED_IMAGE_MARKER) copied += 1
|
|
559
569
|
}
|
|
560
570
|
return copied
|
|
561
571
|
}
|
package/src/types.ts
CHANGED
|
@@ -130,7 +130,7 @@ export interface KnowledgeConfig {
|
|
|
130
130
|
* 组织(表达层)。两步都开着才是完整的「Knowledge Compiler + Wiki Materializer」。
|
|
131
131
|
*/
|
|
132
132
|
cardOrganize: boolean
|
|
133
|
-
/** Python 解释器。空串 =
|
|
133
|
+
/** Python 解释器。空串 = 自动(隔离依赖环境,再退回 `DSH_PYTHON` / `python3`)。 */
|
|
134
134
|
python: string
|
|
135
135
|
embeddingProvider: EmbeddingProvider
|
|
136
136
|
embeddingBaseUrl: string
|
|
@@ -231,10 +231,9 @@ export const DEFAULT_KNOWLEDGE_CONFIG: RuntimeKnowledgeConfig = {
|
|
|
231
231
|
// 卡片组织(模型)默认**开**:Knowledge 只是事实层,Wiki 卡片正文必须经过这一步
|
|
232
232
|
// 物化成可读的表达。只有显式写 `false` / `0` 才关闭兼容路径。
|
|
233
233
|
cardOrganize: !['false', '0'].includes((process.env.DSH_KNOWLEDGE_CARD_ORGANIZE ?? '').trim().toLowerCase()),
|
|
234
|
-
// 0029
|
|
235
|
-
//
|
|
236
|
-
|
|
237
|
-
python: process.env.DSH_PYTHON || '',
|
|
234
|
+
// 0029:空串 = 自动。优先用 `<agentRoot>/resource/python/` 的隔离依赖环境;
|
|
235
|
+
// DSH_PYTHON 只作为创建该环境的基础解释器和未准备时的运行时回退。
|
|
236
|
+
python: '',
|
|
238
237
|
embeddingProvider: 'wemm-python',
|
|
239
238
|
embeddingBaseUrl: process.env.DSH_EMBEDDING_BASE_URL || 'http://127.0.0.1:11434',
|
|
240
239
|
embeddingModel: process.env.DSH_EMBEDDING_MODEL || process.env.DSH_EMBEDDING_MODEL_PATH || '',
|
|
@@ -561,6 +560,12 @@ export interface RecallHit extends SearchHit {
|
|
|
561
560
|
export interface RecallKnowledge {
|
|
562
561
|
id: string
|
|
563
562
|
text: string
|
|
563
|
+
/** Citation anchors are checked; semantic entailment is not established. */
|
|
564
|
+
claimVerification?: 'unverified'
|
|
565
|
+
requiresReview?: boolean
|
|
566
|
+
evidenceStatus?: 'current' | 'historical' | 'unversioned' | 'missing' | 'human'
|
|
567
|
+
sourceVersion?: number
|
|
568
|
+
currentSourceVersion?: number
|
|
564
569
|
/**
|
|
565
570
|
* Knowledge 的真值状态。**如实带出来,不许静默过滤**:Card review 不是 Knowledge 验证,
|
|
566
571
|
* 因此 candidate / disputed / deprecated 都必须让上层看到。
|