dsh-macro-memory 0.0.0-stage → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +457 -2
- package/cordis.patch.yml +6 -0
- package/lib/client.js +3623 -0
- package/lib/index.js +469 -0
- package/lib/macro/brand.js +82 -0
- package/lib/macro/bridge.js +158 -0
- package/lib/macro/chain.js +206 -0
- package/lib/macro/contract.js +471 -0
- package/lib/macro/cost.js +164 -0
- package/lib/macro/defs/admin.js +439 -0
- package/lib/macro/defs/dream.js +24 -0
- package/lib/macro/defs/link.js +257 -0
- package/lib/macro/defs/project.js +25 -0
- package/lib/macro/defs/read.js +145 -0
- package/lib/macro/defs/remember.js +213 -0
- package/lib/macro/defs/search.js +661 -0
- package/lib/macro/defs/update.js +161 -0
- package/lib/macro/dreamledger.js +1398 -0
- package/lib/macro/facade.js +135 -0
- package/lib/macro/fusion.js +182 -0
- package/lib/macro/linkflow.js +391 -0
- package/lib/macro/links.js +448 -0
- package/lib/macro/neighbor.js +209 -0
- package/lib/macro/panel.js +96 -0
- package/lib/macro/prompt.js +139 -0
- package/lib/macro/provenance.js +148 -0
- package/lib/macro/render.js +35 -0
- package/lib/macro/revisions.js +253 -0
- package/lib/macro/shared.js +113 -0
- package/lib/macro/store.js +103 -0
- package/lib/macro/tokencost.js +90 -0
- package/lib/macro/tools.js +67 -0
- package/package.json +54 -4
- package/vendor/VENDOR.json +23 -0
- package/vendor/dsh-memory-nebula/CONTRIBUTING.md +48 -0
- package/vendor/dsh-memory-nebula/LICENSE +21 -0
- package/vendor/dsh-memory-nebula/README.en.md +203 -0
- package/vendor/dsh-memory-nebula/README.md +253 -0
- package/vendor/dsh-memory-nebula/SECURITY.md +43 -0
- package/vendor/dsh-memory-nebula/SUPPORT.md +33 -0
- package/vendor/dsh-memory-nebula/cordis.patch.yml +4 -0
- package/vendor/dsh-memory-nebula/lib/client.js +105 -0
- package/vendor/dsh-memory-nebula/lib/host/adapters/index.js +279 -0
- package/vendor/dsh-memory-nebula/lib/host/adapters/ingest.js +230 -0
- package/vendor/dsh-memory-nebula/lib/host/adapters/meow.js +317 -0
- package/vendor/dsh-memory-nebula/lib/host/adapters/notes.js +305 -0
- package/vendor/dsh-memory-nebula/lib/host/adapters/sqlite.js +287 -0
- package/vendor/dsh-memory-nebula/lib/host/adapters/util.js +231 -0
- package/vendor/dsh-memory-nebula/lib/host/bridge.js +351 -0
- package/vendor/dsh-memory-nebula/lib/host/compress.js +97 -0
- package/vendor/dsh-memory-nebula/lib/host/demo.js +357 -0
- package/vendor/dsh-memory-nebula/lib/host/diagnostics.js +125 -0
- package/vendor/dsh-memory-nebula/lib/host/doctor.js +270 -0
- package/vendor/dsh-memory-nebula/lib/host/graph.js +1663 -0
- package/vendor/dsh-memory-nebula/lib/host/graphcache.js +282 -0
- package/vendor/dsh-memory-nebula/lib/host/http.js +993 -0
- package/vendor/dsh-memory-nebula/lib/host/injectcfg.js +95 -0
- package/vendor/dsh-memory-nebula/lib/host/ledger.js +175 -0
- package/vendor/dsh-memory-nebula/lib/host/live.js +204 -0
- package/vendor/dsh-memory-nebula/lib/host/nebula.js +2105 -0
- package/vendor/dsh-memory-nebula/lib/host/page.js +75 -0
- package/vendor/dsh-memory-nebula/lib/host/path.js +83 -0
- package/vendor/dsh-memory-nebula/lib/host/recall.js +451 -0
- package/vendor/dsh-memory-nebula/lib/host/share.js +633 -0
- package/vendor/dsh-memory-nebula/lib/host/sources.js +266 -0
- package/vendor/dsh-memory-nebula/lib/host/store.js +333 -0
- package/vendor/dsh-memory-nebula/lib/host/text.js +410 -0
- package/vendor/dsh-memory-nebula/lib/host/transcripts.js +396 -0
- package/vendor/dsh-memory-nebula/lib/index.js +759 -0
- package/vendor/dsh-memory-nebula/lib/standalone.mjs +315 -0
- package/vendor/dsh-memory-nebula/lib/web/api.js +426 -0
- package/vendor/dsh-memory-nebula/lib/web/app.js +4159 -0
- package/vendor/dsh-memory-nebula/lib/web/i18n.js +580 -0
- package/vendor/dsh-memory-nebula/lib/web/index.html +174 -0
- package/vendor/dsh-memory-nebula/lib/web/nav.js +132 -0
- package/vendor/dsh-memory-nebula/lib/web/render.js +1890 -0
- package/vendor/dsh-memory-nebula/lib/web/space3d.js +1883 -0
- package/vendor/dsh-memory-nebula/lib/web/style.css +1402 -0
- package/vendor/dsh-memory-nebula/locale/en.json +7 -0
- package/vendor/dsh-memory-nebula/locale/zh.json +7 -0
- package/vendor/dsh-memory-nebula/market/assets/icon-180.png +0 -0
- package/vendor/dsh-memory-nebula/market/assets/icon.svg +40 -0
- package/vendor/dsh-memory-nebula/market/index.json +101 -0
- package/vendor/dsh-memory-nebula/package.json +87 -0
- package/vendor/dsh-memory-nebula/sidecar.ps1 +119 -0
- package/vendor/meow-memory/LICENSE +21 -0
- package/vendor/meow-memory/README.en.md +192 -0
- package/vendor/meow-memory/README.md +174 -0
- package/vendor/meow-memory/README.pt-br.md +180 -0
- package/vendor/meow-memory/cordis.patch.yml +7 -0
- package/vendor/meow-memory/lib/client.js +3344 -0
- package/vendor/meow-memory/lib/client.js.map +7 -0
- package/vendor/meow-memory/lib/index.js +9567 -0
- package/vendor/meow-memory/lib/index.js.map +7 -0
- package/vendor/meow-memory/lib/prompts/README.md +65 -0
- package/vendor/meow-memory/lib/prompts/en/dream-atomic.md +36 -0
- package/vendor/meow-memory/lib/prompts/en/dream-header.md +9 -0
- package/vendor/meow-memory/lib/prompts/en/dream-project-summary.md +12 -0
- package/vendor/meow-memory/lib/prompts/en/dream-topic.md +27 -0
- package/vendor/meow-memory/lib/prompts/en/external-memory-notice.md +8 -0
- package/vendor/meow-memory/lib/prompts/en/first-create-notice.md +8 -0
- package/vendor/meow-memory/lib/prompts/en/labels.md +98 -0
- package/vendor/meow-memory/lib/prompts/en/reflect.md +31 -0
- package/vendor/meow-memory/lib/prompts/en/system-guide.md +121 -0
- package/vendor/meow-memory/lib/prompts/en/tools.md +83 -0
- package/vendor/meow-memory/lib/prompts/en/welcome-guide.md +12 -0
- package/vendor/meow-memory/lib/prompts/pt-br/dream-atomic.md +36 -0
- package/vendor/meow-memory/lib/prompts/pt-br/dream-header.md +9 -0
- package/vendor/meow-memory/lib/prompts/pt-br/dream-project-summary.md +12 -0
- package/vendor/meow-memory/lib/prompts/pt-br/dream-topic.md +27 -0
- package/vendor/meow-memory/lib/prompts/pt-br/external-memory-notice.md +8 -0
- package/vendor/meow-memory/lib/prompts/pt-br/first-create-notice.md +8 -0
- package/vendor/meow-memory/lib/prompts/pt-br/labels.md +98 -0
- package/vendor/meow-memory/lib/prompts/pt-br/reflect.md +31 -0
- package/vendor/meow-memory/lib/prompts/pt-br/system-guide.md +121 -0
- package/vendor/meow-memory/lib/prompts/pt-br/tools.md +83 -0
- package/vendor/meow-memory/lib/prompts/pt-br/welcome-guide.md +12 -0
- package/vendor/meow-memory/lib/prompts/zh/dream-atomic.md +36 -0
- package/vendor/meow-memory/lib/prompts/zh/dream-header.md +9 -0
- package/vendor/meow-memory/lib/prompts/zh/dream-project-summary.md +19 -0
- package/vendor/meow-memory/lib/prompts/zh/dream-topic.md +27 -0
- package/vendor/meow-memory/lib/prompts/zh/external-memory-notice.md +8 -0
- package/vendor/meow-memory/lib/prompts/zh/first-create-notice.md +8 -0
- package/vendor/meow-memory/lib/prompts/zh/labels.md +98 -0
- package/vendor/meow-memory/lib/prompts/zh/reflect.md +30 -0
- package/vendor/meow-memory/lib/prompts/zh/system-guide.md +119 -0
- package/vendor/meow-memory/lib/prompts/zh/tools.md +83 -0
- package/vendor/meow-memory/lib/prompts/zh/welcome-guide.md +12 -0
- package/vendor/meow-memory/package.json +64 -0
|
@@ -0,0 +1,279 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 适配器注册表 —— 星图的「外部记忆」入口层
|
|
3
|
+
*
|
|
4
|
+
* 星图不假设记忆长什么样。它只要求一个适配器能回答四件事:
|
|
5
|
+
*
|
|
6
|
+
* meta 这个适配器叫什么、接什么、配置怎么写
|
|
7
|
+
* detect(config) → Store[] 按配置找出可读的「库」(文件/目录/远端缓存)
|
|
8
|
+
* scan(store) → {nodes, ledgers} 把库读成统一节点
|
|
9
|
+
* probe(config) → 报告 给人/AI 看的接入体检(不写、不炸)
|
|
10
|
+
*
|
|
11
|
+
* 于是接入一个新记忆插件有两条路:
|
|
12
|
+
* 1. 有落盘(SQLite / JSONL / Markdown / 目录)→ 在 sources.json 写一条,零代码;
|
|
13
|
+
* 2. 没落盘(远端服务)→ 用 ingest 通道把它「推」进来,或写一个 10 行的新适配器。
|
|
14
|
+
*
|
|
15
|
+
* 详细接入说明见 docs/adapters.md。
|
|
16
|
+
*/
|
|
17
|
+
import { existsSync, readFileSync } from 'node:fs'
|
|
18
|
+
import { join } from 'node:path'
|
|
19
|
+
import { dshHome, nebulaHome, expandHome, slugKey } from './util.js'
|
|
20
|
+
import * as meow from './meow.js'
|
|
21
|
+
import * as notes from './notes.js'
|
|
22
|
+
import * as sqlite from './sqlite.js'
|
|
23
|
+
import * as ingest from './ingest.js'
|
|
24
|
+
|
|
25
|
+
export const ADAPTERS = { meow, notes, sqlite, ingest }
|
|
26
|
+
|
|
27
|
+
/** 适配器清单(给 nebula_sources 工具和文档用)。 */
|
|
28
|
+
export function adapterCatalog() {
|
|
29
|
+
return Object.entries(ADAPTERS).map(([key, a]) => ({
|
|
30
|
+
id: a.meta?.id || key,
|
|
31
|
+
name: a.meta?.name || key,
|
|
32
|
+
kind: a.meta?.kind || 'file',
|
|
33
|
+
description: a.meta?.description || '',
|
|
34
|
+
config: a.meta?.config || {},
|
|
35
|
+
builtin: key === 'meow' || key === 'ingest',
|
|
36
|
+
}))
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export function configPath() {
|
|
40
|
+
return join(nebulaHome(), 'sources.json')
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
const DEFAULT_CONFIG = {
|
|
44
|
+
version: 1,
|
|
45
|
+
autoDetect: true,
|
|
46
|
+
sources: [],
|
|
47
|
+
extraRoots: [],
|
|
48
|
+
disabled: {},
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** 读配置;文件不存在/坏了都用默认值(永远不因为配置炸掉)。 */
|
|
52
|
+
export function readSourcesConfig() {
|
|
53
|
+
const file = configPath()
|
|
54
|
+
if (!existsSync(file)) return { ...DEFAULT_CONFIG, file, exists: false }
|
|
55
|
+
try {
|
|
56
|
+
const raw = JSON.parse(readFileSync(file, 'utf8').replace(/^\uFEFF/, ''))
|
|
57
|
+
if (!raw || typeof raw !== 'object') return { ...DEFAULT_CONFIG, file, exists: false }
|
|
58
|
+
return {
|
|
59
|
+
...DEFAULT_CONFIG,
|
|
60
|
+
...raw,
|
|
61
|
+
sources: Array.isArray(raw.sources) ? raw.sources : [],
|
|
62
|
+
extraRoots: Array.isArray(raw.extraRoots) ? raw.extraRoots : [],
|
|
63
|
+
disabled: raw.disabled && typeof raw.disabled === 'object' ? raw.disabled : {},
|
|
64
|
+
file,
|
|
65
|
+
exists: true,
|
|
66
|
+
}
|
|
67
|
+
} catch (err) {
|
|
68
|
+
return { ...DEFAULT_CONFIG, file, exists: true, error: String(err?.message || err) }
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** 已知的「别人家记忆插件」落盘约定:存在才接,不存在不吭声。 */
|
|
73
|
+
const KNOWN_LOCATIONS = [
|
|
74
|
+
{ path: '~/.hindsight/knowledge', title: 'Hindsight 知识页', level: 'topic', adapter: 'notes' },
|
|
75
|
+
{ path: '~/.hindsight/pages', title: 'Hindsight 页面', level: 'topic', adapter: 'notes' },
|
|
76
|
+
{ path: '~/.hindsight/memories', title: 'Hindsight 记忆', level: 'note', adapter: 'notes' },
|
|
77
|
+
{ path: '~/.openviking/memory', title: 'OpenViking 记忆', level: 'note', adapter: 'notes' },
|
|
78
|
+
{ path: '~/.openviking/skills', title: 'OpenViking 技能', level: 'note', adapter: 'notes' },
|
|
79
|
+
{ path: '~/.dsh/memories', title: 'DSH 记忆', level: 'note', adapter: 'notes' },
|
|
80
|
+
{ path: '~/.claude/memory', title: 'Claude 记忆', level: 'note', adapter: 'notes' },
|
|
81
|
+
]
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* 发现所有可读来源。
|
|
85
|
+
* @param {{extraRoots?: string[], autoDetect?: boolean, disabled?: object}} opts
|
|
86
|
+
* @returns {{stores: object[], reports: object[], errors: object[]}}
|
|
87
|
+
*/
|
|
88
|
+
export function discoverSources(opts = {}) {
|
|
89
|
+
const config = readSourcesConfig()
|
|
90
|
+
const autoDetect = opts.autoDetect ?? config.autoDetect
|
|
91
|
+
const disabled = opts.disabled || config.disabled || {}
|
|
92
|
+
const stores = []
|
|
93
|
+
const reports = []
|
|
94
|
+
const errors = []
|
|
95
|
+
|
|
96
|
+
const push = (list, label, adapterId) => {
|
|
97
|
+
for (const store of list || []) {
|
|
98
|
+
if (stores.some((s) => s.id === store.id)) continue
|
|
99
|
+
stores.push(store)
|
|
100
|
+
reports.push({ adapter: adapterId, label, store: store.id, title: store.title, ok: true })
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
// 1) 内置:meow-memory(本机主存储,永远尝试)
|
|
105
|
+
if (!disabled.meow) {
|
|
106
|
+
try {
|
|
107
|
+
const roots = [
|
|
108
|
+
...(Array.isArray(opts.extraRoots) ? opts.extraRoots : []),
|
|
109
|
+
...(Array.isArray(config.extraRoots) ? config.extraRoots : []),
|
|
110
|
+
]
|
|
111
|
+
const list = meow.detect({ extraRoots: roots })
|
|
112
|
+
push(list, '各工作区 .dsh-meow', 'meow')
|
|
113
|
+
if (list.length === 0) reports.push({ adapter: 'meow', ok: false, note: '没找到 .dsh-meow 记忆库' })
|
|
114
|
+
} catch (err) {
|
|
115
|
+
errors.push({ adapter: 'meow', error: String(err?.message || err) })
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
// 2) 内置:外部推送(本机其他插件推过来的)
|
|
120
|
+
if (!disabled.ingest) {
|
|
121
|
+
try {
|
|
122
|
+
push(ingest.detect({}), '外部推送目录', 'ingest')
|
|
123
|
+
} catch (err) {
|
|
124
|
+
errors.push({ adapter: 'ingest', error: String(err?.message || err) })
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
// 3) 配置声明的来源(用户/AI 自己加的)
|
|
129
|
+
for (const item of config.sources) {
|
|
130
|
+
if (!item || typeof item !== 'object') continue
|
|
131
|
+
const adapterId = String(item.adapter || item.type || '').trim()
|
|
132
|
+
const adapter = ADAPTERS[adapterId]
|
|
133
|
+
if (!adapter) {
|
|
134
|
+
errors.push({ adapter: adapterId || '(未指定)', error: '没有这个适配器;可用:' + Object.keys(ADAPTERS).join(' / ') })
|
|
135
|
+
continue
|
|
136
|
+
}
|
|
137
|
+
if (disabled[adapterId] || item.enabled === false) continue
|
|
138
|
+
try {
|
|
139
|
+
const list = adapter.detect(item)
|
|
140
|
+
push(list, item.title || item.path || adapterId, adapterId)
|
|
141
|
+
if ((list || []).length === 0) {
|
|
142
|
+
reports.push({
|
|
143
|
+
adapter: adapterId,
|
|
144
|
+
ok: false,
|
|
145
|
+
note: `按配置没找到可读内容:${item.path || '(未配置 path)'}`,
|
|
146
|
+
})
|
|
147
|
+
}
|
|
148
|
+
} catch (err) {
|
|
149
|
+
errors.push({ adapter: adapterId, path: item.path, error: String(err?.message || err) })
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
// 4) 自动探测:常见记忆插件的落盘位置
|
|
154
|
+
if (autoDetect) {
|
|
155
|
+
for (const item of KNOWN_LOCATIONS) {
|
|
156
|
+
const path = expandHome(item.path)
|
|
157
|
+
if (!path || !existsSync(path)) continue
|
|
158
|
+
if (stores.some((s) => s.path === path)) continue
|
|
159
|
+
try {
|
|
160
|
+
const list = notes.detect({ path, title: item.title, level: item.level, id: 'auto:' + slugKey(item.title) })
|
|
161
|
+
push(list, '自动发现:' + item.title, 'notes')
|
|
162
|
+
} catch (err) {
|
|
163
|
+
errors.push({ adapter: 'notes', path, error: String(err?.message || err) })
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
return { stores, reports, errors, config }
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* 扫描全部来源,聚合节点与账本。
|
|
173
|
+
* 单个库读坏不影响其它库。
|
|
174
|
+
*/
|
|
175
|
+
/**
|
|
176
|
+
* 扫描所有来源(生成器版):每扫完一个库 yield 一次。
|
|
177
|
+
*
|
|
178
|
+
* 为什么拆:2 万条库上扫描要 250ms,单个 5000 条的库也要几十毫秒。
|
|
179
|
+
* 这段是同步 SQLite 只读,不开线程池就没法在库内部让路;但库与库之间可以,
|
|
180
|
+
* 而「重建一次卡 250ms」正是宿主界面能感觉到的顿。
|
|
181
|
+
*/
|
|
182
|
+
export function* scanAllSteps(stores) {
|
|
183
|
+
const nodes = []
|
|
184
|
+
/** buildGraph 要的是数组:{sessionId, kind, ids[], ws} —— 别拍成 map。 */
|
|
185
|
+
const ledgers = []
|
|
186
|
+
const errors = []
|
|
187
|
+
const warnings = []
|
|
188
|
+
// 报错里写给人看的名字:meow 的 id 是 `meow:D:\…\proj`,直接甩给用户太长;
|
|
189
|
+
// key 是工作区短名(proj),既对得上 sources[].key,也读得懂。
|
|
190
|
+
const labelOf = (store) => store.key || store.title || store.id
|
|
191
|
+
for (const store of stores || []) {
|
|
192
|
+
yield 'store'
|
|
193
|
+
const adapter = ADAPTERS[store.adapter]
|
|
194
|
+
if (!adapter || (typeof adapter.scan !== 'function' && typeof adapter.scanSteps !== 'function')) {
|
|
195
|
+
errors.push({ store: labelOf(store), id: store.id, error: '适配器 ' + store.adapter + ' 不能扫描' })
|
|
196
|
+
continue
|
|
197
|
+
}
|
|
198
|
+
try {
|
|
199
|
+
// 适配器可以只提供 scanSteps(库内按行让路),也可以只提供 scan(老的一口气版)。
|
|
200
|
+
// 有 scanSteps 就走它:一个 12,500 行的库原来是一整段 110ms 不让路,
|
|
201
|
+
// 现在是每 512 行让一次。
|
|
202
|
+
let res
|
|
203
|
+
if (typeof adapter.scanSteps === 'function') {
|
|
204
|
+
const it = adapter.scanSteps(store)
|
|
205
|
+
let step = it.next()
|
|
206
|
+
while (!step.done) {
|
|
207
|
+
yield step.value
|
|
208
|
+
step = it.next()
|
|
209
|
+
}
|
|
210
|
+
res = step.value || {}
|
|
211
|
+
} else {
|
|
212
|
+
res = adapter.scan(store) || {}
|
|
213
|
+
}
|
|
214
|
+
for (const n of res.nodes || []) nodes.push({ ...n, storeId: n.storeId || store.id })
|
|
215
|
+
if (Array.isArray(res.ledgers)) {
|
|
216
|
+
for (const row of res.ledgers) if (row && row.sessionId && Array.isArray(row.ids)) ledgers.push(row)
|
|
217
|
+
} else if (res.ledgers && typeof res.ledgers === 'object') {
|
|
218
|
+
for (const [session, value] of Object.entries(res.ledgers)) {
|
|
219
|
+
if (value && Array.isArray(value.ids)) ledgers.push({ ...value, sessionId: session })
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
if (res.truncated) {
|
|
223
|
+
// 上限可能写在配置里(store.maxFiles),也可能由适配器自己算出来(res.maxFiles)。
|
|
224
|
+
const cap = Number(store.maxFiles) || Number(res.maxFiles) || 0
|
|
225
|
+
warnings.push({
|
|
226
|
+
store: labelOf(store),
|
|
227
|
+
id: store.id,
|
|
228
|
+
code: "truncated",
|
|
229
|
+
maxFiles: cap,
|
|
230
|
+
message: "来源被文件数上限截断:只读了前 " + cap + " 个文件,其余没有进图。",
|
|
231
|
+
})
|
|
232
|
+
}
|
|
233
|
+
if (res.error) errors.push({ store: labelOf(store), id: store.id, error: res.error })
|
|
234
|
+
} catch (err) {
|
|
235
|
+
errors.push({ store: labelOf(store), id: store.id, error: String(err?.message || err) })
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
return { nodes, ledgers, errors, warnings }
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
/** 同步扫描:脚本、测试、小库直接用它。 */
|
|
242
|
+
export function scanAll(stores) {
|
|
243
|
+
const it = scanAllSteps(stores)
|
|
244
|
+
let step = it.next()
|
|
245
|
+
while (!step.done) step = it.next()
|
|
246
|
+
return step.value
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
/** 异步扫描:每扫完一个库让出一次事件循环(宿主重建走这条)。 */
|
|
250
|
+
export async function scanAllAsync(stores) {
|
|
251
|
+
const it = scanAllSteps(stores)
|
|
252
|
+
for (;;) {
|
|
253
|
+
const step = it.next()
|
|
254
|
+
if (step.done) return step.value
|
|
255
|
+
await new Promise((resolve) => setImmediate(resolve))
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
/** 体检:把每个配置项/适配器都问一遍,给人类看的报告。 */
|
|
260
|
+
export function probeAll() {
|
|
261
|
+
const config = readSourcesConfig()
|
|
262
|
+
const out = []
|
|
263
|
+
out.push({ adapter: 'meow', ...(meow.probe ? meow.probe() : { ok: false, note: '无 probe' }) })
|
|
264
|
+
out.push({ adapter: 'ingest', ...(ingest.probe ? ingest.probe() : { ok: false, note: '无 probe' }) })
|
|
265
|
+
for (const item of config.sources || []) {
|
|
266
|
+
const adapterId = String(item.adapter || item.type || '')
|
|
267
|
+
const adapter = ADAPTERS[adapterId]
|
|
268
|
+
if (!adapter) {
|
|
269
|
+
out.push({ adapter: adapterId, ok: false, note: '未知适配器' })
|
|
270
|
+
continue
|
|
271
|
+
}
|
|
272
|
+
try {
|
|
273
|
+
out.push({ adapter: adapterId, path: item.path, ...(adapter.probe ? adapter.probe(item) : { ok: true }) })
|
|
274
|
+
} catch (err) {
|
|
275
|
+
out.push({ adapter: adapterId, path: item.path, ok: false, note: String(err?.message || err) })
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
return { config: { path: config.file, exists: config.exists, error: config.error, autoDetect: config.autoDetect }, probes: out }
|
|
279
|
+
}
|
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 适配器:ingest —— 星图自己的「外部推送」落点
|
|
3
|
+
*
|
|
4
|
+
* 有些记忆插件根本没把记忆落在本地(hindsight、openviking 都是远端服务),
|
|
5
|
+
* 硬读磁盘是读不到的。这一层给一条正路:
|
|
6
|
+
*
|
|
7
|
+
* 任何插件 / 任何 agent → nebula_ingest 工具 或 POST /api/memory-nebula/ingest
|
|
8
|
+
* → $DSH_HOME/memory-nebula/ingest/<provider>.jsonl
|
|
9
|
+
* → 星图重建时读回来,进同一张图
|
|
10
|
+
*
|
|
11
|
+
* 写的是星图自己的目录,从不碰别人的库;同 uid 后写覆盖,重复推送不会长重影。
|
|
12
|
+
*/
|
|
13
|
+
import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, appendFileSync } from 'node:fs'
|
|
14
|
+
import { join } from 'node:path'
|
|
15
|
+
import { nebulaHome, slugKey, toKeywordList, toMs, hashShort } from './util.js'
|
|
16
|
+
|
|
17
|
+
export const meta = {
|
|
18
|
+
id: 'ingest',
|
|
19
|
+
name: '外部推送',
|
|
20
|
+
kind: 'file',
|
|
21
|
+
description:
|
|
22
|
+
'接收其他记忆插件/外部 agent 主动推进来的记忆(星图自己的目录,JSONL)。给「读不到磁盘」的远端记忆服务(hindsight / openviking)留的正路。',
|
|
23
|
+
config: {},
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** 推送目录。 */
|
|
27
|
+
export function ingestDir() {
|
|
28
|
+
return join(nebulaHome(), 'ingest')
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* 探测:有文件就产出一个 store(每个 provider 一文件)。
|
|
33
|
+
* @param {object} config
|
|
34
|
+
*/
|
|
35
|
+
export function detect(config = {}) {
|
|
36
|
+
const dir = ingestDir()
|
|
37
|
+
if (!existsSync(dir)) return []
|
|
38
|
+
let names = []
|
|
39
|
+
try {
|
|
40
|
+
names = readdirSync(dir).filter((n) => n.endsWith('.jsonl'))
|
|
41
|
+
} catch {
|
|
42
|
+
return []
|
|
43
|
+
}
|
|
44
|
+
return names.map((name) => {
|
|
45
|
+
const provider = name.replace(/\.jsonl$/, '')
|
|
46
|
+
let size = 0
|
|
47
|
+
let mtimeMs = 0
|
|
48
|
+
try {
|
|
49
|
+
const st = statSync(join(dir, name))
|
|
50
|
+
size = st.size
|
|
51
|
+
mtimeMs = st.mtimeMs
|
|
52
|
+
} catch {
|
|
53
|
+
/* ignore */
|
|
54
|
+
}
|
|
55
|
+
return {
|
|
56
|
+
id: 'ingest:' + provider,
|
|
57
|
+
provider,
|
|
58
|
+
adapter: 'ingest',
|
|
59
|
+
title: provider,
|
|
60
|
+
path: join(dir, name),
|
|
61
|
+
key: slugKey(provider, 'ingest'),
|
|
62
|
+
size,
|
|
63
|
+
mtimeMs,
|
|
64
|
+
config,
|
|
65
|
+
}
|
|
66
|
+
})
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** 扫描:读所有推送文件,按 uid 去重(后写覆盖先写)。生成器版:每 512 行让一次路。 */
|
|
70
|
+
export function* scanSteps(store) {
|
|
71
|
+
const nodes = []
|
|
72
|
+
const seen = new Map()
|
|
73
|
+
let text = ''
|
|
74
|
+
try {
|
|
75
|
+
text = readFileSync(store.path, 'utf8').replace(/^\uFEFF/, '')
|
|
76
|
+
} catch {
|
|
77
|
+
return { nodes, ledgers: [] }
|
|
78
|
+
}
|
|
79
|
+
let parsed = 0
|
|
80
|
+
for (const line of text.split(/\r?\n/)) {
|
|
81
|
+
if ((++parsed & 511) === 0) yield 'rows'
|
|
82
|
+
const s = line.trim()
|
|
83
|
+
if (!s || s.startsWith('#')) continue
|
|
84
|
+
let row
|
|
85
|
+
try {
|
|
86
|
+
row = JSON.parse(s)
|
|
87
|
+
} catch {
|
|
88
|
+
continue
|
|
89
|
+
}
|
|
90
|
+
if (!row || typeof row !== 'object') continue
|
|
91
|
+
const uid = String(row.uid || row.id || hashShort(String(row.content || row.title || '') + seen.size))
|
|
92
|
+
seen.set(uid, row)
|
|
93
|
+
}
|
|
94
|
+
for (const [uid, row] of seen) {
|
|
95
|
+
const content = String(row.content ?? '').trim()
|
|
96
|
+
const title = String(row.title ?? '').trim()
|
|
97
|
+
if (!content && !title) continue
|
|
98
|
+
const level = String(row.level || 'note').trim().toLowerCase() || 'note'
|
|
99
|
+
const ws = String(row.ws || row.workspace || store.title)
|
|
100
|
+
nodes.push({
|
|
101
|
+
id: `ingest:${store.key}:${level}:${uid}`,
|
|
102
|
+
uid,
|
|
103
|
+
provider: store.provider,
|
|
104
|
+
adapter: 'ingest',
|
|
105
|
+
storeId: store.id,
|
|
106
|
+
ws: slugKey(ws, 'ingest'),
|
|
107
|
+
wsTitle: ws,
|
|
108
|
+
wsPath: String(row.wsPath || ''),
|
|
109
|
+
level,
|
|
110
|
+
title,
|
|
111
|
+
label: (title || content).length > 42 ? (title || content).slice(0, 42) + '…' : title || content,
|
|
112
|
+
content,
|
|
113
|
+
importance: Math.max(0, Math.min(4, Number(row.importance) || 0)),
|
|
114
|
+
status: String(row.status || 'active'),
|
|
115
|
+
project: String(row.project || '').trim(),
|
|
116
|
+
subcategory: String(row.subcategory || ''),
|
|
117
|
+
goal: String(row.goal || ''),
|
|
118
|
+
corrected: false,
|
|
119
|
+
keywords: toKeywordList(row.keywords),
|
|
120
|
+
sessionId: String(row.sessionId || ''),
|
|
121
|
+
hits: Number(row.hits) || 0,
|
|
122
|
+
createdAt: toMs(row.createdAt) || Date.now(),
|
|
123
|
+
updatedAt: toMs(row.updatedAt) || toMs(row.createdAt) || Date.now(),
|
|
124
|
+
accessedAt: toMs(row.accessedAt),
|
|
125
|
+
})
|
|
126
|
+
}
|
|
127
|
+
return { nodes, ledgers: [] }
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/** 同步壳:跑完 scanSteps 拿到结果。 */
|
|
131
|
+
export function scan(store) {
|
|
132
|
+
const it = scanSteps(store)
|
|
133
|
+
let step = it.next()
|
|
134
|
+
while (!step.done) step = it.next()
|
|
135
|
+
return step.value
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* 把一次推送里被丢掉的原因拼成人话。
|
|
140
|
+
* 1.11.68:以前这里什么也不说 —— 三条坏数据换来一句「已推入 0 条」,调用方以为成功了。
|
|
141
|
+
* @param {{notObject:number, empty:number, notArray?:number}} reasons
|
|
142
|
+
*/
|
|
143
|
+
export function describeSkips(reasons = {}) {
|
|
144
|
+
const parts = []
|
|
145
|
+
if (reasons.notArray) parts.push(`${reasons.notArray} 条不是数组`)
|
|
146
|
+
if (reasons.notObject) parts.push(`${reasons.notObject} 条不是对象`)
|
|
147
|
+
if (reasons.empty) parts.push(`${reasons.empty} 条既没有 title 也没有 content(或只有空白)`)
|
|
148
|
+
return parts.join(';')
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* 追加推送。同 uid 后写覆盖(读取端按 uid 去重)。
|
|
153
|
+
*
|
|
154
|
+
* 1.11.68 起,返回值把「丢了几条、为什么丢、批内重复了几条」都说清楚:
|
|
155
|
+
* - written 真正写进文件的条数(按 uid 去重后)
|
|
156
|
+
* - skipped 被跳过的条数
|
|
157
|
+
* - duplicates 批内 uid 重复的条数(后一条覆盖前一条)
|
|
158
|
+
* - reasons { notObject, empty } 各自的原因计数
|
|
159
|
+
* 以前 written 按输入条数算,同一批里两条同 uid 会报 written=2,而图里只有 1 条。
|
|
160
|
+
*
|
|
161
|
+
* @param {string} provider
|
|
162
|
+
* @param {object[]} items
|
|
163
|
+
* @returns {{written:number, skipped:number, duplicates:number, reasons:object, file:string}}
|
|
164
|
+
*/
|
|
165
|
+
export function appendIngest(provider, items) {
|
|
166
|
+
const reasons = { notObject: 0, empty: 0 }
|
|
167
|
+
if (!Array.isArray(items)) {
|
|
168
|
+
return { written: 0, skipped: 0, duplicates: 0, reasons: { notArray: 1 }, file: '', error: 'items 必须是数组' }
|
|
169
|
+
}
|
|
170
|
+
const name = slugKey(provider || 'external', 'external').toLowerCase().replace(/[^a-z0-9_-]/g, '') || 'external'
|
|
171
|
+
const dir = ingestDir()
|
|
172
|
+
mkdirSync(dir, { recursive: true })
|
|
173
|
+
const file = join(dir, name + '.jsonl')
|
|
174
|
+
const byUid = new Map()
|
|
175
|
+
let skipped = 0
|
|
176
|
+
let duplicates = 0
|
|
177
|
+
for (const raw of items || []) {
|
|
178
|
+
if (!raw || typeof raw !== 'object' || Array.isArray(raw)) {
|
|
179
|
+
skipped += 1
|
|
180
|
+
reasons.notObject += 1
|
|
181
|
+
continue
|
|
182
|
+
}
|
|
183
|
+
const content = String(raw.content ?? '').trim()
|
|
184
|
+
const title = String(raw.title ?? '').trim()
|
|
185
|
+
if (!content && !title) {
|
|
186
|
+
skipped += 1
|
|
187
|
+
reasons.empty += 1
|
|
188
|
+
continue
|
|
189
|
+
}
|
|
190
|
+
// 不给 uid 时按内容哈希(不含时间戳、不含随机数):同一段文字重复推送要落回同一条。
|
|
191
|
+
// 原来这里掺了 Math.random(),于是每次推送都造一个新 id —— 「重复推送不会长重影」就是空话。
|
|
192
|
+
const uid = String(raw.uid || raw.id || hashShort(String(title) + '\u0000' + String(content)))
|
|
193
|
+
if (byUid.has(uid)) duplicates += 1
|
|
194
|
+
byUid.set(uid, {
|
|
195
|
+
uid,
|
|
196
|
+
title,
|
|
197
|
+
content,
|
|
198
|
+
level: String(raw.level || 'note'),
|
|
199
|
+
project: String(raw.project || ''),
|
|
200
|
+
keywords: toKeywordList(raw.keywords),
|
|
201
|
+
importance: Number(raw.importance) || 0,
|
|
202
|
+
status: String(raw.status || 'active'),
|
|
203
|
+
ws: String(raw.ws || raw.workspace || provider || 'external'),
|
|
204
|
+
wsPath: String(raw.wsPath || ''),
|
|
205
|
+
sessionId: String(raw.sessionId || ''),
|
|
206
|
+
createdAt: toMs(raw.createdAt),
|
|
207
|
+
updatedAt: toMs(raw.updatedAt),
|
|
208
|
+
at: Date.now(),
|
|
209
|
+
})
|
|
210
|
+
}
|
|
211
|
+
const lines = []
|
|
212
|
+
for (const row of byUid.values()) lines.push(JSON.stringify(row))
|
|
213
|
+
if (lines.length > 0) appendFileSync(file, lines.join('\n') + '\n', 'utf8')
|
|
214
|
+
return { written: lines.length, skipped, duplicates, reasons, file }
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
export function probe() {
|
|
218
|
+
const dir = ingestDir()
|
|
219
|
+
let files = []
|
|
220
|
+
try {
|
|
221
|
+
files = readdirSync(dir).filter((n) => n.endsWith('.jsonl'))
|
|
222
|
+
} catch {
|
|
223
|
+
/* ignore */
|
|
224
|
+
}
|
|
225
|
+
return {
|
|
226
|
+
ok: true,
|
|
227
|
+
stores: files.map((f) => ({ id: 'ingest:' + f.replace(/\.jsonl$/, ''), path: join(dir, f) })),
|
|
228
|
+
note: files.length ? `${files.length} 个推送来源` : '还没有外部推送(目录 ' + dir + ')',
|
|
229
|
+
}
|
|
230
|
+
}
|