@aiwayds/dsh-topics-memory 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +112 -0
- package/README.zh.md +112 -0
- package/cordis.patch.yml +8 -0
- package/lib/commands.d.ts +21 -0
- package/lib/commands.js +368 -0
- package/lib/config.d.ts +122 -0
- package/lib/config.js +123 -0
- package/lib/delegation.d.ts +23 -0
- package/lib/delegation.js +21 -0
- package/lib/digest.d.ts +42 -0
- package/lib/digest.js +101 -0
- package/lib/distill.d.ts +186 -0
- package/lib/distill.js +759 -0
- package/lib/git.d.ts +66 -0
- package/lib/git.js +169 -0
- package/lib/ilog.d.ts +53 -0
- package/lib/ilog.js +70 -0
- package/lib/index.d.ts +73 -0
- package/lib/index.js +504 -0
- package/lib/observer.d.ts +41 -0
- package/lib/observer.js +128 -0
- package/lib/okf.d.ts +112 -0
- package/lib/okf.js +279 -0
- package/lib/onboard.d.ts +205 -0
- package/lib/onboard.js +664 -0
- package/lib/paths.d.ts +48 -0
- package/lib/paths.js +84 -0
- package/lib/retrieval.d.ts +61 -0
- package/lib/retrieval.js +178 -0
- package/lib/service.d.ts +96 -0
- package/lib/service.js +375 -0
- package/lib/store.d.ts +153 -0
- package/lib/store.js +489 -0
- package/lib/sync.d.ts +52 -0
- package/lib/sync.js +154 -0
- package/lib/tools.d.ts +8 -0
- package/lib/tools.js +208 -0
- package/lib/viz.d.ts +33 -0
- package/lib/viz.js +310 -0
- package/lib/yaml.d.ts +18 -0
- package/lib/yaml.js +334 -0
- package/package.json +97 -0
- package/src/commands.ts +397 -0
- package/src/config.ts +149 -0
- package/src/delegation.ts +27 -0
- package/src/digest.ts +125 -0
- package/src/distill.ts +885 -0
- package/src/git.ts +209 -0
- package/src/ilog.ts +99 -0
- package/src/index.ts +574 -0
- package/src/observer.ts +150 -0
- package/src/okf.ts +317 -0
- package/src/onboard.ts +762 -0
- package/src/paths.ts +104 -0
- package/src/retrieval.ts +219 -0
- package/src/service.ts +400 -0
- package/src/store.ts +567 -0
- package/src/sync.ts +170 -0
- package/src/tools.ts +225 -0
- package/src/viz.ts +333 -0
- package/src/yaml.ts +322 -0
package/lib/store.js
ADDED
|
@@ -0,0 +1,489 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Bundle store — reads and writes the local OKF bundle (the Cache).
|
|
3
|
+
*
|
|
4
|
+
* All writes go through a serialized queue: one git commit per topic save
|
|
5
|
+
* (write-through, ADR 0003) plus a regenerated `index.md`. Observations and
|
|
6
|
+
* the injection log are JSONL sidecars under `meta/`; they are committed on
|
|
7
|
+
* the flush cadence (sync layer), not per append.
|
|
8
|
+
*
|
|
9
|
+
* @module store
|
|
10
|
+
*/
|
|
11
|
+
import { readdir, readFile, writeFile, rename, mkdir, appendFile, access, } from 'node:fs/promises';
|
|
12
|
+
import { readFileSync } from 'node:fs';
|
|
13
|
+
import { constants as FsConstants } from 'node:fs';
|
|
14
|
+
import { dirname, join } from 'node:path';
|
|
15
|
+
import { randomUUID } from 'node:crypto';
|
|
16
|
+
import * as okf from "./okf.js";
|
|
17
|
+
import * as gitmod from "./git.js";
|
|
18
|
+
import { actorFor } from "./paths.js";
|
|
19
|
+
export class StoreError extends Error {
|
|
20
|
+
name = 'StoreError';
|
|
21
|
+
}
|
|
22
|
+
function existsSync(path) {
|
|
23
|
+
return access(path, FsConstants.F_OK).then(() => true, () => false);
|
|
24
|
+
}
|
|
25
|
+
async function atomicWrite(file, content) {
|
|
26
|
+
await mkdir(dirname(file), { recursive: true });
|
|
27
|
+
const tmp = `${file}.tmp-${randomUUID()}`;
|
|
28
|
+
await writeFile(tmp, content, 'utf8');
|
|
29
|
+
await rename(tmp, file);
|
|
30
|
+
}
|
|
31
|
+
export class BundleStore {
|
|
32
|
+
root;
|
|
33
|
+
actor;
|
|
34
|
+
gitDisabled;
|
|
35
|
+
/** Serializes all write paths; reads are lock-free. */
|
|
36
|
+
queue = Promise.resolve();
|
|
37
|
+
constructor(root, opts = {}) {
|
|
38
|
+
this.root = root;
|
|
39
|
+
this.actor = opts.actor ?? actorFor();
|
|
40
|
+
this.gitDisabled = opts.gitDisabled === true;
|
|
41
|
+
}
|
|
42
|
+
/** Serialize an async write operation behind the store-wide queue. */
|
|
43
|
+
enqueue(op) {
|
|
44
|
+
const run = this.queue.then(op, op);
|
|
45
|
+
this.queue = run.catch(() => undefined);
|
|
46
|
+
return run;
|
|
47
|
+
}
|
|
48
|
+
topicsDir() {
|
|
49
|
+
return join(this.root, 'topics');
|
|
50
|
+
}
|
|
51
|
+
metaDir() {
|
|
52
|
+
return join(this.root, 'meta');
|
|
53
|
+
}
|
|
54
|
+
topicPath(slug) {
|
|
55
|
+
return join(this.root, 'topics', `${slug}.md`);
|
|
56
|
+
}
|
|
57
|
+
observationsPath() {
|
|
58
|
+
return join(this.metaDir(), 'observations.jsonl');
|
|
59
|
+
}
|
|
60
|
+
injectionsPath() {
|
|
61
|
+
return join(this.metaDir(), 'injections.jsonl');
|
|
62
|
+
}
|
|
63
|
+
conflictsPath() {
|
|
64
|
+
return join(this.metaDir(), 'conflicts.json');
|
|
65
|
+
}
|
|
66
|
+
distillStatePath() {
|
|
67
|
+
return join(this.metaDir(), 'distill-state.json');
|
|
68
|
+
}
|
|
69
|
+
/** Last distill-lane run outcome (diagnostics for /topics status and e2e). */
|
|
70
|
+
async writeDistillState(state) {
|
|
71
|
+
await mkdir(this.metaDir(), { recursive: true });
|
|
72
|
+
await atomicWrite(this.distillStatePath(), `${JSON.stringify(state, null, 2)}\n`);
|
|
73
|
+
}
|
|
74
|
+
async readDistillState() {
|
|
75
|
+
try {
|
|
76
|
+
return JSON.parse(await readFile(this.distillStatePath(), 'utf8'));
|
|
77
|
+
}
|
|
78
|
+
catch {
|
|
79
|
+
return undefined;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
/** Create the directory skeleton, git repo, and initial index (idempotent). */
|
|
83
|
+
async ensure() {
|
|
84
|
+
await mkdir(this.topicsDir(), { recursive: true });
|
|
85
|
+
await mkdir(this.metaDir(), { recursive: true });
|
|
86
|
+
if (!this.gitDisabled && !(await gitmod.isRepo(this.root))) {
|
|
87
|
+
await gitmod.initRepo(this.root);
|
|
88
|
+
}
|
|
89
|
+
if (!(await existsSync(join(this.root, 'index.md')))) {
|
|
90
|
+
await this.enqueue(() => this.regenerateIndex());
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
/** Repo initialized and not gitDisabled. */
|
|
94
|
+
async hasGit() {
|
|
95
|
+
return !this.gitDisabled && gitmod.isRepo(this.root);
|
|
96
|
+
}
|
|
97
|
+
async commit(paths, message) {
|
|
98
|
+
if (this.gitDisabled)
|
|
99
|
+
return false;
|
|
100
|
+
if (!(await gitmod.isRepo(this.root)))
|
|
101
|
+
return false;
|
|
102
|
+
return gitmod.addAndCommit(this.root, paths, message);
|
|
103
|
+
}
|
|
104
|
+
async listTopics() {
|
|
105
|
+
let files;
|
|
106
|
+
try {
|
|
107
|
+
files = await readdir(this.topicsDir());
|
|
108
|
+
}
|
|
109
|
+
catch {
|
|
110
|
+
return [];
|
|
111
|
+
}
|
|
112
|
+
const metas = [];
|
|
113
|
+
for (const f of files) {
|
|
114
|
+
if (!f.endsWith('.md') || f === 'index.md')
|
|
115
|
+
continue;
|
|
116
|
+
try {
|
|
117
|
+
const raw = await readFile(join(this.topicsDir(), f), 'utf8');
|
|
118
|
+
const doc = okf.parseTopicDoc(raw);
|
|
119
|
+
metas.push({
|
|
120
|
+
slug: f.slice(0, -3),
|
|
121
|
+
title: doc.fm.title,
|
|
122
|
+
description: doc.fm.description,
|
|
123
|
+
status: doc.fm.status,
|
|
124
|
+
tags: doc.fm.tags,
|
|
125
|
+
depends: doc.fm.depends,
|
|
126
|
+
generatedAt: doc.fm.generated.at,
|
|
127
|
+
});
|
|
128
|
+
}
|
|
129
|
+
catch {
|
|
130
|
+
// Broken topic files are skipped from rosters; status() surfaces them.
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
return metas;
|
|
134
|
+
}
|
|
135
|
+
/** Topics broken beyond parsing — surfaced by /topics status, never silently dropped. */
|
|
136
|
+
async brokenTopics() {
|
|
137
|
+
let files;
|
|
138
|
+
try {
|
|
139
|
+
files = await readdir(this.topicsDir());
|
|
140
|
+
}
|
|
141
|
+
catch {
|
|
142
|
+
return [];
|
|
143
|
+
}
|
|
144
|
+
const broken = [];
|
|
145
|
+
for (const f of files) {
|
|
146
|
+
if (!f.endsWith('.md') || f === 'index.md')
|
|
147
|
+
continue;
|
|
148
|
+
try {
|
|
149
|
+
okf.parseTopicDoc(await readFile(join(this.topicsDir(), f), 'utf8'));
|
|
150
|
+
}
|
|
151
|
+
catch {
|
|
152
|
+
broken.push(f);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
return broken;
|
|
156
|
+
}
|
|
157
|
+
async readTopic(slug) {
|
|
158
|
+
if (!okf.RESERVED_FILES.has(`${slug}.md`)) {
|
|
159
|
+
try {
|
|
160
|
+
return okf.parseTopicDoc(await readFile(this.topicPath(slug), 'utf8'));
|
|
161
|
+
}
|
|
162
|
+
catch (e) {
|
|
163
|
+
if (e instanceof Error && e.name === 'OkfError')
|
|
164
|
+
throw e;
|
|
165
|
+
return undefined;
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
return undefined;
|
|
169
|
+
}
|
|
170
|
+
async exists(slug) {
|
|
171
|
+
return existsSync(this.topicPath(slug));
|
|
172
|
+
}
|
|
173
|
+
/** First free slug: `foo`, then `foo-2`, `foo-3`, … */
|
|
174
|
+
async uniqueSlug(base) {
|
|
175
|
+
const clean = okf.slugify(base);
|
|
176
|
+
if (!(await this.exists(clean)))
|
|
177
|
+
return clean;
|
|
178
|
+
for (let i = 2;; i += 1) {
|
|
179
|
+
const candidate = `${clean}-${i}`;
|
|
180
|
+
if (!(await this.exists(candidate)))
|
|
181
|
+
return candidate;
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
/**
|
|
185
|
+
* Write a topic (atomic), regenerate the index, and commit both — one
|
|
186
|
+
* commit per conclusion change (ADR 0003). `slug` in the doc's depends is
|
|
187
|
+
* NOT normalized here; callers pass bundle-relative paths.
|
|
188
|
+
*/
|
|
189
|
+
async saveTopic(input, opts) {
|
|
190
|
+
return this.enqueue(async () => {
|
|
191
|
+
const slug = okf.slugify(input.slug);
|
|
192
|
+
const file = this.topicPath(slug);
|
|
193
|
+
const created = opts.created ?? !(await existsSync(file));
|
|
194
|
+
const now = new Date().toISOString();
|
|
195
|
+
const doc = {
|
|
196
|
+
fm: { ...input.doc.fm, generated: { by: opts.actor ?? this.actor, at: now } },
|
|
197
|
+
body: input.doc.body,
|
|
198
|
+
};
|
|
199
|
+
const raw = okf.serializeTopicDoc(doc);
|
|
200
|
+
await atomicWrite(file, raw);
|
|
201
|
+
await this.regenerateIndex();
|
|
202
|
+
const committed = await this.commit([`topics/${slug}.md`, 'index.md'], opts.message);
|
|
203
|
+
return { slug, path: `topics/${slug}.md`, committed, created };
|
|
204
|
+
});
|
|
205
|
+
}
|
|
206
|
+
async regenerateIndex() {
|
|
207
|
+
const metas = await this.listTopics();
|
|
208
|
+
const entries = metas.map((m) => ({
|
|
209
|
+
slug: m.slug,
|
|
210
|
+
title: m.title,
|
|
211
|
+
description: m.description,
|
|
212
|
+
status: m.status,
|
|
213
|
+
tags: m.tags,
|
|
214
|
+
}));
|
|
215
|
+
await atomicWrite(join(this.root, 'index.md'), okf.renderIndex(entries));
|
|
216
|
+
await atomicWrite(join(this.metaDir(), 'backlinks.json'), `${JSON.stringify(await this.computeBacklinks(), null, 2)}\n`);
|
|
217
|
+
}
|
|
218
|
+
/**
|
|
219
|
+
* Reverse-reference index: slug → who references it and how (`depends`
|
|
220
|
+
* edges + body links). Regenerated on every topic write (ADR 0003
|
|
221
|
+
* write-through), so it never drifts from the files.
|
|
222
|
+
*/
|
|
223
|
+
async computeBacklinks() {
|
|
224
|
+
const backlinks = {};
|
|
225
|
+
const push = (target, from, via) => {
|
|
226
|
+
if (target === from)
|
|
227
|
+
return;
|
|
228
|
+
const list = backlinks[target] ?? [];
|
|
229
|
+
if (!list.some((e) => e.slug === from && e.via === via))
|
|
230
|
+
list.push({ slug: from, via });
|
|
231
|
+
backlinks[target] = list;
|
|
232
|
+
};
|
|
233
|
+
let files;
|
|
234
|
+
try {
|
|
235
|
+
files = await readdir(this.topicsDir());
|
|
236
|
+
}
|
|
237
|
+
catch {
|
|
238
|
+
return {};
|
|
239
|
+
}
|
|
240
|
+
for (const f of files) {
|
|
241
|
+
if (!f.endsWith('.md') || f === 'index.md')
|
|
242
|
+
continue;
|
|
243
|
+
const slug = f.slice(0, -3);
|
|
244
|
+
try {
|
|
245
|
+
const doc = okf.parseTopicDoc(await readFile(join(this.topicsDir(), f), 'utf8'));
|
|
246
|
+
for (const dep of okf.dependsSlugs(doc.fm))
|
|
247
|
+
push(dep, slug, 'depends');
|
|
248
|
+
for (const link of okf.bodyLinkSlugs(doc.body))
|
|
249
|
+
push(link, slug, 'link');
|
|
250
|
+
}
|
|
251
|
+
catch {
|
|
252
|
+
// broken files contribute no edges
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
return backlinks;
|
|
256
|
+
}
|
|
257
|
+
/** Read the generated backlinks index (empty map when absent). */
|
|
258
|
+
async readBacklinks() {
|
|
259
|
+
try {
|
|
260
|
+
const raw = await readFile(join(this.metaDir(), 'backlinks.json'), 'utf8');
|
|
261
|
+
const parsed = JSON.parse(raw);
|
|
262
|
+
if (parsed !== null && typeof parsed === 'object' && !Array.isArray(parsed)) {
|
|
263
|
+
return parsed;
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
catch {
|
|
267
|
+
// absent = no backlinks computed yet
|
|
268
|
+
}
|
|
269
|
+
return {};
|
|
270
|
+
}
|
|
271
|
+
// ------------------------------------------------------------------
|
|
272
|
+
// Observations (M2 raw material; tool-written from M1)
|
|
273
|
+
// ------------------------------------------------------------------
|
|
274
|
+
async appendObservation(input) {
|
|
275
|
+
const obs = {
|
|
276
|
+
id: `obs-${Date.now().toString(36)}-${randomUUID().slice(0, 8)}`,
|
|
277
|
+
at: new Date().toISOString(),
|
|
278
|
+
kind: input.kind,
|
|
279
|
+
source: input.source,
|
|
280
|
+
text: input.text,
|
|
281
|
+
distilled: false,
|
|
282
|
+
};
|
|
283
|
+
if (input.sessionId !== undefined)
|
|
284
|
+
obs.sessionId = input.sessionId;
|
|
285
|
+
// Behind the store-wide queue: markDistilled / recordUnconsumed rewrite
|
|
286
|
+
// the whole JSONL file — an unsynchronized append here could interleave
|
|
287
|
+
// with a rewrite and lose either the new observation or the rewrite.
|
|
288
|
+
return this.enqueue(async () => {
|
|
289
|
+
await mkdir(this.metaDir(), { recursive: true });
|
|
290
|
+
await appendFile(this.observationsPath(), `${JSON.stringify(obs)}\n`, 'utf8');
|
|
291
|
+
return obs;
|
|
292
|
+
});
|
|
293
|
+
}
|
|
294
|
+
async allObservations(limit = 500) {
|
|
295
|
+
let raw;
|
|
296
|
+
try {
|
|
297
|
+
raw = await readFile(this.observationsPath(), 'utf8');
|
|
298
|
+
}
|
|
299
|
+
catch {
|
|
300
|
+
return [];
|
|
301
|
+
}
|
|
302
|
+
const out = [];
|
|
303
|
+
for (const line of raw.split('\n')) {
|
|
304
|
+
if (line.trim() === '')
|
|
305
|
+
continue;
|
|
306
|
+
try {
|
|
307
|
+
out.push(JSON.parse(line));
|
|
308
|
+
}
|
|
309
|
+
catch {
|
|
310
|
+
// Tolerate a torn last line (crash mid-append).
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
return out.slice(-limit);
|
|
314
|
+
}
|
|
315
|
+
async undistilledObservations(limit = 40) {
|
|
316
|
+
return (await this.allObservations(2000)).filter((o) => !o.distilled).slice(-limit);
|
|
317
|
+
}
|
|
318
|
+
async markDistilled(ids, intoSlugs) {
|
|
319
|
+
return this.enqueue(async () => {
|
|
320
|
+
const all = await this.allObservations(2000);
|
|
321
|
+
const idSet = new Set(ids);
|
|
322
|
+
let changed = 0;
|
|
323
|
+
for (const o of all) {
|
|
324
|
+
if (idSet.has(o.id) && !o.distilled) {
|
|
325
|
+
o.distilled = true;
|
|
326
|
+
o.distilledInto = [...intoSlugs];
|
|
327
|
+
changed += 1;
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
if (changed > 0) {
|
|
331
|
+
await atomicWrite(this.observationsPath(), all.map((o) => `${JSON.stringify(o)}\n`).join(''));
|
|
332
|
+
await this.commit(['meta/observations.jsonl'], `topics(meta): distill ${ids.length} observation(s) into ${intoSlugs.join(', ')}`);
|
|
333
|
+
}
|
|
334
|
+
return changed;
|
|
335
|
+
});
|
|
336
|
+
}
|
|
337
|
+
/**
|
|
338
|
+
* GC policy for the distill lane: an observation fed to the model but left
|
|
339
|
+
* unconsumed by any op has failed one attempt; after OBSERVATION_MAX_ATTEMPTS
|
|
340
|
+
* failed attempts it is physically deleted. The user explicitly authorized
|
|
341
|
+
* deleting raw observations the lane demonstrably cannot process — the
|
|
342
|
+
* alternative is the same head haunting the backlog forever. Consumed
|
|
343
|
+
* observations leave the pool via markDistilled and never accrue attempts.
|
|
344
|
+
*/
|
|
345
|
+
static OBSERVATION_MAX_ATTEMPTS = 3;
|
|
346
|
+
/**
|
|
347
|
+
* Post-run GC bookkeeping: increment `attempts` for every observation in
|
|
348
|
+
* `fedIds` that no op in `consumedIds` accounts for, then delete those that
|
|
349
|
+
* reached OBSERVATION_MAX_ATTEMPTS. Returns the deletion count (the caller
|
|
350
|
+
* records it in the distill-state detail). Increments persist with the
|
|
351
|
+
* usual flush cadence; a deletion commits immediately — destroying data
|
|
352
|
+
* must be traceable in git.
|
|
353
|
+
*/
|
|
354
|
+
async recordUnconsumed(fedIds, consumedIds) {
|
|
355
|
+
return this.enqueue(async () => {
|
|
356
|
+
const all = await this.allObservations(2000);
|
|
357
|
+
const fed = new Set(fedIds);
|
|
358
|
+
const consumed = new Set(consumedIds);
|
|
359
|
+
let changed = 0;
|
|
360
|
+
let dropped = 0;
|
|
361
|
+
const kept = [];
|
|
362
|
+
for (const o of all) {
|
|
363
|
+
if (fed.has(o.id) && !consumed.has(o.id) && !o.distilled) {
|
|
364
|
+
o.attempts = (o.attempts ?? 0) + 1;
|
|
365
|
+
changed += 1;
|
|
366
|
+
}
|
|
367
|
+
if (!o.distilled && (o.attempts ?? 0) >= BundleStore.OBSERVATION_MAX_ATTEMPTS) {
|
|
368
|
+
dropped += 1;
|
|
369
|
+
continue;
|
|
370
|
+
}
|
|
371
|
+
kept.push(o);
|
|
372
|
+
}
|
|
373
|
+
if (changed > 0 || dropped > 0) {
|
|
374
|
+
await atomicWrite(this.observationsPath(), kept.map((o) => `${JSON.stringify(o)}\n`).join(''));
|
|
375
|
+
}
|
|
376
|
+
if (dropped > 0) {
|
|
377
|
+
await this.commit(['meta/observations.jsonl'], `topics(meta): gc ${dropped} unprocessable observation(s)`);
|
|
378
|
+
}
|
|
379
|
+
return { dropped };
|
|
380
|
+
});
|
|
381
|
+
}
|
|
382
|
+
// ------------------------------------------------------------------
|
|
383
|
+
// Injection log (ADR 0007) — appended per round, committed on flush
|
|
384
|
+
// ------------------------------------------------------------------
|
|
385
|
+
async appendInjectionRecord(record) {
|
|
386
|
+
await mkdir(this.metaDir(), { recursive: true });
|
|
387
|
+
await appendFile(this.injectionsPath(), `${JSON.stringify(record)}\n`, 'utf8');
|
|
388
|
+
await this.compactInjectionsIfNeeded();
|
|
389
|
+
}
|
|
390
|
+
async compactInjectionsIfNeeded() {
|
|
391
|
+
const file = this.injectionsPath();
|
|
392
|
+
let size = 0;
|
|
393
|
+
try {
|
|
394
|
+
size = (await readFile(file, 'utf8')).length;
|
|
395
|
+
}
|
|
396
|
+
catch {
|
|
397
|
+
return;
|
|
398
|
+
}
|
|
399
|
+
// ~512KB cap; keep the most recent quarter when exceeded (headroom for
|
|
400
|
+
// the next debounce window before the next compaction).
|
|
401
|
+
if (size <= 512 * 1024)
|
|
402
|
+
return;
|
|
403
|
+
const lines = (await readFile(file, 'utf8')).split('\n').filter((l) => l.trim() !== '');
|
|
404
|
+
await atomicWrite(file, lines.slice(-Math.max(1, Math.floor(lines.length / 4))).map((l) => `${l}\n`).join(''));
|
|
405
|
+
}
|
|
406
|
+
async readInjectionRecords(limit = 2000) {
|
|
407
|
+
let raw;
|
|
408
|
+
try {
|
|
409
|
+
raw = await readFile(this.injectionsPath(), 'utf8');
|
|
410
|
+
}
|
|
411
|
+
catch {
|
|
412
|
+
return [];
|
|
413
|
+
}
|
|
414
|
+
const out = [];
|
|
415
|
+
for (const line of raw.split('\n')) {
|
|
416
|
+
if (line.trim() === '')
|
|
417
|
+
continue;
|
|
418
|
+
try {
|
|
419
|
+
out.push(JSON.parse(line));
|
|
420
|
+
}
|
|
421
|
+
catch {
|
|
422
|
+
// Torn tail line tolerated.
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
return out.slice(-limit);
|
|
426
|
+
}
|
|
427
|
+
// ------------------------------------------------------------------
|
|
428
|
+
// Conflicted topics (ADR 0003) — retrieval demotes these
|
|
429
|
+
// ------------------------------------------------------------------
|
|
430
|
+
async getConflicts() {
|
|
431
|
+
try {
|
|
432
|
+
const raw = await readFile(this.conflictsPath(), 'utf8');
|
|
433
|
+
const parsed = JSON.parse(raw);
|
|
434
|
+
if (Array.isArray(parsed))
|
|
435
|
+
return new Set(parsed.filter((x) => typeof x === 'string').map((p) => okf.pathToSlug(p)));
|
|
436
|
+
}
|
|
437
|
+
catch {
|
|
438
|
+
// absent = no conflicts
|
|
439
|
+
}
|
|
440
|
+
return new Set();
|
|
441
|
+
}
|
|
442
|
+
/** Sync variant for the injection hot path. */
|
|
443
|
+
getConflictsSync() {
|
|
444
|
+
try {
|
|
445
|
+
const raw = readFileSync(this.conflictsPath(), 'utf8');
|
|
446
|
+
const parsed = JSON.parse(raw);
|
|
447
|
+
if (Array.isArray(parsed))
|
|
448
|
+
return new Set(parsed.filter((x) => typeof x === 'string').map((p) => okf.pathToSlug(p)));
|
|
449
|
+
}
|
|
450
|
+
catch {
|
|
451
|
+
// absent = no conflicts
|
|
452
|
+
}
|
|
453
|
+
return new Set();
|
|
454
|
+
}
|
|
455
|
+
async setConflicts(paths) {
|
|
456
|
+
await this.enqueue(async () => {
|
|
457
|
+
if (paths.length === 0) {
|
|
458
|
+
await atomicWrite(this.conflictsPath(), '[]\n');
|
|
459
|
+
}
|
|
460
|
+
else {
|
|
461
|
+
await atomicWrite(this.conflictsPath(), `${JSON.stringify([...paths], null, 2)}\n`);
|
|
462
|
+
}
|
|
463
|
+
await this.commit(['meta/conflicts.json'], `topics(meta): mark ${paths.length} conflicted topic(s)`);
|
|
464
|
+
});
|
|
465
|
+
}
|
|
466
|
+
// ------------------------------------------------------------------
|
|
467
|
+
// Roster / status
|
|
468
|
+
// ------------------------------------------------------------------
|
|
469
|
+
async status() {
|
|
470
|
+
const metas = await this.listTopics();
|
|
471
|
+
const byStatus = { draft: 0, stable: 0, deprecated: 0 };
|
|
472
|
+
for (const m of metas)
|
|
473
|
+
byStatus[m.status] += 1;
|
|
474
|
+
const obs = await this.allObservations(2000);
|
|
475
|
+
const conflicts = [...(await this.getConflicts())];
|
|
476
|
+
const head = this.gitDisabled ? undefined : await gitmod.headRev(this.root).catch(() => undefined);
|
|
477
|
+
return {
|
|
478
|
+
root: this.root,
|
|
479
|
+
topicCount: metas.length,
|
|
480
|
+
byStatus,
|
|
481
|
+
observationsPending: obs.filter((o) => !o.distilled).length,
|
|
482
|
+
observationsTotal: obs.length,
|
|
483
|
+
conflicts,
|
|
484
|
+
broken: await this.brokenTopics(),
|
|
485
|
+
git: !this.gitDisabled && (await gitmod.isRepo(this.root)),
|
|
486
|
+
head,
|
|
487
|
+
};
|
|
488
|
+
}
|
|
489
|
+
}
|
package/lib/sync.d.ts
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* GitHub sync layer (ADR 0002/0003/0008) — active only when the `repo`
|
|
3
|
+
* setting is non-empty; local-only mode bypasses the whole module.
|
|
4
|
+
*
|
|
5
|
+
* Semantics: pull --rebase (autostash) on session start and before each
|
|
6
|
+
* push; push debounced (config, default 45s) after write-through commits,
|
|
7
|
+
* hard flush on session end / plugin dispose. A rebase conflict aborts the
|
|
8
|
+
* rebase and marks the conflicted topics — demoted in retrieval with a
|
|
9
|
+
* visible warning, resolved by hand (ADR 0003: no automatic smart merging).
|
|
10
|
+
*
|
|
11
|
+
* @module sync
|
|
12
|
+
*/
|
|
13
|
+
import type { BundleStore } from './store.ts';
|
|
14
|
+
export declare function remoteUrlFor(repo: string): string;
|
|
15
|
+
export interface SyncConfig {
|
|
16
|
+
/** `owner/name`; empty string = local-only mode. */
|
|
17
|
+
repo: string;
|
|
18
|
+
pushDebounceSeconds: number;
|
|
19
|
+
}
|
|
20
|
+
export type TokenResolver = () => Promise<string | undefined>;
|
|
21
|
+
export type UrlResolver = (repo: string) => string;
|
|
22
|
+
export declare class Sync {
|
|
23
|
+
private timer;
|
|
24
|
+
private pending;
|
|
25
|
+
private inFlight;
|
|
26
|
+
lastError: string;
|
|
27
|
+
lastPushAt?: string;
|
|
28
|
+
lastPullAt?: string;
|
|
29
|
+
private readonly store;
|
|
30
|
+
private readonly cfg;
|
|
31
|
+
private readonly token;
|
|
32
|
+
private readonly urlFor;
|
|
33
|
+
constructor(store: BundleStore, cfg: () => SyncConfig, token?: TokenResolver, urlFor?: UrlResolver);
|
|
34
|
+
get active(): boolean;
|
|
35
|
+
/** Session start (or `/topics sync pull`): rebase onto remote, mark conflicts. */
|
|
36
|
+
pull(): Promise<{
|
|
37
|
+
ok: boolean;
|
|
38
|
+
conflicted: string[];
|
|
39
|
+
message: string;
|
|
40
|
+
}>;
|
|
41
|
+
/** Called after every write-through commit — schedules the debounced push. */
|
|
42
|
+
schedulePush(): void;
|
|
43
|
+
/** Debounce flush, also used for session end and plugin dispose. */
|
|
44
|
+
flush(): Promise<{
|
|
45
|
+
ok: boolean;
|
|
46
|
+
message: string;
|
|
47
|
+
}>;
|
|
48
|
+
/** Commit the meta sidecars (injection log, observations) — flush cadence. */
|
|
49
|
+
commitMeta(): Promise<void>;
|
|
50
|
+
dispose(): void;
|
|
51
|
+
private safeToken;
|
|
52
|
+
}
|
package/lib/sync.js
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* GitHub sync layer (ADR 0002/0003/0008) — active only when the `repo`
|
|
3
|
+
* setting is non-empty; local-only mode bypasses the whole module.
|
|
4
|
+
*
|
|
5
|
+
* Semantics: pull --rebase (autostash) on session start and before each
|
|
6
|
+
* push; push debounced (config, default 45s) after write-through commits,
|
|
7
|
+
* hard flush on session end / plugin dispose. A rebase conflict aborts the
|
|
8
|
+
* rebase and marks the conflicted topics — demoted in retrieval with a
|
|
9
|
+
* visible warning, resolved by hand (ADR 0003: no automatic smart merging).
|
|
10
|
+
*
|
|
11
|
+
* @module sync
|
|
12
|
+
*/
|
|
13
|
+
import { spawn } from 'node:child_process';
|
|
14
|
+
import * as gitmod from "./git.js";
|
|
15
|
+
export function remoteUrlFor(repo) {
|
|
16
|
+
return `https://github.com/${repo}.git`;
|
|
17
|
+
}
|
|
18
|
+
export class Sync {
|
|
19
|
+
timer;
|
|
20
|
+
pending = false;
|
|
21
|
+
inFlight = Promise.resolve();
|
|
22
|
+
lastError = '';
|
|
23
|
+
lastPushAt;
|
|
24
|
+
lastPullAt;
|
|
25
|
+
store;
|
|
26
|
+
cfg;
|
|
27
|
+
token;
|
|
28
|
+
urlFor;
|
|
29
|
+
constructor(store, cfg, token = defaultTokenResolver, urlFor = remoteUrlFor) {
|
|
30
|
+
this.store = store;
|
|
31
|
+
this.cfg = cfg;
|
|
32
|
+
this.token = token;
|
|
33
|
+
this.urlFor = urlFor;
|
|
34
|
+
}
|
|
35
|
+
get active() {
|
|
36
|
+
return this.cfg().repo !== '';
|
|
37
|
+
}
|
|
38
|
+
/** Session start (or `/topics sync pull`): rebase onto remote, mark conflicts. */
|
|
39
|
+
async pull() {
|
|
40
|
+
if (!this.active)
|
|
41
|
+
return { ok: true, conflicted: [], message: 'local-only 模式,无同步' };
|
|
42
|
+
const token = await this.safeToken();
|
|
43
|
+
await gitmod.setRemote(this.store.root, this.urlFor(this.cfg().repo));
|
|
44
|
+
if (!(await gitmod.remoteBranchExists(this.store.root, token))) {
|
|
45
|
+
this.lastPullAt = new Date().toISOString();
|
|
46
|
+
return { ok: true, conflicted: [], message: '远端还没有分支,跳过拉取' };
|
|
47
|
+
}
|
|
48
|
+
const outcome = await gitmod.pullRebase(this.store.root, token);
|
|
49
|
+
this.lastPullAt = new Date().toISOString();
|
|
50
|
+
if (outcome.ok) {
|
|
51
|
+
// A successful rebase clears previous conflict marks.
|
|
52
|
+
const prior = await this.store.getConflicts();
|
|
53
|
+
if (prior.size > 0)
|
|
54
|
+
await this.store.setConflicts([]);
|
|
55
|
+
return { ok: true, conflicted: [], message: '已与远端同步' };
|
|
56
|
+
}
|
|
57
|
+
const conflictedSlugs = outcome.conflicted.map((p) => p.replace(/^topics\//, '').replace(/\.md$/, ''));
|
|
58
|
+
if (conflictedSlugs.length > 0) {
|
|
59
|
+
await this.store.setConflicts(outcome.conflicted);
|
|
60
|
+
}
|
|
61
|
+
this.lastError = outcome.output.slice(-500);
|
|
62
|
+
return { ok: false, conflicted: conflictedSlugs, message: `rebase 冲突:${conflictedSlugs.join('、') || '未知路径'}(已标记降权,待手工解决)` };
|
|
63
|
+
}
|
|
64
|
+
/** Called after every write-through commit — schedules the debounced push. */
|
|
65
|
+
schedulePush() {
|
|
66
|
+
if (!this.active)
|
|
67
|
+
return;
|
|
68
|
+
this.pending = true;
|
|
69
|
+
if (this.timer !== undefined)
|
|
70
|
+
return;
|
|
71
|
+
const seconds = Math.max(1, this.cfg().pushDebounceSeconds);
|
|
72
|
+
this.timer = setTimeout(() => {
|
|
73
|
+
this.timer = undefined;
|
|
74
|
+
void this.flush();
|
|
75
|
+
}, seconds * 1000);
|
|
76
|
+
// Never hold the process open for a debounced push.
|
|
77
|
+
this.timer.unref?.();
|
|
78
|
+
}
|
|
79
|
+
/** Debounce flush, also used for session end and plugin dispose. */
|
|
80
|
+
async flush() {
|
|
81
|
+
if (!this.active)
|
|
82
|
+
return { ok: true, message: 'local-only 模式,无同步' };
|
|
83
|
+
this.pending = false;
|
|
84
|
+
if (this.timer !== undefined) {
|
|
85
|
+
clearTimeout(this.timer);
|
|
86
|
+
this.timer = undefined;
|
|
87
|
+
}
|
|
88
|
+
const run = this.inFlight.then(async () => {
|
|
89
|
+
if (!(await this.store.hasGit()))
|
|
90
|
+
return { ok: false, message: 'bundle 不是 git 仓库' };
|
|
91
|
+
const token = await this.safeToken();
|
|
92
|
+
await gitmod.setRemote(this.store.root, this.urlFor(this.cfg().repo));
|
|
93
|
+
if (await gitmod.remoteBranchExists(this.store.root, token)) {
|
|
94
|
+
const pre = await gitmod.pullRebase(this.store.root, token);
|
|
95
|
+
if (!pre.ok) {
|
|
96
|
+
const conflicted = pre.conflicted.map((p) => p.replace(/^topics\//, '').replace(/\.md$/, ''));
|
|
97
|
+
if (conflicted.length > 0)
|
|
98
|
+
await this.store.setConflicts(pre.conflicted);
|
|
99
|
+
this.lastError = pre.output.slice(-500);
|
|
100
|
+
return { ok: false, message: `push 前 rebase 失败:${conflicted.join('、') || pre.output.slice(-120)}` };
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
const result = await gitmod.push(this.store.root, token);
|
|
104
|
+
if (result.ok) {
|
|
105
|
+
this.lastPushAt = new Date().toISOString();
|
|
106
|
+
this.lastError = '';
|
|
107
|
+
return { ok: true, message: '已推送到远端' };
|
|
108
|
+
}
|
|
109
|
+
this.lastError = result.output.slice(-500);
|
|
110
|
+
return { ok: false, message: `push 失败:${result.output.slice(-160)}` };
|
|
111
|
+
});
|
|
112
|
+
this.inFlight = run.then(() => undefined, () => undefined);
|
|
113
|
+
return run;
|
|
114
|
+
}
|
|
115
|
+
/** Commit the meta sidecars (injection log, observations) — flush cadence. */
|
|
116
|
+
async commitMeta() {
|
|
117
|
+
if (!this.active)
|
|
118
|
+
return;
|
|
119
|
+
await gitmod.addAndCommit(this.store.root, ['meta'], 'topics(meta): flush injection log / observations').catch(() => undefined);
|
|
120
|
+
}
|
|
121
|
+
dispose() {
|
|
122
|
+
if (this.timer !== undefined) {
|
|
123
|
+
clearTimeout(this.timer);
|
|
124
|
+
this.timer = undefined;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
async safeToken() {
|
|
128
|
+
try {
|
|
129
|
+
return await this.token();
|
|
130
|
+
}
|
|
131
|
+
catch {
|
|
132
|
+
return undefined;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
/** Env first, gh CLI fallback per dsh-vault precedent; login is never our job. */
|
|
137
|
+
async function defaultTokenResolver() {
|
|
138
|
+
const env = process.env.GITHUB_TOKEN?.trim();
|
|
139
|
+
if (env !== undefined && env !== '')
|
|
140
|
+
return env;
|
|
141
|
+
return ghCliToken();
|
|
142
|
+
}
|
|
143
|
+
/** gh CLI fallback per dsh-vault precedent; login is never this plugin's job. */
|
|
144
|
+
async function ghCliToken() {
|
|
145
|
+
return new Promise((resolve) => {
|
|
146
|
+
const child = spawn('gh', ['auth', 'token'], { stdio: ['ignore', 'pipe', 'ignore'], env: process.env });
|
|
147
|
+
let out = '';
|
|
148
|
+
child.stdout?.on('data', (d) => {
|
|
149
|
+
out += d.toString('utf8');
|
|
150
|
+
});
|
|
151
|
+
child.on('error', () => resolve(undefined));
|
|
152
|
+
child.on('close', (code) => resolve(code === 0 ? out.trim() || undefined : undefined));
|
|
153
|
+
});
|
|
154
|
+
}
|